toxic-bert-plain-vanila.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### FusedAdam OPTIMIZER\n",
    "\n",
    "https://www.kaggle.com/c/jigsaw-unintended-bias-in-toxicity-classification/discussion/92505"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['.gitignore', 'csrc', 'apex', 'examples', 'LICENSE', 'tests', '.nojekyll', 'docs', 'README.md', 'setup.py']\n"
     ]
    }
   ],
   "source": [
    "# This Python 3 environment comes with many helpful analytics libraries installed\n",
    "# It is defined by the kaggle/python docker image: https://github.com/kaggle/docker-python\n",
    "# For example, here's several helpful packages to load in \n",
    "\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "# Input data files are available in the \"../input/\" directory.\n",
    "# For example, running this (by clicking run or pressing Shift+Enter) will list the files in the input directory\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"../input/nvidiaapex/repository/NVIDIA-apex-39e153a\"))\n",
    "#print(os.listdir(\"../input/glove-global-vectors-for-word-representation\"))\n",
    "#print(os.listdir(\"../input/jigsaw-unintended-bias-in-toxicity-classification\"))\n",
    "#print(os.listdir(\"../input/fasttext-crawl-300d-2m\"))\n",
    "\n",
    "# Any results you write to the current directory are saved as output."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/site-packages/pip/_internal/commands/install.py:244: UserWarning: Disabling all use of wheels due to the use of --build-options / --global-options / --install-options.\r\n",
      "  cmdoptions.check_install_build_global(options)\r\n",
      "Created temporary directory: /tmp/pip-ephem-wheel-cache-ppgqs5rf\r\n",
      "Created temporary directory: /tmp/pip-req-tracker-q0a_wleh\r\n",
      "Created requirements tracker '/tmp/pip-req-tracker-q0a_wleh'\r\n",
      "Created temporary directory: /tmp/pip-install-73w0ivef\r\n",
      "Processing /kaggle/input/nvidiaapex/repository/NVIDIA-apex-39e153a\r\n",
      "  Created temporary directory: /tmp/pip-req-build-e6p3gnvb\r\n",
      "  Added file:///kaggle/input/nvidiaapex/repository/NVIDIA-apex-39e153a to build tracker '/tmp/pip-req-tracker-q0a_wleh'\r\n",
      "    Running setup.py (path:/tmp/pip-req-build-e6p3gnvb/setup.py) egg_info for package from file:///kaggle/input/nvidiaapex/repository/NVIDIA-apex-39e153a\r\n",
      "    Running command python setup.py egg_info\r\n",
      "    torch.__version__  =  1.1.0\r\n",
      "    running egg_info\r\n",
      "    creating pip-egg-info/apex.egg-info\r\n",
      "    writing pip-egg-info/apex.egg-info/PKG-INFO\r\n",
      "    writing dependency_links to pip-egg-info/apex.egg-info/dependency_links.txt\r\n",
      "    writing top-level names to pip-egg-info/apex.egg-info/top_level.txt\r\n",
      "    writing manifest file 'pip-egg-info/apex.egg-info/SOURCES.txt'\r\n",
      "    reading manifest file 'pip-egg-info/apex.egg-info/SOURCES.txt'\r\n",
      "    writing manifest file 'pip-egg-info/apex.egg-info/SOURCES.txt'\r\n",
      "  Source in /tmp/pip-req-build-e6p3gnvb has version 0.1, which satisfies requirement apex==0.1 from file:///kaggle/input/nvidiaapex/repository/NVIDIA-apex-39e153a\r\n",
      "  Removed apex==0.1 from file:///kaggle/input/nvidiaapex/repository/NVIDIA-apex-39e153a from build tracker '/tmp/pip-req-tracker-q0a_wleh'\r\n",
      "Skipping bdist_wheel for apex, due to binaries being disabled for it.\r\n",
      "Installing collected packages: apex\r\n",
      "  Created temporary directory: /tmp/pip-record-fol60p1h\r\n",
      "    Running command /opt/conda/bin/python -u -c 'import setuptools, tokenize;__file__='\"'\"'/tmp/pip-req-build-e6p3gnvb/setup.py'\"'\"';f=getattr(tokenize, '\"'\"'open'\"'\"', open)(__file__);code=f.read().replace('\"'\"'\\r\\n'\"'\"', '\"'\"'\\n'\"'\"');f.close();exec(compile(code, __file__, '\"'\"'exec'\"'\"'))' --cpp_ext --cuda_ext install --record /tmp/pip-record-fol60p1h/install-record.txt --single-version-externally-managed --compile\r\n",
      "    torch.__version__  =  1.1.0\r\n",
      "\r\n",
      "    Compiling cuda extensions with\r\n",
      "    nvcc: NVIDIA (R) Cuda compiler driver\r\n",
      "    Copyright (c) 2005-2018 NVIDIA Corporation\r\n",
      "    Built on Sat_Aug_25_21:08:01_CDT_2018\r\n",
      "    Cuda compilation tools, release 10.0, V10.0.130\r\n",
      "    from /usr/local/cuda/bin\r\n",
      "\r\n",
      "    Pytorch binaries were compiled with Cuda 10.0.130\r\n",
      "\r\n",
      "    running install\r\n",
      "    running build\r\n",
      "    running build_py\r\n",
      "    creating build\r\n",
      "    creating build/lib.linux-x86_64-3.6\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex\r\n",
      "    copying apex/__init__.py -> build/lib.linux-x86_64-3.6/apex\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex/reparameterization\r\n",
      "    copying apex/reparameterization/__init__.py -> build/lib.linux-x86_64-3.6/apex/reparameterization\r\n",
      "    copying apex/reparameterization/weight_norm.py -> build/lib.linux-x86_64-3.6/apex/reparameterization\r\n",
      "    copying apex/reparameterization/reparameterization.py -> build/lib.linux-x86_64-3.6/apex/reparameterization\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/_process_optimizer.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/compat.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/utils.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/wrap.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/rnn_compat.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/_amp_state.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/_initialize.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/__init__.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/scaler.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/opt.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/frontend.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/__version__.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/handle.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    copying apex/amp/amp.py -> build/lib.linux-x86_64-3.6/apex/amp\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex/normalization\r\n",
      "    copying apex/normalization/__init__.py -> build/lib.linux-x86_64-3.6/apex/normalization\r\n",
      "    copying apex/normalization/fused_layer_norm.py -> build/lib.linux-x86_64-3.6/apex/normalization\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex/multi_tensor_apply\r\n",
      "    copying apex/multi_tensor_apply/multi_tensor_apply.py -> build/lib.linux-x86_64-3.6/apex/multi_tensor_apply\r\n",
      "    copying apex/multi_tensor_apply/__init__.py -> build/lib.linux-x86_64-3.6/apex/multi_tensor_apply\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex/RNN\r\n",
      "    copying apex/RNN/models.py -> build/lib.linux-x86_64-3.6/apex/RNN\r\n",
      "    copying apex/RNN/__init__.py -> build/lib.linux-x86_64-3.6/apex/RNN\r\n",
      "    copying apex/RNN/cells.py -> build/lib.linux-x86_64-3.6/apex/RNN\r\n",
      "    copying apex/RNN/RNNBackend.py -> build/lib.linux-x86_64-3.6/apex/RNN\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex/fp16_utils\r\n",
      "    copying apex/fp16_utils/loss_scaler.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\r\n",
      "    copying apex/fp16_utils/fp16util.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\r\n",
      "    copying apex/fp16_utils/__init__.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\r\n",
      "    copying apex/fp16_utils/fp16_optimizer.py -> build/lib.linux-x86_64-3.6/apex/fp16_utils\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex/parallel\r\n",
      "    copying apex/parallel/optimized_sync_batchnorm_kernel.py -> build/lib.linux-x86_64-3.6/apex/parallel\r\n",
      "    copying apex/parallel/multiproc.py -> build/lib.linux-x86_64-3.6/apex/parallel\r\n",
      "    copying apex/parallel/distributed.py -> build/lib.linux-x86_64-3.6/apex/parallel\r\n",
      "    copying apex/parallel/sync_batchnorm.py -> build/lib.linux-x86_64-3.6/apex/parallel\r\n",
      "    copying apex/parallel/optimized_sync_batchnorm.py -> build/lib.linux-x86_64-3.6/apex/parallel\r\n",
      "    copying apex/parallel/sync_batchnorm_kernel.py -> build/lib.linux-x86_64-3.6/apex/parallel\r\n",
      "    copying apex/parallel/__init__.py -> build/lib.linux-x86_64-3.6/apex/parallel\r\n",
      "    copying apex/parallel/LARC.py -> build/lib.linux-x86_64-3.6/apex/parallel\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex/optimizers\r\n",
      "    copying apex/optimizers/__init__.py -> build/lib.linux-x86_64-3.6/apex/optimizers\r\n",
      "    copying apex/optimizers/fp16_optimizer.py -> build/lib.linux-x86_64-3.6/apex/optimizers\r\n",
      "    copying apex/optimizers/fused_adam.py -> build/lib.linux-x86_64-3.6/apex/optimizers\r\n",
      "    creating build/lib.linux-x86_64-3.6/apex/amp/lists\r\n",
      "    copying apex/amp/lists/tensor_overrides.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\r\n",
      "    copying apex/amp/lists/torch_overrides.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\r\n",
      "    copying apex/amp/lists/__init__.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\r\n",
      "    copying apex/amp/lists/functional_overrides.py -> build/lib.linux-x86_64-3.6/apex/amp/lists\r\n",
      "    running build_ext\r\n",
      "    building 'apex_C' extension\r\n",
      "    creating build/temp.linux-x86_64-3.6\r\n",
      "    creating build/temp.linux-x86_64-3.6/csrc\r\n",
      "    gcc -pthread -B /opt/conda/compiler_compat -Wl,--sysroot=/ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I/opt/conda/lib/python3.6/site-packages/torch/include -I/opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/include/THC -I/opt/conda/include/python3.6m -c csrc/flatten_unflatten.cpp -o build/temp.linux-x86_64-3.6/csrc/flatten_unflatten.o -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=apex_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid for C/ObjC but not for C++\r\n",
      "    g++ -pthread -shared -B /opt/conda/compiler_compat -L/opt/conda/lib -Wl,-rpath=/opt/conda/lib -Wl,--no-as-needed -Wl,--sysroot=/ build/temp.linux-x86_64-3.6/csrc/flatten_unflatten.o -o build/lib.linux-x86_64-3.6/apex_C.cpython-36m-x86_64-linux-gnu.so\r\n",
      "    building 'amp_C' extension\r\n",
      "    gcc -pthread -B /opt/conda/compiler_compat -Wl,--sysroot=/ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I/opt/conda/lib/python3.6/site-packages/torch/include -I/opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/amp_C_frontend.cpp -o build/temp.linux-x86_64-3.6/csrc/amp_C_frontend.o -O3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid for C/ObjC but not for C++\r\n",
      "    /usr/local/cuda/bin/nvcc -I/opt/conda/lib/python3.6/site-packages/torch/include -I/opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/multi_tensor_scale_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_scale_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    /usr/local/cuda/bin/nvcc -I/opt/conda/lib/python3.6/site-packages/torch/include -I/opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/multi_tensor_axpby_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_axpby_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    /usr/local/cuda/bin/nvcc -I/opt/conda/lib/python3.6/site-packages/torch/include -I/opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/multi_tensor_l2norm_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/multi_tensor_l2norm_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -lineinfo -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=amp_C -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    g++ -pthread -shared -B /opt/conda/compiler_compat -L/opt/conda/lib -Wl,-rpath=/opt/conda/lib -Wl,--no-as-needed -Wl,--sysroot=/ build/temp.linux-x86_64-3.6/csrc/amp_C_frontend.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_scale_kernel.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_axpby_kernel.o build/temp.linux-x86_64-3.6/csrc/multi_tensor_l2norm_kernel.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.6/amp_C.cpython-36m-x86_64-linux-gnu.so\r\n",
      "    building 'fused_adam_cuda' extension\r\n",
      "    gcc -pthread -B /opt/conda/compiler_compat -Wl,--sysroot=/ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I/opt/conda/lib/python3.6/site-packages/torch/include -I/opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/fused_adam_cuda.cpp -o build/temp.linux-x86_64-3.6/csrc/fused_adam_cuda.o -O3 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=fused_adam_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid for C/ObjC but not for C++\r\n",
      "    /usr/local/cuda/bin/nvcc -I/opt/conda/lib/python3.6/site-packages/torch/include -I/opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/fused_adam_cuda_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/fused_adam_cuda_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -O3 --use_fast_math -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=fused_adam_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    g++ -pthread -shared -B /opt/conda/compiler_compat -L/opt/conda/lib -Wl,-rpath=/opt/conda/lib -Wl,--no-as-needed -Wl,--sysroot=/ build/temp.linux-x86_64-3.6/csrc/fused_adam_cuda.o build/temp.linux-x86_64-3.6/csrc/fused_adam_cuda_kernel.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.6/fused_adam_cuda.cpython-36m-x86_64-linux-gnu.so\r\n",
      "    building 'syncbn' extension\r\n",
      "    gcc -pthread -B /opt/conda/compiler_compat -Wl,--sysroot=/ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I/opt/conda/lib/python3.6/site-packages/torch/include -I/opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/syncbn.cpp -o build/temp.linux-x86_64-3.6/csrc/syncbn.o -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=syncbn -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid for C/ObjC but not for C++\r\n",
      "    /usr/local/cuda/bin/nvcc -I/opt/conda/lib/python3.6/site-packages/torch/include -I/opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/welford.cu -o build/temp.linux-x86_64-3.6/csrc/welford.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=syncbn -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    g++ -pthread -shared -B /opt/conda/compiler_compat -L/opt/conda/lib -Wl,-rpath=/opt/conda/lib -Wl,--no-as-needed -Wl,--sysroot=/ build/temp.linux-x86_64-3.6/csrc/syncbn.o build/temp.linux-x86_64-3.6/csrc/welford.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.6/syncbn.cpython-36m-x86_64-linux-gnu.so\r\n",
      "    building 'fused_layer_norm_cuda' extension\r\n",
      "    gcc -pthread -B /opt/conda/compiler_compat -Wl,--sysroot=/ -Wsign-compare -DNDEBUG -g -fwrapv -O3 -Wall -Wstrict-prototypes -fPIC -I/opt/conda/lib/python3.6/site-packages/torch/include -I/opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/layer_norm_cuda.cpp -o build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda.o -O3 -DVERSION_GE_1_1 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=fused_layer_norm_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    cc1plus: warning: command line option ‘-Wstrict-prototypes’ is valid for C/ObjC but not for C++\r\n",
      "    /usr/local/cuda/bin/nvcc -I/opt/conda/lib/python3.6/site-packages/torch/include -I/opt/conda/lib/python3.6/site-packages/torch/include/torch/csrc/api/include -I/opt/conda/lib/python3.6/site-packages/torch/include/TH -I/opt/conda/lib/python3.6/site-packages/torch/include/THC -I/usr/local/cuda/include -I/opt/conda/include/python3.6m -c csrc/layer_norm_cuda_kernel.cu -o build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda_kernel.o -D__CUDA_NO_HALF_OPERATORS__ -D__CUDA_NO_HALF_CONVERSIONS__ -D__CUDA_NO_HALF2_OPERATORS__ --compiler-options '-fPIC' -maxrregcount=50 -O3 --use_fast_math -DVERSION_GE_1_1 -DTORCH_API_INCLUDE_EXTENSION_H -DTORCH_EXTENSION_NAME=fused_layer_norm_cuda -D_GLIBCXX_USE_CXX11_ABI=0 -std=c++11\r\n",
      "    g++ -pthread -shared -B /opt/conda/compiler_compat -L/opt/conda/lib -Wl,-rpath=/opt/conda/lib -Wl,--no-as-needed -Wl,--sysroot=/ build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda.o build/temp.linux-x86_64-3.6/csrc/layer_norm_cuda_kernel.o -L/usr/local/cuda/lib64 -lcudart -o build/lib.linux-x86_64-3.6/fused_layer_norm_cuda.cpython-36m-x86_64-linux-gnu.so\r\n",
      "    running install_lib\r\n",
      "    copying build/lib.linux-x86_64-3.6/fused_layer_norm_cuda.cpython-36m-x86_64-linux-gnu.so -> /opt/conda/lib/python3.6/site-packages\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex_C.cpython-36m-x86_64-linux-gnu.so -> /opt/conda/lib/python3.6/site-packages\r\n",
      "    copying build/lib.linux-x86_64-3.6/amp_C.cpython-36m-x86_64-linux-gnu.so -> /opt/conda/lib/python3.6/site-packages\r\n",
      "    copying build/lib.linux-x86_64-3.6/syncbn.cpython-36m-x86_64-linux-gnu.so -> /opt/conda/lib/python3.6/site-packages\r\n",
      "    copying build/lib.linux-x86_64-3.6/fused_adam_cuda.cpython-36m-x86_64-linux-gnu.so -> /opt/conda/lib/python3.6/site-packages\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex/reparameterization\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/reparameterization/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex/reparameterization\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/reparameterization/weight_norm.py -> /opt/conda/lib/python3.6/site-packages/apex/reparameterization\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/reparameterization/reparameterization.py -> /opt/conda/lib/python3.6/site-packages/apex/reparameterization\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/_process_optimizer.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/compat.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/utils.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/wrap.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/rnn_compat.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/_amp_state.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/_initialize.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex/amp/lists\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/lists/tensor_overrides.py -> /opt/conda/lib/python3.6/site-packages/apex/amp/lists\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/lists/torch_overrides.py -> /opt/conda/lib/python3.6/site-packages/apex/amp/lists\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/lists/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex/amp/lists\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/lists/functional_overrides.py -> /opt/conda/lib/python3.6/site-packages/apex/amp/lists\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/scaler.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/opt.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/frontend.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/__version__.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/handle.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/amp/amp.py -> /opt/conda/lib/python3.6/site-packages/apex/amp\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex/normalization\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/normalization/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex/normalization\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/normalization/fused_layer_norm.py -> /opt/conda/lib/python3.6/site-packages/apex/normalization\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex/multi_tensor_apply\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/multi_tensor_apply/multi_tensor_apply.py -> /opt/conda/lib/python3.6/site-packages/apex/multi_tensor_apply\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/multi_tensor_apply/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex/multi_tensor_apply\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex/RNN\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/RNN/models.py -> /opt/conda/lib/python3.6/site-packages/apex/RNN\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/RNN/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex/RNN\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/RNN/cells.py -> /opt/conda/lib/python3.6/site-packages/apex/RNN\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/RNN/RNNBackend.py -> /opt/conda/lib/python3.6/site-packages/apex/RNN\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex/fp16_utils\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/fp16_utils/loss_scaler.py -> /opt/conda/lib/python3.6/site-packages/apex/fp16_utils\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/fp16_utils/fp16util.py -> /opt/conda/lib/python3.6/site-packages/apex/fp16_utils\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/fp16_utils/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex/fp16_utils\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/fp16_utils/fp16_optimizer.py -> /opt/conda/lib/python3.6/site-packages/apex/fp16_utils\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex/parallel\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/parallel/optimized_sync_batchnorm_kernel.py -> /opt/conda/lib/python3.6/site-packages/apex/parallel\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/parallel/multiproc.py -> /opt/conda/lib/python3.6/site-packages/apex/parallel\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/parallel/distributed.py -> /opt/conda/lib/python3.6/site-packages/apex/parallel\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/parallel/sync_batchnorm.py -> /opt/conda/lib/python3.6/site-packages/apex/parallel\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/parallel/optimized_sync_batchnorm.py -> /opt/conda/lib/python3.6/site-packages/apex/parallel\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/parallel/sync_batchnorm_kernel.py -> /opt/conda/lib/python3.6/site-packages/apex/parallel\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/parallel/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex/parallel\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/parallel/LARC.py -> /opt/conda/lib/python3.6/site-packages/apex/parallel\r\n",
      "    creating /opt/conda/lib/python3.6/site-packages/apex/optimizers\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/optimizers/__init__.py -> /opt/conda/lib/python3.6/site-packages/apex/optimizers\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/optimizers/fp16_optimizer.py -> /opt/conda/lib/python3.6/site-packages/apex/optimizers\r\n",
      "    copying build/lib.linux-x86_64-3.6/apex/optimizers/fused_adam.py -> /opt/conda/lib/python3.6/site-packages/apex/optimizers\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/reparameterization/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/reparameterization/weight_norm.py to weight_norm.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/reparameterization/reparameterization.py to reparameterization.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/_process_optimizer.py to _process_optimizer.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/compat.py to compat.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/utils.py to utils.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/wrap.py to wrap.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/rnn_compat.py to rnn_compat.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/_amp_state.py to _amp_state.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/_initialize.py to _initialize.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/lists/tensor_overrides.py to tensor_overrides.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/lists/torch_overrides.py to torch_overrides.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/lists/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/lists/functional_overrides.py to functional_overrides.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/scaler.py to scaler.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/opt.py to opt.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/frontend.py to frontend.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/__version__.py to __version__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/handle.py to handle.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/amp/amp.py to amp.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/normalization/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/normalization/fused_layer_norm.py to fused_layer_norm.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/multi_tensor_apply/multi_tensor_apply.py to multi_tensor_apply.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/multi_tensor_apply/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/RNN/models.py to models.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/RNN/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/RNN/cells.py to cells.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/RNN/RNNBackend.py to RNNBackend.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/fp16_utils/loss_scaler.py to loss_scaler.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/fp16_utils/fp16util.py to fp16util.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/fp16_utils/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/fp16_utils/fp16_optimizer.py to fp16_optimizer.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/parallel/optimized_sync_batchnorm_kernel.py to optimized_sync_batchnorm_kernel.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/parallel/multiproc.py to multiproc.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/parallel/distributed.py to distributed.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/parallel/sync_batchnorm.py to sync_batchnorm.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/parallel/optimized_sync_batchnorm.py to optimized_sync_batchnorm.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/parallel/sync_batchnorm_kernel.py to sync_batchnorm_kernel.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/parallel/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/parallel/LARC.py to LARC.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/optimizers/__init__.py to __init__.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/optimizers/fp16_optimizer.py to fp16_optimizer.cpython-36.pyc\r\n",
      "    byte-compiling /opt/conda/lib/python3.6/site-packages/apex/optimizers/fused_adam.py to fused_adam.cpython-36.pyc\r\n",
      "    running install_egg_info\r\n",
      "    running egg_info\r\n",
      "    creating apex.egg-info\r\n",
      "    writing apex.egg-info/PKG-INFO\r\n",
      "    writing dependency_links to apex.egg-info/dependency_links.txt\r\n",
      "    writing top-level names to apex.egg-info/top_level.txt\r\n",
      "    writing manifest file 'apex.egg-info/SOURCES.txt'\r\n",
      "    reading manifest file 'apex.egg-info/SOURCES.txt'\r\n",
      "    writing manifest file 'apex.egg-info/SOURCES.txt'\r\n",
      "    Copying apex.egg-info to /opt/conda/lib/python3.6/site-packages/apex-0.1-py3.6.egg-info\r\n",
      "    running install_scripts\r\n",
      "    writing list of installed files to '/tmp/pip-record-fol60p1h/install-record.txt'\r\n",
      "  Running setup.py install for apex ... \u001b[?25l\u001b[?25hdone\r\n",
      "  Removing source in /tmp/pip-req-build-e6p3gnvb\r\n",
      "Successfully installed apex-0.1\r\n",
      "Cleaning up...\r\n",
      "Removed build tracker '/tmp/pip-req-tracker-q0a_wleh'\r\n",
      "1 location(s) to search for versions of pip:\r\n",
      "* https://pypi.org/simple/pip/\r\n",
      "Getting page https://pypi.org/simple/pip/\r\n",
      "Starting new HTTPS connection (1): pypi.org:443\r\n",
      "Could not fetch URL https://pypi.org/simple/pip/: connection error: HTTPSConnectionPool(host='pypi.org', port=443): Max retries exceeded with url: /simple/pip/ (Caused by NewConnectionError('<pip._vendor.urllib3.connection.VerifiedHTTPSConnection object at 0x7f386c35abe0>: Failed to establish a new connection: [Errno -3] Temporary failure in name resolution',)) - skipping\r\n",
      "Done\n"
     ]
    }
   ],
   "source": [
    "# Installing Nvidia Apex\n",
    "! pip install -v --no-cache-dir --global-option=\"--cpp_ext\" --global-option=\"--cuda_ext\" ../input/nvidiaapex/repository/NVIDIA-apex-39e153a\n",
    "print('Done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "import pkg_resources\n",
    "import seaborn as sns\n",
    "import time\n",
    "import scipy.stats as stats\n",
    "import gc\n",
    "import re\n",
    "import operator \n",
    "import sys\n",
    "from sklearn import metrics\n",
    "from sklearn import model_selection\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.utils.data\n",
    "import torch.nn.functional as F\n",
    "from nltk.stem import PorterStemmer\n",
    "from sklearn.metrics import roc_auc_score\n",
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline\n",
    "from tqdm import tqdm, tqdm_notebook\n",
    "import os\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\"\n",
    "import warnings\n",
    "warnings.filterwarnings(action='once')\n",
    "import pickle\n",
    "from apex import amp\n",
    "import shutil\n",
    "from nltk.tokenize.treebank import TreebankWordTokenizer\n",
    "\n",
    "\n",
    "import torch.utils.data\n",
    "from tqdm import tqdm\n",
    "import warnings\n",
    "from nltk.tokenize.treebank import TreebankWordTokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nUse a custom loss function where we multiply the result of our loss function by our weight,\\n'"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "Use a custom loss function where we multiply the result of our loss function by our weight,\n",
    "'''\n",
    "def custom_loss(data, targets):\n",
    "    ''' Define custom loss function for weighted BCE on 'target' column '''\n",
    "    '''\n",
    "    data->torch.Size([512, 7])\n",
    "    targets->torch.Size([512, 8])\n",
    "    '''\n",
    "    bce_loss_1 = nn.BCEWithLogitsLoss(weight=targets[:,1:2])(data[:,:1],targets[:,:1])\n",
    "    bce_loss_2 = nn.BCEWithLogitsLoss()(data[:,1:],targets[:,2:])\n",
    "    return (bce_loss_1 * loss_weight) + bce_loss_2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "device=torch.device('cuda')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX_SEQUENCE_LENGTH = 220#220\n",
    "SEED = 5060#1234\n",
    "EPOCHS = 1\n",
    "Data_dir=\"../input/jigsaw-unintended-bias-in-toxicity-classification\"\n",
    "Input_dir = \"../input\"\n",
    "WORK_DIR = \"../working/\"\n",
    "num_to_load= 1250000                         #Train size to match time limit\n",
    "valid_size= 100000                          #Validation Size\n",
    "TOXICITY_COLUMN = 'target'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "#! pip install ../input/gpt2-pytorch/gpt2-pytorch/pytorch-pretrained-BERT-master"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.6/importlib/_bootstrap.py:219: ImportWarning: can't resolve package from __spec__ or __package__, falling back on __name__ and __path__\n",
      "  return f(*args, **kwds)\n",
      "/opt/conda/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "# Add the Bart Pytorch repo to the PATH\n",
    "# using files from: https://github.com/huggingface/pytorch-pretrained-BERT\n",
    "package_dir_a = \"../input/ppbert/pytorch-pretrained-bert/pytorch-pretrained-BERT\"\n",
    "#package_dir_a = \"../input/gpt2-pytorch/gpt2-pytorch/pytorch-pretrained-BERT-master\"\n",
    "sys.path.insert(0, package_dir_a)\n",
    "\n",
    "from pytorch_pretrained_bert import convert_tf_checkpoint_to_pytorch\n",
    "from pytorch_pretrained_bert import BertTokenizer, BertForSequenceClassification,BertAdam\n",
    "#from apex import optimizers\n",
    "#from apex.optimizers import fused_adam as FusedAdam\n",
    "#from pytorch_pretrained_bert.optimization import WarmupLinearSchedule\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Building PyTorch model from configuration: {\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"vocab_size\": 30522\n",
      "}\n",
      "\n",
      "Converting TensorFlow checkpoint from /kaggle/input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/bert_model.ckpt\n",
      "Loading TF weight bert/embeddings/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/embeddings/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/embeddings/position_embeddings with shape [512, 768]\n",
      "Loading TF weight bert/embeddings/token_type_embeddings with shape [2, 768]\n",
      "Loading TF weight bert/embeddings/word_embeddings with shape [30522, 768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_0/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_0/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_0/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_0/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_1/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_1/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_1/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_1/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_10/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_10/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_10/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_10/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_11/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_11/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_11/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_11/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_2/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_2/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_2/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_2/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_3/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_3/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_3/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_3/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_4/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_4/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_4/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_4/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_5/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_5/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_5/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_5/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_6/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_6/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_6/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_6/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_7/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_7/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_7/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_7/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_8/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_8/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_8/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_8/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/output/dense/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/self/key/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/self/key/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/self/query/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/self/query/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/self/value/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/attention/self/value/kernel with shape [768, 768]\n",
      "Loading TF weight bert/encoder/layer_9/intermediate/dense/bias with shape [3072]\n",
      "Loading TF weight bert/encoder/layer_9/intermediate/dense/kernel with shape [768, 3072]\n",
      "Loading TF weight bert/encoder/layer_9/output/LayerNorm/beta with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/output/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/output/dense/bias with shape [768]\n",
      "Loading TF weight bert/encoder/layer_9/output/dense/kernel with shape [3072, 768]\n",
      "Loading TF weight bert/pooler/dense/bias with shape [768]\n",
      "Loading TF weight bert/pooler/dense/kernel with shape [768, 768]\n",
      "Loading TF weight cls/predictions/output_bias with shape [30522]\n",
      "Loading TF weight cls/predictions/transform/LayerNorm/beta with shape [768]\n",
      "Loading TF weight cls/predictions/transform/LayerNorm/gamma with shape [768]\n",
      "Loading TF weight cls/predictions/transform/dense/bias with shape [768]\n",
      "Loading TF weight cls/predictions/transform/dense/kernel with shape [768, 768]\n",
      "Loading TF weight cls/seq_relationship/output_bias with shape [2]\n",
      "Loading TF weight cls/seq_relationship/output_weights with shape [2, 768]\n",
      "Initialize PyTorch weight ['bert', 'embeddings', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'embeddings', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'embeddings', 'position_embeddings']\n",
      "Initialize PyTorch weight ['bert', 'embeddings', 'token_type_embeddings']\n",
      "Initialize PyTorch weight ['bert', 'embeddings', 'word_embeddings']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_0', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_1', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_10', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_11', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_2', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_3', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_4', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_5', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_6', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_7', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_8', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'self', 'key', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'self', 'key', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'self', 'query', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'self', 'query', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'self', 'value', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'attention', 'self', 'value', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'intermediate', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'intermediate', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'output', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'output', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'output', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'encoder', 'layer_9', 'output', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['bert', 'pooler', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['bert', 'pooler', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['cls', 'predictions', 'output_bias']\n",
      "Initialize PyTorch weight ['cls', 'predictions', 'transform', 'LayerNorm', 'beta']\n",
      "Initialize PyTorch weight ['cls', 'predictions', 'transform', 'LayerNorm', 'gamma']\n",
      "Initialize PyTorch weight ['cls', 'predictions', 'transform', 'dense', 'bias']\n",
      "Initialize PyTorch weight ['cls', 'predictions', 'transform', 'dense', 'kernel']\n",
      "Initialize PyTorch weight ['cls', 'seq_relationship', 'output_bias']\n",
      "Initialize PyTorch weight ['cls', 'seq_relationship', 'output_weights']\n",
      "Save PyTorch model to ../working/pytorch_model.bin\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'../working/bert_config.json'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Translate model from tensorflow to pytorch\n",
    "BERT_MODEL_PATH = '../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'\n",
    "#BERT_MODEL_PATH = '../input/bert-pretrained-models/uncased_l-24_h-1024_a-16/uncased_L-24_H-1024_A-16/'\n",
    "convert_tf_checkpoint_to_pytorch.convert_tf_checkpoint_to_pytorch(\n",
    "    BERT_MODEL_PATH + 'bert_model.ckpt',\n",
    "BERT_MODEL_PATH + 'bert_config.json',\n",
    "WORK_DIR + 'pytorch_model.bin')\n",
    "\n",
    "shutil.copyfile(BERT_MODEL_PATH + 'bert_config.json', WORK_DIR + 'bert_config.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['pytorch_model.bin',\n",
       " '__output__.json',\n",
       " 'bert_config.json',\n",
       " '__notebook__.ipynb']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "os.listdir(\"../working\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done\n"
     ]
    }
   ],
   "source": [
    "# This is the Bert configuration file\n",
    "from pytorch_pretrained_bert import BertConfig\n",
    "\n",
    "bert_config = BertConfig('../input/bert-pretrained-models/uncased_l-12_h-768_a-12/uncased_L-12_H-768_A-12/'+'bert_config.json')\n",
    "#bert_config = BertConfig('../input/bert-pretrained-models/uncased_l-24_h-1024_a-16/uncased_L-24_H-1024_A-16/'+'bert_config.json')\n",
    "print('Done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Converting the lines to BERT format\n",
    "# Thanks to https://www.kaggle.com/httpwwwfszyc/bert-in-keras-taming\n",
    "def convert_lines(example, max_seq_length,tokenizer):\n",
    "    max_seq_length -=2\n",
    "    all_tokens = []\n",
    "    longer = 0\n",
    "    for text in tqdm_notebook(example):\n",
    "        tokens_a = tokenizer.tokenize(text)\n",
    "        if len(tokens_a)>max_seq_length:\n",
    "            tokens_a = tokens_a[:max_seq_length]\n",
    "            longer += 1\n",
    "        one_token = tokenizer.convert_tokens_to_ids([\"[CLS]\"]+tokens_a+[\"[SEP]\"])+[0] * (max_seq_length - len(tokens_a))\n",
    "        all_tokens.append(one_token)\n",
    "    print(longer)\n",
    "    return np.array(all_tokens)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49ecb07c390d4acbb1bd2022b73de7a2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Leyendo datos\n",
      "loaded 1350000 records\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a9f17d64b0a8461ca506b14bc993d8ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1350000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%time\n",
    "from tqdm._tqdm_notebook import tqdm_notebook\n",
    "tqdm_notebook().pandas()\n",
    "\n",
    "tokenizer = BertTokenizer.from_pretrained(BERT_MODEL_PATH, cache_dir=None,do_lower_case=True)\n",
    "\n",
    "print('Leyendo datos')\n",
    "train_df = pd.read_csv(os.path.join(Data_dir,\"train.csv\")).sample(num_to_load+valid_size,random_state=SEED)\n",
    "#num_to_load = int(len(train_df)/2) #++\n",
    "#train_df = train_df.loc[:num_to_load+valid_size, :]#++\n",
    "print('loaded %d records' % len(train_df))\n",
    "\n",
    "# Make sure all comment_text values are strings\n",
    "train_df['comment_text'] = train_df['comment_text'].astype(str) \n",
    "\n",
    "sequences = convert_lines(train_df[\"comment_text\"].fillna(\"DUMMY_VALUE\"),MAX_SEQUENCE_LENGTH,tokenizer)\n",
    "train_df=train_df.fillna(0)\n",
    "\n",
    "# List all identities\n",
    "identity_columns = [\n",
    "    'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',\n",
    "    'muslim', 'black', 'white', 'psychiatric_or_mental_illness']\n",
    "\n",
    "y_columns =['target', 'severe_toxicity', 'obscene', 'identity_attack', 'insult', 'threat']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#++ TODA LA CELDA\n",
    "y_aux_train = train_df[y_columns] #++\n",
    "\n",
    "# Overall\n",
    "weights = np.ones((len(train_df),)) / 4\n",
    "\n",
    "# Subgroup\n",
    "weights += (train_df[identity_columns].fillna(0).values>=0.5).sum(axis=1).astype(bool).astype(np.int) / 4\n",
    "\n",
    "# Background Positive, Subgroup Negative\n",
    "weights += (( (train_df['target'].values>=0.5).astype(bool).astype(np.int) +\n",
    "   (train_df[identity_columns].fillna(0).values<0.5).sum(axis=1).astype(bool).astype(np.int) ) > 1 ).astype(bool).astype(np.int) / 4\n",
    "\n",
    "# Background Negative, Subgroup Positive\n",
    "weights += (( (train_df['target'].values<0.5).astype(bool).astype(np.int) +\n",
    "   (train_df[identity_columns].fillna(0).values>=0.5).sum(axis=1).astype(bool).astype(np.int) ) > 1 ).astype(bool).astype(np.int) / 4\n",
    "loss_weight = 1.0 / weights.mean()\n",
    "\n",
    "y_train = np.vstack([(train_df['target'].values>=0.5).astype(np.int),weights]).T #//\n",
    "\n",
    "max_features = 410047"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = train_df.drop(['comment_text'],axis=1)\n",
    "# convert target to 0,1\n",
    "train_df['target']=(train_df['target']>=0.5).astype(float)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nX = sequences[:num_to_load]                \\n#y = train_df[y_columns].values[:num_to_load]\\ny = np.hstack([y_train[:num_to_load], y_aux_train[:num_to_load]])\\nX_val = sequences[-valid_size:]                \\n#y_val = train_df[y_columns].values[num_to_load:]\\ny_val = np.hstack([y_train[-valid_size:], y_aux_train[-valid_size:]])\\n'"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "X = sequences[:num_to_load]                \n",
    "#y = train_df[y_columns].values[:num_to_load]\n",
    "y = np.hstack([y_train[:num_to_load], y_aux_train[:num_to_load]])\n",
    "X_val = sequences[num_to_load:]                \n",
    "#y_val = train_df[y_columns].values[num_to_load:]\n",
    "y_val = np.hstack([y_train[num_to_load:], y_aux_train[num_to_load:]])\n",
    "'''\n",
    "X = sequences[:num_to_load]                \n",
    "#y = train_df[y_columns].values[:num_to_load]\n",
    "y = np.hstack([y_train[:num_to_load], y_aux_train[:num_to_load]])\n",
    "X_val = sequences[-valid_size:]                \n",
    "#y_val = train_df[y_columns].values[num_to_load:]\n",
    "y_val = np.hstack([y_train[-valid_size:], y_aux_train[-valid_size:]])\n",
    "'''\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# AQUI se separan\n",
    "test_df=train_df.tail(valid_size).copy() #// +1\n",
    "train_df=train_df.head(num_to_load)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = torch.utils.data.TensorDataset(torch.tensor(X,dtype=torch.long), torch.tensor(y,dtype=torch.float))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 6.3 s, sys: 1.39 s, total: 7.69 s\n",
      "Wall time: 8.33 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "output_model_file = \"bert_pytorch.bin\"\n",
    "#3rd part: num_train_optimization_steps = 1/2*int(2*len(train)/batch_size/num_accumulation_steps), lr=1/2*lr, wu =0\n",
    "lr= 2.5e-5 #--> https://arxiv.org/pdf/1905.05583.pdf\n",
    "batch_size = 32\n",
    "accumulation_steps=1\n",
    "np.random.seed(SEED)\n",
    "torch.manual_seed(SEED)\n",
    "torch.cuda.manual_seed(SEED)\n",
    "torch.backends.cudnn.deterministic = True\n",
    "\n",
    "numero_de_labels = len(y_columns) + 1 # añadido + 1 para custom_loss\n",
    "model = BertForSequenceClassification.from_pretrained(\"../working\",cache_dir=None,num_labels=numero_de_labels) #//y_columns\n",
    "\n",
    "model.zero_grad()\n",
    "model = model.to(device)\n",
    "param_optimizer = list(model.named_parameters())\n",
    "no_decay = ['bias', 'LayerNorm.bias', 'LayerNorm.weight']\n",
    "optimizer_grouped_parameters = [\n",
    "    {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)], 'weight_decay': 0.01},\n",
    "    {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}\n",
    "    ]\n",
    "train = train_dataset\n",
    "\n",
    "num_train_optimization_steps = int(EPOCHS*len(train)/batch_size/accumulation_steps)\n",
    "#num_train_optimization_steps = int(2*len(train)/batch_size/accumulation_steps) # para 2 EPOCHS\n",
    "\n",
    "\n",
    "optimizer = BertAdam(optimizer_grouped_parameters,\n",
    "                     lr=lr,\n",
    "                     warmup=0.05,#0.05 -------- 0.1 -->https://arxiv.org/pdf/1905.05583.pdf\n",
    "                     t_total=num_train_optimization_steps)\n",
    "                     \n",
    "model, optimizer = amp.initialize(model, optimizer, opt_level=\"O1\",verbosity=0)\n",
    "\n",
    "model=model.train()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e472ecbd0c44b199c2c7cbd4a830ebd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e4c320d949e3462b8565a7b544dc2062",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=39063), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%%time\n",
    "tq = tqdm_notebook(range(EPOCHS))\n",
    "for epoch in tq:\n",
    "    train_loader = torch.utils.data.DataLoader(train, batch_size=batch_size, shuffle=True)\n",
    "    avg_loss = 0.\n",
    "    avg_accuracy = 0.\n",
    "    lossf=None\n",
    "    tk0 = tqdm_notebook(enumerate(train_loader),total=len(train_loader),leave=False)\n",
    "    optimizer.zero_grad() #https://www.kaggle.com/c/jigsaw-unintended-bias-in-toxicity-classification/discussion/92505#latest-541682\n",
    "    for i,(x_batch, y_batch) in tk0: #[32, 8]\n",
    "        #optimizer.zero_grad()\n",
    "        y_pred = model(x_batch.to(device), attention_mask=(x_batch>0).to(device), labels=None) #[32, 6]\n",
    "        #loss =  F.binary_cross_entropy_with_logits(y_pred,y_batch.to(device))\n",
    "        loss = custom_loss(y_pred, y_batch.to(device)) #++\n",
    "        with amp.scale_loss(loss, optimizer) as scaled_loss:\n",
    "            scaled_loss.backward()\n",
    "        if (i+1) % accumulation_steps == 0:             # Wait for several backward steps\n",
    "            optimizer.step()                            # Now we can do an optimizer step\n",
    "            optimizer.zero_grad()\n",
    "        if lossf:\n",
    "            lossf = 0.98*lossf+0.02*loss.item()\n",
    "        else:\n",
    "            lossf = loss.item()\n",
    "        tk0.set_postfix(loss = lossf)\n",
    "        avg_loss += loss.item() / len(train_loader)\n",
    "        avg_accuracy += torch.mean(((torch.sigmoid(y_pred[:,0])>0.5) == (y_batch[:,0]>0.5).to(device)).to(torch.float) ).item()/len(train_loader)\n",
    "    tq.set_postfix(avg_loss=avg_loss,avg_accuracy=avg_accuracy)\n",
    "\n",
    "\n",
    "torch.save(model.state_dict(), output_model_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "IncompatibleKeys(missing_keys=[], unexpected_keys=[])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1)\n",
       "  (classifier): Linear(in_features=768, out_features=7, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "BertForSequenceClassification(\n",
       "  (bert): BertModel(\n",
       "    (embeddings): BertEmbeddings(\n",
       "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
       "      (position_embeddings): Embedding(512, 768)\n",
       "      (token_type_embeddings): Embedding(2, 768)\n",
       "      (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "      (dropout): Dropout(p=0.1)\n",
       "    )\n",
       "    (encoder): BertEncoder(\n",
       "      (layer): ModuleList(\n",
       "        (0): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (1): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (2): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (3): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (4): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (5): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (6): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (7): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (8): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (9): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (10): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "        (11): BertLayer(\n",
       "          (attention): BertAttention(\n",
       "            (self): BertSelfAttention(\n",
       "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "            (output): BertSelfOutput(\n",
       "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "              (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "              (dropout): Dropout(p=0.1)\n",
       "            )\n",
       "          )\n",
       "          (intermediate): BertIntermediate(\n",
       "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
       "          )\n",
       "          (output): BertOutput(\n",
       "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
       "            (LayerNorm): FusedLayerNorm(torch.Size([768]), eps=1e-12, elementwise_affine=True)\n",
       "            (dropout): Dropout(p=0.1)\n",
       "          )\n",
       "        )\n",
       "      )\n",
       "    )\n",
       "    (pooler): BertPooler(\n",
       "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
       "      (activation): Tanh()\n",
       "    )\n",
       "  )\n",
       "  (dropout): Dropout(p=0.1)\n",
       "  (classifier): Linear(in_features=768, out_features=7, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "19c3329196ec4b078e4beab0c0761a53",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3125), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Run validation\n",
    "# The following 2 lines are not needed but show how to download the model for prediction\n",
    "model = BertForSequenceClassification(bert_config,num_labels=len(y_columns) + 1)#++\n",
    "model.load_state_dict(torch.load(output_model_file ))\n",
    "model.to(device)\n",
    "for param in model.parameters():\n",
    "    param.requires_grad=False\n",
    "model.eval()\n",
    "valid_preds = np.zeros((len(X_val)))\n",
    "valid = torch.utils.data.TensorDataset(torch.tensor(X_val,dtype=torch.long))\n",
    "valid_loader = torch.utils.data.DataLoader(valid, batch_size=batch_size, shuffle=False)\n",
    "\n",
    "tk0 = tqdm_notebook(valid_loader)\n",
    "for i,(x_batch,)  in enumerate(tk0):\n",
    "    pred = model(x_batch.to(device), attention_mask=(x_batch>0).to(device), labels=None)\n",
    "    valid_preds[i*batch_size:(i+1)*batch_size]=pred[:,0].detach().cpu().squeeze().numpy()\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# From baseline kernel\n",
    "\n",
    "def calculate_overall_auc(df, model_name):\n",
    "    true_labels = df[TOXICITY_COLUMN]>0.5\n",
    "    predicted_labels = df[model_name]\n",
    "    return metrics.roc_auc_score(true_labels, predicted_labels)\n",
    "\n",
    "def power_mean(series, p):\n",
    "    total = sum(np.power(series, p))\n",
    "    return np.power(total / len(series), 1 / p)\n",
    "\n",
    "def get_final_metric(bias_df, overall_auc, POWER=-5, OVERALL_MODEL_WEIGHT=0.25):\n",
    "    bias_score = np.average([\n",
    "        power_mean(bias_df[SUBGROUP_AUC], POWER),\n",
    "        power_mean(bias_df[BPSN_AUC], POWER),\n",
    "        power_mean(bias_df[BNSP_AUC], POWER)\n",
    "    ])\n",
    "    return (OVERALL_MODEL_WEIGHT * overall_auc) + ((1 - OVERALL_MODEL_WEIGHT) * bias_score)\n",
    "\n",
    "\n",
    "\n",
    "SUBGROUP_AUC = 'subgroup_auc'\n",
    "BPSN_AUC = 'bpsn_auc'  # stands for background positive, subgroup negative\n",
    "BNSP_AUC = 'bnsp_auc'  # stands for background negative, subgroup positive\n",
    "\n",
    "def compute_auc(y_true, y_pred):\n",
    "    try:\n",
    "        return metrics.roc_auc_score(y_true, y_pred)\n",
    "    except ValueError:\n",
    "        return np.nan\n",
    "\n",
    "def compute_subgroup_auc(df, subgroup, label, model_name):\n",
    "    subgroup_examples = df[df[subgroup]>0.5]\n",
    "    return compute_auc((subgroup_examples[label]>0.5), subgroup_examples[model_name])\n",
    "\n",
    "def compute_bpsn_auc(df, subgroup, label, model_name):\n",
    "    \"\"\"Computes the AUC of the within-subgroup negative examples and the background positive examples.\"\"\"\n",
    "    subgroup_negative_examples = df[(df[subgroup]>0.5) & (df[label]<=0.5)]\n",
    "    non_subgroup_positive_examples = df[(df[subgroup]<=0.5) & (df[label]>0.5)]\n",
    "    examples = subgroup_negative_examples.append(non_subgroup_positive_examples)\n",
    "    return compute_auc(examples[label]>0.5, examples[model_name])\n",
    "\n",
    "def compute_bnsp_auc(df, subgroup, label, model_name):\n",
    "    \"\"\"Computes the AUC of the within-subgroup positive examples and the background negative examples.\"\"\"\n",
    "    subgroup_positive_examples = df[(df[subgroup]>0.5) & (df[label]>0.5)]\n",
    "    non_subgroup_negative_examples = df[(df[subgroup]<=0.5) & (df[label]<=0.5)]\n",
    "    examples = subgroup_positive_examples.append(non_subgroup_negative_examples)\n",
    "    return compute_auc(examples[label]>0.5, examples[model_name])\n",
    "\n",
    "def compute_bias_metrics_for_model(dataset,\n",
    "                                   subgroups,\n",
    "                                   model,\n",
    "                                   label_col,\n",
    "                                   include_asegs=False):\n",
    "    \"\"\"Computes per-subgroup metrics for all subgroups and one model.\"\"\"\n",
    "    records = []\n",
    "    for subgroup in subgroups:\n",
    "        record = {\n",
    "            'subgroup': subgroup,\n",
    "            'subgroup_size': len(dataset[dataset[subgroup]>0.5])\n",
    "        }\n",
    "        record[SUBGROUP_AUC] = compute_subgroup_auc(dataset, subgroup, label_col, model)\n",
    "        record[BPSN_AUC] = compute_bpsn_auc(dataset, subgroup, label_col, model)\n",
    "        record[BNSP_AUC] = compute_bnsp_auc(dataset, subgroup, label_col, model)\n",
    "        records.append(record)\n",
    "    return pd.DataFrame(records).sort_values('subgroup_auc', ascending=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.metrics import roc_auc_score\n",
    "\n",
    "\n",
    "class JigsawEvaluator:\n",
    "\n",
    "    def __init__(self, y_true, y_identity, power=-5, overall_model_weight=0.25):\n",
    "        self.y = (y_true >= 0.5).astype(int)\n",
    "        self.y_i = (y_identity >= 0.5).astype(int)\n",
    "        self.n_subgroups = self.y_i.shape[1]\n",
    "        self.power = power\n",
    "        self.overall_model_weight = overall_model_weight\n",
    "\n",
    "    @staticmethod\n",
    "    def _compute_auc(y_true, y_pred):\n",
    "        try:\n",
    "            return roc_auc_score(y_true, y_pred)\n",
    "        except ValueError:\n",
    "            return np.nan\n",
    "\n",
    "    def _compute_subgroup_auc(self, i, y_pred):\n",
    "        mask = self.y_i[:, i] == 1\n",
    "        return self._compute_auc(self.y[mask], y_pred[mask])\n",
    "\n",
    "    def _compute_bpsn_auc(self, i, y_pred):\n",
    "        mask = self.y_i[:, i] + self.y == 1\n",
    "        return self._compute_auc(self.y[mask], y_pred[mask])\n",
    "\n",
    "    def _compute_bnsp_auc(self, i, y_pred):\n",
    "        mask = self.y_i[:, i] + self.y != 1\n",
    "        return self._compute_auc(self.y[mask], y_pred[mask])\n",
    "\n",
    "    def compute_bias_metrics_for_model(self, y_pred):\n",
    "        records = np.zeros((3, self.n_subgroups))\n",
    "        for i in range(self.n_subgroups):\n",
    "            records[0, i] = self._compute_subgroup_auc(i, y_pred)\n",
    "            records[1, i] = self._compute_bpsn_auc(i, y_pred)\n",
    "            records[2, i] = self._compute_bnsp_auc(i, y_pred)\n",
    "        return records\n",
    "\n",
    "    def _calculate_overall_auc(self, y_pred):\n",
    "        return roc_auc_score(self.y, y_pred)\n",
    "\n",
    "    def _power_mean(self, array):\n",
    "        total = sum(np.power(array, self.power))\n",
    "        return np.power(total / len(array), 1 / self.power)\n",
    "\n",
    "    def get_final_metric(self, y_pred):\n",
    "        bias_metrics = self.compute_bias_metrics_for_model(y_pred)\n",
    "        bias_score = np.average([\n",
    "            self._power_mean(bias_metrics[0]),\n",
    "            self._power_mean(bias_metrics[1]),\n",
    "            self._power_mean(bias_metrics[2])\n",
    "        ])\n",
    "        overall_score = self.overall_model_weight * self._calculate_overall_auc(y_pred)\n",
    "        bias_score = (1 - self.overall_model_weight) * bias_score\n",
    "        return overall_score + bias_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bnsp_auc</th>\n",
       "      <th>bpsn_auc</th>\n",
       "      <th>subgroup</th>\n",
       "      <th>subgroup_auc</th>\n",
       "      <th>subgroup_size</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.961928</td>\n",
       "      <td>0.892207</td>\n",
       "      <td>homosexual_gay_or_lesbian</td>\n",
       "      <td>0.843069</td>\n",
       "      <td>549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.962506</td>\n",
       "      <td>0.903963</td>\n",
       "      <td>white</td>\n",
       "      <td>0.861263</td>\n",
       "      <td>1281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.964227</td>\n",
       "      <td>0.897970</td>\n",
       "      <td>black</td>\n",
       "      <td>0.866665</td>\n",
       "      <td>764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.966792</td>\n",
       "      <td>0.918288</td>\n",
       "      <td>muslim</td>\n",
       "      <td>0.904612</td>\n",
       "      <td>1060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.948588</td>\n",
       "      <td>0.947208</td>\n",
       "      <td>jewish</td>\n",
       "      <td>0.904987</td>\n",
       "      <td>398</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.964820</td>\n",
       "      <td>0.936285</td>\n",
       "      <td>psychiatric_or_mental_illness</td>\n",
       "      <td>0.922136</td>\n",
       "      <td>213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.938936</td>\n",
       "      <td>0.966325</td>\n",
       "      <td>christian</td>\n",
       "      <td>0.923644</td>\n",
       "      <td>1901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.962372</td>\n",
       "      <td>0.952446</td>\n",
       "      <td>female</td>\n",
       "      <td>0.935634</td>\n",
       "      <td>2727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.965045</td>\n",
       "      <td>0.949850</td>\n",
       "      <td>male</td>\n",
       "      <td>0.938437</td>\n",
       "      <td>2186</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   bnsp_auc  bpsn_auc      ...       subgroup_auc  subgroup_size\n",
       "2  0.961928  0.892207      ...           0.843069            549\n",
       "7  0.962506  0.903963      ...           0.861263           1281\n",
       "6  0.964227  0.897970      ...           0.866665            764\n",
       "5  0.966792  0.918288      ...           0.904612           1060\n",
       "4  0.948588  0.947208      ...           0.904987            398\n",
       "8  0.964820  0.936285      ...           0.922136            213\n",
       "3  0.938936  0.966325      ...           0.923644           1901\n",
       "1  0.962372  0.952446      ...           0.935634           2727\n",
       "0  0.965045  0.949850      ...           0.938437           2186\n",
       "\n",
       "[9 rows x 5 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0.9383906199117588"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "MODEL_NAME = 'model1'\n",
    "test_df[MODEL_NAME]=torch.sigmoid(torch.tensor(valid_preds)).numpy()\n",
    "TOXICITY_COLUMN = 'target'\n",
    "bias_metrics_df = compute_bias_metrics_for_model(test_df, identity_columns, MODEL_NAME, 'target')\n",
    "bias_metrics_df\n",
    "get_final_metric(bias_metrics_df, calculate_overall_auc(test_df, MODEL_NAME))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cv score: 0.93951 \n"
     ]
    }
   ],
   "source": [
    "# target and subgroup columns\n",
    "identity_columns = [\n",
    "    'male', 'female', 'homosexual_gay_or_lesbian', 'christian', 'jewish',\n",
    "    'muslim', 'black', 'white', 'psychiatric_or_mental_illness']\n",
    "y_true = test_df['target'].values\n",
    "y_identity = test_df[identity_columns].values\n",
    "\n",
    "# predict\n",
    "y_pred = valid_preds\n",
    "\n",
    "# evaluate\n",
    "evaluator = JigsawEvaluator(y_true, y_identity)\n",
    "auc_score = evaluator.get_final_metric(y_pred)\n",
    "print(f'cv score: {auc_score:<8.5f}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "0fb3f8386c5243a784505fdea6e38e4e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ad5d85b9128a4fa885424ab541014800",
       "max": 39063,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_ca9b4958e1224a96bf241b89eba6103e",
       "value": 3881
      }
     },
     "102f99a3bd9a4512846b13e22a4bd6b3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "10ff0c09fe0543cc80e6a3272a974456": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_1c570e6e98204028818bd8e540467a2a",
       "max": 3125,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_ed36d08e1c1340179147df8d72de1690",
       "value": 3125
      }
     },
     "132194d876424090be5b8e88f5030c01": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "152cdb1ea25a4ff88f8a1a8979f7f391": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "19c3329196ec4b078e4beab0c0761a53": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_10ff0c09fe0543cc80e6a3272a974456",
        "IPY_MODEL_813e9d4144fe44ee8e574a4cbe7268df"
       ],
       "layout": "IPY_MODEL_6d8177ca4d4c4b4a8eb9468fdcc63505"
      }
     },
     "1c570e6e98204028818bd8e540467a2a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "2620ee6450dd432eb980c7e607f78660": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_d3c0afd359b342cbad6de84f22e3c648",
       "max": 1,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_ca0b26d6d38a4dec87c7e2d91043fb5b",
       "value": 1
      }
     },
     "4950d07388d94a6d9d632719a4e344a1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "49ecb07c390d4acbb1bd2022b73de7a2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_2620ee6450dd432eb980c7e607f78660",
        "IPY_MODEL_6d43c8d3d0f2469993a98564c810f271"
       ],
       "layout": "IPY_MODEL_6b25d4277cbc443b997db023e2261acc"
      }
     },
     "4ac24ca6f76741da94b1f82cef57e41f": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "5e552ed389514b51bb88badc38b1e160": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_d4827d37a77d491c9d347faab7a2b885",
       "placeholder": "​",
       "style": "IPY_MODEL_4950d07388d94a6d9d632719a4e344a1",
       "value": " 21% 287443/1350000 [05:37&lt;19:51, 892.12it/s]"
      }
     },
     "60b4c3a6c90b4d74aa1e2c04e7f3fd4c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6b25d4277cbc443b997db023e2261acc": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6bc7e75fcc7f4f08b135579ddbdfd8c7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "6d43c8d3d0f2469993a98564c810f271": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_90ed163d6c9c40f18c4bd0b225bb9167",
       "placeholder": "​",
       "style": "IPY_MODEL_8d9b8ab0f7dc4025882f2b93b09efe46",
       "value": "0/|/| 0/? [00:00&lt;?, ?it/s]"
      }
     },
     "6d8177ca4d4c4b4a8eb9468fdcc63505": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "76b2ab79fe97434f949b47c65db43df2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_db1692f1a7b241b59b677377b1bf368a",
       "max": 1,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_9897d93a5cfc4ec09f85bdcf0be596ed",
       "value": 0
      }
     },
     "813e9d4144fe44ee8e574a4cbe7268df": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_60b4c3a6c90b4d74aa1e2c04e7f3fd4c",
       "placeholder": "​",
       "style": "IPY_MODEL_a19c2e565c7e49319ad965538a113966",
       "value": "100% 3125/3125 [12:09&lt;00:00,  4.28it/s]"
      }
     },
     "89c5aec0bbc0415fb76a1cca591a2fe2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_102f99a3bd9a4512846b13e22a4bd6b3",
       "placeholder": "​",
       "style": "IPY_MODEL_96bccf73fbb04917a3dfe8b1717f9d9b",
       "value": " 10% 3881/39063 [47:26&lt;7:08:40,  1.37it/s, loss=0.303]"
      }
     },
     "8d9b8ab0f7dc4025882f2b93b09efe46": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "8e472ecbd0c44b199c2c7cbd4a830ebd": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_76b2ab79fe97434f949b47c65db43df2",
        "IPY_MODEL_fa288537e44841039cc7a9da6e53829d"
       ],
       "layout": "IPY_MODEL_132194d876424090be5b8e88f5030c01"
      }
     },
     "90ed163d6c9c40f18c4bd0b225bb9167": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "96bccf73fbb04917a3dfe8b1717f9d9b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "9897d93a5cfc4ec09f85bdcf0be596ed": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "a19c2e565c7e49319ad965538a113966": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "a9f17d64b0a8461ca506b14bc993d8ff": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_c45d408eadca4331bd452feaf31891a4",
        "IPY_MODEL_5e552ed389514b51bb88badc38b1e160"
       ],
       "layout": "IPY_MODEL_152cdb1ea25a4ff88f8a1a8979f7f391"
      }
     },
     "ad5d85b9128a4fa885424ab541014800": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ae891c493bce4b10bdbcf40668fa0255": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c45d408eadca4331bd452feaf31891a4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "IntProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "IntProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "ProgressView",
       "bar_style": "",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_ae891c493bce4b10bdbcf40668fa0255",
       "max": 1350000,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_4ac24ca6f76741da94b1f82cef57e41f",
       "value": 287533
      }
     },
     "ca0b26d6d38a4dec87c7e2d91043fb5b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "ca9b4958e1224a96bf241b89eba6103e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "d3c0afd359b342cbad6de84f22e3c648": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "d4827d37a77d491c9d347faab7a2b885": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "da0b949ecffa4dd1bcd18dff92a84ae2": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "db1692f1a7b241b59b677377b1bf368a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e4c320d949e3462b8565a7b544dc2062": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_0fb3f8386c5243a784505fdea6e38e4e",
        "IPY_MODEL_89c5aec0bbc0415fb76a1cca591a2fe2"
       ],
       "layout": "IPY_MODEL_ea521802bd8446fea2ccb29ad85f3e73"
      }
     },
     "ea521802bd8446fea2ccb29ad85f3e73": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.1.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.1.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ed36d08e1c1340179147df8d72de1690": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.1.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "fa288537e44841039cc7a9da6e53829d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.4.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.4.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.4.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_6bc7e75fcc7f4f08b135579ddbdfd8c7",
       "placeholder": "​",
       "style": "IPY_MODEL_da0b949ecffa4dd1bcd18dff92a84ae2",
       "value": "  0% 0/1 [00:00&lt;?, ?it/s]"
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}