LSTM_Model.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eoJT68CUE5JE"
      },
      "outputs": [],
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.layers import BatchNormalization, Flatten, Dense, Input, Conv1D, LSTM, Concatenate\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras.initializers import glorot_uniform\n",
        "from tensorflow.keras.callbacks import ReduceLROnPlateau, EarlyStopping\n",
        "from sklearn.metrics import mean_squared_error\n",
        "import numpy as np\n",
        "import gc\n",
        "import pickle"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "B7I6P4vvH52k",
        "outputId": "6686134d-3f05-45aa-d1f8-171bdc82dd86"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting pytorch_lightning\n",
            "  Downloading pytorch_lightning-1.8.3.post1-py3-none-any.whl (798 kB)\n",
            "\u001b[K     |████████████████████████████████| 798 kB 7.0 MB/s \n",
            "\u001b[?25hRequirement already satisfied: torch>=1.9.* in /usr/local/lib/python3.7/dist-packages (from pytorch_lightning) (1.12.1+cu113)\n",
            "Requirement already satisfied: fsspec[http]>2021.06.0 in /usr/local/lib/python3.7/dist-packages (from pytorch_lightning) (2022.11.0)\n",
            "Requirement already satisfied: tqdm>=4.57.0 in /usr/local/lib/python3.7/dist-packages (from pytorch_lightning) (4.64.1)\n",
            "Collecting torchmetrics>=0.7.0\n",
            "  Downloading torchmetrics-0.10.3-py3-none-any.whl (529 kB)\n",
            "\u001b[K     |████████████████████████████████| 529 kB 44.9 MB/s \n",
            "\u001b[?25hCollecting tensorboardX>=2.2\n",
            "  Downloading tensorboardX-2.5.1-py2.py3-none-any.whl (125 kB)\n",
            "\u001b[K     |████████████████████████████████| 125 kB 71.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: packaging>=17.0 in /usr/local/lib/python3.7/dist-packages (from pytorch_lightning) (21.3)\n",
            "Collecting lightning-utilities==0.3.*\n",
            "  Downloading lightning_utilities-0.3.0-py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: typing-extensions>=4.0.0 in /usr/local/lib/python3.7/dist-packages (from pytorch_lightning) (4.1.1)\n",
            "Requirement already satisfied: PyYAML>=5.4 in /usr/local/lib/python3.7/dist-packages (from pytorch_lightning) (6.0)\n",
            "Requirement already satisfied: numpy>=1.17.2 in /usr/local/lib/python3.7/dist-packages (from pytorch_lightning) (1.21.6)\n",
            "Collecting fire\n",
            "  Downloading fire-0.4.0.tar.gz (87 kB)\n",
            "\u001b[K     |████████████████████████████████| 87 kB 8.6 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from fsspec[http]>2021.06.0->pytorch_lightning) (2.23.0)\n",
            "Requirement already satisfied: aiohttp!=4.0.0a0,!=4.0.0a1 in /usr/local/lib/python3.7/dist-packages (from fsspec[http]>2021.06.0->pytorch_lightning) (3.8.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch_lightning) (1.3.1)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch_lightning) (2.1.1)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch_lightning) (1.3.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch_lightning) (1.8.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.7/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch_lightning) (4.0.2)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch_lightning) (6.0.2)\n",
            "Requirement already satisfied: asynctest==0.13.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch_lightning) (0.13.0)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch_lightning) (22.1.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=17.0->pytorch_lightning) (3.0.9)\n",
            "Requirement already satisfied: protobuf<=3.20.1,>=3.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorboardX>=2.2->pytorch_lightning) (3.19.6)\n",
            "Requirement already satisfied: idna>=2.0 in /usr/local/lib/python3.7/dist-packages (from yarl<2.0,>=1.0->aiohttp!=4.0.0a0,!=4.0.0a1->fsspec[http]>2021.06.0->pytorch_lightning) (2.10)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from fire->lightning-utilities==0.3.*->pytorch_lightning) (1.15.0)\n",
            "Requirement already satisfied: termcolor in /usr/local/lib/python3.7/dist-packages (from fire->lightning-utilities==0.3.*->pytorch_lightning) (2.1.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]>2021.06.0->pytorch_lightning) (2022.9.24)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]>2021.06.0->pytorch_lightning) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->fsspec[http]>2021.06.0->pytorch_lightning) (3.0.4)\n",
            "Building wheels for collected packages: fire\n",
            "  Building wheel for fire (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for fire: filename=fire-0.4.0-py2.py3-none-any.whl size=115943 sha256=053e0144a028fd055623378c78b18b88b4b254312d2aafa70912dabc0a65718a\n",
            "  Stored in directory: /root/.cache/pip/wheels/8a/67/fb/2e8a12fa16661b9d5af1f654bd199366799740a85c64981226\n",
            "Successfully built fire\n",
            "Installing collected packages: fire, torchmetrics, tensorboardX, lightning-utilities, pytorch-lightning\n",
            "Successfully installed fire-0.4.0 lightning-utilities-0.3.0 pytorch-lightning-1.8.3.post1 tensorboardX-2.5.1 torchmetrics-0.10.3\n"
          ]
        }
      ],
      "source": [
        "!pip install pytorch_lightning"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "46pnhn0vHtXk"
      },
      "outputs": [],
      "source": [
        "import numpy as np \n",
        "import pandas as pd \n",
        "import os, gc, re, warnings\n",
        "warnings.filterwarnings(\"ignore\")\n",
        "from keras.preprocessing.text import Tokenizer\n",
        "from tqdm.notebook import tqdm\n",
        "from sklearn.model_selection import train_test_split\n",
        "import gc\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import pytorch_lightning as pl\n",
        "from pytorch_lightning.callbacks.early_stopping import EarlyStopping\n",
        "from nltk.tokenize import word_tokenize"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uITV28OMFA2N",
        "outputId": "7d1c1388-9418-4a8c-dcbb-421e7c377337"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ]
        }
      ],
      "source": [
        "from google.colab import drive\n",
        "drive.mount(\"/content/gdrive\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NpZ9Lv38FBt8"
      },
      "outputs": [],
      "source": [
        "with open('/content/gdrive/MyDrive/ProjectEmbeddings/train_sent_emb.pkl', 'rb') as f:\n",
        "    train_sent_emb = pickle.load(f)\n",
        "with open('/content/gdrive/MyDrive/ProjectEmbeddings/test_sent_emb.pkl', 'rb') as f:\n",
        "    test_sent_emb = pickle.load(f)\n",
        "\n",
        "with open('/content/gdrive/MyDrive/ProjectEmbeddings/y_train.pkl', 'rb') as f:\n",
        "    y = pickle.load(f)\n",
        "\n",
        "with open('/content/gdrive/MyDrive/ProjectEmbeddings/y_test.pkl', 'rb') as f:\n",
        "    y_test = pickle.load(f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D-0qB0kIGhF3",
        "outputId": "00987af9-a8c6-4a7a-dbfc-324c42c0c78e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(3128, 768)\n",
            "(783, 768)\n"
          ]
        }
      ],
      "source": [
        "print(train_sent_emb.shape)\n",
        "print(test_sent_emb.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "X7TzdoLQIRIq"
      },
      "outputs": [],
      "source": [
        "\n",
        "classes = ['cohesion', 'syntax', 'vocabulary', 'phraseology', 'grammar', 'conventions',]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lFlh0hXCHNYN"
      },
      "outputs": [],
      "source": [
        "class EssayDataset:\n",
        "    def __init__(self,x_embeddings, labels = False, test = False):\n",
        "        self.test = test\n",
        "        self.texts = x_embeddings\n",
        "        self.classes = ['cohesion', 'syntax', 'vocabulary', 'phraseology', 'grammar', 'conventions',]\n",
        "        if not test:\n",
        "            self.labels = labels.loc[:,self.classes].values\n",
        "            \n",
        "            \n",
        "    def __len__(self):\n",
        "        return len(self.texts)\n",
        "    \n",
        "    def __getitem__(self, index):\n",
        "        text = self.texts[index]\n",
        "        text = torch.tensor(text, dtype=torch.long)\n",
        "        if self.test is False:\n",
        "            label = self.labels[index,:] / 5.\n",
        "            label = torch.tensor(label, dtype=torch.float32)\n",
        "            return text, label\n",
        "        \n",
        "        return text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "u84wWKT8Sc37"
      },
      "outputs": [],
      "source": [
        "train = train_sent_emb[:int(0.8*len(train_sent_emb))]\n",
        "validation = train_sent_emb[int(0.8*len(train_sent_emb)):]\n",
        "y_train = y[:int(0.8*len(y))]\n",
        "y_valid = y[int(0.8*len(y)):]\n",
        "# print(len(train_ds) + len(test_ds))\n",
        "# print(len(train_ds))\n",
        "# print(len(test_ds))\n",
        "# print(len(train_sent_emb))\n",
        "\n",
        "train_ds = EssayDataset(train, labels= y_train)\n",
        "valid_ds = EssayDataset(validation, labels = y_valid)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "V8UddmeHTQIF"
      },
      "outputs": [],
      "source": [
        "config = {\n",
        "    'embed_dim': 15,\n",
        "    'hidden_dim': 8,\n",
        "    'seq_len': 768,\n",
        "    'vocab': 768,\n",
        "    'n_layers': 4,\n",
        "    'output_dim': len(classes),\n",
        "    'lr': 0.025,\n",
        "    'epochs': 25,\n",
        "    'batch_size': 16,\n",
        "    'model_name': 'lstm-embeddings'\n",
        "}\n",
        "train_loader = torch.utils.data.DataLoader(train_ds,batch_size=config['batch_size'],shuffle=True)\n",
        "val_loader = torch.utils.data.DataLoader(valid_ds,batch_size=config['batch_size'])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "qEDj0KNAHlfa"
      },
      "outputs": [],
      "source": [
        "class RNNModel(pl.LightningModule):\n",
        "    def __init__(self, config):\n",
        "        super(RNNModel, self).__init__()\n",
        "        self.save_hyperparameters()\n",
        "        self.config = config\n",
        "        self.vocab_size = self.config['vocab']\n",
        "        self.embed_dim = self.config['embed_dim']\n",
        "        self.hidden_dim = self.config['hidden_dim']\n",
        "        self.seq_len = self.config['seq_len']\n",
        "        self.n_layers = self.config['n_layers']\n",
        "        self.output_dim = self.config['output_dim']\n",
        "        \n",
        "        self.lr = config['lr']\n",
        "        \n",
        "        self.embedding = nn.Embedding(self.vocab_size, self.embed_dim)\n",
        "        \n",
        "        self.lstm = nn.LSTM(input_size=self.embed_dim, \n",
        "                            hidden_size=self.hidden_dim,\n",
        "                            num_layers = self.n_layers,\n",
        "                            batch_first = True,\n",
        "                            dropout = 0.3\n",
        "                           )\n",
        "        self.mean_pool = nn.AvgPool1d(4)\n",
        "        self.linear = nn.Linear(self.hidden_dim//4, self.output_dim)\n",
        "        \n",
        "        \n",
        "        self.test_preds = []\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.embedding(x)\n",
        "        x, (h,c) = self.lstm(x)\n",
        "        #print(\"Type of h[-1] is:\")\n",
        "        #print(type(h[-1]))\n",
        "        pooled = self.mean_pool(h[-1])\n",
        "        x = self.linear(pooled)\n",
        "        return x\n",
        "    \n",
        "    def loss_fn(self, outputs, targets):\n",
        "        colwise_mse = torch.mean(torch.square(targets - outputs), dim=0)\n",
        "        loss = torch.mean(torch.sqrt(colwise_mse), dim=0)\n",
        "        return loss\n",
        "\n",
        "    def configure_optimizers(self):\n",
        "        optimizer = torch.optim.AdamW(self.parameters(), lr=self.lr)\n",
        "        scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,T_max=20,eta_min=1e-6)\n",
        "        return [optimizer], [scheduler]\n",
        "    \n",
        "    def training_step(self, batch, batch_idx):\n",
        "        \n",
        "        x, y = batch\n",
        "        outputs = self(x)\n",
        "        \n",
        "        loss = self.loss_fn(outputs, y)\n",
        "\n",
        "        self.log('train_loss', loss.item(), on_epoch=True)\n",
        "        \n",
        "        return loss\n",
        "    \n",
        "    def validation_step(self, batch, batch_idx):\n",
        "        \n",
        "        x,y = batch\n",
        "        \n",
        "        outputs = self(x)\n",
        "        \n",
        "        loss = self.loss_fn(outputs, y)\n",
        "        \n",
        "        self.log('val_loss', loss.item(), on_epoch=True)\n",
        "        \n",
        "    def test_step(self, batch, batch_idx):\n",
        "        sample = batch\n",
        "        preds = self(sample) * 5.\n",
        "        self.test_preds.append(preds.detach().cpu())\n",
        "        \n",
        "    def get_predictions(self):\n",
        "        return torch.cat(self.test_preds).numpy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PFKjneABIDEH",
        "outputId": "9e7800ff-e962-401a-da29-babcaaff0a15"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:pytorch_lightning.utilities.rank_zero:GPU available: False, used: False\n",
            "INFO:pytorch_lightning.utilities.rank_zero:TPU available: False, using: 0 TPU cores\n",
            "INFO:pytorch_lightning.utilities.rank_zero:IPU available: False, using: 0 IPUs\n",
            "INFO:pytorch_lightning.utilities.rank_zero:HPU available: False, using: 0 HPUs\n"
          ]
        }
      ],
      "source": [
        "model = RNNModel(config)\n",
        "trainer = pl.Trainer(\n",
        "                     callbacks=[\n",
        "                         EarlyStopping(monitor=\"val_loss\", \n",
        "                                       mode=\"min\",\n",
        "                                       patience=5,\n",
        "                                      )\n",
        "                     ],\n",
        "                     max_epochs = config['epochs']\n",
        "                    )\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/",
          "height": 377,
          "referenced_widgets": [
            "6d983cdf48714fdb9a62b44098fcb6ea",
            "4cb80d31f0be416592a16e710f847a58",
            "af225ea194b4400496d0c040bbda1991",
            "de97f57699c7435591ec87d9a9ca2bd8",
            "2795f838118945efb68f43fe83b0a761",
            "770bbbec135146a7abb6cba1ce963a73",
            "6fb20b89d33a437d8c24524a86c4e297",
            "f5d57f03cc9a4fc3891a8c43a0c2c1e8",
            "beb8e49310be4958b7f2d798fb20adb1",
            "57bcd3e477e6491e863bf2c4a36613bc",
            "ecc21748c1a7451b992b3416eb4f985d",
            "c29bc77a53894787a640abd51dc5c42f",
            "b1e3e4257ae249aca75b7b3e5c9efcec",
            "30d9ac4ee3f8429d9c31bcde41256198",
            "0d89d959ccf94aaa8bd9bd2f1ac98f30",
            "0ccaabeaa28d41bf8a1843d6b7c80f34",
            "1c5e4b76097f43319f0edd35fdf86f77",
            "05ffd43b986744f7b3b3c64a7fdd578b",
            "9c63f97834274b0aa3b6b399610a29e4",
            "39d8069db3a34e308ea6c0512161ec54",
            "c8d13cfc2e53469f8c479d8d1ca3cef7",
            "137fd837d47e40ae956bb5ee5adfac5f",
            "3bd6bc3118a744bbb37b3a55392ccb2d",
            "2fc9deaf35034fc5b082846a7ffa0da0",
            "fb02ca4339a344aeb400b9ac2746f332",
            "9a7439bd232344d08b9a64a5ac132923",
            "dabd8ab3991341e1b63ca4e5f3588a5b",
            "b63a893ddd8245d4b84b58346a754984",
            "f40c1f977ab14c5e8287274525884ead",
            "00f4e1fa805c49d08df1bd10b1eae624",
            "0c1e8f55f5b0443a8a0fd1501f9e2cba",
            "9d5a3610178241b3802d632b1283fc68",
            "7b80e49dbff649d38363bb85bb762672",
            "7ff7c555714e4d7a900665f141d5c219",
            "c73cb96808e14b7db690d4972e48d8c8",
            "ea7d4d54772341efbd6ae6523659a811",
            "f364ba35efee480d9bbee75caa78ba3b",
            "0ed12ff0b51e455682af2f4286855da5",
            "3b969951e8534884851450f5f70dd649",
            "a5523cb11b8e4bfdbe36013d53b1c182",
            "c465786437c54840b5acd59e9421b416",
            "8effbcbebeef40ea9aaddcfd2d552141",
            "d942235ae5304861b1ae2658f11f0247",
            "c9c691c77d674f9dbcd2f8be2eaf3206",
            "677ac3dac3404ec8b7ec110f5cb63477",
            "be9dc20a4a734534ae55295b6cb3ab8e",
            "eee82ba3650a4c008e5e954b87a3f09f",
            "e36547747e634fa2bb99fa9e35e92e85",
            "9a76f8632fbf469c8e80b2b7e7877607",
            "6a1060dfc6684b20849745b98e988000",
            "cc3a4985dfda477aa581568b55c7a09f"
          ]
        },
        "id": "kX2SpxQEQkLS",
        "outputId": "a178603f-c1d6-4f8e-eb72-629d8df00c13"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "INFO:pytorch_lightning.callbacks.model_summary:\n",
            "  | Name      | Type      | Params\n",
            "----------------------------------------\n",
            "0 | embedding | Embedding | 11.5 K\n",
            "1 | lstm      | LSTM      | 2.5 K \n",
            "2 | mean_pool | AvgPool1d | 0     \n",
            "3 | linear    | Linear    | 18    \n",
            "----------------------------------------\n",
            "14.1 K    Trainable params\n",
            "0         Non-trainable params\n",
            "14.1 K    Total params\n",
            "0.056     Total estimated model params size (MB)\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6d983cdf48714fdb9a62b44098fcb6ea",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Sanity Checking: 0it [00:00, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c29bc77a53894787a640abd51dc5c42f",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Training: 0it [00:00, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3bd6bc3118a744bbb37b3a55392ccb2d",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Validation: 0it [00:00, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7ff7c555714e4d7a900665f141d5c219",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Validation: 0it [00:00, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "677ac3dac3404ec8b7ec110f5cb63477",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Validation: 0it [00:00, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "be9dc20a4a734534ae55295b6cb3ab8e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Validation: 0it [00:00, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "eee82ba3650a4c008e5e954b87a3f09f",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Validation: 0it [00:00, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e36547747e634fa2bb99fa9e35e92e85",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Validation: 0it [00:00, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9a76f8632fbf469c8e80b2b7e7877607",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Validation: 0it [00:00, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "6a1060dfc6684b20849745b98e988000",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Validation: 0it [00:00, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "cc3a4985dfda477aa581568b55c7a09f",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Validation: 0it [00:00, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "{'train_loss_step': tensor(0.1240), 'val_loss': tensor(0.1276), 'train_loss_epoch': tensor(0.1336)}\n"
          ]
        }
      ],
      "source": [
        "trainer.fit(model, train_loader, val_loader)\n",
        "metrics = trainer.logged_metrics\n",
        "\n",
        "logs = {\n",
        "    'train_loss': metrics['train_loss_epoch'].item(),\n",
        "    'val_loss': metrics['val_loss'].item()\n",
        "}\n",
        "print(metrics)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "referenced_widgets": [
            "3ab462624f3d4def9350829323d9d799"
          ]
        },
        "id": "qzlM36fLIxMO",
        "outputId": "802616c6-ac13-4f13-da8a-8eb04b91e00f"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "3ab462624f3d4def9350829323d9d799",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Testing: 0it [00:00, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "test_ds = EssayDataset(test_sent_emb, test = True)\n",
        "test_loader = torch.utils.data.DataLoader(test_ds,batch_size=1,shuffle=False)\n",
        "trainer.test(model,test_loader)\n",
        "p = model.get_predictions()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "0KGJhjC5Vma4",
        "outputId": "228a4482-47fe-495c-f41d-e5be6fbb51bd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(783, 768)\n"
          ]
        }
      ],
      "source": [
        "print(test_sent_emb.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "YJL2L6dihZZb",
        "outputId": "6504e116-560a-4c99-adef-bacf411bca28"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "tf.Tensor(0.6493355784899156, shape=(), dtype=float64)\n"
          ]
        }
      ],
      "source": [
        "def mcrmse(y_true, y_pred):\n",
        "    colwise_mse = tf.reduce_mean(tf.square(y_true - y_pred), axis=0)\n",
        "    return tf.reduce_mean(tf.sqrt(colwise_mse), axis=0)\n",
        "\n",
        "print(mcrmse(p,y_test))"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "00f4e1fa805c49d08df1bd10b1eae624": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": "2",
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "05ffd43b986744f7b3b3c64a7fdd578b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0c1e8f55f5b0443a8a0fd1501f9e2cba": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0ccaabeaa28d41bf8a1843d6b7c80f34": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "inline-flex",
            "flex": null,
            "flex_flow": "row wrap",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "100%"
          }
        },
        "0d89d959ccf94aaa8bd9bd2f1ac98f30": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c8d13cfc2e53469f8c479d8d1ca3cef7",
            "placeholder": "​",
            "style": "IPY_MODEL_137fd837d47e40ae956bb5ee5adfac5f",
            "value": " 18/197 [00:12&lt;02:04,  1.43it/s, loss=0.132, v_num=1]"
          }
        },
        "0ed12ff0b51e455682af2f4286855da5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "inline-flex",
            "flex": null,
            "flex_flow": "row wrap",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": "100%"
          }
        },
        "137fd837d47e40ae956bb5ee5adfac5f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1c5e4b76097f43319f0edd35fdf86f77": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2795f838118945efb68f43fe83b0a761": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "inline-flex",
            "flex": null,
            "flex_flow": "row wrap",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": "100%"
          }
        },
        "2fc9deaf35034fc5b082846a7ffa0da0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b63a893ddd8245d4b84b58346a754984",
            "placeholder": "​",
            "style": "IPY_MODEL_f40c1f977ab14c5e8287274525884ead",
            "value": "Validation DataLoader 0: 100%"
          }
        },
        "30d9ac4ee3f8429d9c31bcde41256198": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9c63f97834274b0aa3b6b399610a29e4",
            "max": 197,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_39d8069db3a34e308ea6c0512161ec54",
            "value": 18
          }
        },
        "39d8069db3a34e308ea6c0512161ec54": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3b969951e8534884851450f5f70dd649": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3bd6bc3118a744bbb37b3a55392ccb2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2fc9deaf35034fc5b082846a7ffa0da0",
              "IPY_MODEL_fb02ca4339a344aeb400b9ac2746f332",
              "IPY_MODEL_9a7439bd232344d08b9a64a5ac132923"
            ],
            "layout": "IPY_MODEL_dabd8ab3991341e1b63ca4e5f3588a5b"
          }
        },
        "4cb80d31f0be416592a16e710f847a58": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_770bbbec135146a7abb6cba1ce963a73",
            "placeholder": "​",
            "style": "IPY_MODEL_6fb20b89d33a437d8c24524a86c4e297",
            "value": "Sanity Checking DataLoader 0: 100%"
          }
        },
        "57bcd3e477e6491e863bf2c4a36613bc": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6d983cdf48714fdb9a62b44098fcb6ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_4cb80d31f0be416592a16e710f847a58",
              "IPY_MODEL_af225ea194b4400496d0c040bbda1991",
              "IPY_MODEL_de97f57699c7435591ec87d9a9ca2bd8"
            ],
            "layout": "IPY_MODEL_2795f838118945efb68f43fe83b0a761"
          }
        },
        "6fb20b89d33a437d8c24524a86c4e297": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "770bbbec135146a7abb6cba1ce963a73": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7b80e49dbff649d38363bb85bb762672": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7ff7c555714e4d7a900665f141d5c219": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c73cb96808e14b7db690d4972e48d8c8",
              "IPY_MODEL_ea7d4d54772341efbd6ae6523659a811",
              "IPY_MODEL_f364ba35efee480d9bbee75caa78ba3b"
            ],
            "layout": "IPY_MODEL_0ed12ff0b51e455682af2f4286855da5"
          }
        },
        "8effbcbebeef40ea9aaddcfd2d552141": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9a7439bd232344d08b9a64a5ac132923": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9d5a3610178241b3802d632b1283fc68",
            "placeholder": "​",
            "style": "IPY_MODEL_7b80e49dbff649d38363bb85bb762672",
            "value": " 40/40 [00:04&lt;00:00,  8.17it/s]"
          }
        },
        "9c63f97834274b0aa3b6b399610a29e4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": "2",
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9d5a3610178241b3802d632b1283fc68": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a5523cb11b8e4bfdbe36013d53b1c182": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "af225ea194b4400496d0c040bbda1991": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f5d57f03cc9a4fc3891a8c43a0c2c1e8",
            "max": 2,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_beb8e49310be4958b7f2d798fb20adb1",
            "value": 2
          }
        },
        "b1e3e4257ae249aca75b7b3e5c9efcec": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1c5e4b76097f43319f0edd35fdf86f77",
            "placeholder": "​",
            "style": "IPY_MODEL_05ffd43b986744f7b3b3c64a7fdd578b",
            "value": "Epoch 3:   9%"
          }
        },
        "b63a893ddd8245d4b84b58346a754984": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "beb8e49310be4958b7f2d798fb20adb1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c29bc77a53894787a640abd51dc5c42f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b1e3e4257ae249aca75b7b3e5c9efcec",
              "IPY_MODEL_30d9ac4ee3f8429d9c31bcde41256198",
              "IPY_MODEL_0d89d959ccf94aaa8bd9bd2f1ac98f30"
            ],
            "layout": "IPY_MODEL_0ccaabeaa28d41bf8a1843d6b7c80f34"
          }
        },
        "c465786437c54840b5acd59e9421b416": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": "2",
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c73cb96808e14b7db690d4972e48d8c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3b969951e8534884851450f5f70dd649",
            "placeholder": "​",
            "style": "IPY_MODEL_a5523cb11b8e4bfdbe36013d53b1c182",
            "value": "Validation DataLoader 0: 100%"
          }
        },
        "c8d13cfc2e53469f8c479d8d1ca3cef7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c9c691c77d674f9dbcd2f8be2eaf3206": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d942235ae5304861b1ae2658f11f0247": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dabd8ab3991341e1b63ca4e5f3588a5b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "inline-flex",
            "flex": null,
            "flex_flow": "row wrap",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": "hidden",
            "width": "100%"
          }
        },
        "de97f57699c7435591ec87d9a9ca2bd8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_57bcd3e477e6491e863bf2c4a36613bc",
            "placeholder": "​",
            "style": "IPY_MODEL_ecc21748c1a7451b992b3416eb4f985d",
            "value": " 2/2 [00:00&lt;00:00,  8.82it/s]"
          }
        },
        "ea7d4d54772341efbd6ae6523659a811": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c465786437c54840b5acd59e9421b416",
            "max": 40,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8effbcbebeef40ea9aaddcfd2d552141",
            "value": 40
          }
        },
        "ecc21748c1a7451b992b3416eb4f985d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f364ba35efee480d9bbee75caa78ba3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d942235ae5304861b1ae2658f11f0247",
            "placeholder": "​",
            "style": "IPY_MODEL_c9c691c77d674f9dbcd2f8be2eaf3206",
            "value": " 40/40 [00:04&lt;00:00,  8.53it/s]"
          }
        },
        "f40c1f977ab14c5e8287274525884ead": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f5d57f03cc9a4fc3891a8c43a0c2c1e8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": "2",
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fb02ca4339a344aeb400b9ac2746f332": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_00f4e1fa805c49d08df1bd10b1eae624",
            "max": 40,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0c1e8f55f5b0443a8a0fd1501f9e2cba",
            "value": 40
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}