Character-based language model from scratch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Character-based language model from scratch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from fastai.io import *\n",
    "from fastai.conv_learner import *\n",
    "from fastai.column_data import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset - Nietzsche"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "PATH = 'data/nietzsche/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = open(f'{PATH}nietzsche.txt', encoding=\"utf-8\").read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "600893"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'PREFACE\\n\\n\\nSUPPOSING that Truth is a woman--what then? Is there not ground\\nfor suspecting that all philosophers, in so far as they have been\\ndogmatists, have failed to understand women--that the terrible\\nseriousness and clumsy importunity with which they have usually paid\\ntheir addresses to Truth, ha'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text[:300]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "chars = sorted(list(set(text)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total number of chars:  85\n"
     ]
    }
   ],
   "source": [
    "chars.insert(0, \"\\0\")  # zeror value, e.g. for padding\n",
    "voc_size = len(chars)\n",
    "print('total number of chars: ', voc_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n !\"\\'(),-.0123456789:;=?ABCDEFGHIJKLMNOPQRSTUVWXYZ[]_abcdefghijklmnopqrstuvwxy'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''.join(chars[1:-6])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Mapping from char to id and vice versa:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "char_indices = {c: i for i, c in enumerate(chars)}\n",
    "indices_char = {i: c for i, c in enumerate(chars)}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Convert dataset to indices**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx = [char_indices[c] for c in text]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[40, 42, 29, 30, 25, 27, 29, 1, 1, 1]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Quick test to make sure this is working as expected:**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'PREFACE\\n\\n\\nSUPPOSING that Truth is a woman--what th'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''.join(indices_char[i] for i in idx[:50])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predicting the 4th char from the first 3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Creating the inputs and output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "cs = 3\n",
    "# Inputs:\n",
    "c1_dat = [idx[i] for i in range(0, len(idx) - 1 - cs, cs)]  # datasets: every 3 chars, shifted by one\n",
    "c2_dat = [idx[i+1] for i in range(0, len(idx) - 1 - cs, cs)]\n",
    "c3_dat = [idx[i+2] for i in range(0, len(idx) - 1 - cs, cs)]\n",
    "# Output:\n",
    "c4_dat = [idx[i+3] for i in range(0, len(idx) - 1 - cs, cs)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[40, 42, 29, 30, 25, 27, 29, 1, 1, 1]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([40, 30, 29], [42, 25, 1], [29, 27, 1], [30, 29, 1])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "c1_dat[:3], c2_dat[:3], c3_dat[:3], c4_dat[:3]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Inputs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "x1 = np.stack(c1_dat[:-2])\n",
    "x2 = np.stack(c2_dat[:-2])\n",
    "x3 = np.stack(c3_dat[:-2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([40, 30, 29, ..., 67, 68, 72]),\n",
       " array([42, 25,  1, ..., 72, 59, 62]),\n",
       " array([29, 27,  1, ...,  2,  2, 67]))"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x1, x2, x3"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Output:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = np.stack(c4_dat[:-2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([30, 29,  1, ..., 68, 72, 59])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**We will try to predicht 30 from 40, 42, and 29 etc.**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create and train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_hidden = 256"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_emb = 42"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Char3Model(nn.Module):\n",
    "    def __init__(self, voc_size, n_emb):\n",
    "        super().__init__()\n",
    "        self.e = nn.Embedding(voc_size, n_emb)\n",
    "        \n",
    "        # input to hidden\n",
    "        self.l_in = nn.Linear(n_emb, n_hidden)\n",
    "\n",
    "        # hidden to hidden\n",
    "        self.l_hid = nn.Linear(n_hidden, n_hidden)\n",
    "\n",
    "        # hidden to output\n",
    "        self.l_out = nn.Linear(n_hidden, voc_size)\n",
    "        \n",
    "    def forward(self, c1, c2, c3):\n",
    "        in1 = F.relu(self.l_in(self.e(c1)))\n",
    "        in2 = F.relu(self.l_in(self.e(c2)))\n",
    "        in3 = F.relu(self.l_in(self.e(c3)))\n",
    "        \n",
    "        h = V(torch.zeros(in1.size())).cuda()\n",
    "        h = torch.tanh(self.l_hid(h + in1))\n",
    "        h = torch.tanh(self.l_hid(h + in2))\n",
    "        h = torch.tanh(self.l_hid(h + in3))\n",
    "        \n",
    "        return F.log_softmax(self.l_out(h))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "modeldata = ColumnarModelData.from_arrays('.', [-1], np.stack([x1, x2, x3], axis=1), y, bs=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Char3Model(voc_size, n_emb).cuda()  # standard pytorch model, not fastai"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n",
      "torch.Size([512, 85])\n"
     ]
    }
   ],
   "source": [
    "# Let's look at the output from the model\n",
    "it = iter(modeldata.trn_dl)\n",
    "*xs, yt = next(it)\n",
    "print(len(xs))\n",
    "t = model(*V(xs))\n",
    "print(t.shape)  # 85 probs for chars, bs = 512"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = optim.Adam(model.parameters(), 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a228df4b0414474bf2eb5cd3790842e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                               \n",
      "    0      2.11806    4.820609  \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[4.820609092712402]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit(model, modeldata, 1, opt, F.nll_loss)  # negative log likelihood loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "set_lrs(opt, 0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fe1be26c827b4feaa61726a74d1453fd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                               \n",
      "    0      1.866076   5.203579  \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[5.203579425811768]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit(model, modeldata, 1, opt, F.nll_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_next_char(inp):\n",
    "    idxs = T(np.array([char_indices[c] for c in inp]))\n",
    "    p = model(*V(idxs))  # log probability\n",
    "    predicted_char = np.argmax(p.data.cpu().numpy())\n",
    "    return indices_char[predicted_char]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'T'"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_next_char('y. ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'e'"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_next_char(' th')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'e'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_next_char('ppl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_next_char('wom')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'n'"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_next_char('oma')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Seems to work quite well!**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating a first RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "cs = 8  # size of the RNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[40, 42, 29, 30, 25, 27, 29, 1, 1, 1, 43, 45, 40, 40, 39, 43, 33, 38, 31, 2]"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idx[:20]  # The first 20 chars converted to indices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "char_inp_data = [[idx[i + j] for i in range(cs)] for j in range(len(idx) - cs)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[40, 42, 29, 30, 25, 27, 29, 1],\n",
       " [42, 29, 30, 25, 27, 29, 1, 1],\n",
       " [29, 30, 25, 27, 29, 1, 1, 1],\n",
       " [30, 25, 27, 29, 1, 1, 1, 43]]"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_inp_data[:4]  # Sequences of cs = 8 chars, shifted by 1 (0 to 7, 1 to 8, 2 to 9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "char_outp_data = [idx[j + cs] for j in range(len(idx) - cs)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 1, 43, 45]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "char_outp_data[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "xs = np.stack(char_inp_data, axis=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(600885, 8)"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[40, 42, 29, 30, 25, 27, 29,  1],\n",
       "       [42, 29, 30, 25, 27, 29,  1,  1],\n",
       "       [29, 30, 25, 27, 29,  1,  1,  1],\n",
       "       [30, 25, 27, 29,  1,  1,  1, 43]])"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs[:4]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = np.stack(char_outp_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 1,  1, 43, 45])"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y[:4]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create and train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_idxs = get_cv_idxs(len(idx) - cs - 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "modeldata = ColumnarModelData.from_arrays('.', val_idxs, xs, y, bs=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CharLoopModel(nn.Module):\n",
    "    # RNN\n",
    "    def __init__(self, voc_size, n_emb, n_hidden):\n",
    "        super().__init__()\n",
    "        self.e = nn.Embedding(voc_size, n_emb)\n",
    "        self.l_in = nn.Linear(n_emb, n_hidden)\n",
    "        self.l_hid =  nn.Linear(n_hidden, n_hidden)\n",
    "        self.l_out = nn.Linear(n_hidden, voc_size)\n",
    "        \n",
    "    def forward(self, *cs):\n",
    "        bs = cs[0].size(0)\n",
    "        h = V(torch.zeros(bs, n_hidden).cuda())\n",
    "        for c in cs:\n",
    "            inp = F.relu(self.l_in(self.e(c)))\n",
    "            h = torch.tanh(self.l_hid(inp + h))  # tanh is good here becaus it limits the hidden state to -1, 1\n",
    "            \n",
    "        return F.log_softmax(self.l_out(h), dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CharLoopModel(voc_size, n_emb, n_hidden).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = optim.Adam(model.parameters(), 1e-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68fd29bfd0a34689bf479835c0e054a5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                              \n",
      "    0      1.984019   1.985254  \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.9852543738214967]"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit(model, modeldata, 1, opt, F.nll_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = optim.Adam(model.parameters(), 1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4c4006d4b3b5450bb08eda8c436f084e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                              \n",
      "    0      1.683578   1.683077  \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.6830767515162435]"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit(model, modeldata, 1, opt, F.nll_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Concat the input and hidden state together instead of adding them\n",
    "Concatenating is better than adding since input and hidden state are qualitatively different and you loose information by adding them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CharLoopConcatModel(nn.Module):\n",
    "    def __init__(self, voc_size, n_emb, n_hidden):\n",
    "        super().__init__()\n",
    "        self.e = nn.Embedding(voc_size, n_emb)\n",
    "        self.l_in = nn.Linear(n_emb + n_hidden, n_hidden)\n",
    "        self.l_hid = nn.Linear(n_hidden, n_hidden)\n",
    "        self.l_out = nn.Linear(n_hidden, voc_size)\n",
    "        \n",
    "    def forward(self, *cs):\n",
    "        bs = cs[0].size(0)\n",
    "        h = V(torch.zeros(bs, n_hidden).cuda())\n",
    "        for c in cs:\n",
    "            inp = torch.cat((h, self.e(c)), dim=1)\n",
    "            inp = F.relu(self.l_in(inp))\n",
    "            h = torch.tanh(self.l_hid(inp))\n",
    "            \n",
    "        return F.log_softmax(self.l_out(h), dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CharLoopConcatModel(voc_size, n_emb, n_hidden).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = optim.Adam(model.parameters(), 1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "it = iter(modeldata.trn_dl)\n",
    "*xs,yt = next(it)\n",
    "t = model(*V(xs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "512"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs[0].size(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ea068588df264283b765a1c7d3049036",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                              \n",
      "    0      1.867465   1.834486  \n",
      "    1      1.656378   1.647034                              \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.6470337330842681]"
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit(model, modeldata, 2, opt, F.nll_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = optim.Adam(model.parameters(), 1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6fc3d610a32f4e99a16eda140023cd92",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                              \n",
      "    0      1.565058   1.584099  \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.5840994116311193]"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit(model, modeldata, 1, opt, F.nll_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_next_char(inp):\n",
    "    idxs = T(np.array([char_indices[c] for c in inp])).unsqueeze(1)\n",
    "    p = model(*VV(idxs))  # log probability\n",
    "    predicted_char = np.argmax(p.data.cpu().numpy())\n",
    "    return indices_char[predicted_char]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'e'"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_next_char('for thos')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'t'"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_next_char('part of ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a'"
      ]
     },
     "execution_count": 67,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_next_char(' and wom')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Now the same in pytorch\n",
    "### pytorch creates the input layers and the loop automatically"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CharRnn(nn.Module):\n",
    "    def __init__(self, voc_size, n_emb):\n",
    "        super().__init__()\n",
    "        self.e = nn.Embedding(voc_size, n_emb)\n",
    "        self.rnn = nn.RNN(n_emb, n_hidden)\n",
    "        self.l_out = nn.Linear(n_hidden, voc_size)\n",
    "        \n",
    "    def forward(self, *cs):\n",
    "        bs = cs[0].size(0)\n",
    "        # hidden state = rank 3 tensor in pytorch\n",
    "        # needed for bidirectional rnn and multilayer rnns \n",
    "        h = V(torch.zeros(1, bs, n_hidden).cuda())\n",
    "        inp = self.e(torch.stack(cs))\n",
    "        outp, h = self.rnn(inp, h)\n",
    "        \n",
    "        # pytorch appends the output to a list => -1\n",
    "        return F.log_softmax(self.l_out(outp[-1]), dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CharRnn(voc_size, n_emb).cuda()\n",
    "opt = optim.Adam(model.parameters(), 1e-3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's look at the dimensionality of the output:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [],
   "source": [
    "it = iter(modeldata.trn_dl)\n",
    "*xs, yt = next(it)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "embedding:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = model.e(V(torch.stack(xs)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 512, 42])"
      ]
     },
     "execution_count": 72,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.size()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Hidden state for input:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "ht = V(torch.zeros(1, 512, n_hidden))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Output of the rnn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "outp, hn = model.rnn(t, ht)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(torch.Size([8, 512, 256]), torch.Size([1, 512, 256]))"
      ]
     },
     "execution_count": 75,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "outp.size(), hn.size()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Dimensionality of output:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = model(*V(xs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([512, 85])"
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t.size()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fbe2d9a703f443dab257cd101f646343",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                              \n",
      "    0      1.871305   1.846348  \n",
      "    1      1.680553   1.671313                              \n",
      "    2      1.588609   1.597636                              \n",
      "    3      1.541642   1.553672                              \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.5536721405184963]"
      ]
     },
     "execution_count": 78,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit(model, modeldata, 4, opt, F.nll_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = optim.Adam(model.parameters(), 1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cbab9983a835484586e49b4876f8aca8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                              \n",
      "    0      1.473815   1.511779  \n",
      "    1      1.46036    1.506225                              \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.5062247376168334]"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit(model, modeldata, 2, opt, F.nll_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Test the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_next_char(inp):\n",
    "    idxs = T(np.array([char_indices[c] for c in inp])).unsqueeze(1)\n",
    "    p = model(*VV(idxs))  # log probability\n",
    "    predicted_char = np.argmax(p.data.cpu().numpy())\n",
    "    return indices_char[predicted_char]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_next_n_chars(inp, n):\n",
    "    result = inp\n",
    "    for i in range(n):\n",
    "        c = get_next_char(inp)\n",
    "        result += c\n",
    "        inp = inp[1:] + c\n",
    "    return result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Humanity of the such and and and and and and '"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_next_n_chars('Human', 40)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Multi-output model\n",
    "Until now, we used 8 chars to predict the next one. Then, we shifted by one char to predict the following char. However, this means that 7 chars overlapped!\n",
    "\n",
    "**Let's take a non-overlapping set of characters this time!**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's create the input:\n",
    "\n",
    "(`idx` is the entire text converted to indices.)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "c_in_dat = [[idx[i + j] for i in range(cs)] for j in range (0, len(idx) - cs - 1, cs)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And the output (everything shifted by one char):"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "c_out_dat= [[idx[i + j] for i in range(cs)] for j in range(1, len(idx) - cs, cs)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((75111, 8), (75111, 8))"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs = np.stack(c_in_dat)\n",
    "ys = np.stack(c_out_dat)\n",
    "\n",
    "xs.shape, ys.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[40, 42, 29, 30, 25, 27, 29,  1],\n",
       "       [ 1,  1, 43, 45, 40, 40, 39, 43],\n",
       "       [33, 38, 31,  2, 73, 61, 54, 73],\n",
       "       [ 2, 44, 71, 74, 73, 61,  2, 62]])"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xs[:4,]  # Reading top left to bottom right this is the nietzsche text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[42, 29, 30, 25, 27, 29,  1,  1],\n",
       "       [ 1, 43, 45, 40, 40, 39, 43, 33],\n",
       "       [38, 31,  2, 73, 61, 54, 73,  2],\n",
       "       [44, 71, 74, 73, 61,  2, 62, 72]])"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ys[:4,]  # the same apart from the missing first character"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Build the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_idxs = get_cv_idxs(len(xs) - cs - 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "modeldata = ColumnarModelData.from_arrays('.', val_idxs, xs, ys, bs=512)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CharSeqRnn(nn.Module):\n",
    "    def __init__(self, voc_size, n_emb, n_hidden):\n",
    "        super().__init__()\n",
    "        self.e = nn.Embedding(voc_size, n_emb)\n",
    "        self.rnn = nn.RNN(n_emb, n_hidden)\n",
    "        self.l_out = nn.Linear(n_hidden, voc_size)\n",
    "        \n",
    "    def forward(self, *cs):\n",
    "        bs = cs[0].size(0)\n",
    "        h = V(torch.zeros(1, bs, n_hidden))\n",
    "        inp = self.e(torch.stack(cs))\n",
    "        outp, h = self.rnn(inp, h)\n",
    "\n",
    "        # only difference is that in the last model \n",
    "        # we only returned ... of outp[-1]\n",
    "        return F.log_softmax(self.l_out(outp), dim=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CharSeqRnn(voc_size, n_emb, n_hidden).cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = optim.Adam(model.parameters(), 1e-3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Negative log likelyhood expects two rank two tensors, but because we are considering the timestep, we have a rank 3 tensor."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "def nll_loss_seq(pred, target):\n",
    "    # pytorch rnn appends the outputs, sequence length sl is the size of the list = 8 timesteps\n",
    "    sl, bs, n_char = pred.size()  # torch.Size([8, 512, 85])\n",
    "    \n",
    "    # torch.Size([8, 512, 85]) => torch.Size([4096, 85])\n",
    "    pred = pred.view(-1, n_char)\n",
    "    \n",
    "    # targ has one label for each of the 8 timesteps\n",
    "    # torch.Size([512, 8]) => torch.Size([4096])\n",
    "    target = target.transpose(0, 1).contiguous().view(-1)\n",
    "    # when transposing, pytorch actually does not change \n",
    "    # the memory, just updates the stride..\n",
    "    # contiguous() makes pytorch update the memory so that\n",
    "    # the transposed object is contiguous in memroy\n",
    "    \n",
    "    return F.nll_loss(pred.view(-1, n_char), target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e2ef3c4725e4c85b1bb47e0f1a035a2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                              \n",
      "    0      2.582693   2.399541  \n",
      "    1      2.284425   2.195281                              \n",
      "    2      2.133081   2.081026                              \n",
      "    3      2.045372   2.011189                              \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[2.0111888978833683]"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit(model, modeldata, 4, opt, nll_loss_seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "opt = optim.Adam(model.parameters(), 1e-4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c4a104a067364891b8975f737d020a64",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=2), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                              \n",
      "    0      1.994704   1.993445  \n",
      "    1      1.985808   1.986614                              \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.9866136107400318]"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit(model, modeldata, 2, opt, nll_loss_seq)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Note: this model is not any better than the previous, but it is more efficient because we use 8 chars to predict 8 chars and now 8 chars to predicht one char.**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Improve the model: better initialization\n",
    "\n",
    "Problem: if the weights of the hidden to hidden layer are significantly smaller than 1, the gradients might vanish, if they are significantly larger, they might explode.\n",
    "\n",
    "Solution: initialize with identity matrix\n",
    "\n",
    "(*A simple way to initialize recurrent networks of rectified linear units*, Navdeep and Geoffrey Hinton, 2015)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CharSeqRnn(voc_size, n_emb, n_hidden).cuda()\n",
    "opt = optim.Adam(model.parameters(), 1e-2)\n",
    "\n",
    "# We can even use a larger learning rate now"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1., 0., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 1., 0.,  ..., 0., 0., 0.],\n",
       "        [0., 0., 1.,  ..., 0., 0., 0.],\n",
       "        ...,\n",
       "        [0., 0., 0.,  ..., 1., 0., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 1., 0.],\n",
       "        [0., 0., 0.,  ..., 0., 0., 1.]], device='cuda:0')"
      ]
     },
     "execution_count": 99,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.rnn.weight_hh_l0.data.copy_(torch.eye(n_hidden))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6cffe207abbf47e5893de0c5ee8ccbfa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                              \n",
      "    0      2.39149    2.205234  \n",
      "    1      2.111292   2.03962                               \n",
      "    2      2.002707   1.982047                              \n",
      "    3      1.949826   1.936901                              \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.9369013573929728]"
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit(model, modeldata, 4, opt, nll_loss_seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d1e75084b4b9467ca1bc664be971f9d9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, description='Epoch', max=4), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch      trn_loss   val_loss                             \n",
      "    0      1.902322   1.925076  \n",
      "    1      1.893846   1.916647                              \n",
      "    2      1.870862   1.879582                              \n",
      "    3      1.854034   1.881128                              \n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[1.8811284509701989]"
      ]
     },
     "execution_count": 101,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fit(model, modeldata, 4, opt, nll_loss_seq)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:deeplearning]",
   "language": "python",
   "name": "conda-env-deeplearning-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}