05b_early_stopping_jcat.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "from exp.nb_05 import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Jump_to notebook introduction in lesson 10 video](https://course.fast.ai/videos/?lesson=10&t=3167)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Early stopping"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Better callback cancellation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Jump_to lesson 10 video](https://course.fast.ai/videos/?lesson=10&t=3230)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the data, set the loss function\n",
    "x_train,y_train,x_valid,y_valid = get_data()\n",
    "train_ds,valid_ds = Dataset(x_train, y_train),Dataset(x_valid, y_valid)\n",
    "n_hidden,batch_size = 50,512\n",
    "n_out = y_train.max().item()+1\n",
    "loss_func = F.cross_entropy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a DataBunch\n",
    "data = DataBunch(*get_dls(train_ds, valid_ds, batch_size), n_out)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Slightly refactor Callback() and add three Cancellation callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "\n",
    "# Callback() class is slightly refactored from notebook 04_callbacks\n",
    "class Callback():\n",
    "    \n",
    "    # initialize _order to zero. \n",
    "    _order=0\n",
    "    \n",
    "    # set_runner() method takes a callback as an input\n",
    "    #     note that initially self.run is unset -- there is no default value \n",
    "    def set_runner(self, run): \n",
    "        self.run=run\n",
    "    def __getattr__(self, callback_name): \n",
    "        return getattr(self.run, callback_name)\n",
    "    \n",
    "    # set the callback name property\n",
    "    #     if the callback doesn't have a name, set the callback name property to 'callback'\n",
    "    @property\n",
    "    def name(self):\n",
    "        name = re.sub(r'Callback$', '', self.__class__.__name__)\n",
    "        return camel2snake(name or 'callback')\n",
    "    \n",
    "    # this is the only modification to the 04_callbacks notebook\n",
    "    #     it allows the Callback() class to be called as a function\n",
    "    def __call__(self, callback_name):\n",
    "        f = getattr(self, callback_name, None)\n",
    "        # check this callback name, and return True if it is the requested callback\n",
    "        if f and f(): return True\n",
    "        return False\n",
    "\n",
    "# this helper callback is used in Runner()\n",
    "class TrainEvalCallback(Callback):\n",
    "    \n",
    "    # initialize the epoch, batch, and iteration counters\n",
    "    def begin_fit(self):\n",
    "        self.run.n_epoch_float=0.\n",
    "        self.run.n_batch = 0\n",
    "        self.run.n_iter=0\n",
    "    \n",
    "    # if we are in the training phase, update the epoch and batch counters\n",
    "    def after_batch(self):\n",
    "        if not self.in_train: \n",
    "            return\n",
    "        # each batch represents a fraction of an epoch\n",
    "        self.run.n_epoch_float += 1./self.n_batches\n",
    "        self.run.n_batch   += 1\n",
    "        \n",
    "    # execute the training phase\n",
    "    def begin_epoch(self):\n",
    "        self.run.n_epoch_float=self.n_epoch_float\n",
    "        self.model.train()\n",
    "        self.run.in_train=True\n",
    "\n",
    "    # execute the prediction phase\n",
    "    def begin_validate(self):\n",
    "        self.model.eval()\n",
    "        self.run.in_train=False\n",
    "\n",
    "# add three cancellation callbacks\n",
    "class CancelTrainException(Exception): pass\n",
    "class CancelEpochException(Exception): pass\n",
    "class CancelBatchException(Exception): pass"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Refactor Runner() to use the cancellation callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class Runner():\n",
    "\n",
    "    # initialize by setting the stop Flag to False, and constructing a list of callbacks from the inputs\n",
    "    def __init__(self, callbacks=None, callback_funcs=None):\n",
    "        # inputs are two lists: callbacks and callback_funcs\n",
    "        # Q: it's not clear why we need two lists rather than one\n",
    "        # create a list of callbacks from the input callbacks\n",
    "        self.in_train = False\n",
    "        callbacks = listify(callbacks)\n",
    "        # associate each callback_func() to its snake case callback name then append it to the callbacks list\n",
    "        for callback_func in listify(callback_funcs):\n",
    "            callback = callback_func()\n",
    "            setattr(self, callback.name, callback)\n",
    "            callbacks.append(callback)\n",
    "        # set the stopping flag to `False` and append TrainEvalCallback() to the callbacks list\n",
    "        self.stop,self.callbacks = False,[TrainEvalCallback()]+callbacks\n",
    "\n",
    "    # get the properties of the Learner object\n",
    "    @property\n",
    "    def opt(self):       return self.learn.opt\n",
    "    @property\n",
    "    def model(self):     return self.learn.model\n",
    "    @property\n",
    "    def loss_func(self): return self.learn.loss_func\n",
    "    @property\n",
    "    def data(self):      return self.learn.data\n",
    "    \n",
    "    \n",
    "    # method to process a single batch\n",
    "    def one_batch(self, xb, yb):\n",
    "        try:\n",
    "            self.xb,self.yb = xb,yb\n",
    "            self('begin_batch')\n",
    "            self.pred = self.model(self.xb)\n",
    "            self('after_pred')\n",
    "            self.loss = self.loss_func(self.pred, self.yb)\n",
    "            self('after_loss')\n",
    "            if not self.in_train: return\n",
    "            self.loss.backward()\n",
    "            self('after_backward')\n",
    "            self.opt.step()\n",
    "            self('after_step')\n",
    "            self.opt.zero_grad()\n",
    "        except CancelBatchException: self('after_cancel_batch')\n",
    "        finally: self('after_batch')\n",
    "\n",
    "    # method to process all batches\n",
    "    def all_batches(self, dataloader):\n",
    "        # total number of batches in an epoch\n",
    "        # self.n_epoch_float = 0.\n",
    "        self.n_batches = len(dataloader)\n",
    "        try:\n",
    "            for xb,yb in dataloader: self.one_batch(xb, yb)\n",
    "        except CancelEpochException: self('after_cancel_epoch')\n",
    "\n",
    "    # method to process training or validation data\n",
    "    def fit(self, learn, n_epochs,):\n",
    "        self.n_epochs,self.learn,self.loss = n_epochs,learn,tensor(0.)\n",
    "\n",
    "        try:\n",
    "            for callback in self.callbacks: \n",
    "                callback.set_runner(self)\n",
    "            self('begin_fit')\n",
    "            for epoch_number in range(n_epochs):\n",
    "                self.epoch_number = epoch_number\n",
    "                \n",
    "                \n",
    "                # training phase\n",
    "                if not self('begin_epoch'): \n",
    "                    self.all_batches(self.data.train_dl)\n",
    "\n",
    "                # validation phase\n",
    "                with torch.no_grad(): \n",
    "                    if not self('begin_validate'): \n",
    "                        self.all_batches(self.data.valid_dl)\n",
    "                self('after_epoch')\n",
    "            \n",
    "        except CancelTrainException: \n",
    "            self('after_cancel_train')\n",
    "            \n",
    "        finally:\n",
    "            # set the `after_fit` state to `True`\n",
    "            self('after_fit')\n",
    "            # erase the Learner object\n",
    "            self.learn = None\n",
    "\n",
    "    def __call__(self, callback_name):\n",
    "        # __call__ allows an instance of this class to be called as a function\n",
    "        # Q: note clear what this loop is trying to do; it always returns result = False\n",
    "        result = False\n",
    "        for callback in sorted(self.callbacks, key=lambda x: x._order): \n",
    "            result = callback(callback_name) and result\n",
    "        return result"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Refactor TestCallback() to use a cancellation callback"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TestCallback(Callback):\n",
    "    _order=1\n",
    "    def after_step(self):\n",
    "        self.n_iter += 1\n",
    "        print(self.n_iter)\n",
    "        if self.n_iter>=10: \n",
    "            raise CancelTrainException()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "learn = create_learner(get_model, loss_func, data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "run = Runner(callback_funcs=TestCallback)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "2\n",
      "3\n",
      "4\n",
      "5\n",
      "6\n",
      "7\n",
      "8\n",
      "9\n",
      "10\n"
     ]
    }
   ],
   "source": [
    "run.fit(learn, n_epochs = 3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. AvgStatsCallback, Recorder, and ParamScheduler Callbacks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#export\n",
    "class AvgStatsCallback(Callback):\n",
    "    def __init__(self, metrics):\n",
    "        self.train_stats,self.valid_stats = AvgStats(metrics,in_train = True),AvgStats(metrics,in_train = False)\n",
    "        \n",
    "    # initialize train_stats and valid_stats\n",
    "    def begin_epoch(self):\n",
    "        self.train_stats.reset()\n",
    "        self.valid_stats.reset()\n",
    "        \n",
    "    # compute and accumulate stats after the loss function has been evaluated\n",
    "    def after_loss(self):\n",
    "        stats = self.train_stats if self.in_train else self.valid_stats\n",
    "        with torch.no_grad(): stats.accumulate(self.run)\n",
    "    \n",
    "    # print stats after the epoch has been processed\n",
    "    def after_epoch(self):\n",
    "        print(self.train_stats)\n",
    "        print(self.valid_stats)\n",
    "        \n",
    "class Recorder(Callback):\n",
    "    def begin_fit(self):\n",
    "        self.lrs = [[] for _ in self.opt.param_groups]\n",
    "        self.losses = []\n",
    "\n",
    "    def after_batch(self):\n",
    "        if not self.in_train: return\n",
    "        for pg,lr in zip(self.opt.param_groups,self.lrs): lr.append(pg['lr'])\n",
    "        self.losses.append(self.loss.detach().cpu())        \n",
    "\n",
    "    def plot_lr  (self, pgid=-1): \n",
    "        plt.plot(self.lrs[pgid])\n",
    "        plt.xlabel('iteration')\n",
    "        plt.ylabel('loss')\n",
    "    def plot_loss(self, skip_last=0): # !!!!! not used\n",
    "        plt.plot(self.losses[:len(self.losses)-skip_last])\n",
    "        plt.xlabel('iteration')\n",
    "        plt.ylabel('loss')\n",
    "\n",
    "        \n",
    "    def plot(self, skip_last=0, pgid=-1):\n",
    "        losses = [o.item() for o in self.losses]\n",
    "        lrs    = self.lrs[pgid]\n",
    "        n = len(losses)-skip_last\n",
    "        plt.xscale('log')\n",
    "        plt.plot(lrs[:n], losses[:n])\n",
    "        plt.xlabel('learning rate')\n",
    "        plt.ylabel('loss')\n",
    "\n",
    "class ParamScheduler(Callback):\n",
    "    _order=1\n",
    "    def __init__(self, pname, sched_funcs): \n",
    "        self.pname,self.sched_funcs = pname,sched_funcs\n",
    "        \n",
    "    def begin_fit(self):\n",
    "        if not isinstance(self.sched_funcs, (list,tuple)):\n",
    "            self.sched_funcs = [self.sched_funcs] * len(self.opt.param_groups)\n",
    "\n",
    "    def set_param(self):\n",
    "        assert len(self.opt.param_groups)==len(self.sched_funcs)\n",
    "        for pg,f in zip(self.opt.param_groups,self.sched_funcs):\n",
    "            # !!!!! this is wrong -- \n",
    "            pg[self.pname] = f(self.n_epochs/self.n_epochs)\n",
    "            \n",
    "    def begin_batch(self): \n",
    "        if self.in_train: self.set_param()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Learning Rate Finder"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "NB: You may want to also add something that saves the model before running this, and loads it back after running - otherwise you'll lose your weights!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "[Jump_to lesson 10 video](https://course.fast.ai/videos/?lesson=10&t=3545)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LR_Find(Callback):\n",
    "    _order=1\n",
    "    def __init__(self, max_iter=100, min_lr=1e-6, max_lr=10):\n",
    "        self.max_iter,self.min_lr,self.max_lr = max_iter,min_lr,max_lr\n",
    "        self.best_loss = 1e9\n",
    "        \n",
    "    def begin_batch(self): \n",
    "        if not self.in_train: \n",
    "            return\n",
    "        pos = self.n_iter/self.max_iter\n",
    "        lr = self.min_lr * (self.max_lr/self.min_lr) ** pos\n",
    "        for pg in self.opt.param_groups: pg['lr'] = lr\n",
    "            \n",
    "    def after_step(self):\n",
    "        self.n_iter += 1\n",
    "        if self.n_iter >= self.max_iter or self.loss > self.best_loss*10:\n",
    "            raise CancelTrainException()\n",
    "        if self.loss < self.best_loss: \n",
    "            self.best_loss = self.loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "NB: In fastai we also use exponential smoothing on the loss. For that reason we check for `best_loss*3` instead of `best_loss*10`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instantiate a Learner object with data, loss_func, opt and model\n",
    "learn = Learner(*get_model(data), loss_func, data) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instantiate a Runner object using the callback_funcs() input\n",
    "run = Runner(callback_funcs=[LR_Find, Recorder])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run a training/validation loop\n",
    "run.fit(learn, n_epochs=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAELCAYAAADTK53JAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deZhcZ3Xn8e+5tfWulq3WZtkWjo3XsY2jAYwxMTthMQk4EBIyQAgOmZBtskwYMiyTJxPCZMiwTABBgGGCSbANtnEcljyJYxzAIHmRFxkGW7ItS7JaQuq99jN/3Ful6qV6c9/qqtu/z/PU01W3btV7qlQ69da5731fc3dERCR5gtUOQERE4qEELyKSUErwIiIJpQQvIpJQSvAiIgmlBC8iklCxJngz+z0ze9DMHjCzL5lZV5ztiYjISRbXOHgzOw24E7jA3afM7MvAbe7++WaP2bBhg2/fvj2WeEREkmj37t1H3X1orvvSMbedBrrNrAT0AAfn23n79u3s2rUr5pBERJLDzB5rdl9sJRp3fxL4S+Bx4BAw4u7fjKs9ERGZLrYEb2brgdcCzwC2Ar1m9uY59rvWzHaZ2a7h4eG4whERWXPiPMj6EmCfuw+7ewn4CvC8mTu5+0533+HuO4aG5iwjiYjIMsSZ4B8HnmtmPWZmwIuBvTG2JyIiDeKswd8F3ADcDdwftbUzrvZERGS6WEfRuPv7gPfF2YaIiMxNZ7KKiLTAj4+MUyxXW9qmEryISMzGC2Ve+ZFvc9O9T7a0XSV4EZGYTRbKFCtVTkwWW9quEryISMyKlbA0U6q0dolUJXgRkZiVo8ReVoIXEUmWUr0Hr4OsIiKJUi/RVJXgRUQSpaQSjYhIMqlEIyKSUCWNohERSaaTJRr14EVEEqVUVolGRCSR6iWaqko0IiKJUhsmqRKNiEjC1GrwOsgqIpIwZQ2TFBFJplK9RKMevIhIohTrJRr14EVEEkWjaEREEqo2Dl6jaEREEqbWc09MicbMzjWzexsuo2b2u3G1JyLSrlbrIGs6rid29x8ClwKYWQp4EvhqXO2JiLSr+lQFCZ0P/sXAI+7+WIvaExFpG/WDrOVkHmT9ReBLLWpLRKSt1IZJlpPWgzezLHA1cH2T+681s11mtmt4eDjucEREWq6c4Pngfxa4292fmutOd9/p7jvcfcfQ0FALwhERaa0kr+j0JlSeEZE1LJFrsppZD/BS4CtxtiMi0s6KlZOjaNxbl+RjGyYJ4O6TwKlxtiEi0u5qpRl3qFSddMpa0q7OZBURiVljaabcwvlolOBFRGJWbDi42soDrUrwIiIxK01L8OrBi4gkRmOCb+WMkkrwIiIxa5yioKgELyKSHI2TjLVyLLwSvIhIzEqVKploaGQr56NRghcRiVmp7PRkw9OOii2cUVIJXkQkZqVKlZ5sClAPXkQkUYqVKt1RgtcwSRGRBGnswetEJxGRBClXnJ5Mun69VZTgRURiVK065aqfLNGoBi8ikgy1hN6bixJ8WQleRCQRagdVu2slGs0mKSKSDLUee3c2TLc6yCoikhD1Ek10opOGSYqIJES9RFM70Uk9eBGRZKiVaOrj4FWDFxFJhlrNvTYXjUbRiIgkRLEyvQevuWhERBKiduZqT9LmojGzQTO7wcweNrO9ZnZ5nO2JiLSbWommuz6KpnU9+HTMz/8R4Ovufo2ZZYGemNsTEWkrtRJNLh0QWGvnooktwZvZAPAC4K0A7l4EinG1JyLSjmolmUwqIJMKEjMXzVnAMPA5M7vHzD5jZr0zdzKza81sl5ntGh4ejjEcEZHWq42ayaQsTPAJWdEpDVwGfMLdnwVMAH88cyd33+nuO9x9x9DQUIzhiIi0Xm3UTCYVkE5ZYkbRHAAOuPtd0e0bCBO+iMiaUZxZoknCKBp3Pww8YWbnRpteDDwUV3siIu2oVqLJpgIygSVqFM1vAV+MRtA8Crwt5vZERNpKLaFn0kY6FbR0LppYE7y73wvsiLMNEZF2Vk/wqYBMyjQXjYhIUtSHSQZRDV5z0YiIJMP0Eo1pRScRkaSYXqIJtKKTiEhS1IZJpgMjEyjBi4gkRqlSJZsKMItKNEkYBy8iIuE4+HTKAKK5aJTgRUQSoVx1Mqkw1WZSplE0IiJJUaxU6wk+HQSJmYtGRGTNK5WrZGslmnRC5qIREZHwIGsmHZVoWjwXjRK8iEiMShUnHYQ9eI2iERFJkFJDDV4nOomIJEipUiWbVoIXEUmcUsUbRtFoLhoRkcQIh0k2jqJRD15EJBGm1eADo1Rx3FvTi1eCFxGJUbmxRBP9rbSoTKMELyISo1JjiSZK8K062UkJXkQkRsVpwyTDRF9q0XQFSvAiIjGqTRcM1E94atXJTkrwIiIxKpUbZpNM10o0renBp+N8cjPbD4wBFaDs7jvibE9EpN2Uqw3zwQcJSvCRF7r70Ra0IyLSdorlhumCUyrRiIgkRqni06YqCLcl4yCrA980s91mdm3MbYmItJ3pwySjUTQt6sHHXaK5wt0PmtlG4Ftm9rC739G4Q5T4rwU444wzYg5HRKR13H3akn3pqAbfqlWdYu3Bu/vB6O8R4KvAs+fYZ6e773D3HUNDQ3GGIyLSUrWe+mqNooktwZtZr5n1164DLwMeiKs9EZF2U0vk9RJNkJwSzSbgq2ZWa+c6d/96jO2JiLSVkwl++lw0rRpFE1uCd/dHgUvien4RkXZXnJHgTx5k7fASjYjIWlcrxWRTyRwmKSKyZpXKUQ8+fXLRbaBlqzotKsGb2e+Y2YCF/sbM7jazl8UdnIhIJ6sNh6wNj2zXHvyvuvso4UiYIeBtwAdji0pEJAGK5RnDJOtz0bRRDx6w6O8rgc+5+30N20REZA61nnp2ZommzXrwu83sm4QJ/hvR+PbWrRwrItKBZg6TbHWJZrHDJN8OXAo86u6TZnYKYZlGRESaaD5Msr1KNJcDP3T3E2b2ZuBPgJH4whIR6Xzl+lQFtRJNe85F8wlg0swuAf4IeAz4QmxRiYgkwOwSTXv24Mvu7sBrgY+4+0eA/vjCEhHpfLMSfJuu6DRmZu8GfgW40sxSQCa+sEREOl9xxmySQWAE1n4rOr0RKBCOhz8MnAb8j9iiEhFJgNqZrLWpCiBM9m11olOU1L8IrDOzVwN5d1cNXkRkHrWDqbWpCqCW4NuoB29mbwC+D/wC8AbgLjO7Js7AREQ6Xa1EU5uqAMKTnVo1imaxNfj3AP8+WpkJMxsC/gm4Ia7AREQ6XUeUaICgltwjx5bwWBGRNak+iqaxRBNY263o9HUz+wbwpej2G4Hb4glJRCQZZg6ThPBkp7YaJunuf2hmrweuIJxkbKe7fzXWyEREOlypXoNvPMhq7bdkn7vfCNwYYywiIolSqlTJpIxobWqgtTX4eRO8mY0Bc33VGODuPhBLVCIiCRAm+OmHK9Mpa48E7+6ajkBEZJlKFZ+V4DOpoL2W7Hs6zCxlZveY2a1xtyUi0k6Kc/TgM0H7DZN8On4H2NuCdkRE2kq5UiWbmr74XViiSUAP3sy2Aa8CPhNnOyIi7ahU8foc8DWZVNB2S/Yt1/8inD9ey/uJyJpTjEbRNMokoQcfTUp2xN13L7DftWa2y8x2DQ8PxxWOiEjLlcpzjKJJSA3+CuBqM9sP/B3wIjP725k7uftOd9/h7juGhoZiDEdEpLVKlSrZ9IwSTToBo2jc/d3uvs3dtwO/CPyzu785rvZERNrNnMMkg9aNg9eEYSIiMSnNUYNvmxOdVoq73w7c3oq2RETaRalSpTc3Pc2Go2g6vEQjIrLWNTuTVSUaEZEON2eJpoXzwSvBi4jEZM6pCtLBtCX7/uXhI3z2zn2xtK8ELyISk3LFpy3XBydXdHIPe/G37jnEp7/9aCztK8GLiMSkVKmSnnUma5h2a2Phj4zl2TjQFUv7SvAiIjGZez74KMFHdfgjowU29udiaV8JXkQkJsU5piqoHXQtRXX4I2N5Ng0owYuIdJRSxWdPVRAl/FK5SqFc4fhkiU39KtGIiHSUcnXuM1nD+5zhsQIAG9WDFxHpHO7eZC6aqAdfqfLUaC3BqwcvItIxaiczzR4Hb/X7h8fyADrIKiLSSWrTEcw+k7U2iuZkD36TevAiIp3jZIJvMoqm4hwZy5MOjFN6srHEoAQvIhKDpiWa1PQa/FB/jiCwWY9fCUrwIiIxqPXgZ05VUD/RqVrlyFh8JzmBEryISCxqCX7WVAVBQ4lmNL5pCkAJXkQkFk1r8OnGEk18Z7GCEryISCyK5blr8OmoBz9ZDM9i3RjTWaygBC8iEovanO/Z9NyzSR48MQWgHryISKdpPkwyvP3k8TDBqwcvItJhaiWa2olNNbWDrk9GPfi45qEBJXgRkSX79v8b5vhEcd596sMkZ5ZoooRfT/Cd2IM3sy4z+76Z3WdmD5rZB+JqS0SkVY6NF/gPn/0+//d7j827X/NRNGHCP3hiilRgnNobz1msAOnYnhkKwIvcfdzMMsCdZvaP7v69GNsUEYnVngMjuMPh0fy8+41MlQDo78pM214r2RwdL7J5oCu2s1ghxgTv4Yqy49HNTHTxuNoTEWmF+w6cAKjP5d5MbZTMlnXTSzCNk4/FOYIGYq7Bm1nKzO4FjgDfcve75tjnWjPbZWa7hoeH4wxHRORp23NgBICj4wsk+JE8p/Zm6cqkpm1vLNnEeRYrxJzg3b3i7pcC24Bnm9lFc+yz0913uPuOoaGhOMMREXla3J09S+jBbxmcncAbpy6Icx4aaNEoGnc/AdwOvKIV7YmIxOHgSJ6j40X6u9IMjxUIK9FzO3Qiz5Z13bO2ZxqGTcY1D3xNnKNohsxsMLreDbwEeDiu9kRE4rbnibD3/jPPHKJQrjJWKDfd9+DIFKcNzk7wQWCkogOrndyD3wL8i5ntAX5AWIO/Ncb2RERide+BE2RSxgueGZaTm5VpxvIlxvLlWQdYa2rz0cTdg49zFM0e4FlxPb+ISKvteWKE8zYP1Hvmw2MFfmqob9Z+h0bCIZRb5ujBQ3igtVCuMtTBPXgRkcSoVp0HnhzhktPX1RNzsx58bYjkaXMcZIWTQyU7tgYvIpIkjx6dYKxQ5uJtgwz1LZTgox78HAdZIVzVKe6zWEEJXkRkUWrDIy/ZNsi67gyZlDHcZCz8oZEpAmt+EDUTGEN98a3FWhPnVAUiIomx58AIPdkUZ2/sIwiMDX25pj34J09MsXmgq77+6kyZdMBgd2bO+1aSevAiIotw7xMnuGjruvoQx6H+XNOzWQ+dyDc9wArQnUmxuckIm5WkHryIyAKK5SoPHRrlLZefWd821JdrOuHYoZEpLjptXdPn+9A1F8+ahCwO6sGLiCzgR0+NUSxXuXjbYH3bUP/cJRp35+BIfs6TnGou3jbIMzb0xhJrIyV4EZEFPHgwnGCssVe+oS/HsYkiler06QqOTRQplqtNT3JqJSV4EZEF7D00Rk82xZmn9NS3DfXnqFSd45PTV3Y6dGL+k5xaSQleRGQBew+Ncu7m/mnDGpud7PRk/SQnJXgRkbbm7uw9NMr5WwambW+W4A+NzL3Qx2pQghcRmcfBkTyj+fLsBN/kbNZDI3ly6YBTYj5LdTGU4EVE5rH34CgAF2zpn7a93oMfn12i2TrYjVm8Z6kuxpoYB1+qVJkolMmXqmTTAT3ZFLl0gJnh7lSqjtnJOZqXqlp1StUquXRq3v0K5Qr5UrXeZmBGV6YWC0wWK4zmS4zny2RSYZzd2RQ92XTT2KpVp+JO1Z1sKljwQ1WuVJkoVpgslpkqVsimA7oyKbozKXqyqQUfX606k6Xpj+/JpunNppqetbcU7k656hTKVarueBWqfvI1ukNgRjYdkE0FNKydgHt4qUaLMHRnUtNqptWqM1YoU6k6vbnUgv9eIhDW3wHO3Ty9B9+bS9OTTXF0Zg/+xFRblGcgIQn+pR/+V/LlSv12pRImiGK5Gv6tVGc9ppbHGhdkMQtXW8llAga6Mgx0Z+jLpZgsVpgolJksVujvSrOxv4uNAzmmihX2H5vgsWOTFMpVerIp1vdkGejOYIQrjFerzli+xPHJElOlyqw4alKBzRpu1SibDujNpkgFRiF6XaVKlZkLymTTAV3pgO5sit5smu5siqrDyGSRkakSE8XmMQQGA90ZBroyBNEXzlSxQqFcnZZg53sNqcDIBBYmVj/5vvbl0vR1penLhR+5SvTFVChVmSqF7UyVKuRLFeZ5G5asL5emN5eiWK4yMlWa9tzZVPhvjYehujtdmVR0CShXvf4eFCvhF3PVw/0CMwIz0iljsDvD+t4sgz0ZyhWvv56qO+kgnFQqmw7ozYVf1t3RGp2193OqFH6+pkoV3MMl3TKpgJQZZuH7lwoC+nNp+qP3sOJOMfqMVx08erNTZuQyAbl0+BoGujL0d2UY6E7T35WhvyvNQFeawKz2z0M2FTDYk6Evl26LXme72Xt4lDNP7al/dhsN9edm9eAPnsjz/HM2tCq8eSUiwV92xnpKDUk8CE728HKZgL5smt5cmq5MimK5wmSpQr5YwQl7g6nAcIdytUqp4uRLYU96dKrERKHCpoFM+G2dSTGaL3FkrMDdjx8nl06x/dReXnDOEIM9GU5MlvjJZJHRqRIAZoYB/V0Z1vdkGOzJ0JUJk3RgRtWdfKlKvlShUvXwP193+B+tVKkyGfW0a0lmslihXHVy6fB1ZaMZ6VIWJtQw8VfCpFmsMBE9NjC4YMsAgz1h8u7NpeirvR+VsP2pYoXxQpmRqVI9EfZGvyBy6RSpgHo7PdmTiapUqTJeKDNRqFCshPGVK7VfRWCEr3O8UGY8X2a8UMbs5Puei76MuhsSa1c6RS4T1JNomOBOXq9WnWLFKVWqs74Uw8eECXuyWInaLJFLp1jfk2FdT5aUwUSxwli+TL5UqcfjfvJXVr5UIZ0KX2t3Jk02HRBE+0GYUCvV8BfRiakSxyeKHJ8skg4CBnuybF0X/juXq2GMhXL473lsfDJqM/xsmFH/lXZKb5bAjFIl/PIuV8MvcAfKlTIHjk8ylg/fx3T0Gc9EnwGi56pUvd6xmSyWl/RlmQqs3okI32uL3pvwdZ822M15W/rr86HXPgcD3WmG+nP0ZBORTmbZe2iM82f03mtmzkdTrlQ5MpZnq3rwK+cvrrl4tUMQaTvu4S+QsXyZ0XyJsXyJ0XyZsXyZavQFDFAohb9uTkwVmSiEvzyqHn6BQfgro1RxHv/JBDffc5C/LTw+Z3u92RQbB7oY6s+xsT/Hxv4uNg3k2DQQ/uJ95qZ+NvTFu8DFSpssltl/bIKfu/S0Oe8f6svxyPB4/fZTYwWq3h5j4CEhCV5EZjMzenPhr9eVmtjK3XnyxBRHxgrhr8RCmdF8meGxAsNjBY6M5TkyVuDBg6P88+gRJmeUBM/b3M/lP3Uqz95+Cs/c3M+Zp/SsyLGbuDx8eAx3OG/GAdaaof4c39t3rH67ttDHViV4Eek0Zsa29T1sW9+z8M7AeKHM4ZE8h0fy3HfgBN995BjX3fU4n/u3/UBY//+pjX284JwNXHXuRnZsX0+mjRJ+7QDrBVvmLtEM9ec4MVmiUK6QS6dOJniVaEQk6fpyac7e2MfZG/t4/jkb+M0Xnk2+VOGHh8f48ZFxfnRkjPsPjPDZf9vHp+54lP5cmhedv5GfvWgLV507RFdmdUc67T00Sn8uzbb1c/fIa0Mlj40X2TrYveBarK2mBC8iLdWVSXHJ6YNccvrJmRnHC2X+7cdH+ee9R/jmQ4e5+d6D9GRTvPj8Tfz8s7Zy5TlDq9Kzf/jQGOdt6W86uqjxZKct67r4xoOH2bqua84RN6shtijM7HTgC8BmoArsdPePxNWeiHSuvlyal1+4mZdfuJk/q1zEXft+wj/cf4jb7j/E1+47yCm9Wa756W385lVns64n/nnUIRyt9fDhMV532dwHWGH6dAW33HeQex4/wYde3z6DPuL8mikDv+/ud5tZP7DbzL7l7g/F2KaIdLh0KuCKszdwxdkbeP9rLuRffzTMTfc8yWe+/Shf3vUE/+mlz+SXnn1G7AdnDxyfYrwwe4qCRhuiBP/E8Uk+fcejXLh1gNf/9LZY41qK2N4hdz/k7ndH18eAvUDzr0IRkRmy6YCXXrCJ//3Ll3Hrb13J+ZsHeO/ND/Kqj97J48cmY237oegA67wJvi+cb+avb3+EgyN5/uurL1j2GfFxaElRy8y2A88C7prjvmvNbJeZ7RoeHm5FOCLSgS7YOsB173gOn3zzT/PUWJ7XfeI79YU44vDDw2OYwbmb5h4iCZBLp1jXnWF4rMArLtzMc886NbZ4liP2BG9mfcCNwO+6++jM+919p7vvcPcdQ0NDcYcjIh3MzHjFRZu5/tcvJ5My3vip7/GdR47G0ta+o+NsXddNd3b+kTxD/TmyqYB3v/K8WOJ4OmJN8GaWIUzuX3T3r8TZloisHeds6ufG33geW9Z18dbP/oAbdx9Y8Tb2HZtc1Lqpb3nedj7w2gs589T411hdqtgSvIXjiv4G2OvuH46rHRFZm7YOdnP9Oy/nsjMH+f3r7+O9Nz9AsTx7YsHl2n90gjNPXfiErl957pm86dlnrFi7KynOHvwVwK8ALzKze6PLK2NsT0TWmMGeLH/79ufwjiufwRe++xhv+vT3eGo0/7Sf9/hEOPvqYnrw7SzOUTR3uru5+8Xufml0uS2u9kRkbUqnAt7zqgv42Juexd5Do1z98TvZc+DE03rO/ccmANjehmWXpWifSR9ERJ6G11yylRt/43mkg4A3fOq73Lrn4LKfq57g1YMXEWkP528Z4OZ3XcFFW9fxruvu4RO3P7Ks59l3dJLA4PRT2mNOmeVSgheRRNnQl+OL73gOr754C3/x9YeXNYxy/9EJtg52d/yyjkrwIpI4uXSKD11zMWdt6OUPvnwfo/nSkh6//9hExx9gBSV4EUmonmyaD7/xUp4aK/D+mx9c9OPcnX1HJzr+ACsowYtIgl16+iDveuHZfOWeJ7nt/kOLeszxyRJj+fKixsC3OyV4EUm0d73obC7Zto7/8tX7OTZeWHD/fUfDETQq0YiItLlMKuAvf+ESxvNlPviPDy+4//6jyRgiCUrwIrIGnLOpn1+78iyu332AXft/Mu++jx2bCIdILnLd2XamBC8ia8Jvv/hstq7r4k9ueoBypfmcNfuOTXLa+m6y6c5Pj53/CkREFqEnm+a9r7mQhw+P8fnv7G+63/6EjKABJXgRWUNefuEmXnjuEH/1rR9xeGT2pGTuzv6jyRgDD0rwIrKGmBkfuPoiKu78yU334+7T7v/JRJGxQlk9eBGRTnTGqT38wcvO5Z/2HuGW+6ZPSHZykrHOP8AKSvAisga97YpncNkZg7zvlgcZHjs5Nn7f0XAhb/XgRUQ6VCowPnTNJUwWK7zvlgfq2/cfnSAVGKefkowefHq1AxARWQ1nb+zjd19yDh/6+g9582fuojeX4qFDo2xb300mlYy+rxK8iKxZ1155Fo8fm+TBg6MMjxVImfGaS7audlgrRgleRNasdCrgg6+/eLXDiE0yfoeIiMgsSvAiIgkVW4I3s8+a2REze2DhvUVEZKXF2YP/PPCKGJ9fRETmEVuCd/c7gPnn5RQRkdioBi8iklCrnuDN7Foz22Vmu4aHh1c7HBGRxFj1BO/uO919h7vvGBoaWu1wREQSo61OdNq9e/dRMzsBjDRsXtdwu3Z9rm0bgKPLaLbxuZZy/8zt892e7zUo7sXdP9f2hWJtvN64bTmxLxT3YmJstk2f8aXFtdD9SYl7MbECnNO0JXeP5QJ8CTgElIADwNsX+bidzW7XrjfZtmuZce5czv3zxdksxrleg+JeXtyLiXWez8ySY18o7sXEuJT3XJ9xxb2YWBdqL7YevLu/aZkP/do8t782z7blWujxze6fL86Ztxd6Dcux1uOeuW214262z2K26TO+uPYXe39S4p65bTHXp7HoG6Djmdkud9+x2nEsleJuvU6NXXG3VqfG3WjVD7KuoJ2rHcAyKe7W69TYFXdrdWrcdYnpwYuIyHRJ6sGLiEgDJXgRkYRSghcRSajEJ3gzC8zsz8zsY2b2ltWOZynM7Coz+7aZfdLMrlrteJbCzHrNbLeZvXq1Y1ksMzs/eq9vMLPfWO14lsLMfs7MPm1mN5vZy1Y7nsUys7PM7G/M7IbVjmUh0Wf6/0Tv8y+vdjyL0dYJvtmc8mb2CjP7oZn92Mz+eIGneS1wGidPuGqJFYrdgXGgixbFvkJxA/xn4MvxRDnbSsTt7nvd/Z3AG4CWDY9bodhvcvd3AG8F3hhjuI3xrUTcj7r72+ONtLklvobXATdE7/PVLQ92OZZzplarLsALgMuABxq2pYBHgLOALHAfcAHw74BbZ1w2An8M/Hr02Bs6LPYgetwm4IsdFPdLgF8kTDav7pS4o8dcDXwH+KVO+qw0PO5/Apd1YNwt+7/5NF7Du4FLo32uW414l3ppq7loZnL3O8xs+4zNzwZ+7O6PApjZ3wGvdfc/B2aVA8zsAFCMblbii3a6lYi9wXEgF0ecM63Qe/5CoJfwP8WUmd3m7tV2jzt6nluAW8zsH4Dr4ot4Wpsr8Z4b8EHgH9397ngjDq3wZ3xVLOU1EP6K3gbcS5tXP2raOsE3cRrwRMPtA8Bz5tn/K8DHzOxK4I44A1uEJcVuZq8DXg4MAh+PN7R5LSlud38PgJm9FTgad3Kfx1Lf76sIf4bngNtijWxhS/2c/xbhL6d1Zna2u38yzuDmsdT3/FTgz4Bnmdm7oy+C1dbsNXwU+LiZvYqnP51BS3Rigrc5tjU9W8vdJ4FVq/HNsNTYv0L4BbXalhR3fQf3z698KEuy1Pf7duD2uIJZoqXG/lHCBLTalhr3MeCd8YWzLHO+BnefAN7W6mCejo74mTHDAeD0htvbgIOrFMtSdWrsirv1OjX2To27URJeA9CZCf4HwDlm9gwzyxIezLtllWNarE6NXXG3XqfG3qlxN0rCawit9lHeBY5wzzmnPPBK4EeER7rfs9pxJil2xa3Ykx530l7DfBdNNiYiklCdWKIREZFFUIIXEUkoJXgRkYRSghcRSSgleBGRhFKCFxFJKCV46dNd90gAAALaSURBVBhmNt6CNq5e5HTIK9nmVWb2vFa2KWtDJ85FI/K0mFnK3eecWdSj2SRjaDPt7uUmd19FOO//d1a6XVnb1IOXjmRmf2hmPzCzPWb2gYbtN0UrST1oZtc2bB83s/9mZncBl5vZfjP7gJndbWb3m9l50X5vNbOPR9c/b2YfNbPvmNmjZnZNtD0ws7+O2rjVzG6r3TcjxtvN7L+b2b8Cv2NmrzGzu8zsHjP7JzPbFE1V+07g98zsXjO70syGzOzG6PX9wMyuiPO9lORSD146joVL0p1DOG+3Ec7f/gJ3vwP4VXf/iZl1Az8wsxs9nLGwl3BRh/dGzwHhVMaXmdl/BP4A+LU5mtsCPB84j7BnfwPhlMLbCRex2AjsBT7bJNxBd/+ZqM31wHPd3c3s14A/cvffN7NPAuPu/pfRftcBf+Xud5rZGcA3gPOX/YbJmqUEL53oZdHlnuh2H2HCvwP4bTP7+Wj76dH2Y4SLvdw443lqUzHvJkzac7nJw/nsHzKzTdG25wPXR9sPm9m/zBPr3zdc3wb8vZltIVwpaF+Tx7wEuCD6EgIYMLN+dx+bpx2RWZTgpRMZ8Ofu/qlpG8MFO14CXO7uk2Z2O+F6tgD5Oeruhehvheb/FwoN123G38WYaLj+MeDD7n5LFOv7mzwmIHwNU0toR2QW1eClE30D+FUz6wMws9PMbCOwDjgeJffzgOfG1P6dwOujWvwmwoOki7EOeDK6/paG7WNAf8PtbwLvqt0ws0uXH6qsZUrw0nHc/ZuE66V+18zuJ6yL9wNfB9Jmtgf4U+B7MYVwI+HUsg8AnwLuAkYW8bj3A9eb2beBow3bvwb8fO0gK/DbwI7oAPJDtN+KR9IhNF2wyDKYWZ+7j0drin4fuMLdD692XCKNVIMXWZ5bzWyQ8GDpnyq5SztSD15EJKFUgxcRSSgleBGRhFKCFxFJKCV4EZGEUoIXEUkoJXgRkYT6/3POtC9EqnRNAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot learning rate vs. loss\n",
    "run.recorder.plot(skip_last=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEGCAYAAABvtY4XAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAaS0lEQVR4nO3de3Rc5Xnv8e+j0c2yJFu2ZBksG9lgm4s53GzCPdzKoZCWNicJ5DRpS+hxk9Uk0JU2h5zkrJ529Q/OSlZPaJqmywuakJSQtkAuzQVIg7m4FGMbDBiMsbFkkGzdbFmSJY0uM8/5Y/YY2bFsWdaePbPn91lLS7P37Jn32Wz7x+t33nm3uTsiIhI/JVEXICIi4VDAi4jElAJeRCSmFPAiIjGlgBcRianSqAuYqL6+3pubm6MuQ0SkYGzZsqXH3RuO9VxeBXxzczObN2+OugwRkYJhZnsme05DNCIiMaWAFxGJKQW8iEhMKeBFRGJKAS8iElMKeBGRmFLAi4jElAJeRCRCv3yzk3949p1Q3lsBLyISoV9s28d3X2gN5b0V8CIiEersT9I4pzKU91bAi4hEqLN/hMYaBbyISOx09iVZqB68iEi8DI6MMzAyTmOtAl5EJFY6+pMALJxTEcr7K+BFRCLSGQS8xuBFRGLmcMBrDF5EJF46+kYAWFiIY/Bm9qdm9oaZbTOzR8wsnLMQESlAnf1JaipKmV0Rzs31Qgt4M1sEfB5Y7e6rgARwR1jtiYgUms7+JAtqw/mAFcIfoikFZplZKVAF7A25PRGRgtHRH94ceAgx4N29Hfga8C6wD+hz96eOPs7M1prZZjPb3N3dHVY5IiJ5p7MvGdoceAh3iKYOuA1YCpwOzDazTxx9nLuvc/fV7r66oaEhrHJERPJKOu10DYyE9gErhDtEcyPQ4u7d7j4GPA5cEWJ7IiIFY//gKONpL8wePJmhmcvMrMrMDLgB2B5ieyIiBePwHPhCDHh33wg8CrwMvB60tS6s9kRECklHX3aZgvACPpzJlwF3/wvgL8JsQ0SkEHUc7sEX7jRJERE5hq7+JCUGDdUKeBGRWOnoT1JfXUFpIrwYVsCLiESgo38k1PF3UMCLiESisy/JgpCWCc5SwIuIRKBzIBnajT6yFPAiIjmWHEtxcGgs1G+xggJeRCTncvElJ1DAi4jkXPZLTgp4EZGY6RwI7uSkWTQiIvHSqR68iEg8dfQnmVWWoLYy1NViFPAiIrnW0Z+ksbaCzEK74VHAi4jkWHvvMIvqZoXejgJeRCTH2g8Os2iuAl5EJFaSYym6B0ZoqqsKvS0FvIhIDu09OAygHryISNy0ZwNeY/AiIvHS1psJ+CYFvIhIvLT3DpMosdAXGgMFvIhITrUfHGZhbWWod3LKUsCLiORQW+9QTsbfQQEvIpJT7b3DORl/BwW8iEjOjKXSdPQnacrBFElQwIuI5ExHX5K0k5MvOYECXkQkZ7JTJDUGLyISM229Q0BuvsUKCngRkZxpPziMGZw2N/w58KCAFxHJmbbeYRbUVFBRmshJewp4EZEcyUyRzM0HrKCAFxHJmVytA5+lgBcRyYFU2tl7MHdfcgIFvIhITnQNJBlPe86mSIICXkQkJ95fJlhj8CIisdLem7s7OWUp4EVEciB7JyeNwYuIxExb7xD11eVUluVmDjwo4EVEcqKtN7dTJCHkgDezuWb2qJm9ZWbbzezyMNsTEclXrfsHWTJ/dk7bDLsHfz/whLufDVwAbA+5PRGRvDM6nqa9d5il83M3gwagNKw3NrNa4BrgDwHcfRQYDas9EZF89V7vEGmH5vr49OCXAd3At83sFTN7wMxye3YiInmgtWcQiFfAlwIXA99y94uAQeDeow8ys7VmttnMNnd3d4dYjohINFqCgF8aozH4NqDN3TcG24+SCfwjuPs6d1/t7qsbGhpCLEdEJBqt+weZM6uMutnlOW03tIB39w7gPTNbGey6AXgzrPZERPJVa89QzodnIMQPWQOfAx42s3JgN3BnyO2JiOSdlp5B1jTX5bzdUAPe3bcCq8NsQ0QknyXHUuztG6a5vinnbeubrCIiIXrvwBDu0JzjD1hBAS8iEqqWiKZIggJeRCRUrfujmSIJCngRkVC19AxRV1XGnKqynLetgBcRCVFrz2AkwzOggBcRCVXr/sFIhmdAAS8iEprh0RT7+pLqwYuIxM2eA9HNoAEFvIhIaA6vIpnjdeCzFPAiIiFp6RkC1IMXEYmd1p5B5s8up7Yy91MkQQEvIhKalv3RTZEEBbyISGh2dx9imQJeRCRe9h8aoefQKCsX1kRWgwJeRCQEb3ceAmB5owJeRCRWdnYNALBSAS8iEi87OgaoqSylsbYishoU8CIiIdjZeYgVjTWYWWQ1KOBFRGaYu/N21wArIhyeAQW8iMiM6x4Y4eDQGCsaqyOtQwEvIjLDsjNoovyAFRTwIiIz7u3OzAyaKKdIggJeRGTGvd05QF1VGfXV5ZHWoYAXEZlhb3cORD6DBhTwIiIzyt0PT5GMmgJeRGQG7etLMjAyHvkMGlDAi4jMqOwHrOrBi4jEjAJeRCSm3u48RENNBXWzo51BA1MMeDO728xqLeNBM3vZzG4KuzgRkUKzs3MgL8bfYeo9+E+5ez9wE9AA3AncF1pVIiIFaDyVZkfnACsba6MuBZh6wGcnc94CfNvdX52wT0REgHe6B0mOpTm/qbACfouZPUUm4J80sxogHV5ZIiKF5/X2PgBWnT4n4koySqd43F3AhcBudx8ys3lkhmlERCSwrb2PqvIEyxoKawz+cmCHux80s08AXwH6witLRKTwvN7ex7mn1ZIoyY8R7KkG/LeAITO7APgisAf4bmhViYgUmFTaeXNvP6sW5cfwDEw94Mfd3YHbgPvd/X4g+ln8IiJ54p3uQwyPpTg/jwJ+qmPwA2b2JeCTwNVmlgDKwitLRKSwvN6WGbU+vyl/An6qPfjbgREy8+E7gEXAV6fyQjNLmNkrZvbTadYoIpL3Xm/vY1ZZgjPz5ANWmGLAB6H+MDDHzD4EJN19qmPwdwPbp1mfiEhB2Nbex7mn588HrDD1pQo+BrwEfBT4GLDRzD4yhdc1AbcCD5xKkSIi+SyVdt7Y259X4+8w9TH4LwNr3L0LwMwagH8HHj3B675OZtbNpB/ImtlaYC3AkiVLpliOiEj+2B18wJpPM2hg6mPwJdlwD+w/0WuDoZwud99yvOPcfZ27r3b31Q0NDVMsR0Qkf2S/wVqoPfgnzOxJ4JFg+3bg5yd4zZXAb5vZLUAlUGtm/+Tun5heqSIi+en19j4qy0o4s2F21KUcYUoB7+5/bmb/jUxoG7DO3X94gtd8CfgSgJldC/yZwl1E4mhb8A3W0kR+3WJjqj143P0x4LEQaxERKTjjqTRv7O3no5c0RV3KrzluwJvZAODHegpwd5/Smpju/gzwzMkWJyKS797qGGBoNMXFZ9RFXcqvOW7Au7uWIxAROY5NrQcAWNM8L+JKfl1+DRiJiBSYzXt6WTR3FqfPnRV1Kb9GAS8iMk3uzubWA1ySh8MzoIAXEZm2tt5hOvtHWNOsgBcRiZXs+PvqPBx/BwW8iMi0bWrtpaaylBWN+TkfRQEvIjJN2fH3fFpBciIFvIjINBwcGmVn16G8nB6ZpYAXEZmGLXt6AVidpzNoQAEvIjItm1p7KUsYFyyeG3Upk1LAi4hMw+bWA5y/aA6VZYmoS5mUAl5E5CQlx1K81taXt9MjsxTwIiIn6aWWA4ym0lx+5vyoSzkuBbyIyEl6fmc35YkSPrBUPXgRkVh5fmcPa5bWUVU+5VtqREIBLyJyEjr7k7zVMcDVy/P/HtIKeBGRk/D8zh4Arl5eH3ElJ6aAFxE5Cc/v7Ka+uoJzFk7phnaRUsCLiExROu08v7OHq5fXU5Kn689MpIAXEZmiN/f1c2BwlGtW5P/wDCjgRUSm7Lmd3QBceZYCXkQkVp57u5tzTqtlQU1l1KVMiQJeRGQKBkfG2bKnt2CGZ0ABLyIyJc/s6GYs5Vy3ckHUpUyZAl5EZAp+sW0f82eX5/UNPo6mgBcROYHkWIr1b3Vx03mNeXt7vmNRwIuInMCGnT0Mjqa4edVpUZdyUhTwIiIn8IttHdRWlnL5svxeHvhoCngRkeMYS6X59+2d3HhOI+WlhRWZhVWtiEiOvbh7P33DY9y8amHUpZw0BbyIyHE8sa2DqvIE16zI/+WBj6aAFxGZRCrtPPlGJ9etXJDXN9eejAJeRGQSG1v203NopCCHZ0ABLyIyqUc3t1FTWcpvnNsYdSnTooAXETmGgeQYP9+2j9+64PSCHJ4BBbyIyDH97LV9JMfSfPSSpqhLmTYFvIjIMfzrljbOWlDNhYvnRl3KtIUW8Ga22MzWm9l2M3vDzO4Oqy0RkZm0q+sQW/b08tFLmjArnLVnjlYa4nuPA19w95fNrAbYYma/dPc3Q2xTROSUPbqljUSJ8bsXL4q6lFMSWg/e3fe5+8vB4wFgO1DY/7VEJPbGU2kef7mN61Y2FMydmyaTkzF4M2sGLgI2HuO5tWa22cw2d3d356IcEZFJrd/RTdfACB+5ZHHUpZyy0APezKqBx4B73L3/6OfdfZ27r3b31Q0NhfdVYBGJlwc37GbR3FnceE7h3LlpMqEGvJmVkQn3h9398TDbEhE5Vdva+3hx9wH+8IpmShOFP8kwzFk0BjwIbHf3vwmrHRGRmfLghhZmlye4/dLCH56BcHvwVwKfBK43s63Bzy0hticiMm0dfUn+7dW93L5mCbWVZVGXMyNCmybp7huAwp1AKiJF5aH/bCXtzp1XNkddyowp/EEmEZFTNDgyzsMv7uHmVQtZPK8q6nJmjAJeRIre9ze+S39ynLuuWhZ1KTNKAS8iRW0gOcbfP7OLq5fXc8kZdVGXM6MU8CJS1B7c0ELv0Bh//l9XRl3KjFPAi0jROjA4ygPPt3DzeQv5L02Fu2rkZBTwIlK0/uHZdxgcHecLN62IupRQKOBFpCh19id56IVWfveiRSxvrIm6nFAo4EWkKN33i7dIu3PPDfHsvYMCXkSK0Au7evjhK+18+oNnsmR+fOa9H00BLyJFZWQ8xVd+vI0l86r4k+vOirqcUIV5RycRkbyz7tnd7O4e5Dt3rqGyLBF1OaFSD15Eisae/YN8Y/0ubj3/NK5dWfjrvZ+IAl5EisJ4Ks0X/uVVyhMl/O8PnRt1OTmhIRoRKQrfeHoXm/f0cv8dF7JwTmHfa3Wq1IMXkdh7qeUA33h6Jx++eBG3Xbgo6nJyRgEvIrHWNzTGPT94hSXzqvir21ZFXU5OaYhGRGJrPJXmnn9+ha6BER77zBVUVxRX5BXX2YpIUfnrn21n/Y5u/vp3VnHB4vgtJnYiGqIRkVj63n+28p0XWvnUlUv5xGVnRF1OJBTwIhI763d08X/+7U1uOHsBX771nKjLiYwCXkRiZcPOHj79vS2sbKzhbz9+EYkSi7qkyCjgRSQ2Nuzs4a6HNrG0fjb/9EcfYHaRfah6NAW8iMTC8zu7D4f79//HZcybXR51SZFTwItIwfuXze9x57cz4f7wH31A4R4o7n+/iEhBS6edrz21g79/5h2uOqueb/7excyZVRZ1WXlDAS8iBalvaIwvPvYqT77RyccvXcJf3XYeZQkNSkykgBeRgrO59QB3/2Arnf1JvnLrOdx11VLMine2zGQU8CJSMEbGU3xz/Tv83dM7WTyvisc+c0VRfkN1qhTwIlIQXtjVw1d+tI3dPYN8+KJF/OVt51FTqfH241HAi0hee+/AEF97agc/3rqXJfOqeOhTl/LBFQ1Rl1UQFPAikpe6BpL83dO7eOSldykx47PXncVnrz8r9vdRnUkKeBHJK7u6DvHghhYef7mNVNq5fc1iPnf98qK5C9NMUsCLSOTGUmmefquLH7z0Lut3dFNRWsKHL27ij69ZRnP97KjLK1gKeBGJRDrtbG07yM9e28ePt7bTc2iUBTUV3HPjcj552RnMr66IusSCp4AXkZwZHk3xYst+nt3RzRPbOujoT1KWMG44u5GPrWnimuUNlOrLSjNGAS8ioRkeTbH1vYNsaj3Axpb9bGrpZTSVpqK0hA+uaOB/nr+S689u1PICIVHAi8iM6E+OsbNzgO37BtjW3sdrbX283TnAeNoxg5WNNfz+5WdwzYoGLl06T7NhckABLyJT1jc8RlvvEO29w7x7YIiWnkFa9w+yu3uQfX3Jw8fNrSrj/EVzWLtyGWua53HxkjrmVKmXnmuhBryZ3QzcDySAB9z9vjDbE5GTMzqepj85Rt/wGAeHxugdHOXA0CgHBkfpGRih59AIXQMjdPQn6exLMjiaOuL1tZWlLG2o5rJl81neWM3KxhpWNNbQVDdLa8PkgdAC3swSwDeB3wDagE1m9hN3fzOsNkXymbuTSjspd9whlXbS7qTTMJ5OH35uPJU5bjztjKfTjKeCx6k0o6nM9lgqzVgqzch4mtHx938nx1Mkx9KMjKVIjqUYHksxNJpieDTF4Og4gyMpBkfGGRgZZyA5RnIsPWm9s8oS1NeU01BdwdkLa/jgigZOm1NJU10VTXWzaKqroq6qTEGex8LswV8K7HL33QBm9gPgNmDGA/63vrGB5FjqxAcWEI+6gGlyP7nKJz16kicm7p6sLT/8fHb7/eMmvuTIx37E67JbmcfZY33Cth/enw52poN9aXfSzuF9h7dzqLy0hKryBLPKMj+zK0qpKk9QX11Oc/1sqitKqa5IUFtZxpyqMmory5hbVUZdVTl1VeXMry4v+tvdxUGYV3AR8N6E7TbgA0cfZGZrgbUAS5YsmVZDZzbMZjQ1eU+kUBkF2jM6ybInO3yynqEdcczx3zP7HnasJ8n8N86+h014vyP2W7DH3j8m+3xJcFCJvf98oiTzwDASJdnnjIQZJQYlJUaJvf9ciRmliew+o7Tk/e2yRMnh7dKSEsoSJZQljPLSzOPy0hLKEyVUlGYeV5YlKE+UUFLEN5qW94UZ8Mf6E/Zr/Rh3XwesA1i9evW0+jlfv+Oi6bxMRCTWwvxGQRuweMJ2E7A3xPZERGSCMAN+E7DczJaaWTlwB/CTENsTEZEJQhuicfdxM/ss8CSZaZL/6O5vhNWeiIgcKdSPyd3958DPw2xDRESOTav6iIjElAJeRCSmFPAiIjGlgBcRiSk72a+Wh8nMuoE903x5PdAzg+UUgmI8ZyjO8y7Gc4biPO+TPecz3L3hWE/kVcCfCjPb7O6ro64jl4rxnKE4z7sYzxmK87xn8pw1RCMiElMKeBGRmIpTwK+LuoAIFOM5Q3GedzGeMxTnec/YOcdmDF5ERI4Upx68iIhMoIAXEYmpgg94M7vZzHaY2S4zuzfqesJiZovNbL2ZbTezN8zs7mD/PDP7pZntDH7XRV3rTDOzhJm9YmY/DbaXmtnG4Jz/OViOOlbMbK6ZPWpmbwXX/PK4X2sz+9Pgz/Y2M3vEzCrjeK3N7B/NrMvMtk3Yd8xraxl/G+Tba2Z28cm0VdABP+HG3r8JnAt83MzOjbaq0IwDX3D3c4DLgD8JzvVe4Ffuvhz4VbAdN3cD2yds/1/g/wXn3AvcFUlV4bofeMLdzwYuIHP+sb3WZrYI+Dyw2t1XkVli/A7iea2/A9x81L7Jru1vAsuDn7XAt06moYIOeCbc2NvdR4Hsjb1jx933ufvLweMBMn/hF5E534eCwx4CfieaCsNhZk3ArcADwbYB1wOPBofE8ZxrgWuABwHcfdTdDxLza01m+fJZZlYKVAH7iOG1dvfngANH7Z7s2t4GfNczXgTmmtlpU22r0AP+WDf2XhRRLTljZs3ARcBGoNHd90HmfwLAgugqC8XXgS8C2buqzwcOuvt4sB3Ha74M6Aa+HQxNPWBms4nxtXb3duBrwLtkgr0P2EL8r3XWZNf2lDKu0AN+Sjf2jhMzqwYeA+5x9/6o6wmTmX0I6HL3LRN3H+PQuF3zUuBi4FvufhEwSIyGY44lGHO+DVgKnA7MJjM8cbS4XesTOaU/74Ue8EV1Y28zKyMT7g+7++PB7s7sP9mC311R1ReCK4HfNrNWMsNv15Pp0c8N/hkP8bzmbUCbu28Mth8lE/hxvtY3Ai3u3u3uY8DjwBXE/1pnTXZtTynjCj3gi+bG3sHY84PAdnf/mwlP/QT4g+DxHwA/znVtYXH3L7l7k7s3k7m2T7v77wHrgY8Eh8XqnAHcvQN4z8xWBrtuAN4kxteazNDMZWZWFfxZz55zrK/1BJNd258Avx/MprkM6MsO5UyJuxf0D3AL8DbwDvDlqOsJ8TyvIvNPs9eArcHPLWTGpH8F7Ax+z4u61pDO/1rgp8HjZcBLwC7gX4GKqOsL4XwvBDYH1/tHQF3crzXwl8BbwDbge0BFHK818AiZzxnGyPTQ75rs2pIZovlmkG+vk5llNOW2tFSBiEhMFfoQjYiITEIBLyISUwp4EZGYUsCLiMSUAl5EJKYU8BJLZvZC8LvZzP77DL/3/zpWWyL5RtMkJdbM7Frgz9z9QyfxmoS7p47z/CF3r56J+kTCpB68xJKZHQoe3gdcbWZbg/XGE2b2VTPbFKyv/cfB8dcG6+1/n8wXSjCzH5nZlmCN8rXBvvvIrHi41cwenthW8G3Drwbrmb9uZrdPeO9nJqzv/nDwbU2RUJWe+BCRgnYvE3rwQVD3ufsaM6sA/sPMngqOvRRY5e4twfan3P2Amc0CNpnZY+5+r5l91t0vPEZbHybzDdQLgPrgNc8Fz10EnEdmHZH/ILPOzoaZP12R96kHL8XmJjJre2wls9zyfDI3UwB4aUK4A3zezF4FXiSz4NNyju8q4BF3T7l7J/AssGbCe7e5e5rMMhPNM3I2IsehHrwUGwM+5+5PHrEzM1Y/eNT2jcDl7j5kZs8AlVN478mMTHicQn/3JAfUg5e4GwBqJmw/CXwmWHoZM1sR3EzjaHOA3iDczyZzm8Sssezrj/IccHswzt9A5q5ML83IWYhMg3oREnevAePBUMt3yNzrtBl4Ofigs5tj3wbuCeDTZvYasIPMME3WOuA1M3vZM8sXZ/0QuBx4lczKn190947gfxAiOadpkiIiMaUhGhGRmFLAi4jElAJeRCSmFPAiIjGlgBcRiSkFvIhITCngRURi6v8DZjreNhZ7yjUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# plot iteration number vs. loss\n",
    "run.recorder.plot_lr()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Export"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Converted 05b_early_stopping_jcat.ipynb to exp\\nb_05b.py\n"
     ]
    }
   ],
   "source": [
    "!python notebook2script.py 05b_early_stopping_jcat.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}