msxcn-multilabel-classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "4107cc14",
   "metadata": {
    "papermill": {
     "duration": 0.023901,
     "end_time": "2021-09-02T19:31:09.716190",
     "exception": false,
     "start_time": "2021-09-02T19:31:09.692289",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<h2>Movsisyans Singularity eXtracting Convolutional Network (MSXCN)</h2>\n",
    "By <a href=\"https://movsisyan.info/\">Mher Movsisyan</a> and <a href=\"https://www.linkedin.com/in/tigran-avetisyan/\">Tigran Avetisyan</a>\n",
    "\n",
    "1. Importing Tools and Data\n",
    "2. Joining Data\n",
    "3. Creating Custom Layer and Callback\n",
    "4. Building the model\n",
    "5. Evaluating the model\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "65ad0dc3",
   "metadata": {
    "papermill": {
     "duration": 0.02266,
     "end_time": "2021-09-02T19:31:09.761723",
     "exception": false,
     "start_time": "2021-09-02T19:31:09.739063",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<h2 id=\"1\">1 - Importing Tools and Data</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b8a2d97d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:31:09.825674Z",
     "iopub.status.busy": "2021-09-02T19:31:09.825085Z",
     "iopub.status.idle": "2021-09-02T19:31:25.171364Z",
     "shell.execute_reply": "2021-09-02T19:31:25.170766Z",
     "shell.execute_reply.started": "2021-09-02T18:38:02.758419Z"
    },
    "papermill": {
     "duration": 15.386864,
     "end_time": "2021-09-02T19:31:25.171508",
     "exception": false,
     "start_time": "2021-09-02T19:31:09.784644",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting keract\r\n",
      "  Downloading keract-4.5.0-py2.py3-none-any.whl (12 kB)\r\n",
      "Installing collected packages: keract\r\n",
      "Successfully installed keract-4.5.0\r\n",
      "\u001b[33mWARNING: Running pip as root will break packages and permissions. You should install packages reliably by using venv: https://pip.pypa.io/warnings/venv\u001b[0m\r\n",
      "[name: \"/device:CPU:0\"\n",
      "device_type: \"CPU\"\n",
      "memory_limit: 268435456\n",
      "locality {\n",
      "}\n",
      "incarnation: 8138356278534030125\n",
      ", name: \"/device:GPU:0\"\n",
      "device_type: \"GPU\"\n",
      "memory_limit: 15683598464\n",
      "locality {\n",
      "  bus_id: 1\n",
      "  links {\n",
      "  }\n",
      "}\n",
      "incarnation: 3775516010650611401\n",
      "physical_device_desc: \"device: 0, name: Tesla P100-PCIE-16GB, pci bus id: 0000:00:04.0, compute capability: 6.0\"\n",
      "]\n"
     ]
    }
   ],
   "source": [
    "! pip install keract\n",
    "\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import tensorflow as tf\n",
    "from keras import backend as K\n",
    "import matplotlib.pyplot as plt\n",
    "from keras.models import Sequential\n",
    "import matplotlib.patches as patches\n",
    "from skimage.filters import threshold_otsu\n",
    "from keras.optimizers import RMSprop, SGD, Adam\n",
    "from keras.utils.np_utils import to_categorical\n",
    "from sklearn.model_selection import train_test_split\n",
    "from keract import get_activations, display_activations\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from sklearn.metrics import classification_report, confusion_matrix\n",
    "from keras.callbacks import EarlyStopping, ReduceLROnPlateau, ModelCheckpoint, Callback\n",
    "from keras.layers.experimental.preprocessing import RandomTranslation, RandomRotation, RandomWidth, RandomHeight, RandomZoom, Resizing\n",
    "from keras.layers import Dense, Flatten, Dropout, Conv2D, MaxPool2D, BatchNormalization, Reshape, GlobalAveragePooling2D, LeakyReLU, Layer, Concatenate, Input\n",
    "\n",
    "seed = 173\n",
    "np.random.seed(seed)\n",
    "tf.random.set_seed(seed)\n",
    "\n",
    "from tensorflow.python.client import device_lib\n",
    "print(device_lib.list_local_devices())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f15d2f87",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:31:25.223755Z",
     "iopub.status.busy": "2021-09-02T19:31:25.223256Z",
     "iopub.status.idle": "2021-09-02T19:31:28.969570Z",
     "shell.execute_reply": "2021-09-02T19:31:28.968967Z",
     "shell.execute_reply.started": "2021-09-02T18:38:11.100271Z"
    },
    "papermill": {
     "duration": 3.773781,
     "end_time": "2021-09-02T19:31:28.969711",
     "exception": false,
     "start_time": "2021-09-02T19:31:25.195930",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>775</th>\n",
       "      <th>776</th>\n",
       "      <th>777</th>\n",
       "      <th>778</th>\n",
       "      <th>779</th>\n",
       "      <th>780</th>\n",
       "      <th>781</th>\n",
       "      <th>782</th>\n",
       "      <th>783</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5505</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5506</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5507</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5508</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5509</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5510 rows × 785 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      0  1  2  3  4  5  6  7  8  9  ...  775  776  777  778  779  780  781  \\\n",
       "0     0  0  0  0  0  0  0  0  0  0  ...    0    0    0    0    0    0    0   \n",
       "1     0  0  0  0  0  0  0  0  0  0  ...    0    0    0    0    0    0    0   \n",
       "2     0  0  0  0  0  0  0  0  0  0  ...    0    0    0    0    0    0    0   \n",
       "3     0  0  0  0  0  0  0  0  0  0  ...    0    0    0    0    0    0    0   \n",
       "4     0  0  0  0  0  0  0  0  0  0  ...    0    0    0    0    0    0    0   \n",
       "...  .. .. .. .. .. .. .. .. .. ..  ...  ...  ...  ...  ...  ...  ...  ...   \n",
       "5505  0  0  0  0  0  0  0  0  0  0  ...    0    0    0    0    0    0    0   \n",
       "5506  0  0  0  0  0  0  0  0  0  0  ...    0    0    0    0    0    0    0   \n",
       "5507  0  0  0  0  0  0  0  0  0  0  ...    0    0    0    0    0    0    0   \n",
       "5508  0  0  0  0  0  0  0  0  0  0  ...    0    0    0    0    0    0    0   \n",
       "5509  0  0  0  0  0  0  0  0  0  0  ...    0    0    0    0    0    0    0   \n",
       "\n",
       "      782  783  label  \n",
       "0       0    0      0  \n",
       "1       0    0      0  \n",
       "2       0    0      0  \n",
       "3       0    0      0  \n",
       "4       0    0      0  \n",
       "...   ...  ...    ...  \n",
       "5505    0    0      9  \n",
       "5506    0    0      9  \n",
       "5507    0    0      9  \n",
       "5508    0    0      9  \n",
       "5509    0    0      9  \n",
       "\n",
       "[5510 rows x 785 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"../input/digits-mini-dataset-5500/drawings_non_binary.csv\")\n",
    "mnist = pd.read_csv(\"../input/digit-recognizer/train.csv\")\n",
    "df"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "56545020",
   "metadata": {
    "papermill": {
     "duration": 0.024387,
     "end_time": "2021-09-02T19:31:29.018891",
     "exception": false,
     "start_time": "2021-09-02T19:31:28.994504",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<h2 id=\"2\">2 - Joining Data</h2>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "8264f5c5",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:31:29.078413Z",
     "iopub.status.busy": "2021-09-02T19:31:29.077303Z",
     "iopub.status.idle": "2021-09-02T19:31:29.639485Z",
     "shell.execute_reply": "2021-09-02T19:31:29.639020Z",
     "shell.execute_reply.started": "2021-09-02T18:38:15.378556Z"
    },
    "papermill": {
     "duration": 0.595029,
     "end_time": "2021-09-02T19:31:29.639617",
     "exception": false,
     "start_time": "2021-09-02T19:31:29.044588",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "joint = pd.DataFrame(mnist.iloc[:, 1:].to_numpy())\n",
    "joint[\"label\"] = mnist.label\n",
    "jointDf = pd.DataFrame(np.concatenate((joint, df), axis=0)).rename({784: \"label\"}, axis=1).sample(frac=1, random_state=seed)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0bbbfe9a",
   "metadata": {
    "papermill": {
     "duration": 0.024615,
     "end_time": "2021-09-02T19:31:29.689053",
     "exception": false,
     "start_time": "2021-09-02T19:31:29.664438",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Making sure the classes are balanced"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "552699cd",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:31:29.743211Z",
     "iopub.status.busy": "2021-09-02T19:31:29.742417Z",
     "iopub.status.idle": "2021-09-02T19:31:29.916626Z",
     "shell.execute_reply": "2021-09-02T19:31:29.917048Z",
     "shell.execute_reply.started": "2021-09-02T18:38:16.057369Z"
    },
    "papermill": {
     "duration": 0.203489,
     "end_time": "2021-09-02T19:31:29.917205",
     "exception": false,
     "start_time": "2021-09-02T19:31:29.713716",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEGCAYAAACUzrmNAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAASjklEQVR4nO3df9BmdV3/8eeLXRTRFIQ7vriLLVOMiVYqO0hRVvAV0UzIWQ1L3YyGmi8aVlNfrZkwiybnm5lZOsO46KIkIWhS44Q7QFhOgruI8mMjN3+xG7qbIEh+FRff/XF9Fi/3B5+buK5zXbv38zFzzX3O55zr+rzve3bv133O+ZzPSVUhSdJDOWjWBUiS5p9hIUnqMiwkSV2GhSSpy7CQJHUtn3UB03DkkUfWqlWrZl2GJO1XNm3a9J9VtbC3bQdkWKxatYqNGzfOugxJ2q8k+cK+tnkaSpLUZVhIkroMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQuw0KS1HVA3sE9j774xh8arK8n//7Ng/UlaWnwyEKS1GVYSJK6DAtJUpdhIUnqMiwkSV2GhSSpy7CQJHUZFpKkLsNCktRlWEiSugwLSVKXc0NJmgtveMMbDsi+DhQeWUiSujyy0OCue85PDtbXT370usH6kg5kHllIkroMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQu77NYYk5+28mD9POx13xskH6kA9GPXH7VYH19as3zFrWfRxaSpK4lcWRxwm9fPEg/m/7fKwfpR5q0zRdcM0g/T/29UwbpR5PnkYUkqcuwkCR1Tf00VJJlwEZgW1W9MMmxwKXAEcAm4BVVdX+SRwMXAycAXwF+vqo+3z7j9cDZwAPAr1fVcFd/dMD6y9/6u0H6efWbf3aQfjQZl73/xEH6eelLbhikn0kZ4sjiPGDz2PqbgLdU1Q8AdzMKAdrXu1v7W9p+JDkeOAt4GnA68PYWQJKkgUw1LJKsBH4GeGdbD3AKcHnbZT1wZls+o63Ttp/a9j8DuLSqvllVnwO2AMNEvyQJmP6RxZ8DvwN8u60fAXy1qna29a3Aira8ArgDoG2/p+3/YPte3vOgJOck2Zhk444dOyb8bUjS0ja1sEjyQmB7VW2aVh/jqurCqlpdVasXFhaG6FKSloxpXuA+GXhRkhcAhwCPB94KHJZkeTt6WAlsa/tvA44BtiZZDjyB0YXuXe27jL9HkjSAqR1ZVNXrq2plVa1idIH6mqr6ReBaYE3bbS3wobZ8ZVunbb+mqqq1n5Xk0W0k1XHA/jWMQJL2c7O4g/v/Apcm+SPgk8C61r4OeE+SLcBdjAKGqro1yWXAbcBO4NyqemD4siVp6RokLKrqH4F/bMufZS+jmarqG8BL9vH+C4ALplehJOmheAe3JKnLsJAkdRkWkqSuJTFFuTSvLnj5mv5OE/J77728v5O0Dx5ZSJK6DAtJUpdhIUnqMiwkSV2GhSSpy7CQJHUZFpKkLsNCktRlWEiSugwLSVKXYSFJ6jIsJEldhoUkqcuwkCR1GRaSpC7DQpLUZVhIkroMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQuw0KS1GVYSJK6DAtJUpdhIUnqMiwkSV2GhSSpy7CQJHUZFpKkrqmFRZJDktyQ5FNJbk3yB6392CTXJ9mS5G+SPKq1P7qtb2nbV4191utb++1JnjetmiVJezfNI4tvAqdU1Y8AzwBOT3IS8CbgLVX1A8DdwNlt/7OBu1v7W9p+JDkeOAt4GnA68PYky6ZYtyRpN1MLixq5r60e3F4FnAJc3trXA2e25TPaOm37qUnS2i+tqm9W1eeALcCJ06pbkrSnqV6zSLIsyU3AdmAD8O/AV6tqZ9tlK7CiLa8A7gBo2+8Bjhhv38t7xvs6J8nGJBt37Ngxhe9GkpauqYZFVT1QVc8AVjI6GvjBKfZ1YVWtrqrVCwsL0+pGkpakQUZDVdVXgWuBHwUOS7K8bVoJbGvL24BjANr2JwBfGW/fy3skSQOY5miohSSHteXHAM8FNjMKjTVtt7XAh9rylW2dtv2aqqrWflYbLXUscBxww7TqliTtaXl/l/+xo4H1beTSQcBlVfX3SW4DLk3yR8AngXVt/3XAe5JsAe5iNAKKqro1yWXAbcBO4NyqemCKdUuSdjO1sKiqTwPP3Ev7Z9nLaKaq+gbwkn181gXABZOuUZK0ON7BLUnqMiwkSV2GhSSpy7CQJHUZFpKkLsNCktRlWEiSuhYVFkmuXkybJOnA9JA35SU5BDgUODLJ4UDapsezl5lfJUkHpt4d3L8KvBZ4ErCJ74TFvcBfTq8sSdI8eciwqKq3Am9N8pqqettANUmS5syi5oaqqrcl+TFg1fh7quriKdUlSZojiwqLJO8Bvh+4Cdg142sBhoUkLQGLnXV2NXB8e76EJGmJWex9FrcA/2uahUiS5tdijyyOBG5LcgPwzV2NVfWiqVQlSZoriw2LN0yzCEnSfFvsaKjrpl2IJGl+LXY01NcYjX4CeBRwMPBfVfX4aRUmSZofiz2y+J5dy0kCnAGcNK2iJEnz5WHPOlsjfws8b/LlSJLm0WJPQ714bPUgRvddfGMqFUmS5s5iR0P97NjyTuDzjE5FSZKWgMVes3jVtAuRJM2vxT78aGWSDybZ3l5XJFk57eIkSfNhsRe43wVcyei5Fk8C/q61SZKWgMWGxUJVvauqdrbXu4GFKdYlSZojiw2LryR5eZJl7fVy4CvTLEySND8WGxa/DLwU+BJwJ7AG+KUp1SRJmjOLHTr7RmBtVd0NkOSJwJ8yChFJ0gFusUcWP7wrKACq6i7gmdMpSZI0bxYbFgclOXzXSjuyWOxRiSRpP7fYX/hvBv4lyfvb+kuAC6ZTkiRp3iz2Du6Lk2wETmlNL66q26ZXliRpniz6VFILBwNCkpaghz1F+WIlOSbJtUluS3JrkvNa+xOTbEjymfb18NaeJH+RZEuSTyd51thnrW37fybJ2mnVLEnau6mFBaPZaX+rqo5n9KCkc5McD7wOuLqqjgOubusAzweOa69zgHfAgxfTzweeDZwInD9+sV2SNH1TC4uqurOqbmzLXwM2AysYTW2+vu22HjizLZ8BXNwervRx4LAkRzN6yNKGqrqrDd/dAJw+rbolSXua5pHFg5KsYnRfxvXAUVV1Z9v0JeCotrwCuGPsbVtb277ad+/jnCQbk2zcsWPHZL8BSVriph4WSR4HXAG8tqruHd9WVQXUJPqpqguranVVrV5YcI5DSZqkqYZFkoMZBcUlVfWB1vzldnqJ9nV7a98GHDP29pWtbV/tkqSBTHM0VIB1wOaq+rOxTVcCu0Y0rQU+NNb+yjYq6iTgnna66irgtCSHtwvbp7U2SdJApjllx8nAK4Cbk9zU2n4X+BPgsiRnA19gNJstwIeBFwBbgK8Dr4LRPFRJ/hD4RNvvjW1uKknSQKYWFlX1z0D2sfnUvexfwLn7+KyLgIsmV50k6eEYZDSUJGn/ZlhIkroMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQuw0KS1GVYSJK6DAtJUpdhIUnqMiwkSV2GhSSpy7CQJHUZFpKkLsNCktRlWEiSugwLSVKXYSFJ6jIsJEldhoUkqcuwkCR1GRaSpC7DQpLUZVhIkroMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQuw0KS1GVYSJK6DAtJUtfUwiLJRUm2J7llrO2JSTYk+Uz7enhrT5K/SLIlyaeTPGvsPWvb/p9JsnZa9UqS9m2aRxbvBk7fre11wNVVdRxwdVsHeD5wXHudA7wDRuECnA88GzgROH9XwEiShjO1sKiqjwJ37dZ8BrC+La8Hzhxrv7hGPg4cluRo4HnAhqq6q6ruBjawZwBJkqZs6GsWR1XVnW35S8BRbXkFcMfYfltb277aJUkDmtkF7qoqoCb1eUnOSbIxycYdO3ZM6mMlSQwfFl9up5doX7e39m3AMWP7rWxt+2rfQ1VdWFWrq2r1wsLCxAuXpKVs6LC4Etg1omkt8KGx9le2UVEnAfe001VXAaclObxd2D6ttUmSBrR8Wh+c5H3ATwFHJtnKaFTTnwCXJTkb+ALw0rb7h4EXAFuArwOvAqiqu5L8IfCJtt8bq2r3i+aSpCmbWlhU1cv2senUvexbwLn7+JyLgIsmWJok6WHyDm5JUpdhIUnqMiwkSV2GhSSpy7CQJHUZFpKkLsNCktRlWEiSugwLSVKXYSFJ6jIsJEldhoUkqcuwkCR1GRaSpC7DQpLUZVhIkroMC0lSl2EhSeoyLCRJXYaFJKnLsJAkdRkWkqQuw0KS1GVYSJK6DAtJUpdhIUnqMiwkSV2GhSSpy7CQJHUZFpKkLsNCktRlWEiSugwLSVKXYSFJ6jIsJEldhoUkqWu/CYskpye5PcmWJK+bdT2StJTsF2GRZBnwV8DzgeOBlyU5frZVSdLSsV+EBXAisKWqPltV9wOXAmfMuCZJWjJSVbOuoSvJGuD0qvqVtv4K4NlV9eqxfc4BzmmrTwFuf4TdHgn85yP8jEmYhzrmoQaYjzqs4TvmoY55qAHmo45J1PB9VbWwtw3LH+EHz42quhC4cFKfl2RjVa2e1Oftz3XMQw3zUoc1zFcd81DDvNQx7Rr2l9NQ24BjxtZXtjZJ0gD2l7D4BHBckmOTPAo4C7hyxjVJ0pKxX5yGqqqdSV4NXAUsAy6qqlun3O3ETmk9QvNQxzzUAPNRhzV8xzzUMQ81wHzUMdUa9osL3JKk2dpfTkNJkmbIsJAkdRkWezHrqUWSXJRke5Jbhu57tzqOSXJtktuS3JrkvBnUcEiSG5J8qtXwB0PXMFbLsiSfTPL3M6zh80luTnJTko0zrOOwJJcn+dckm5P86MD9P6X9DHa97k3y2iFraHX8Rvt3eUuS9yU5ZOgaWh3ntRpundbPwWsWu2lTi/wb8FxgK6ORWC+rqtsGrOE5wH3AxVX19KH63UsdRwNHV9WNSb4H2AScOfDPIsBjq+q+JAcD/wycV1UfH6qGsVp+E1gNPL6qXjh0/62GzwOrq2qmN4AlWQ/8U1W9s41QPLSqvjqjWpYxGkr/7Kr6woD9rmD07/H4qvr/SS4DPlxV7x6qhlbH0xnNanEicD/wD8CvVdWWSfbjkcWeZj61SFV9FLhryD73UcedVXVjW/4asBlYMXANVVX3tdWD22vwv3CSrAR+Bnjn0H3PmyRPAJ4DrAOoqvtnFRTNqcC/DxkUY5YDj0myHDgU+I8Z1PBU4Pqq+npV7QSuA1486U4Miz2tAO4YW9/KwL8g51GSVcAzgetn0PeyJDcB24ENVTV4DcCfA78DfHsGfY8r4CNJNrUpbmbhWGAH8K52Wu6dSR47o1pgdN/V+4butKq2AX8KfBG4E7inqj4ydB3ALcBPJDkiyaHAC/jum5gnwrBQV5LHAVcAr62qe4fuv6oeqKpnMLpz/8R22D2YJC8EtlfVpiH73Ycfr6pnMZqB+dx2ynJoy4FnAe+oqmcC/wXM5LEB7RTYi4D3z6DvwxmddTgWeBLw2CQvH7qOqtoMvAn4CKNTUDcBD0y6H8NiT04tMqZdJ7gCuKSqPjDLWtqpjmuB0wfu+mTgRe16waXAKUneO3ANwIN/zVJV24EPMjptOrStwNaxI7zLGYXHLDwfuLGqvjyDvv838Lmq2lFV3wI+APzYDOqgqtZV1QlV9RzgbkbXXSfKsNiTU4s07eLyOmBzVf3ZjGpYSHJYW34Mo4EH/zpkDVX1+qpaWVWrGP17uKaqBv8LMslj20AD2mmf0xidghhUVX0JuCPJU1rTqcBggx528zJmcAqq+SJwUpJD2/+VUxld1xtcku9tX5/M6HrFX0+6j/1iuo8hzWhqke+S5H3ATwFHJtkKnF9V64asoTkZeAVwc7tmAPC7VfXhAWs4GljfRrwcBFxWVTMbujpjRwEfHP1eYjnw11X1DzOq5TXAJe0Pqs8Crxq6gBaYzwV+dei+Aarq+iSXAzcCO4FPMrtpP65IcgTwLeDcaQw4cOisJKnL01CSpC7DQpLUZVhIkroMC0lSl2EhSeoyLKQJSHJfZ/uqhzuLcJJ3J1nzyCqTJsOwkCR1GRbSBCV5XJKrk9zYnjsxPmPx8iSXtOc/XN4mfSPJCUmua5MDXtWmhpfmimEhTdY3gJ9rk/39NPDmNhUEwFOAt1fVU4F7gf/T5t56G7Cmqk4ALgIumEHd0kNyug9psgL8cZsN9tuMprc/qm27o6o+1pbfC/w6o1lCnw5saJmyjNF019JcMSykyfpFYAE4oaq+1Waq3fWozd3n1ilG4XJrVQ36WFLp4fI0lDRZT2D07ItvJflp4PvGtj157FnVv8DokZy3Awu72pMcnORpg1YsLYJhIU3WJcDqJDcDr+S7p1O/ndEDizYDhzN6eND9wBrgTUk+xejBNTN5JoL0UJx1VpLU5ZGFJKnLsJAkdRkWkqQuw0KS1GVYSJK6DAtJUpdhIUnq+m9eHxfIXbJz0QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.countplot(x = joint.label);"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "342c7e33",
   "metadata": {
    "papermill": {
     "duration": 0.024878,
     "end_time": "2021-09-02T19:31:29.967865",
     "exception": false,
     "start_time": "2021-09-02T19:31:29.942987",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "One-hot encoding labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1054cb20",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:31:30.026749Z",
     "iopub.status.busy": "2021-09-02T19:31:30.025674Z",
     "iopub.status.idle": "2021-09-02T19:31:30.029366Z",
     "shell.execute_reply": "2021-09-02T19:31:30.028943Z",
     "shell.execute_reply.started": "2021-09-02T18:38:16.275101Z"
    },
    "papermill": {
     "duration": 0.036944,
     "end_time": "2021-09-02T19:31:30.029481",
     "exception": false,
     "start_time": "2021-09-02T19:31:29.992537",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "y = to_categorical(jointDf.label)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc879f07",
   "metadata": {
    "papermill": {
     "duration": 0.024841,
     "end_time": "2021-09-02T19:31:30.079228",
     "exception": false,
     "start_time": "2021-09-02T19:31:30.054387",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Extracting inference and holdout datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f27f2a70",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:31:30.135508Z",
     "iopub.status.busy": "2021-09-02T19:31:30.135008Z",
     "iopub.status.idle": "2021-09-02T19:31:30.373424Z",
     "shell.execute_reply": "2021-09-02T19:31:30.372971Z",
     "shell.execute_reply.started": "2021-09-02T18:38:16.285068Z"
    },
    "papermill": {
     "duration": 0.269132,
     "end_time": "2021-09-02T19:31:30.373551",
     "exception": false,
     "start_time": "2021-09-02T19:31:30.104419",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((42759, 784), (4751, 784), (42759, 10), (4751, 10))"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train, x_test, y_train, y_test = train_test_split(jointDf.iloc[:,:-1], y, train_size=0.9, random_state=seed) # no need to stratify for large balanced datasets\n",
    "x_train.shape, x_test.shape, y_train.shape, y_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "25d1315d",
   "metadata": {
    "papermill": {
     "duration": 0.02604,
     "end_time": "2021-09-02T19:31:30.425993",
     "exception": false,
     "start_time": "2021-09-02T19:31:30.399953",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Displaying a random observation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "44ebd86d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:31:30.496857Z",
     "iopub.status.busy": "2021-09-02T19:31:30.489822Z",
     "iopub.status.idle": "2021-09-02T19:31:30.608030Z",
     "shell.execute_reply": "2021-09-02T19:31:30.608447Z",
     "shell.execute_reply.started": "2021-09-02T18:38:16.572789Z"
    },
    "papermill": {
     "duration": 0.15671,
     "end_time": "2021-09-02T19:31:30.608602",
     "exception": false,
     "start_time": "2021-09-02T19:31:30.451892",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAANaUlEQVR4nO3dcaid9X3H8c+nWXLFtGVJ7EKapta1diCDxXqbGhTrkJZEC7H/iNlWMlCvHQoWHEzcMHZ/yZgVmdIZNTQbrUXWWjOmrmnokBZNcw2ZJroZlYQmxGRBaKPQeJN898d9LFe95/fcnOc55zn2+37B5ZzzfM9zn+99Tj55zjnP+Z2fI0IAfvd9qOsGAAwHYQeSIOxAEoQdSIKwA0n83jA3tsBjcZYWDnOTQCq/0Vt6O054tlqjsNteI+leSfMkPRQRd5Xuf5YW6gu+oskmARTsiO09a30/jbc9T9L9ktZKukDSetsX9Pv7AAxWk9fsqyS9EhGvRcTbkr4vaV07bQFoW5OwL5f0yxm3D1bL3sX2hO1J25NTOtFgcwCaGPi78RGxKSLGI2J8vsYGvTkAPTQJ+yFJK2bc/kS1DMAIahL2nZLOt32e7QWSrpW0tZ22ALSt71NvEXHS9s2S/lPTp942R8Te1joD0KpG59kj4glJT7TUC4AB4uOyQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQaTdlse7+k45JOSToZEeNtNAWgfY3CXvnTiDjWwu8BMEA8jQeSaBr2kPRj28/ZnpjtDrYnbE/anpzSiYabA9Cvpk/jL42IQ7b/QNI22/8TEU/PvENEbJK0SZI+6sXRcHsA+tToyB4Rh6rLo5Iek7SqjaYAtK/vsNteaPsj71yX9GVJe9pqDEC7mjyNXyrpMdvv/J7vRcRTrXQ1go7duLpn7dk77iuu+6Xrv16sjz25s6+eUHZsovCYbSw/ZrcevrhYf+XPPlmsn3r51WK9C32HPSJek/QnLfYCYIA49QYkQdiBJAg7kARhB5Ig7EASbQyESWH19bt61qbiVHHdt276VbE+9mRfLaHG6hv6f8xOxry22+kcR3YgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSILz7HN03/IdPWtTNedkd37u0WJ9/Ia/KtaXPPhMsZ7VibWfL9bv/fgDPWunVX7MJo+uKNYXvbyvWB9FHNmBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnOs89Rafxz3djoWsyT05e67wk4XdixdY9ZhPvqaZRxZAeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJDjPPkfz3f/3iF+y+9pifclDjFfvh13+gMKH1Ptced3jWfe7P4hqj+y2N9s+anvPjGWLbW+zva+6XDTYNgE0NZen8d+RtOY9y26TtD0izpe0vboNYITVhj0inpb0xnsWr5O0pbq+RdLV7bYFoG39vmZfGhGHq+uvS1ra6462JyRNSNJZOrvPzQFoqvG78RERKgzliIhNETEeEePzNdZ0cwD61G/Yj9heJknV5dH2WgIwCP2GfaukDdX1DZIeb6cdAINS+5rd9iOSLpd0ju2DkjZKukvSo7avk3RA0jWDbHIYDvz96mJ9Kp4r1PKNjR4FdfuV8ezvVhv2iFjfo3RFy70AGCA+LgskQdiBJAg7kARhB5Ig7EASDHGtfPSiY8U6wyVHD0NczwxHdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IIs159hNrP1+s3/7ZR4p1hkuOnkEOcf27z/5HsX7Pmj8v1hc8tbNY7wJHdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IIs159uOfLP+pV539q2K9ydjo47uWFOuLi9Vm5v3RZ4r1I1/82AC3Lv1i4/09a6Xz4FJ5n0+v3/vrvevWr3vMrjr7zWJ97cP/XKx/ZflFxXoXOLIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBJpzrOvvn5XsV43vrl0XrZu3XPveKZYr1M3Fv+tm3p/RuALSw8U1/23Zd8r1uvOR9f97afV/35ruu0mj1nTbY+i2iO77c22j9reM2PZnbYP2d5d/Vw52DYBNDWXp/HfkbRmluX3RMTK6ueJdtsC0LbasEfE05LeGEIvAAaoyRt0N9t+vnqav6jXnWxP2J60PTmlEw02B6CJfsP+bUmflrRS0mFJd/e6Y0RsiojxiBifr7E+Nwegqb7CHhFHIuJURJyW9KCkVe22BaBtfYXd9rIZN78qaU+v+wIYDbXn2W0/IulySefYPihpo6TLba+UFJL2S7pxcC22477lO4r1qSifV20yNvrYxOpiffUN5c8A3PvxB4r10rjw+jHh/f/dUv3f3mS/jfK2L/vmLcX6EjX7bMUg1IY9ItbPsvjhAfQCYID4uCyQBGEHkiDsQBKEHUiCsANJpBniWjckcZDDJZ/deF+jbdedHiutP8hhonNZ/68PX9az9osHLiyuW3P2SzXfRK1nvtl7vzf9u+u2PYo4sgNJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEmnOs9eeN60xqkM169a/ZPe1xXUX/tPvF+sLntpZrNf7Tc/KoIeB3vr1i3vW7l72bHHdun1e+xmAEcSRHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSSHOe/bx/v6FY33vV/cV6l9P/1n1tcWls9ZKHRu8rjYfl55vGe9am7vh5cV3GswP4wCLsQBKEHUiCsANJEHYgCcIOJEHYgSTSnGc/90fl+heX/0Wxbvc+sRpRHtx8fNeSYv3cO8rnwkdx+t8PuqbfIVD6TnpJ+spDF51xT4NWe2S3vcL2T22/aHuv7Vuq5Yttb7O9r7pcNPh2AfRrLk/jT0q6NSIukHSxpJtsXyDpNknbI+J8Sdur2wBGVG3YI+JwROyqrh+X9JKk5ZLWSdpS3W2LpKsH1COAFpzRa3bbn5J0oaQdkpZGxOGq9LqkpT3WmZA0IUln6ey+GwXQzJzfjbf9YUk/kPSNiPj1zFpEhHoMDYiITRExHhHj8zXWqFkA/ZtT2G3P13TQvxsRP6wWH7G9rKovk3R0MC0CaEPt03jblvSwpJci4lszSlslbZB0V3X5+EA6bMnYk+WvRB57cnDbXjy4X42Cpf/V+/iz+vTN5ZUbThc9iqdL5/Ka/RJJX5P0gu3d1bLbNR3yR21fJ+mApGsG0iGAVtSGPSJ+pt7/z13RbjsABoWPywJJEHYgCcIOJEHYgSQIO5BEmiGuyOfUy6/2rC0p1H5XcWQHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkasNue4Xtn9p+0fZe27dUy++0fcj27urnysG3C6Bfc5kk4qSkWyNil+2PSHrO9raqdk9E/OPg2gPQlrnMz35Y0uHq+nHbL0laPujGALTrjF6z2/6UpAsl7agW3Wz7edubbS/qsc6E7Unbk1M60axbAH2bc9htf1jSDyR9IyJ+Lenbkj4taaWmj/x3z7ZeRGyKiPGIGJ+vseYdA+jLnMJue76mg/7diPihJEXEkYg4FRGnJT0oadXg2gTQ1FzejbekhyW9FBHfmrF82Yy7fVXSnvbbA9CWubwbf4mkr0l6wfbuatntktbbXikpJO2XdOMA+gPQkrm8G/8zSZ6l9ET77QAYFD5BByRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSMIRMbyN2f8n6cCMRedIOja0Bs7MqPY2qn1J9NavNns7NyI+NlthqGF/38btyYgY76yBglHtbVT7kuitX8PqjafxQBKEHUii67Bv6nj7JaPa26j2JdFbv4bSW6ev2QEMT9dHdgBDQtiBJDoJu+01tv/X9iu2b+uih15s77f9QjUN9WTHvWy2fdT2nhnLFtveZntfdTnrHHsd9TYS03gXphnvdN91Pf350F+z254n6WVJX5J0UNJOSesj4sWhNtKD7f2SxiOi8w9g2L5M0puS/iUi/rha9g+S3oiIu6r/KBdFxN+MSG93Snqz62m8q9mKls2cZlzS1ZL+Uh3uu0Jf12gI+62LI/sqSa9ExGsR8bak70ta10EfIy8inpb0xnsWr5O0pbq+RdP/WIauR28jISIOR8Su6vpxSe9MM97pviv0NRRdhH25pF/OuH1QozXfe0j6se3nbE903cwslkbE4er665KWdtnMLGqn8R6m90wzPjL7rp/pz5viDbr3uzQiPidpraSbqqerIymmX4ON0rnTOU3jPSyzTDP+W13uu36nP2+qi7AfkrRixu1PVMtGQkQcqi6PSnpMozcV9ZF3ZtCtLo923M9vjdI03rNNM64R2HddTn/eRdh3Sjrf9nm2F0i6VtLWDvp4H9sLqzdOZHuhpC9r9Kai3ippQ3V9g6THO+zlXUZlGu9e04yr433X+fTnETH0H0lXavod+Vcl/W0XPfTo6w8l/Xf1s7fr3iQ9oumndVOafm/jOklLJG2XtE/STyQtHqHe/lXSC5Ke13SwlnXU26Wafor+vKTd1c+VXe+7Ql9D2W98XBZIgjfogCQIO5AEYQeSIOxAEoQdSIKwA0kQdiCJ/wdF4Tbab8EzRAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(np.array(x_test.sample(1, random_state=seed)).reshape(28, 28));"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7568eded",
   "metadata": {
    "papermill": {
     "duration": 0.025524,
     "end_time": "2021-09-02T19:31:30.660374",
     "exception": false,
     "start_time": "2021-09-02T19:31:30.634850",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<h2 id=\"3\">3 - Creating Custom Layer and Callback</h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd77cac8",
   "metadata": {
    "papermill": {
     "duration": 0.02616,
     "end_time": "2021-09-02T19:31:30.712221",
     "exception": false,
     "start_time": "2021-09-02T19:31:30.686061",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Creating a custom callback for the convolutional neural network.   \n",
    "This callback works like the inverse of ReduceLROnPlateau. In case the model converges to a local extremum, the GateOfLearning callback kicks the learning rate up to help it overcome barriers and hopefully converge to a global extremum.\n",
    "  \n",
    "  \n",
    "Guess where the name 'GateOfLearning' is from :D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "47ec53d2",
   "metadata": {
    "_kg_hide-input": false,
    "execution": {
     "iopub.execute_input": "2021-09-02T19:31:30.780129Z",
     "iopub.status.busy": "2021-09-02T19:31:30.776544Z",
     "iopub.status.idle": "2021-09-02T19:31:30.782524Z",
     "shell.execute_reply": "2021-09-02T19:31:30.782115Z",
     "shell.execute_reply.started": "2021-09-02T18:38:16.761202Z"
    },
    "papermill": {
     "duration": 0.044757,
     "end_time": "2021-09-02T19:31:30.782631",
     "exception": false,
     "start_time": "2021-09-02T19:31:30.737874",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class GateOfLearning(Callback):\n",
    "    \"\"\"Increases learning rate when stuck at extrema, a friend to ReduceLROnPlateau, ModelCheckpoint callbacks.\\n\n",
    "\n",
    "    \\n\n",
    "    Args:\\n\n",
    "        `monitor`: quantity to be monitored.\\n\n",
    "        `factor`: factor by which the learning rate will be reduced. Must be multitudes greater than that of the ReduceLROnPlateau\\n\n",
    "            `new_lr = lr * factor`.\\n\n",
    "        `patience`: number of epochs with no improvement after which learning rate will be increased. Must be greater \\\n",
    "            than that of the ReduceLROnPlateau `(6 by default)`\\n\n",
    "        `verbose`: int. 0: quiet, 1: update messages. `(1 by default)`\\n\n",
    "        `mode`: one of `{'min', 'max'}`. In `'min'` mode, the learning rate will be increased when the quantity\\\n",
    "             monitored has stopped decreasing; in `'max'` mode it will be increased when the quantity monitored has stopped increasing.\\n\n",
    "        `cooldown`: number of epochs to wait before resuming normal operation afterlr has been reduced. `(0 by default)`\\n\n",
    "        `max_lr`: upper bound on the learning rate. `(initial value * 50 by default)`\\n\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, monitor=\"val_loss\", factor=15.0, patience=6, verbose=1, mode=\"min\", cooldown=0, max_lr=999):\n",
    "        # Sanity check\n",
    "        if factor <= 1.0:\n",
    "            raise ValueError(\n",
    "                \"GateOfLearning does not support a factor <= 1.0.\")\n",
    "\n",
    "        if mode not in [\"min\", \"max\"]:\n",
    "            raise ValueError(\n",
    "                f\"GateOfLearning does not support a mode '{mode}'. Use 'min' or 'max' instead.\")\n",
    "\n",
    "        # Init\n",
    "        super(GateOfLearning, self).__init__()\n",
    "        self.monitor = monitor\n",
    "        self.factor = factor\n",
    "        self.patience = patience\n",
    "        self.verbose = verbose\n",
    "        self.objective = min if mode == \"min\" else max\n",
    "        self.cooldown = cooldown\n",
    "        self.max_lr = max_lr\n",
    "\n",
    "        self.backup = (monitor, factor, patience,\n",
    "                       verbose, mode, cooldown, max_lr)\n",
    "\n",
    "        self.observations = []\n",
    "        self.lr_history = []\n",
    "        self.last_opened = 0\n",
    "\n",
    "    def _reset(self):\n",
    "        \"\"\"Reset state\"\"\"\n",
    "        self.monitor, self.factor, self.patience, self.verbose, self.mode, self.cooldown, self.max_lr = self.backup\n",
    "\n",
    "        self.observations = []\n",
    "        self.lr_history = []\n",
    "        self.last_opened = 0\n",
    "\n",
    "    def on_train_begin(self, logs=None):\n",
    "        \"\"\"Training start handler\"\"\"\n",
    "        self._reset()\n",
    "\n",
    "    def open_gate(self):\n",
    "        \"\"\"Increases learning rate\"\"\"\n",
    "        new_lr = self.lr_history[-1] * self.factor\n",
    "\n",
    "        assert new_lr > self.lr_history[-1], f\"old: {self.lr_history[-1]}, new: {new_lr}\"\n",
    "\n",
    "        if new_lr > self.max_lr:\n",
    "\n",
    "            if self.verbose:\n",
    "                print(\"Learning rate diverged. You can solve this problem by using a faster ReduceLROnPlateau, \\\n",
    "                    a smaller factor, or a bigger patience/cooldown. Make sure the objective is appropriate.\")\n",
    "        else:\n",
    "            old_lr = float(self.model.optimizer.learning_rate)\n",
    "            self.model.optimizer.learning_rate = new_lr\n",
    "            if self.verbose:\n",
    "                print(\n",
    "                    f\"\\nGateOfLearning: Learning rate increased from {old_lr} to {float(self.model.optimizer.learning_rate)}\")\n",
    "\n",
    "    def on_epoch_end(self, epoch, logs=None):\n",
    "        \"\"\"Epoch end handler\"\"\"\n",
    "        # Log learning rate.\n",
    "        self.lr_history.append(logs[\"lr\"])\n",
    "\n",
    "        # Set the maximum learning rate to the initial or otherwise specified maximum learning rate\n",
    "        if len(self.lr_history) <= 1:\n",
    "            self.max_lr = min(self.max_lr, 50 * self.lr_history[0])\n",
    "\n",
    "        # Check if the metric is reported, otherwise use default metrics.\n",
    "        if self.monitor not in logs.keys():\n",
    "            initMetric = self.monitor\n",
    "            self.monitor = \"val_loss\" if \"val_loss\" in logs.keys() else \"loss\"\n",
    "            if self.verbose:\n",
    "                print(\n",
    "                    f\"\\nGateOfLearning: The '{initMetric}' metric was never reported. Using '{self.monitor}' instead.\\n\")\n",
    "\n",
    "        # Log metric\n",
    "        self.observations.append(logs[self.monitor])\n",
    "\n",
    "        # Check if it is too early for an opening\n",
    "        if len(self.observations) <= self.patience:\n",
    "            return\n",
    "\n",
    "        # Check if there is no improvement\n",
    "        if self.objective(self.observations[-self.patience:]) == self.observations[-self.patience]:\n",
    "            if epoch - self.last_opened > self.cooldown:\n",
    "                self.open_gate()\n",
    "                self.last_opened = epoch\n",
    "                self.observations = [self.observations[-self.patience]]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a223431",
   "metadata": {
    "papermill": {
     "duration": 0.025649,
     "end_time": "2021-09-02T19:31:30.833931",
     "exception": false,
     "start_time": "2021-09-02T19:31:30.808282",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Singularity Extractor is a layer that explores the feature space taking into account the grid-like structure of it. It accentuates non-uniform feature localities that otherwise a convolutional block might miss. An example of this would be the array `[10, 10, 2, 10, 10]`, where the feature in the middle differs from the rest significantly, this significant difference would be accentuated by the singularity extractor and it would output an array that would look like this: `[0.13, 0.08, 0.94, 0.08, 0.13]`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "aa69505e",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:31:30.903113Z",
     "iopub.status.busy": "2021-09-02T19:31:30.902413Z",
     "iopub.status.idle": "2021-09-02T19:31:30.905557Z",
     "shell.execute_reply": "2021-09-02T19:31:30.905944Z",
     "shell.execute_reply.started": "2021-09-02T18:38:16.784557Z"
    },
    "papermill": {
     "duration": 0.046388,
     "end_time": "2021-09-02T19:31:30.906072",
     "exception": false,
     "start_time": "2021-09-02T19:31:30.859684",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class SingularityExtractor2D(Layer):\n",
    "    \"\"\"\n",
    "    Accentuates pixels that differ from its surrounding pixels. \n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, degree=6, kernel_size=3, padding=\"SYMMETRIC\", margin=1, **kwargs):\n",
    "        if not ((kernel_size % 2) and kernel_size >= 3 and isinstance(kernel_size, int)):\n",
    "            raise ValueError(\"kernel_size: value must be odd, >= 3\")\n",
    "\n",
    "        if not (margin >= 1 and isinstance(margin, int)):\n",
    "            raise ValueError(\"margin: must be integer >= 1\")\n",
    "\n",
    "        if padding not in [\"CONSTANT\", \"SYMMETRIC\", \"REFLECT\"]:\n",
    "            raise ValueError(\n",
    "                \"padding: must be one of ['CONSTANT', 'SYMMETRIC', 'REFLECT']\")\n",
    "\n",
    "        self.degree = degree\n",
    "        self.kernel_size = kernel_size\n",
    "        self.padding = padding\n",
    "        self.margin = margin\n",
    "\n",
    "        self.radius = int((kernel_size - 1)/2)\n",
    "        \n",
    "        if \"trainable\" in kwargs.keys():\n",
    "            super(SingularityExtractor2D, self).__init__(**kwargs)\n",
    "        else:\n",
    "            super(SingularityExtractor2D, self).__init__(trainable = False, **kwargs)\n",
    "\n",
    "    # region Keras api\n",
    "\n",
    "    def build(self, input_shape):\n",
    "        if len(input_shape) != 4:\n",
    "            raise ValueError(\"input must be 4D (batch_size, x, y, channels)\")\n",
    "\n",
    "        super(SingularityExtractor2D, self).build(input_shape)\n",
    "\n",
    "        # Lazy load conv kernel\n",
    "        self.ones = tf.ones((self.kernel_size, self.kernel_size, input_shape[-1], input_shape[-1]))\n",
    "\n",
    "    def get_config(self):\n",
    "        \"\"\"Layer to dict = serializability\"\"\"\n",
    "\n",
    "        config = super(SingularityExtractor2D, self).get_config()\n",
    "        config[\"degree\"] = self.degree\n",
    "        config[\"kernel_size\"] = self.kernel_size\n",
    "        config[\"padding\"] = self.padding\n",
    "        config[\"margin\"] = self.margin\n",
    "        config[\"dtype\"] = self.dtype\n",
    "\n",
    "        return config\n",
    "\n",
    "    @classmethod\n",
    "    def from_config(cls, config):\n",
    "        return cls(\n",
    "            config[\"degree\"],\n",
    "            config[\"kernel_size\"],\n",
    "            config[\"padding\"],\n",
    "            config[\"margin\"],\n",
    "            dtype=config[\"dtype\"]\n",
    "        )\n",
    "\n",
    "    def call(self, input_data):\n",
    "        return self.extract(input_data, \"keras\")\n",
    "\n",
    "    # endregion Keras api\n",
    "\n",
    "    # region Scikit api\n",
    "\n",
    "    def fit(self, x, *args, **kwargs):\n",
    "        if len(x.shape) != 4:\n",
    "            raise ValueError(\n",
    "                \"input must be 4D (number_of_observations, x, y, 1)\")\n",
    "\n",
    "    def transform(self, input_data, *args, **kwargs):\n",
    "        return self.extract(input_data, \"sk\")\n",
    "\n",
    "    # endregion Scikit api\n",
    "\n",
    "    def extract(self, input_data, api):\n",
    "        if api == \"sk\":\n",
    "            self.extract(input_data, \"keras\").numpy()\n",
    "        else:\n",
    "            # Pad the incoming tensor\n",
    "            matpad = tf.pad(input_data, [\n",
    "                [0, 0],\n",
    "                [self.radius, self.radius],\n",
    "                [self.radius, self.radius],\n",
    "                [0, 0]\n",
    "            ], mode=self.padding)\n",
    "\n",
    "            # Sum all the pixels in the kernel\n",
    "            conv = tf.nn.convolution(matpad, self.ones, 1, padding=\"SAME\")\n",
    "\n",
    "            # Calculate the change in shape\n",
    "            mrgn_0 = int((conv.shape[1] - input_data.shape[1])/2) + self.margin\n",
    "            mrgn_1 = int((conv.shape[2] - input_data.shape[2])/2) + self.margin\n",
    "\n",
    "            # Select appropriate regions\n",
    "            selection_conv = conv[:, mrgn_0:-mrgn_0, mrgn_1:-mrgn_1, :]\n",
    "            selection_input = input_data[:, self.margin:-\n",
    "                                         self.margin, self.margin:-self.margin, :]\n",
    "\n",
    "            # Compute the output\n",
    "            return ((selection_conv - selection_input) / selection_conv) ** self.degree\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c9898400",
   "metadata": {
    "papermill": {
     "duration": 0.025572,
     "end_time": "2021-09-02T19:31:30.957456",
     "exception": false,
     "start_time": "2021-09-02T19:31:30.931884",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<h2 id=\"4\">4 - Building the model:</h2>"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "200276a5",
   "metadata": {
    "papermill": {
     "duration": 0.02583,
     "end_time": "2021-09-02T19:31:31.009431",
     "exception": false,
     "start_time": "2021-09-02T19:31:30.983601",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Randomly change the width, position and zoom of the training batch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d4ac1728",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:31:31.070664Z",
     "iopub.status.busy": "2021-09-02T19:31:31.070108Z",
     "iopub.status.idle": "2021-09-02T19:31:31.073223Z",
     "shell.execute_reply": "2021-09-02T19:31:31.072748Z",
     "shell.execute_reply.started": "2021-09-02T18:38:16.811025Z"
    },
    "papermill": {
     "duration": 0.037813,
     "end_time": "2021-09-02T19:31:31.073325",
     "exception": false,
     "start_time": "2021-09-02T19:31:31.035512",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class PreprocessingBlock(Layer):\n",
    "    \"\"\"Preprocessing block (width, translation, zoom, seed=173)\"\"\"\n",
    "\n",
    "    def __init__(self, width, translation, zoom, seed=173, **kwargs):\n",
    "        super(PreprocessingBlock, self).__init__(**kwargs)\n",
    "        self.w, self.t, self.z, self.s = width, translation, zoom, seed\n",
    "        self.reshape = Reshape((28, 28, 1))\n",
    "        self.width = RandomWidth(width, seed=seed, interpolation=\"bicubic\")\n",
    "        self.translation = RandomTranslation(\n",
    "            *translation, seed=seed, fill_mode=\"constant\", fill_value=0)\n",
    "        self.zoom = RandomZoom(*zoom, seed=seed)\n",
    "        self.resize = Resizing(28, 28, interpolation=\"bicubic\")\n",
    "\n",
    "    def call(self, input_data, training=None):\n",
    "        x = self.reshape(input_data)\n",
    "\n",
    "        if training:\n",
    "            x = self.width(x)\n",
    "            x = self.translation(x)\n",
    "            x = self.zoom(x)\n",
    "            x = self.resize(x)\n",
    "\n",
    "        return x\n",
    "    \n",
    "    def get_config(self):\n",
    "        config = super(PreprocessingBlock, self).get_config()\n",
    "        config[\"width\"] = self.w\n",
    "        config[\"translation\"] = self.t\n",
    "        config[\"zoom\"] = self.z\n",
    "        config[\"seed\"] = self.s\n",
    "        \n",
    "        return config\n",
    "        \n",
    "    @classmethod\n",
    "    def from_config(cls, config):\n",
    "        cls(**config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "72afb610",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:31:31.134649Z",
     "iopub.status.busy": "2021-09-02T19:31:31.134098Z",
     "iopub.status.idle": "2021-09-02T19:31:31.137078Z",
     "shell.execute_reply": "2021-09-02T19:31:31.136665Z",
     "shell.execute_reply.started": "2021-09-02T18:38:16.830195Z"
    },
    "papermill": {
     "duration": 0.038101,
     "end_time": "2021-09-02T19:31:31.137193",
     "exception": false,
     "start_time": "2021-09-02T19:31:31.099092",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class ConvBlock(Layer):\n",
    "    \"\"\"A block of convolutional layers\"\"\"\n",
    "\n",
    "    def __init__(self, units, kernel_size, depth, pool=False, seed=173, **kwargs):\n",
    "        super(ConvBlock, self).__init__(**kwargs)\n",
    "        self.units = units\n",
    "        self.kernel_size = kernel_size\n",
    "        self.depth = depth\n",
    "        self.pool = pool\n",
    "        self.seed = seed\n",
    "        self.layers = []\n",
    "\n",
    "        for i in range(depth - 1):\n",
    "            self.layers.append(\n",
    "                Conv2D(units, kernel_size=kernel_size, padding=\"same\", activation=\"relu\"))\n",
    "            self.layers.append(BatchNormalization(axis=1))\n",
    "\n",
    "        if pool:\n",
    "            self.layers.append(\n",
    "                Conv2D(units, kernel_size=kernel_size, padding=\"same\", activation=\"relu\"))\n",
    "            self.layers.append(MaxPool2D())\n",
    "        else:\n",
    "            self.layers.append(Conv2D(\n",
    "                units, kernel_size=kernel_size, padding=\"same\", strides=2, activation=\"relu\"))\n",
    "\n",
    "    def call(self, input_data):\n",
    "        out = input_data\n",
    "        for layer in self.layers:\n",
    "            out = layer(out)\n",
    "        return out\n",
    "    \n",
    "    def get_config(self):\n",
    "        config = super(ConvBlock, self).get_config()\n",
    "        config[\"units\"] = self.units\n",
    "        config[\"kernel_size\"] = self.kernel_size\n",
    "        config[\"depth\"] = self.depth\n",
    "        config[\"pool\"] = self.pool\n",
    "        config[\"seed\"] = self.seed\n",
    "        \n",
    "        return config\n",
    "        \n",
    "    @classmethod\n",
    "    def from_config(cls, config):\n",
    "        cls(**config)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0aaf8336",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:31:31.203224Z",
     "iopub.status.busy": "2021-09-02T19:31:31.202616Z",
     "iopub.status.idle": "2021-09-02T19:31:32.266327Z",
     "shell.execute_reply": "2021-09-02T19:31:32.266996Z",
     "shell.execute_reply.started": "2021-09-02T18:43:07.099385Z"
    },
    "papermill": {
     "duration": 1.10414,
     "end_time": "2021-09-02T19:31:32.267229",
     "exception": false,
     "start_time": "2021-09-02T19:31:31.163089",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            [(None, 784)]        0                                            \n",
      "__________________________________________________________________________________________________\n",
      "preprocessing_block (Preprocess (None, 28, 28, 1)    0           input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "conv_block (ConvBlock)          (None, 14, 14, 32)   26576       preprocessing_block[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_1 (BatchNor (None, 14, 14, 32)   56          conv_block[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "singularity_extractor2d (Singul (None, 26, 26, 1)    0           preprocessing_block[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "dropout (Dropout)               (None, 14, 14, 32)   0           batch_normalization_1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu (LeakyReLU)         (None, 26, 26, 1)    0           singularity_extractor2d[0][0]    \n",
      "__________________________________________________________________________________________________\n",
      "conv_block_1 (ConvBlock)        (None, 7, 7, 64)     153784      dropout[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "singularity_extractor2d_1 (Sing (None, 24, 24, 1)    0           leaky_re_lu[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_3 (BatchNor (None, 7, 7, 64)     28          conv_block_1[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_1 (LeakyReLU)       (None, 24, 24, 1)    0           singularity_extractor2d_1[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)             (None, 7, 7, 64)     0           batch_normalization_3[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "singularity_extractor2d_2 (Sing (None, 22, 22, 1)    0           leaky_re_lu_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "conv_block_2 (ConvBlock)        (None, 3, 3, 128)    614684      dropout_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_2 (LeakyReLU)       (None, 22, 22, 1)    0           singularity_extractor2d_2[0][0]  \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d (Globa (None, 128)          0           conv_block_2[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv2d_6 (Conv2D)               (None, 22, 22, 64)   7808        leaky_re_lu_2[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "flatten (Flatten)               (None, 128)          0           global_average_pooling2d[0][0]   \n",
      "__________________________________________________________________________________________________\n",
      "max_pooling2d_3 (MaxPooling2D)  (None, 11, 11, 64)   0           conv2d_6[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_5 (BatchNor (None, 128)          512         flatten[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "flatten_1 (Flatten)             (None, 7744)         0           max_pooling2d_3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)             (None, 128)          0           batch_normalization_5[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_6 (BatchNor (None, 7744)         30976       flatten_1[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "concatenate (Concatenate)       (None, 7872)         0           dropout_2[0][0]                  \n",
      "                                                                 batch_normalization_6[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense (Dense)                   (None, 1024)         8061952     concatenate[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "leaky_re_lu_3 (LeakyReLU)       (None, 1024)         0           dense[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_7 (BatchNor (None, 1024)         4096        leaky_re_lu_3[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)             (None, 1024)         0           batch_normalization_7[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 2048)         2099200     dropout_3[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "batch_normalization_8 (BatchNor (None, 2048)         8192        dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 2048)         0           batch_normalization_8[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 10)           20490       dropout_4[0][0]                  \n",
      "==================================================================================================\n",
      "Total params: 11,028,354\n",
      "Trainable params: 11,006,326\n",
      "Non-trainable params: 22,028\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "def get_model(seed=173):\n",
    "    \"\"\"Generate MSXCN\"\"\"\n",
    "    \n",
    "    x = Input((28 * 28))\n",
    "    y = PreprocessingBlock(0.18, (0.14, 0.2), ((-0.05, -0.1), (-0.05, -0.02)), seed=seed)(x)\n",
    "\n",
    "\n",
    "    # Convolutional chain\n",
    "    y_0_0 = ConvBlock(32, 5, 2, True, seed=seed)(y)\n",
    "    y_0 = BatchNormalization(axis=1)(y_0_0) # Some papers suggest to stay away from\n",
    "    y_0 = Dropout(0.3, seed=seed)(y_0)      # using batch-norm and dropout together\n",
    "                                            # https://link.springer.com/article/10.1007/s11042-019-08453-9\n",
    "        \n",
    "    y_0 = ConvBlock(64, 5, 2, True, seed=seed)(y_0)\n",
    "    y_0 = BatchNormalization(axis=1)(y_0)\n",
    "    y_0 = Dropout(0.3, seed=seed)(y_0)\n",
    "\n",
    "    \n",
    "    y_0 = ConvBlock(128, 5, 2, True, seed=seed)(y_0)\n",
    "    y_0 = GlobalAveragePooling2D()(y_0)\n",
    "    y_0 = Flatten()(y_0)\n",
    "    y_0 = BatchNormalization(axis=1)(y_0)\n",
    "    y_0 = Dropout(0.4, seed=seed)(y_0)\n",
    "    \n",
    "    \n",
    "    \n",
    "    # Singularity Extractor chain 1\n",
    "    y_1 = SingularityExtractor2D(6, 5, margin=1)(y)\n",
    "    y_1 = LeakyReLU(0)(y_1)\n",
    "    y_1 = SingularityExtractor2D(6, 5, margin=1)(y_1)\n",
    "    y_1 = LeakyReLU(0)(y_1)\n",
    "    y_1 = SingularityExtractor2D(6, 5, margin=1)(y_1)\n",
    "    y_1 = LeakyReLU(0)(y_1)\n",
    "    y_1 = Conv2D(64, kernel_size=11, padding=\"same\", activation=\"relu\")(y_1)\n",
    "    y_1 = MaxPool2D()(y_1)\n",
    "    y_1 = Flatten()(y_1)\n",
    "    y_1 = BatchNormalization(axis=1)(y_1)\n",
    "    \n",
    "\n",
    "\n",
    "    # Cognitive block\n",
    "    y = Concatenate()([y_0, y_1])\n",
    "    y = Dense(1024)(y)\n",
    "    y = LeakyReLU(0.1)(y)\n",
    "    y = BatchNormalization(axis=1)(y)\n",
    "    y = Dropout(0.5, seed=seed)(y)\n",
    "\n",
    "    y = Dense(2048, activation=\"tanh\")(y)\n",
    "    y = BatchNormalization(axis=1)(y)\n",
    "    y = Dropout(0.65, seed=seed)(y)\n",
    "\n",
    "    y = Dense(10, activation=\"softmax\")(y)\n",
    "\n",
    "    opt = RMSprop(learning_rate = 0.002, decay = 0)\n",
    "    model = tf.keras.Model(x, y)\n",
    "    model.compile(opt, \"categorical_crossentropy\", metrics=[\"accuracy\"])\n",
    "    \n",
    "    return model\n",
    "    \n",
    "lr_cut = ReduceLROnPlateau(monitor=\"val_loss\", factor=0.3, patience=3, verbose=1, min_lr=0.000000001, mode=\"min\", cooldown=3)\n",
    "quicksave = ModelCheckpoint(monitor=\"val_loss\", filepath=\"data/checkpoint.hdf5\", verbose=1, save_best_only=True)\n",
    "lr_boost = GateOfLearning(monitor=\"val_loss\", factor=100, patience = 17, mode = \"min\", cooldown=19)\n",
    "\n",
    "model = get_model(seed)\n",
    "model.summary()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "cdb40696",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:31:32.365341Z",
     "iopub.status.busy": "2021-09-02T19:31:32.364573Z",
     "iopub.status.idle": "2021-09-02T19:42:13.126822Z",
     "shell.execute_reply": "2021-09-02T19:42:13.127302Z",
     "shell.execute_reply.started": "2021-09-02T18:43:07.892390Z"
    },
    "papermill": {
     "duration": 640.816113,
     "end_time": "2021-09-02T19:42:13.127460",
     "exception": false,
     "start_time": "2021-09-02T19:31:32.311347",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/75\n",
      "335/335 [==============================] - 18s 28ms/step - loss: 2.3565 - accuracy: 0.5879 - val_loss: 3.8668 - val_accuracy: 0.4262\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 3.86684, saving model to data/checkpoint.hdf5\n",
      "Epoch 2/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.2342 - accuracy: 0.9486 - val_loss: 0.6665 - val_accuracy: 0.9148\n",
      "\n",
      "Epoch 00002: val_loss improved from 3.86684 to 0.66654, saving model to data/checkpoint.hdf5\n",
      "Epoch 3/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.1694 - accuracy: 0.9663 - val_loss: 0.3319 - val_accuracy: 0.9339\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.66654 to 0.33190, saving model to data/checkpoint.hdf5\n",
      "Epoch 4/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.1317 - accuracy: 0.9726 - val_loss: 0.2277 - val_accuracy: 0.9653\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.33190 to 0.22773, saving model to data/checkpoint.hdf5\n",
      "Epoch 5/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.1240 - accuracy: 0.9755 - val_loss: 0.1537 - val_accuracy: 0.9741\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.22773 to 0.15375, saving model to data/checkpoint.hdf5\n",
      "Epoch 6/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.1138 - accuracy: 0.9776 - val_loss: 0.1387 - val_accuracy: 0.9766\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.15375 to 0.13872, saving model to data/checkpoint.hdf5\n",
      "Epoch 7/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.1071 - accuracy: 0.9792 - val_loss: 0.0791 - val_accuracy: 0.9846\n",
      "\n",
      "Epoch 00007: val_loss improved from 0.13872 to 0.07912, saving model to data/checkpoint.hdf5\n",
      "Epoch 8/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0920 - accuracy: 0.9817 - val_loss: 0.0682 - val_accuracy: 0.9895\n",
      "\n",
      "Epoch 00008: val_loss improved from 0.07912 to 0.06818, saving model to data/checkpoint.hdf5\n",
      "Epoch 9/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0771 - accuracy: 0.9836 - val_loss: 0.1443 - val_accuracy: 0.9766\n",
      "\n",
      "Epoch 00009: val_loss did not improve from 0.06818\n",
      "Epoch 10/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0745 - accuracy: 0.9854 - val_loss: 0.0866 - val_accuracy: 0.9830\n",
      "\n",
      "Epoch 00010: val_loss did not improve from 0.06818\n",
      "Epoch 11/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0706 - accuracy: 0.9847 - val_loss: 0.0400 - val_accuracy: 0.9914\n",
      "\n",
      "Epoch 00011: val_loss improved from 0.06818 to 0.03998, saving model to data/checkpoint.hdf5\n",
      "Epoch 12/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0698 - accuracy: 0.9863 - val_loss: 0.0754 - val_accuracy: 0.9878\n",
      "\n",
      "Epoch 00012: val_loss did not improve from 0.03998\n",
      "Epoch 13/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0594 - accuracy: 0.9880 - val_loss: 0.0543 - val_accuracy: 0.9905\n",
      "\n",
      "Epoch 00013: val_loss did not improve from 0.03998\n",
      "Epoch 14/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0633 - accuracy: 0.9872 - val_loss: 0.0642 - val_accuracy: 0.9895\n",
      "\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 0.0006000000284984708.\n",
      "\n",
      "Epoch 00014: val_loss did not improve from 0.03998\n",
      "Epoch 15/75\n",
      "335/335 [==============================] - 9s 25ms/step - loss: 0.0461 - accuracy: 0.9904 - val_loss: 0.0352 - val_accuracy: 0.9945\n",
      "\n",
      "Epoch 00015: val_loss improved from 0.03998 to 0.03522, saving model to data/checkpoint.hdf5\n",
      "Epoch 16/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0363 - accuracy: 0.9924 - val_loss: 0.0356 - val_accuracy: 0.9943\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 0.03522\n",
      "Epoch 17/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0287 - accuracy: 0.9927 - val_loss: 0.0532 - val_accuracy: 0.9914\n",
      "\n",
      "Epoch 00017: val_loss did not improve from 0.03522\n",
      "Epoch 18/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0290 - accuracy: 0.9931 - val_loss: 0.0364 - val_accuracy: 0.9935\n",
      "\n",
      "Epoch 00018: val_loss did not improve from 0.03522\n",
      "Epoch 19/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0293 - accuracy: 0.9941 - val_loss: 0.0337 - val_accuracy: 0.9945\n",
      "\n",
      "Epoch 00019: val_loss improved from 0.03522 to 0.03369, saving model to data/checkpoint.hdf5\n",
      "Epoch 20/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0255 - accuracy: 0.9942 - val_loss: 0.0391 - val_accuracy: 0.9931\n",
      "\n",
      "Epoch 00020: val_loss did not improve from 0.03369\n",
      "Epoch 21/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0261 - accuracy: 0.9942 - val_loss: 0.0402 - val_accuracy: 0.9937\n",
      "\n",
      "Epoch 00021: val_loss did not improve from 0.03369\n",
      "Epoch 22/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0263 - accuracy: 0.9942 - val_loss: 0.0314 - val_accuracy: 0.9941\n",
      "\n",
      "Epoch 00022: val_loss improved from 0.03369 to 0.03143, saving model to data/checkpoint.hdf5\n",
      "Epoch 23/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0229 - accuracy: 0.9942 - val_loss: 0.0435 - val_accuracy: 0.9924\n",
      "\n",
      "Epoch 00023: val_loss did not improve from 0.03143\n",
      "Epoch 24/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0242 - accuracy: 0.9944 - val_loss: 0.0396 - val_accuracy: 0.9931\n",
      "\n",
      "Epoch 00024: val_loss did not improve from 0.03143\n",
      "Epoch 25/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0232 - accuracy: 0.9950 - val_loss: 0.0456 - val_accuracy: 0.9933\n",
      "\n",
      "Epoch 00025: ReduceLROnPlateau reducing learning rate to 0.00018000000854954124.\n",
      "\n",
      "Epoch 00025: val_loss did not improve from 0.03143\n",
      "Epoch 26/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0239 - accuracy: 0.9946 - val_loss: 0.0375 - val_accuracy: 0.9939\n",
      "\n",
      "Epoch 00026: val_loss did not improve from 0.03143\n",
      "Epoch 27/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0173 - accuracy: 0.9959 - val_loss: 0.0390 - val_accuracy: 0.9941\n",
      "\n",
      "Epoch 00027: val_loss did not improve from 0.03143\n",
      "Epoch 28/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0192 - accuracy: 0.9954 - val_loss: 0.0365 - val_accuracy: 0.9937\n",
      "\n",
      "Epoch 00028: val_loss did not improve from 0.03143\n",
      "Epoch 29/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0153 - accuracy: 0.9965 - val_loss: 0.0358 - val_accuracy: 0.9943\n",
      "\n",
      "Epoch 00029: val_loss did not improve from 0.03143\n",
      "Epoch 30/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0157 - accuracy: 0.9960 - val_loss: 0.0347 - val_accuracy: 0.9943\n",
      "\n",
      "Epoch 00030: ReduceLROnPlateau reducing learning rate to 5.400000081863254e-05.\n",
      "\n",
      "Epoch 00030: val_loss did not improve from 0.03143\n",
      "Epoch 31/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0156 - accuracy: 0.9964 - val_loss: 0.0344 - val_accuracy: 0.9941\n",
      "\n",
      "Epoch 00031: val_loss did not improve from 0.03143\n",
      "Epoch 32/75\n",
      "335/335 [==============================] - 9s 26ms/step - loss: 0.0131 - accuracy: 0.9966 - val_loss: 0.0343 - val_accuracy: 0.9943\n",
      "\n",
      "Epoch 00032: val_loss did not improve from 0.03143\n",
      "Epoch 33/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0181 - accuracy: 0.9961 - val_loss: 0.0352 - val_accuracy: 0.9939\n",
      "\n",
      "Epoch 00033: val_loss did not improve from 0.03143\n",
      "Epoch 34/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0190 - accuracy: 0.9955 - val_loss: 0.0358 - val_accuracy: 0.9937\n",
      "\n",
      "Epoch 00034: val_loss did not improve from 0.03143\n",
      "Epoch 35/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0159 - accuracy: 0.9966 - val_loss: 0.0337 - val_accuracy: 0.9949\n",
      "\n",
      "Epoch 00035: ReduceLROnPlateau reducing learning rate to 1.6200000027311033e-05.\n",
      "\n",
      "Epoch 00035: val_loss did not improve from 0.03143\n",
      "Epoch 36/75\n",
      "335/335 [==============================] - 9s 25ms/step - loss: 0.0156 - accuracy: 0.9963 - val_loss: 0.0334 - val_accuracy: 0.9949\n",
      "\n",
      "Epoch 00036: val_loss did not improve from 0.03143\n",
      "Epoch 37/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0178 - accuracy: 0.9962 - val_loss: 0.0335 - val_accuracy: 0.9949\n",
      "\n",
      "Epoch 00037: val_loss did not improve from 0.03143\n",
      "Epoch 38/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0121 - accuracy: 0.9968 - val_loss: 0.0341 - val_accuracy: 0.9947\n",
      "\n",
      "Epoch 00038: val_loss did not improve from 0.03143\n",
      "\n",
      "GateOfLearning: Learning rate increased from 1.6199999663513154e-05 to 0.0016199999954551458\n",
      "Epoch 39/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0321 - accuracy: 0.9929 - val_loss: 0.0554 - val_accuracy: 0.9914\n",
      "\n",
      "Epoch 00039: val_loss did not improve from 0.03143\n",
      "Epoch 40/75\n",
      "335/335 [==============================] - 9s 26ms/step - loss: 0.0368 - accuracy: 0.9921 - val_loss: 0.0401 - val_accuracy: 0.9928\n",
      "\n",
      "Epoch 00040: ReduceLROnPlateau reducing learning rate to 0.0004859999986365437.\n",
      "\n",
      "Epoch 00040: val_loss did not improve from 0.03143\n",
      "Epoch 41/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0266 - accuracy: 0.9938 - val_loss: 0.0461 - val_accuracy: 0.9943\n",
      "\n",
      "Epoch 00041: val_loss did not improve from 0.03143\n",
      "Epoch 42/75\n",
      "335/335 [==============================] - 9s 25ms/step - loss: 0.0219 - accuracy: 0.9949 - val_loss: 0.0337 - val_accuracy: 0.9949\n",
      "\n",
      "Epoch 00042: val_loss did not improve from 0.03143\n",
      "Epoch 43/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0202 - accuracy: 0.9954 - val_loss: 0.0422 - val_accuracy: 0.9937\n",
      "\n",
      "Epoch 00043: val_loss did not improve from 0.03143\n",
      "Epoch 44/75\n",
      "335/335 [==============================] - 9s 26ms/step - loss: 0.0179 - accuracy: 0.9958 - val_loss: 0.0323 - val_accuracy: 0.9954\n",
      "\n",
      "Epoch 00044: val_loss did not improve from 0.03143\n",
      "Epoch 45/75\n",
      "335/335 [==============================] - 9s 25ms/step - loss: 0.0185 - accuracy: 0.9958 - val_loss: 0.0467 - val_accuracy: 0.9941\n",
      "\n",
      "Epoch 00045: ReduceLROnPlateau reducing learning rate to 0.00014580000133719294.\n",
      "\n",
      "Epoch 00045: val_loss did not improve from 0.03143\n",
      "Epoch 46/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0193 - accuracy: 0.9958 - val_loss: 0.0372 - val_accuracy: 0.9947\n",
      "\n",
      "Epoch 00046: val_loss did not improve from 0.03143\n",
      "Epoch 47/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0175 - accuracy: 0.9960 - val_loss: 0.0361 - val_accuracy: 0.9952\n",
      "\n",
      "Epoch 00047: val_loss did not improve from 0.03143\n",
      "Epoch 48/75\n",
      "335/335 [==============================] - 9s 26ms/step - loss: 0.0149 - accuracy: 0.9966 - val_loss: 0.0348 - val_accuracy: 0.9949\n",
      "\n",
      "Epoch 00048: val_loss did not improve from 0.03143\n",
      "Epoch 49/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0145 - accuracy: 0.9965 - val_loss: 0.0339 - val_accuracy: 0.9952\n",
      "\n",
      "Epoch 00049: val_loss did not improve from 0.03143\n",
      "Epoch 50/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0159 - accuracy: 0.9965 - val_loss: 0.0363 - val_accuracy: 0.9954\n",
      "\n",
      "Epoch 00050: ReduceLROnPlateau reducing learning rate to 4.37400012742728e-05.\n",
      "\n",
      "Epoch 00050: val_loss did not improve from 0.03143\n",
      "Epoch 51/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0127 - accuracy: 0.9969 - val_loss: 0.0338 - val_accuracy: 0.9954\n",
      "\n",
      "Epoch 00051: val_loss did not improve from 0.03143\n",
      "Epoch 52/75\n",
      "335/335 [==============================] - 9s 26ms/step - loss: 0.0109 - accuracy: 0.9972 - val_loss: 0.0347 - val_accuracy: 0.9952\n",
      "\n",
      "Epoch 00052: val_loss did not improve from 0.03143\n",
      "Epoch 53/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0098 - accuracy: 0.9975 - val_loss: 0.0334 - val_accuracy: 0.9954\n",
      "\n",
      "Epoch 00053: val_loss did not improve from 0.03143\n",
      "Epoch 54/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0108 - accuracy: 0.9974 - val_loss: 0.0333 - val_accuracy: 0.9954\n",
      "\n",
      "Epoch 00054: val_loss did not improve from 0.03143\n",
      "Epoch 55/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0112 - accuracy: 0.9974 - val_loss: 0.0340 - val_accuracy: 0.9960\n",
      "\n",
      "Epoch 00055: ReduceLROnPlateau reducing learning rate to 1.312200038228184e-05.\n",
      "\n",
      "Epoch 00055: val_loss did not improve from 0.03143\n",
      "Epoch 56/75\n",
      "335/335 [==============================] - 9s 26ms/step - loss: 0.0117 - accuracy: 0.9974 - val_loss: 0.0339 - val_accuracy: 0.9958\n",
      "\n",
      "Epoch 00056: val_loss did not improve from 0.03143\n",
      "Epoch 57/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0133 - accuracy: 0.9973 - val_loss: 0.0341 - val_accuracy: 0.9956\n",
      "\n",
      "Epoch 00057: val_loss did not improve from 0.03143\n",
      "Epoch 58/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0142 - accuracy: 0.9970 - val_loss: 0.0333 - val_accuracy: 0.9958\n",
      "\n",
      "Epoch 00058: val_loss did not improve from 0.03143\n",
      "Epoch 59/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0115 - accuracy: 0.9972 - val_loss: 0.0339 - val_accuracy: 0.9956\n",
      "\n",
      "Epoch 00059: val_loss did not improve from 0.03143\n",
      "Epoch 60/75\n",
      "335/335 [==============================] - 9s 27ms/step - loss: 0.0107 - accuracy: 0.9975 - val_loss: 0.0342 - val_accuracy: 0.9958\n",
      "\n",
      "Epoch 00060: ReduceLROnPlateau reducing learning rate to 3.936600114684552e-06.\n",
      "\n",
      "Epoch 00060: val_loss did not improve from 0.03143\n",
      "\n",
      "GateOfLearning: Learning rate increased from 3.936600023735082e-06 to 0.001312199980020523\n",
      "Epoch 61/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0227 - accuracy: 0.9948 - val_loss: 0.0542 - val_accuracy: 0.9928\n",
      "\n",
      "Epoch 00061: val_loss did not improve from 0.03143\n",
      "Epoch 62/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0270 - accuracy: 0.9936 - val_loss: 0.0390 - val_accuracy: 0.9937\n",
      "\n",
      "Epoch 00062: val_loss did not improve from 0.03143\n",
      "Epoch 63/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0285 - accuracy: 0.9939 - val_loss: 0.0521 - val_accuracy: 0.9897\n",
      "\n",
      "Epoch 00063: val_loss did not improve from 0.03143\n",
      "Epoch 64/75\n",
      "335/335 [==============================] - 9s 26ms/step - loss: 0.0282 - accuracy: 0.9941 - val_loss: 0.0269 - val_accuracy: 0.9960\n",
      "\n",
      "Epoch 00064: val_loss improved from 0.03143 to 0.02686, saving model to data/checkpoint.hdf5\n",
      "Epoch 65/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0300 - accuracy: 0.9932 - val_loss: 0.0375 - val_accuracy: 0.9941\n",
      "\n",
      "Epoch 00065: val_loss did not improve from 0.02686\n",
      "Epoch 66/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0249 - accuracy: 0.9944 - val_loss: 0.0380 - val_accuracy: 0.9937\n",
      "\n",
      "Epoch 00066: val_loss did not improve from 0.02686\n",
      "Epoch 67/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0253 - accuracy: 0.9936 - val_loss: 0.0381 - val_accuracy: 0.9941\n",
      "\n",
      "Epoch 00067: ReduceLROnPlateau reducing learning rate to 0.00039365999400615693.\n",
      "\n",
      "Epoch 00067: val_loss did not improve from 0.02686\n",
      "Epoch 68/75\n",
      "335/335 [==============================] - 9s 27ms/step - loss: 0.0206 - accuracy: 0.9952 - val_loss: 0.0338 - val_accuracy: 0.9947\n",
      "\n",
      "Epoch 00068: val_loss did not improve from 0.02686\n",
      "Epoch 69/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0148 - accuracy: 0.9963 - val_loss: 0.0358 - val_accuracy: 0.9945\n",
      "\n",
      "Epoch 00069: val_loss did not improve from 0.02686\n",
      "Epoch 70/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0132 - accuracy: 0.9972 - val_loss: 0.0300 - val_accuracy: 0.9943\n",
      "\n",
      "Epoch 00070: val_loss did not improve from 0.02686\n",
      "Epoch 71/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0160 - accuracy: 0.9961 - val_loss: 0.0304 - val_accuracy: 0.9947\n",
      "\n",
      "Epoch 00071: val_loss did not improve from 0.02686\n",
      "Epoch 72/75\n",
      "335/335 [==============================] - 9s 28ms/step - loss: 0.0128 - accuracy: 0.9971 - val_loss: 0.0312 - val_accuracy: 0.9949\n",
      "\n",
      "Epoch 00072: ReduceLROnPlateau reducing learning rate to 0.00011809799470938741.\n",
      "\n",
      "Epoch 00072: val_loss did not improve from 0.02686\n",
      "Epoch 73/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0175 - accuracy: 0.9963 - val_loss: 0.0293 - val_accuracy: 0.9952\n",
      "\n",
      "Epoch 00073: val_loss did not improve from 0.02686\n",
      "Epoch 74/75\n",
      "335/335 [==============================] - 8s 25ms/step - loss: 0.0127 - accuracy: 0.9973 - val_loss: 0.0279 - val_accuracy: 0.9949\n",
      "\n",
      "Epoch 00074: val_loss did not improve from 0.02686\n",
      "Epoch 75/75\n",
      "335/335 [==============================] - 8s 24ms/step - loss: 0.0101 - accuracy: 0.9972 - val_loss: 0.0279 - val_accuracy: 0.9945\n",
      "\n",
      "Epoch 00075: val_loss did not improve from 0.02686\n"
     ]
    }
   ],
   "source": [
    "with tf.device(\"/device:GPU:0\"):\n",
    "    hist = model.fit(x_train/255, y_train, epochs=75, batch_size=128, validation_data = (x_test/255, y_test), shuffle=True, callbacks=[lr_cut, quicksave, lr_boost])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d34cabb",
   "metadata": {
    "papermill": {
     "duration": 2.502129,
     "end_time": "2021-09-02T19:42:18.271521",
     "exception": false,
     "start_time": "2021-09-02T19:42:15.769392",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<h2>5 - Evaluatong the model</h2>\n",
    "Note: in the context of the Digit Recognition competition, we are only using a validation set to evaluate the model. The test set will be used to calculate the leaderboard score."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "144f8439",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:42:23.688494Z",
     "iopub.status.busy": "2021-09-02T19:42:23.678315Z",
     "iopub.status.idle": "2021-09-02T19:42:23.836792Z",
     "shell.execute_reply": "2021-09-02T19:42:23.836191Z",
     "shell.execute_reply.started": "2021-09-02T18:16:52.061727Z"
    },
    "papermill": {
     "duration": 2.634818,
     "end_time": "2021-09-02T19:42:23.836926",
     "exception": false,
     "start_time": "2021-09-02T19:42:21.202108",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAABHhElEQVR4nO2dd3xb1dnHv8e2bHnHM8NOYpvsHcgmQAIEwgozQKAUKPNllJa3ZbXQFmhp+wItFFqaMtMCKYQVViAJgQAZZJDtOAmZtuMV723L5/3jXNmyI9uy4yFbz/fz0UfSXTrXlu7vPs95htJaIwiCIPgeft09AEEQBKF7EAEQBEHwUUQABEEQfBQRAEEQBB9FBEAQBMFHCejuAbSF2NhYnZSU1N3DEARB6FFs2rQpT2sd13R5jxKApKQkNm7c2N3DEARB6FEopQ65Wy4uIEEQBB9FBEAQBMFHEQEQBEHwUUQABEEQfBQRAEEQBB9FBEAQBMFHEQEQBEHwUUQABEHwSny1VP2q3Tl8vO0oVbWOTv+sHpUIJghC76esqpYH393OtvRClt97BjZ/37lPXZWWw02vbaBOQ3RoIFecksjVkweSEhfWKZ8nAiAIQqdS46ijvLrx3WxQgB92m/9x2x7MK+O2f28iLbsEgPSCCpJjQ7tknN3NnuwS7n7je4b3i+AX5wxjyaZ0Xv7mAAtX72dqcjRPXDa2w4VABEAQhHZRXl1LTnEV0WGBRNhtx63PLKzgtTUHeeO7w5RU1jZaZ/NXnD40jnkTBnD2yL6EBgXwxe5s7lm8BX8/xb1zhvH08j0czCvzCQE4VlrFTa9tIDjQn5eun8SAPsGcNbIvOSWVLNmUzgffZxITFtThnysCIAg+jNaajYcKePO7w6zZd4zxAyM5a2RfZg+PJy7cXHBKq2rZeDCfdfvz2ZZeSFZxJTnFVZRWmYu6n4JRAyKYmhzDtJQYokJsLFp7iI+3H0VrzXlj+3PyoKhGn5tZWMEn24+ycncOwTZ/JiVF8c2+PEb1j+CFH51CaFAATy/fw/68MmZ3+V+la6mqdXD7fzaRU1zFf2+bzoA+wfXr4sPt3DFrCHfMGtIpny0CIAguFJXXEBrkT0Av9zsXldewZHM6b353mH05pYQFBTBzSCxb0wv5bGc2AOMH9gFgR0YRjjpNgJ9i9IAIRvaL4PShQcRHBBEXFkR6QQXr9h/j3+sO8dI3BwAICwrgxhlJXD8jiYHRIW7H8KvzR7LxUAFLt2bwRWoOV5ycyGOXjMFu80drTWSwjQN5pV3y9+gutNY89O4ONhws4G8LJjLB+pt3FSIAgmCx6VABP35pPbeefhL3nD20u4fTaWQXV3L5P9aQXlDBxEF9+PMV47hwXH9CAgPQWpN6tISVqdmsSsvB30/xP2ecxLSUGE4e3IeQwOYvGZU1DrYeKSSjsII5o/oS7sYt5Iqfn2JKcjRTkqPhksbrlFIkx4ZyIK+sA864a9h6pJDKGgdTU2Ja3baq1sHSLZm89M0BdmeV8LOzh3LR+AFdMMrGiAAIAuYu94aXv6Os2tGr7zqLymv48UvfUVBWzVu3TTcXXxeUUowaEMGoARHcfVbbRNBu8/fo4ucpKbGhrNt/rMOO15kUlldzwyvm+/PeHTMYPSDS7XYFZdX8Z90hXlt7iLzSKkb0C+ep+eO57OSELh6xQQRA6JVU1TrYkVFMUkxIq5NnaVklXPfSeiKCbcRFBJFdXNVFozxxSipreO/7DFJiw5iSHE1gQPOuq4pqBze9toEDeWW8fMPk4y7+3kZybCjvfp9BRbWD4MDjI4a8ib+u2EtRRQ3RoYHc9cb3fHj3TMKCGl9eMworuPKFtWQUVnDGsDhuOS2FU4fEoJTqplGLAAheRmZhBYvWHuLH0wc3mgxrCwfzyrjzjc3szCwGYECknTEJkYxJiGSs9eyc4NyfW8q1L64nMMCPN26Zyp+XpZGaVdxh59OZ7Mgo4s43NnPoWDlg/O6nD4vlzBF9mTkkln6R9vptaxx13PXGZjYdLuC5BSczc2hsdw3bY5LjTPTPwWNljOwf0c2jaZ692SX8e90hFkwZxMUTErh64Voeenc7z1w9of7inl1cyTX/WkdxZQ3v3jHjuEnx7kIEQPAaMgorWLBwHYfzy1m84TBPzR/PWSP7tukYS7dm8tC72/H3Uzxx2VhKK2vZnlHEjowiPt+VXb9dvwg7YxIi2JFRjNaa12+ezuCYUOIjgli9x7stAK01r605yB8+2U1MWCD/uWkqlTUOVu7O4Yvd2XyyPQuAuPAgxgyIYGxCJD/klrFydw6PXTKGC8b17+Yz8Axn+OeBPO8VAK01j32cSkigP/fOGUZMWBD3zhnGk5/vYfpJMSyYMohjpVVc++J68kqq+PfNU73m4g8iAIKXkF5QzoJ/raOwvIbnrpnIP778gZte28jNM5O5b+6IFl0bYCYgf/fhLt787jCnDI7i2QUTSWhiQZRU1rArs7heELZnFBFk8+PlGyYzJN4k2MSH2ympqqWsqpbQIO/6eWitySis4PGPUlm2M4uzRsTz5PzxRIUGAnD2qL5oPYadmcVsOJjP9owidmYU89WeXOo0/PzsYVw3bXA3n4XnJMU0CIC3sioth9V7cnn4wlH1rsY7Zg1h/YF8frt0Jymxofzuw10cyS/ntZ9M8aqLP4gACF2I1prNhwtIjg0j2rpoARzJNxf/4ooaXr95KuMS+3D2yL784ZNUXvzmABsOFfDrC0YyND6MPiEN+1XWOFh/IJ+Vqdks35XN0aJK/mfWSdw7Z5jb8gHhdhtTU2JanKjsG2F+xDklVSR3swDklFTy1oYj7M4q4UBeGQfyyiivdhDgp/j1BSO5aWbycf5jpVS9u8tJRbWD/PLq4wTR2wkNCqBfhJ39ud4pANW1dTz2USopcaH8eHqDsPr5KZ6+cgLnP/s1Vy1cR6C/H/+6fhLTOnCCvKMQARC6jJe+OcDjH6cCMLxvONNSopkwqA9PfraH0qpaXr95GmMTzYXLbvPn0YvHMC0lhvuXbGP+C2sBUx8lOTaUsKAANhzMp7zagd3mx8whcfz5inGcNjTuhMbYN8L4zXOKK7stAzWnuJIXvtrP6+sPUe2oY1B0CMmxoUxJjiYlNpRpKTEM7Rvu8fGCA/1JCOxZF38nJhTUO6OyFq09yIG8Ml65cfJxNxxx4UE8e/VEfrlkK49cOIozhp3Y97KzEAEQuoQdGUX8adluZg2PY3JSNOv2H+Otjem8tvYQkcE2Xr95aqO7Vifnj+3PlORoth4p5EBeGT/klnEgr5SjRRVcfnIiZ46MZ3pKjNu6Mu0h3poczi7p+nmAzMIKFq7ez5vfHaa2TnPpxATumj2EJG8thaA1fPEY1Dlgyq0Q2fGhjMlxoXy6/Wj7D7D/Szi8Hmbd32FjAsgqquSZFXuZNTyO2cPj3W4z/aQYvrn/zA793I5GBEDodMqra/np4u+JDg3kL1dOICo0kDtnD6HGUcfOzGIGRNqJj7A3u39sWFCbJ4PbS7yLBdAV1Drq+GJ3Dos3HOHLtByUUlx+cgJ3zh7C4BgvvfA72fAifP2Ueb32ORh9KUy/EwZM7LCPSI4JpaC8hoKy6vq5jjax7S3Y8gZMuQVCOibstby6lpsXbaBOax65cFSHHLO7EAEQOp3HPtrFgbwyXr9paqMfsc3fr8tT31sjwh6A3eZHTidbANnFlfxn3SHe2niE7OIq4sODuGPWEK6eMpDEKPelE7yKo1vhs4dgyBw4///gu4WweRFsfxsGzzRCMGwu+DWZi6ksgs3/NvuPuwqGnAVN4+DL82HTq1CWS/LAewA4cKysfQJQng9oOLQGRl7YrlN1pa5Oc+9/t7Irs5gXr5/UaWWauwoRAKFDKK+u5ZkVexk1III5o/rWlwz4ZPtR3vzuCP8z6yRmDPH+2HOlFPHhdrI7yQLYlVnMi9/s58OtmdTWaWYPj+exiwdy5oj4nlN/qLIY3r4BQmLh0n9CaAzMfQJmPWBEYN0LsHgBRJ8E0++A8ddAeZ5ZvnkRVJdAUARsfwviRhixGHslFGfA+hfg+/9AjcltGDL8J4DJ7WhXBE25lUl88JsOEYCnlqexbGcWv75gJGeO6BqrtDMRARA6hH9+tZ9/rt4PQLDNn7NH9eXMEXH85oOdjE+M5N45w7p5hJ7TNyKowwVgT3YJv/twJ9/uO0ZIoD/XTh3Mjacmeb+bpylaw0c/g4JDcMNH5uLvxB4JM+6GqbdD6lJY8xx8/L+w4lFz0Vd+DW6i+NGw812zzdK74fOHjXXgFwDjroT+4+HT+0ioOYK/n2p/KGhFvnk+9M0Jn/q7m9N5ftUPLJgykJtmJp/w8bwBEQDhhMkpqeRfX+/nvDH9uGFGEku3ZvLJ9qN8uDWTkEB/nrl6Yo/q6hQfYSf1aMdlA2ut+d+3tpJeUM4D541gweRBRIa0XCjNa9n0Kux4B858GAbPcL+Nvw3GXA6jL4PD68w+4X2tieLEhu3GX23cQAe/NpZBn8HGVx/eD4qPwqf3YSvYy8CoYexvrwCU5xvhydphXrdzHmDToXweeGc701Ki+d28Md1avqEjEQEQTphnVuyluraO++aOIDk2lKkpMfx23mjW/HCMPsE2741iaYb48CC+Suu4OYAVqTlszyjiz1eM48pJA83C2mr4+F7rbnhkh33WcdRWw873YMcSqG1i1cSNhKm3QcxJx++XvRPW/xMKDjRefng9nHQmzLy39c9WCgZPN4+Wtkk+3TxcCe9n3ES5aSTHTuRAe3IB6uqgshAGzTAWwOG1MOKCth8H+POyNOLCg3jhR6e0mpTYk/DoTJRSc5VSaUqpfUqpB9ysH6yUWqmU2qaU+lIpleiy7k9KqR3W4yqX5a8qpQ4opbZYjwkdckZCl/JDbimLNxzhmqmDGsXN2/z9OGNYXH1N+Z5E3wg7pVY28ImiteavK/YwOCaEyya6hEke+ha+/zd889cT/gy3lOfD10/DM+PgvVshby84ahoetVWw6RX42ynw5jVw8Fvj3tm3AhZdAv+YYSJoaqsb7zfsHLh04fGTux2NUhA7DPL2kBwbxoG8srY3ia8sBF0HQ8+GALuZB2gHJZU1bDpUwEXjBzRKROwNtGoBKKX8geeBOUA6sEEptVRrvctlsyeBRVrr15RSZwJPANcppS4ATgYmAEHAl0qpT7XWTvv6l1rrJR13OkJX8+dlu7EH+PHTNpYO9mY6Mhv4813Z7Mws5sn54xtP8u5bYZ53fWCiaOwe1LrRGtb9HXJ3G3dKv7HHb3PsB1j3D9jyuplITZkN855zH21Tkm1COTe8CGkfQ3C08ZmH9YOzHoFTbuyw0Ml2ETcc9i4neUQoFTUOsourGhW4a5WKAvMc3h8SJ7dbANb+cIzaOt3xyVw1lbDtv7B/FYy4EEZdbNxnXYgn3+4pwD6t9X4ApdRi4GLAVQBGAU6bcBXwvsvy1VrrWqBWKbUNmAu8deJDF7qbTYfy+WxnNvfOGUZsJ/Qr7S7iw81FJvsEs4Hr6jR/XbGX5NhQLpnQpNnHD1+YC1PJUSMCJ1/XysEc8MkvYeNLoPyNzzxlFky/C046C46sg7XPw+6PzUTq2PnGvdRvTPPHDO8LZ/4KZv4cti2GfSvNhWjM5RDgBXe6scNgy+sMDTeW2P680rYJQLk1ARwcDUkz4cs/GlEIbls00Vd7cgkN9OeUwR1Ux6c01/wfv/uXiY4KijRuuuW/MS65k38MwX065rNawRMBSACOuLxPB6Y22WYrcBnwDHApEK6UirGW/0Yp9RQQAsymsXD8Xin1CLASeEBr7d1lGIV6tNb84ZPdxIUHcfNpXhgRkZNq4tQdNS1vZ4+Ek6+HIWfXuzWcFkC7I4Gqy2H5w6yNvJDUo+U8fWWTu/+iDMjZBXMeNRfyrW+2LAA1FfDOzbD7Izj1Z3DqPbD5NeOjf/2Khjv34Cg47X8bJlI9JTAEJv3EPLyJuOEAnKQyAVMUbsZJbQgldkYAhVgCgIZDa2HE+R4fQmvNV3tymX5S7In7/nPTjEhvXQyOKhh6rhHppNNg7+cmmW75w/DVn+D0X5r/cydPNnfUJPAvgOeUUjcAq4EMwKG1/lwpNRlYA+QCawGHtc+DQBYQCCwE7gcebXpgpdStwK0AgwYN6qDhCu1Fa01WcSUfbzvKpkMF/OHSsS22Cew21jxnkn8GnNzydukbzYU1driJWR93VX02cG57k8GW3W8u7LYdpMQ+yLymrf5+WGmeh8wxAvXFY5B/AKLdCGl5Prx5NRz5Ds77s7lDBHPXPu1Oc+eYuhROmm3i7QN7QBKZp8Sa0OGYigPYbXFtnwiutwCiICIB/IPM3EsbBOBAXhnpBRXcdoabiXJP0BoOfGW+j/uWm7mI8VebC78lcAAMn2seR7fCl3+CFb+BgoNw/pPg33m/L0+OnAEMdHmfaC2rR2udibEAUEqFAZdrrQutdb8Hfm+tewPYYy13FvioUkq9ghGR49BaL8QIBJMmTWrjLJDQERRX1vDqtwfZdKiAnZlF5JVWAzB6QARXTkpsZe9uoLoMdr0PY6+Ai59vedvaarPtmr/Bh/fAyscIv/I17Da/9lkA296GzYuoCO7LlPIN/OKsmOMTvPathPABJvrHHgFfPG7uCmc/2Hi78nx4ea65EMx/FUZf0nh9QCCMv8o8eiNRSeAfhN+xPSTFJLU9F8DVArDZYeAUE3LaBr7akwvA+VWfwdIdxrpyN/fijuxdZgI+azuExsHsXxkrK7QFK6b/eLj6dXNT8PVTUJoNl7/UacLuiU2zARiqlEpWSgUCVwNLXTdQSsUqpZzHehB42Vrub7mCUEqNA8YBn1vv+1vPCtMSescJn43QLvJKq6hx1Lldt3xXNnOe/oq/rNhDdnEls4fH87t5o3nnf2bw7h0zvDN7NfVDqC6FCde2vm1AoEk8um01XP8R1NWitrxB3wh7m1tDVmWl4Vh6D9l9JnKXvh+bcjBXN5l4dNSaSb8hZxrzPjIRUs6ArW+YsEUnWsP7/2PCMH/0zvEXf1/Azx9ihkDuHlLi2tEgvvyYyQEIsooMDj4Vjm6DikKPD/HVnlwuiTpAzJf3G7fbCzNh0cWwd3nj/1dTDn4Lr8w1/v55z8HPdsAZ97V88XeilJmEP/9JSPvUfJ7TmulgWrUAtNa1Sqm7gM8Af+BlrfVOpdSjwEat9VJgFvCEUkpjXEB3WrvbgK+tpIli4EfWhDDA60qpOEABW4DbO+ysvJ3VT5q71Cm3QMSA1rd3R10dfHofjLms+YScViiprOFPy3bzn3WHiQqxcd7Y/swbP4ApSdEcK6vmtx/u5ONtRxnRL5yF103qOSGdW143d4+DWog/b4pSkHwa9B0N+fvpG24np8QzC2DNvjye+HALfyq4l35KcXHWT6gO7U9J9BjCt70J0/+nYeOMTSbjdcjZDcsmXAvv3gKH11i+aow/eM8y4/ZJPs3z8+htxA2DzO9JHhHK5zuzqXHUeZ5UWG7NizhDVpNmwld/NPkAw89rdffKGgd79u/nM/tfIDoFrnvf5FM4515c3IbYXMpt73zf/D+jkuFHS6BPO13XU26BsHh45xZ46Ry47t32H6sZPHIuaa0/AT5psuwRl9dLgOPCObXWlZhIIHfH9O46qZ3JN38xd6hrnoUxVxh/YP9xbTvGvuWw4V9QdKRdArAyNZtfv7+DrOJKfjRtEMUVtby3OYM31h+mX4SdyloH5VUOfnHOMG4746Sek8lbeBgOfA2zHmzfBFp0MuxdTtyAIFIzPcsGfmdzBj8qfIFRfofYeOo/WTJpHgl9glHf7Tcinb3TCAsY/7/yMxE8TkZcCIHhpmpl0kw4sgFW/BZGXmTCPX2Z2OGw831S+vhTW6dJL6jwPDKrIh9CXEpVJE428wAHv/FIADYcyOOPPEeIoxjmvw99BjbMvTRxGzL5ZvPY+S58er9xNy1YfOJhtKMuNu6jT+8D/46PzPLC2bteTlWpufhPvsWE621eZELwRlwIVy4yZq8nrH3OPP/wBVSVQJBnDULyy6r57dKdLN2ayfC+4fz92pOZaBXZKq+uZUVqDh9uzURrzQPnjaxvldghFKWbH0dlUePlccNN/ZhYN7kEWTtMpcnoFM+iIrb+F9Bmoq09RKdAaTaJIZovPZwDiDryOVep5TDjp0ya4/K5Y66Az35lLuzn/t4s27cCEiY1DkUMDDEunh3vwuyHYMlPjGU477lOjwLxeuKGAZqRgaaf84G8Us8FoDzfREg5sdkhcZLH+QA1Xz3Nmf7bqT7nKfxd/f5Ot+HY+eZYa58zlsXXT0FdjfktX/5iY6vgRBg8A25d3SnJdz3ktq4XUZZjnhNOhvP+CPfuMncOuz8y2ZqecHQbHFgNw88HRzXsXY7Wmhe/3s91L61nf677DkppWSVc9Ldv+HTHUX5+9jA+vHtm/cUfICQwgHnjB/CvH0/ixesnd+zF31FjKkj+sMpkZzofdbWmPPBzk+CNq83de10d7F1hfJ8vnGpcOit+Y4qGOVrIztXa+NKTToOodva+jU4B4CRbDmXVDkpbyQZ21GlmFS8lPyjB+G1dCY2BYeeajFpHLZQdg4zNjd0/TiZcCzVlxtQvOQpXvNplseBeTayJlBnkMHEnOzLaUKOpouD4O/CkmZC1rdFNyDd783jpmwONM40PreWMjIWsDT6DwKk3uT++0214zX/hro1wyvXGQrhyUcdd/J10Uua1CEBXU2oJQJjVRSi4D0z8kXmdl+bZMdY+B4FhJsIlNI661A/53Ye7ePzjVNbtP8ZFf/uGD7Y0CtRi1e4cLv/HGs6u+YJvpqzlnrOHdlxNE0cNbF8C/73O+LjdsfJRSN8AlzwPN37S8PjJMvj5DjjjAUj/Dl67EP4vBV6/3MRNn/Ub+MVeExf9/b9h8TVm/sQdR9ZD/n4Yv6D952IJwGCygNYbw2RkZjKVneQknOs+i3PCNUb0f1hpJn/RJiu3KYOmGZ9xcQbM+R0kntL+c+hNxAwB5Udo8T5mDonlha9+aPYG5zjK88muDWn8W0iaaW48dhuP9g+5pdz674089tEu7n9nG446DQWHcLx9I4fr4kib/LhnVljsULjgKTj7t55b8V6ACEBXU2pMWcJcaolb8c7k7ml9/+JMU41x4nUQEk3tsPOp3vUpb67Zwy2nJfPVL2czakAE9yzewv1LtlFR7eCVbw9w02vf8WDwe/zO8Tf6fv+sOc6JUlkE3z4Lz0yAd26CtE/g1Qthz+eNt9vzmZnvmHSTKQfclLB4EwL5851w0bPmDv7ShXDPNjjtXnMXd+av4YKnzdzHa/PM3XRTtrwBtlDjN20vUSYWv2+t+fu0FglUuO1DbMqB/+h57jcYeo6pm7/ldRP+GRzlvmOWUqam/mm/gGl3tH/8vQ2bHfoMRuXt4cn54wkK8OOni7+nuraFCBxM34Xq0jzeT6vgnsVbePFrU6qcgVOh71hYehfV3y/mztc3ExTgx09OTeatjen8+dW30S/NwVFZyp019zBjtBcmOXYgIgBdTb0F4CIAgaEQOdAzC2D9P80dzLTbKSqv4U+HhmLXFTw7tYhfXTCKAX2CefOWadw5+yTe2nSE0/78BY9/uJ3XYv/DtZVvmkqOYOYOToSNL8PTo0zmYnSymfD6+U5zJ/Tm1aapBxi//3u3mdjpc//Q8jFtwcaMvurfJra9aTmCyTfBlf+G7B3w0tnGZ+50CVWXm6SoUfMg6ARcV/YICIklqiodoNVIoNAflnFUR5Mw5lT3G/jbjK847VPY+5n5+zd3hzj8PDjrYfH7NyVuOOTtoV+knT9dPo4dGcU89bn730pGYQX3vrWFy/+2gkBdzcRhKZw3ph+Pf5zKu5vTISDI9DEYNJ3AD27j9Nw3ePrK8Txy0Sj+Pr2Yuw79lPyKOh6N/yv54SMY2pFuUC9EBKCrKc0xUSCu0QlgrIDcVgSgqtRUcBw5j8qwgVy1cC1vZCdRYwvnXLWhfrMAfz9+ee4IXrtxClG2Gj7r9wKnlXxq3CjXvmOKfTmLkbWHg9+aRh8Jp8CtX5kf1PDzTPmBGz42ce0f3GkyGpfcZFxE818zd3MnysgL4cdWGsqSG+HZiSa9fttiqCo2LpcTJTqF0NLDAOS0ZAFUlzEwfw1rbNMICWohQmPCNWaupvyYqdsjtI3YYXBsHzhqOWd0P340bRD/XL2fr/fm1m9SVFHDE5+mMvvJL/lo21HunGJ8/1NGD+GvV09gxkkx/HLJNr7YnQ3Bffh43HN85JjGQ7Y3mX3gL7DtLc7fejeO8AQuLH+E/+wP5oxhcb2m7n9zSBRQV1OabcK6mt4Fxg03pQvq6pqf8NnyunG7TL+Lvyzfw+6sEl6+YQq2Xeebao6OZxqljZ+eFMrymKeNX/6Cp80dNBgf9O6PTYGxtvory/KMuycqCa76z/FVLIPCYcF/zYTtl9Yd/+Uvua85314GTTWTbnuWmYv/Zw+Z5ZGDTD/aEyU6Bf9D3xBs8285G3jfSgJ1FftjW4lo7j8O+o4xlos7/7/QMnHDjYAWHoKYk/jV+aNYvz+fxYsXMXF8Noujbue5Vfsoqqjh0okJ3DtnGImV+0wlsuBoggL8+ed1p7DgX+u44/XN/P6SsfxmaRoj+j/Meckf4b/+H+ZzBs+kz9Wv8+Cech58ZxsXNS3h0QsRAehqSnMaJoBdiR0GtRUmrt9dBEudw5QCHjiVbWoo//r6WxZMGWj6ktZdaMrKHvrW3H07WXa/mXid/1rjTNIhZxkxydgMAyd7Pva6OuPOKT8GN69ovoRxQCBc+oL54eo6U5Kho/HzN809RlxgzmPjS5Z7pQOM2ugU1Lb/khiuWmwO79i1lCIdhkryIOHsrN+YRK+2FGkTDFYkELlpEHMSwYH+/O2KEUS+dANhm/P5v8rpTB2WwANzRzBqgPWd/MGaI7KigMLtNl69cQrzX1jL/769lchgG89ccwr+kaeam5Nj++Ds34HNzrzxfbhgbH/8/Xr33T+IAHQ9pdmN/f9OnIWh8va4F4C0T6DgIDVnPcp9S7YRFx7Eg+dbnaSGnAUBwaYEglMArJo0zLz3+DICKbONG2rfirYJwJpnzD7nP2lqlrSEUmYCtytIONk8OoroFEAzJqSQzOJmRK62GvYsY7ljEkP7eVAmeNg55iG0nTgrSCIvDTCF3EYcXATKlEd4fX4Ck06Z0ngfZx0glzyA2LAgFv1kCve/s43bzjiJhD5WqOaUW477SF+4+IPMAXQ9pTnuBcD1Lscdez8Hex9eyBrO7qwSfn/JWCLsVthhYKiLW6cO8vaZxt2DppsCVE0JiTZVMp1VKT3h8DqT8TjqEpO30JuxqnIOD8xt3gI4sBr/6hKW1U1meD/PkvCEdmKPNPNWzii5kmzTSS3SlEWY1MdNWKizdk6TubaB0SG8ccu0jm/u0kMRAehKtG6YA2hKaIz5sjYXCXRkA2XxE3l21X4uGj+As0c1EZGR86Ak07gZ3r7BpI1f/lLzpWSHnG3mBlorMuWoNbVN3r7RpMLPe7b3R6lYuQDJftnN5wGkLqXaL4T1jCEltndHingFccMafhtf/sH0N573jHlflH789s5uYG1s/uJriAuoK6koMKni7iwAMFZA7h7SC8rZdKgArU2D8n5BVaTkpvJ+yCTCggL47UVuyisNO9eUlvjvdcb8veZtiEw4fjsnQ8426ev7V5kOUE2pKjEZuuv/YerrRCWbDEd7ZPvOvScRHAX2SBJ0Vn02cJhra8g6B+z+mC3BU0gIi+pVTcK9ltjhpmR29k7j2pxym8kXQbkXgPJ8U1/JGzqbeTEiAF1JmRW21mQSuNZRx/tbMkkojmFU4Spm/ukLTJFUw+l+W1kUCB8XDuS3V44mxl37xeA+kHyGcevM+Gnr/uaEk8HeB/Z9cbwAbHnD1OypKjZupHOfMGGePSjD8YRQCqJTiKtxJoNVEhbncpd/eB2U5/Fx4CSGJYj7p0uIGw7VJaZEdlCEKa3sbzNtNd1aAPkQInf/rSEC0JW4ywIGFn69nz8vS+Ou4BimU8oT5/Zn3PAhBAX4k1NSSfR331CX5sdF5190fHcpV2b+zJSLbVqTxh1+/qaL1A8rjWvK6dbJ2ARLf2oqJ57zuO+WJIhOIeKgya3IKa7iJFcBSP0Q7R/EkuKR3NZXBKBLcGbLH91qEgqdNX4iE03kXFOaFoIT3CK2a1fiLgsY+GjrUSYO6sO9Cy4CYEFyBaMHRDIkPowZJ8UyoiYVv76jWTBzVMuJKcmnw0V/dV+Txh1DzjaFx3KsNs0VhcbXH97PdCXy1Ys/QHQK9rIMAqhtnA2sNez+iOKE0ygjmGEiAF2DM0ouKtlU0nXSZ2ALFoAIQGuIAHQl9RZAgwvo0LEydh0t5oKx/fGLdxMJVFdn7srbEq7pKc6yEPtWmAvb0rtNMbIrXpYfT1QySjtIUHmNk8GOboGiI6RFmXBbiQDqIsL6mgv/xc839utHJhoBaNqdSywAjxAXUFdSmm0aUrhMpH66w1SdPHd0P4iwm2JmeS5F4XJ3G1984pSmRztxIgZA/GgjALYQ01x8zqOmmYWvY0UCDQ3IbVwOIvVDUP6sVpOx2woZFN2LmrB7M0rBBU8evzxyoMkSLsuFcBfLWiwAjxALoCtx5gC4uHE+3ZHF2IRIBkaHmCzW2CGNBSD9O/PcWRflIWfBobWmnMLQc2D63Z3zOT0NSwDGBB8ju6SJACTNZOsxP4bGh/tMwpDXEplonl3dQI5aUzJFLIBWEQHoSkqzG7l/Mgsr2HqkkPPGupQHsEJB6znynckPsC5IHc6Qs0xoakgsXPJCpzWe6HGExYMtlKG23AYXUG6aEeeRF5GWVSL+f2+gXgBcJoKdOQBiAbSK/Nq7kiZ1gJZZ7p/zxvRv2CZuGBSnm8qfYAQgcUrnJV8NmgHjrzGF3UJjWt/eV7BCQQeRRa7TAkg1VUgLB51DTkkVw/tJAli3484CcFMGQnCPCEBX0sQC+HTHUUb0C2/c4zTWpSZQeT4c29s5E8BOAgLh0n/4dsRPc0Qn09+RSVZRJbWOOuP+SZxMWrm58IsF4AXY+5iEL1cBqC8DIQLQGiIAXUWdw1TRtEJAc0oq2XiogLljmlSHdC0Kl27V+B84tQsHKtQTnUxUdSZVNTXc99JHJgZ95EXsyS4BJALIK1Dq+FyAChEATxEB6CrK8kxpZMsC+GxnNlo3cf+A8fX7BRh/85HvQPm7byEodD7RKfjX1fCX8+KJOWzaXB7tfzZp2SWE2wPoF9EBDW6EE6epAJSLC8hTJAy0q2iSBbxsx1FS4kIZ1reJH9nfZkQgb48J/+w3xlT7FLoea+L94oGVzO6fSlruYBa8nklYUADD+4b3+m5RPYbIRMjc3PBeLACPEQugq3DJAs4vq2bd/nzOG9PP/UUkdpjJzk3f1Dnx/4JnOCOvDq8jIncTcZOvIDLYxuH8coaJ+8d7iEw07tXqcvO+PB/8bBAok/StIQLQVbhkAS/flYWjTh/v/nESNxzy90NNmfj/u5PwASZxb8O/AE30pMt5744Z/GjaIK6aNLC7Ryc4ibT+F8UZ5tmZBCYWWquIC6ircApAaDyf7thBYlQwo53t65rijASCzo0AElrGz8/0Ps5LM9ZA/Cj6KMXjl4zt7pEJrrjmAsQOlTIQbUAsgK6iNMeEqwWGsOlgAbOGxzXvQ3a2wAuNhz5u2kMKXYfTDTTyIrmj9Fb6WBaAMxS0XMpAeIoIQFtYvxC2vdW+fa0cgOLKGkqqaluuIeMsfTuwExPABM+oF4B53TsOoXnC+5se14VWJFBFvnQC8xBxAbWFDS+axivjrmz7vlYdoMzCCgAGOBtSuyMwFE77RUODd6H7GHsF1NWaHsqCd9K0MUx5PgwUC8ATRADaQlUJVBa2b9+yHIgf5ZkAAJz1cPs+R+hYEk42D8G7ceYCaG1ZACIAniAuoLZQVWJcOc5ws7ZQmg1hfckoNIXFEloTAEEQPMfZF6CqxFhsIVLXyhNEADylzmF6koJpkt4WaipNedqweDILK7D5K+Lc9fUVBKF9RCaaMNDyY+a9TAJ7hAiAp1SXNrwuONi2fcsaksAyCyvoF2nHT+rIC0LH4WwM4+ymJy4gjxAB8JSqkobXbRWA0sYCMCBS3D+C0KE4k8GytplnsQA8QgTAUyqLG163WQAasoAzCyvF/y8IHY0zGezoVvMsFoBHiAB4iqsFUHio+e1W/9/xuQKWANQGx5JVXNl6BJAgCG2jXgDEAmgLEgbqKU4BCIlt3gKoq4Nv/goBQSZz1GZd6C0XUE5dBI46LQIgCB2NPdJqDGMFaNj7dOtwegpiAXhKVZF57j/OCIDWx29TdMRMFpcfg62LG5aXZkNIDJkltQAM6CN15AWhQ3E2hgEjBv5yb+sJHgmAUmquUipNKbVPKfWAm/WDlVIrlVLblFJfKqUSXdb9SSm1w3pc5bI8WSm13jrmf5VSgR1zSp2E0wLoNxZqyk2Dl6bkpJrnoEhY93djEUB9FnCGlQQmcwCC0Ak4BUD8/x7TqgAopfyB54HzgFHAAqXUqCabPQks0lqPAx4FnrD2vQA4GZgATAV+oZRylsD8E/AXrfUQoAC46YTPpjOpF4Bx5tmdGyhnl3k++xHT0GXfcvPeagafaSWB9RcBEISOx1kUTvz/HuOJBTAF2Ke13q+1rgYWAxc32WYU8IX1epXL+lHAaq11rda6DNgGzFWmDOaZwBJru9eAS9p9Fl1BZTGgoO9o896tAKSacLSTr4eIBFjzN7PcygLOLKwgMthGWJCYp4LQ4YgF0GY8EYAEwKXhJunWMle2ApdZry8FwpVSMdbyuUqpEKVULDAbGAjEAIVa69oWjgmAUupWpdRGpdTG3NxcT86pc6gqgaDwhvLMzQlA/EhTnGrqbXDwaxOWVm8BVMgEsCB0Fs5cACkD4TEdNQn8C+AMpdT3wBlABuDQWn8OfAKsAd4E1gKOthxYa71Qaz1Jaz0pLi6ug4bbDpwCEBgCYf2g8GDj9Y5a0zgkfqR5f/L1piXdl3+E2or6OYAEmQAWhM7BaQGIC8hjPBGADMxdu5NEa1k9WutMrfVlWuuJwK+sZYXW8++11hO01nMABewBjgF9lFIBzR3T66gqgiBr+iJqMBQ0yQXI329S0eOt6ZHgPjDxOkj7xLx3ZgGLBSAInYO4gNqMJwKwARhqRe0EAlcDS103UErFKqWcx3oQeNla7m+5glBKjQPGAZ9rrTVmruAKa5/rgQ9O9GQ6FacFAKZNYFMXkHMC2GkBAEy73TSqAMoDoymurBUBEITOIiIRpt0BIy7o7pH0GFoVAMtPfxfwGZAKvKW13qmUelQp5WyTNAtIU0rtAfoCv7eW24CvlVK7gIXAj1z8/vcD9yql9mHmBF7qoHNqnYpCWHKTaRzhKU0FoDgDaqsb1uekmou9s5uXczurk1ROXSTgQR8AQRDah58fzH0C+jYNUhSaw6NwFK31JxhfvuuyR1xeL6Ehosd1m0pMJJC7Y+7HRBh1PUe+gx1LYNxVMOwcz/apLIY+g8zrqCTQdSbxK+Yksyxnp2kfaGtygT/z12CP4KAaAOTIHIAgCF6Db2YCl2aZ59oKz/dxtQDcRQLlpDb4/12JHQrz/kZmsZn7FgtAEARvwTcFoMSqzlnTVgFwTgInmWenANRUmElgdwJgkVlYgb+fIj5cLABBELwD3xSA0jYKgKMWasoaBCC8P/gHNlQFzdtjXEKuE8BNyCysoF+EHX9pBCMIgpfgowJguYA8FQBnK0inC8jPz8wHOC0AZw2gFiwAkwMg7h9BELwH3xQApwvI0zmAqiYCAI1DQXN2GYsgOqXZQ2QWVUgVUEEQvArfFIB6C6DSs+2d3cDsEQ3LGglAKsQOb7YEraNOk1UkjWAEQfAufE8AtHaZBC73bB93FkCfwVBZBBUFDTWAmiGvtIoahzSCEQTBu/A9AagsBEeVeV3roQVQLwBNLACArB0mH6AFAZA+AIIgeCO+JwDOu3/w3AVUZbmA3AnAnmXmuZUQUJAcAEEQvAvfEwCn/x/a4AJyCoDrJLCVDOYs9tZKCChIK0hBELwL3xMApwUQENwOF5CLANgjITjKJIAFhjWUiXBDZmEl4fYAwu22dg5aEASh4/E9AXBaAFFJnlsAlcWm0FtgaOPlTjdQ/EjTlLoZJAdAEARvxPcEoCQbbCEQGtuGOQCrDlDTi7yrALRAZmEF/SPF/SMIgnfhewJQmgVhfY0ItCURzHUC2ImzKFwLE8CANIIRBMEr8T0BKMmG8H6mbLOnpSCqit0LgAcWQHl1LQXlNSIAgiB4HR71A+hVlGZB3zGWALQhDNR1AtjJ8PPg6BYYOLXZXTMLzWfIHIAgCN6G71oAAfa2ZQK7E4DwfnDRM8c3gXFBcgAEQfBWfEsAqstMZc/6OYA21AKyu3EBecAn24+iFCTFhLRrf0EQhM7CtwTA2QcgvB/Y7GYOQOvW92vOAmiFlanZLN5whFtPTyE+QqKABEHwLnxLAJxJYGF9jdtGO8BR0/p+7RCAY6VV3P/ONkb0C+feOcNa30EQBKGL8S0BcCaBhfczmcDQeiioo8ZsExTp8cdorXnw3e0UV9Ty16snEBTg384BC4IgdB6+JQCNLADLJdNaKKi7MhCt8PamdD7flc0vzx3OiH7tmzsQBEHobHxLAEqzwC8AgqPNJDB4IABuCsG1wJH8cn63dCfTUqK5aWbyCQxWEAShc/EtASjJNnf/fn4mDBRajwRy1w2sGbTW3LdkG35K8eT88fhJA3hBELwY3xIAZxkIcLEAWskFaIMLaP2BfNbuP8a95wwjMUrCPgVB8G58SwCcSWDgMgfQigXQBgF4ftU+YsOCWDCl+dLQgiAI3oJvCYCrBeCMAvJ4DqDlKKCtRwr5em8eN5+WjN0mUT+CIHg/viMAtdVQfszFAvAwDNTDSeDnVu0jwh7AtVPl7l8QhJ6B7whAWY55rp8DcFoAJ+4CSssqYfmubG44NVm6fgmC0GPwHQEocSkDAS4C0MokcGWxCR1toeDb37/cR0igPzfOSDrxcQqCIHQRviMAzizgpnMArYWBNtcNzOJgXhkfbs3kR9MGExUa2EGDFQRB6Hx8RwBKXMpAgEsUkAdhoC24f1746gcC/P24WZK+BEHoYfiOAJRmAwpC48z7AE/nAIqbjQA6WlTBO5vTuWrSQKn2KQhCj8N3BKAkC0JiwN+apPXzA/8gD6KAmrcA3t2cQY1Dc+vpKR08WEEQhM7HdwSgNKfB/ePEk77AzbWDBD7adpRTBkcxMFqyfgVB6Hn4kAC4JIE58UQAmukG9kNuKalHi7lgbP8OHKQgCELX4TsC4FoGwkmA3bNy0G4sgE+2HQXgfBEAQRB6KL4hAHV1JhHsOAvAg77AzQjAx9uPMjkpin6RMvkrCELPxCMBUErNVUqlKaX2KaUecLN+sFJqpVJqm1LqS6VUosu6PyuldiqlUpVSzyplAuqt7dKUUlusR3zHnVYTyo9BXa2bOYBWLIDaKnBUQVBjF9C+nBJ2Z5WI+0cQhB5NqwKglPIHngfOA0YBC5RSo5ps9iSwSGs9DngUeMLadwZwKjAOGANMBs5w2e9arfUE65FzoifTLE2TwJzYQloWgPoyEI0F4ONtWSgF54kACILQg/HEApgC7NNa79daVwOLgYubbDMK+MJ6vcplvQbsQCAQBNiA7BMddJtpWgbCSYC95TDQZgrBfbw9k8lJ0fSV2H9BEHownghAAnDE5X26tcyVrcBl1utLgXClVIzWei1GEI5aj8+01qku+71iuX8edrqGmqKUulUptVEptTE3N9eD4bqhWQugFReQm25ge7JL2JNdyoXj5O5fEISeTUdNAv8COEMp9T3GxZMBOJRSQ4CRQCJGNM5USp1m7XOt1noscJr1uM7dgbXWC7XWk7TWk+Li4to3upITdQE1WAAfbzuKUjB3TL9mdhIEQegZeCIAGcBAl/eJ1rJ6tNaZWuvLtNYTgV9Zywox1sA6rXWp1roU+BSYbq3PsJ5LgDcwrqbOoTTb+PEDmyRsBdhbjgJqIgBaaz7efpSpydHEh4v7RxCEno0nArABGKqUSlZKBQJXA0tdN1BKxSqlnMd6EHjZen0YYxkEKKVsGOsg1Xofa+1rAy4Edpz46TRDiZskMGjVAigrzjcvrEngPdml7Msp5YJxAzpjlIIgCF1KQGsbaK1rlVJ3AZ8B/sDLWuudSqlHgY1a66XALOAJpZQGVgN3WrsvAc4EtmMmhJdprT9USoUCn1kXf39gBfCvjj01F+JHQUTTaQtanAP4YEsGGz/YwGM2uO2tvQweXE1GQQV+CuaOFvePIAg9n1YFAEBr/QnwSZNlj7i8XoK52DfdzwHc5mZ5GXBKWwfbbmY/6H55QDDU1YCjFvwb/hTZxZU88sFOfh5RBxVwrDaQVWsOUl1bx2lDY4kLD+qigQuCIHQeHglAr8W1L7B/g5//oXe3U1nj4OKJ4bA1kCV3n0mNo459OaX0k9BPQRB6Cb5RCqI53PQFfndzBit35/DLc4cT5VdZPwFs8/djZP8I6folCEKvQQQA6ruCZRVV8tsPdzI5KYobT01utRuYIAhCT8a3BSDAcufUVqK15oF3t1HjqOP/rhiPv5+yegEcXwpaEAShN+DbAlBvAVTw9sZ0vkzL5YG5I0iKDTXLq0pEAARB6LWIAADUVPDKmoOMS4zkx9OTGtZXuW8GIwiC0BvwbQGwGsNXV5axJ7uE04bG4ufnUpKosvl2kIIgCD0d3xYAywJIzzmGo04zNiGy8XqZBBYEoRcjAgBk5JqSD2NcBUBrmQMQBKFX49sCYEUBZeYVEh0aSEKf4IZ1tZUmS1gsAEEQeim+LQA2Ux00r6CQMQmRNGpJ4KYUtCAIQm/CxwXAWADFJSWMTWji6nEKgL3JvIAgCEIvwbdrAVlRQEG6ilFNJ4Ari8yzWACCIPRSfNsC8A/AoQKwq+rGE8AgLiBBEHo9vi0AQLUKIjKgtvEEMEBloXm29+nqIQmCIHQJPi8AFTqQAaGa43rSlx8zzyHRXT8oQRCELsCnBaCyxkFpnY34YH38ynKrHWSwCIAgCL0TnxaA3VklVOhAYgIdx6+sKDBhojZpACMIQu/EpwVge0YRlQQSaXMjAOX5cvcvCEKvxqcFYEd6EQ6/IOyq+viVFfkQEtX1gxIEQegifFoAtmcUEWAPRdVUHL9SLABBEHo5PisAlTUO9mSXYA8ObdQTuJ6KfIkAEgShV+OzApCWVUJtnSYsLLy+J3AjyvMhJKbrByYIgtBF+KwAbM8wpR4iwiNM5U9X6hwmEUxcQIIg9GJ8VgB2ZBTRJ8RGaGgoNJ0DqCwCXScuIEEQejU+KwDbM4oYmxCJCgw5XgAkCUwQBB/AJwWgssZBWlaJKQAXEAyOKqira9igwhIAsQAEQejF+KQAbEsvotbZA9hqC0mtixUgFoAgCD6ATwrAorUHCQ8K4LShsQ0C4BoKWm8BSCKYIAi9F58TgPSCcj7dkcWCqYMIt9vq+wKLBSAIgq/hcwLw6rcHUcANM5LMAqsvcKOJ4Ip8UP7SDlIQhF6NTwlAcWUNizcc4YJx/RngbADjrPZZ08QCCImGpj0CBEEQehE+JQD//e4IpVW13DwzpWFh/SRwkzkAcf8IgtDL8RkBqHXU8cq3B5iaHM3YRBfXToBzEtilHES51AESBKH34zMC8MmOLDKLKrn5tJTGK+pdQC4WgFQCFQTBB/AJAdBa8+LX+0mJDeWsEfGNV9ZPArtYANILQBAEH8AnBGDDwQK2pRfxk5nJ+Pk1mditDwO1LACtxQIQBMEn8AkB+NfX+4kKsXH5yYnHr2waBlpTbkpDyByAIAi9nIDuHkBXcM3UQcwd3Y/gQP/jVzYNA5UkMEEQfASPLACl1FylVJpSap9S6gE36wcrpVYqpbYppb5USiW6rPuzUmqnUipVKfWsUia4Xil1ilJqu3XM+uWdwezh8Vx+ipu7f2iIAnJmAkshOEEQfIRWBUAp5Q88D5wHjAIWKKVGNdnsSWCR1noc8CjwhLXvDOBUYBwwBpgMnGHt8w/gFmCo9Zh7oifTLvxtJutXLABBEHwMTyyAKcA+rfV+rXU1sBi4uMk2o4AvrNerXNZrwA4EAkGADchWSvUHIrTW67TWGlgEXHIiJ9JulDLJYM4w0HoLQNpBCoLQu/FEABKAIy7v061lrmwFLrNeXwqEK6VitNZrMYJw1Hp8prVOtfZPb+WYACilblVKbVRKbczNzfVguO3AFtzgAioXF5AgCL5BR0UB/QI4Qyn1PcbFkwE4lFJDgJFAIuYCf6ZS6rS2HFhrvVBrPUlrPSkuLq6DhtuEgGA3LiDJAxAEoXfjSRRQBjDQ5X2itawerXUmlgWglAoDLtdaFyqlbgHWaa1LrXWfAtOBf1vHafaYXYrN3iAAFfkQFGHmBgRBEHoxnlgAG4ChSqlkpVQgcDWw1HUDpVSsUsp5rAeBl63XhzGWQYBSyoaxDlK11keBYqXUNCv658fABx1wPu3D1sQCkLt/QRB8gFYtAK11rVLqLuAzwB94WWu9Uyn1KLBRa70UmAU8oZTSwGrgTmv3JcCZwHbMhPAyrfWH1ro7gFeBYOBT69E9BAQ3DgMV/78gtImamhrS09OprKxsfWOh07Db7SQmJmKzeebB8CgRTGv9CfBJk2WPuLxegrnYN93PAdzWzDE3YkJDux9bMFSXmddSBkIQ2kx6ejrh4eEkJSXRiSk9QgtorTl27Bjp6ekkJyd7tI9PlIJoFVcXkFgAgtBmKisriYmJkYt/N6KUIiYmpk1WmAgAmIJw9WGgBWIBCEI7kIt/99PW/4EIAJiCcDUV4KiFqiJJAhMEwScQAYCGMNCKAvNeXECCIPgAIgBgZQJXQvkx817CQAVBcENtbW13D6FD8Yly0K0SEGz6ADgFQCwAQWg3v/twJ7syizv0mKMGRPCbi0a3uM0ll1zCkSNHqKys5J577uHWW29l2bJlPPTQQzgcDmJjY1m5ciWlpaXcfffdbNy4EaUUv/nNb7j88ssJCwujtLQUgCVLlvDRRx/x6quvcsMNN2C32/n+++859dRTufrqq7nnnnuorKwkODiYV155heHDh+NwOLj//vtZtmwZfn5+3HLLLYwePZpnn32W999/H4Dly5fz97//nffee69D/z7tRQQAGnoClBw1zzIJLAg9jpdffpno6GgqKiqYPHkyF198MbfccgurV68mOTmZ/HxT5uWxxx4jMjKS7du3A1BQUNDqsdPT01mzZg3+/v4UFxfz9ddfExAQwIoVK3jooYd45513WLhwIQcPHmTLli0EBASQn59PVFQUd9xxB7m5ucTFxfHKK6/wk5/8pFP/Dm1BBAAauoIVZ5pnsQAEod20dqfeWTz77LP1d9ZHjhxh4cKFnH766fUx8dHR5ne9YsUKFi9eXL9fVFTrLt/58+fj728aShUVFXH99dezd+9elFLU1NTUH/f2228nICCg0eddd911/Oc//+HGG29k7dq1LFq0qIPO+MQRAYCGvsBOARALQBB6FF9++SUrVqxg7dq1hISEMGvWLCZMmMDu3bs9PoZrCGXTWPrQ0ND61w8//DCzZ8/mvffe4+DBg8yaNavF4954441cdNFF2O125s+fXy8Q3oBMAoOLBZAO/oEQGNry9oIgeBVFRUVERUUREhLC7t27WbduHZWVlaxevZoDBw4A1LuA5syZw/PPP1+/r9MF1LdvX1JTU6mrq2vRR19UVERCgqle/+qrr9YvnzNnDv/85z/rJ4qdnzdgwAAGDBjA448/zo033thxJ90BiABAwxxAcaa5+5eEFkHoUcydO5fa2lpGjhzJAw88wLRp04iLi2PhwoVcdtlljB8/nquuugqAX//61xQUFDBmzBjGjx/PqlWrAPjjH//IhRdeyIwZM+jfv3+zn3Xffffx4IMPMnHixEZRQTfffDODBg1i3LhxjB8/njfeeKN+3bXXXsvAgQMZOXJkJ/0F2ocyDbl6BpMmTdIbN27s+APv+RzemA9h/UwS2B1rOv4zBKEXk5qa6nUXN2/irrvuYuLEidx0002d/lnu/hdKqU1a60lNt/UeZ1R3YrMaw5dmQ+zQ7h2LIAi9ilNOOYXQ0FCeeuqp7h7KcYgAQIMAoCUJTBCEDmXTpk3dPYRmkTkAcBEAJARUEASfQQQAGsJAQUJABUHwGUQAQCwAQRB8EhEAaCwAYgEIguAjiACAKQbnRCwAQfAJwsLCunsI3Y4IAEBAEGAlf4kFIAhCF+ANpaUlDBRM5q/NKgktFoAgnBifPgBZ2zv2mP3Gwnl/bHb1Aw88wMCBA7nzzjsB+O1vf0tYWBi33347F198MQUFBdTU1PD4449z8cUXt/hR7spKA72ytLQIgJMAuyUA0g5SEHoaV111FT/72c/qBeCtt97is88+w26389577xEREUFeXh7Tpk1j3rx5LfbObVpW+vLLL6eurq5XlpYWAXBiCzEtIe2R3T0SQejZtHCn3llMnDiRnJwcMjMzyc3NJSoqioEDB1JTU8NDDz3E6tWr8fPzIyMjg+zsbPr169fssZqWld67dy+5ubm9srS0CIATmx2C+4Cff3ePRBCEdjB//nyWLFlCVlZWfeG3119/ndzcXDZt2oTNZiMpKem4Us+uuCsr3dL2zdFTSkvLJLATW7BMAAtCD+aqq65i8eLFLFmyhPnz5wPmDjs+Ph6bzcaqVas4dOhQi8dwV1YaYNq0ab2ytLQIgJOAYJkAFoQezOjRoykpKSEhIaG+nPO1117Lxo0bGTt2LIsWLWLEiBEtHsNdWWmg15aWlnLQTlI/BOUHIy7onOMLQi9GykF3Ha2VlpZy0O1h5EXdPQJBEIQW6ejS0iIAgiAIPYSOLi0tcwCCIHQIPcmd3Ftp6/9ABEAQhBPGbrdz7NgxEYFuRGvNsWPHsNvtrW9sIS4gQRBOmMTERNLT08nNze3uofg0drudxMREj7cXARAE4YSx2Wz1WbJCz0FcQIIgCD6KCIAgCIKPIgIgCILgo/SoTGClVC7QcjGP5okF8jpwOF2BjLnz6WnjBRlzV9HTxtzSeAdrreOaLuxRAnAiKKU2ukuF9mZkzJ1PTxsvyJi7ip425vaMV1xAgiAIPooIgCAIgo/iSwKwsLsH0A5kzJ1PTxsvyJi7ip425jaP12fmAARBEITG+JIFIAiCILggAiAIguCj+IQAKKXmKqXSlFL7lFIPdPd43KGUelkplaOU2uGyLFoptVwptdd6jurOMbqilBqolFqllNqllNqplLrHWu7NY7Yrpb5TSm21xvw7a3myUmq99f34r1IqsLvH6opSyl8p9b1S6iPrvbeP96BSartSaotSaqO1zGu/FwBKqT5KqSVKqd1KqVSl1HRvHrNSarj193U+ipVSP2vrmHu9ACil/IHngfOAUcACpdSo7h2VW14F5jZZ9gCwUms9FFhpvfcWaoH/1VqPAqYBd1p/V28ecxVwptZ6PDABmKuUmgb8CfiL1noIUAC477XXfdwDpLq89/bxAszWWk9wiUv35u8FwDPAMq31CGA85u/ttWPWWqdZf98JwClAOfAebR2z1rpXP4DpwGcu7x8EHuzucTUz1iRgh8v7NKC/9bo/kNbdY2xh7B8Ac3rKmIEQYDMwFZM9GeDu+9LdDyDR+iGfCXwEKG8erzWmg0Bsk2Ve+70AIoEDWEExPWHMTcZ5DvBte8bc6y0AIAE44vI+3VrWE+irtT5qvc4C+nbnYJpDKZUETATW4+VjttwpW4AcYDnwA1Cota61NvG278dfgfuAOut9DN49XgANfK6U2qSUutVa5s3fi2QgF3jFcrW9qJQKxbvH7MrVwJvW6zaN2RcEoFegjaR7XcyuUioMeAf4mda62HWdN45Za+3QxmxOBKYAI7p3RM2jlLoQyNFad2wj2M5nptb6ZIzb9U6l1OmuK73wexEAnAz8Q2s9ESijievEC8cMgDX/Mw94u+k6T8bsCwKQAQx0eZ9oLesJZCul+gNYzzndPJ5GKKVsmIv/61rrd63FXj1mJ1rrQmAVxoXSRynlbI7kTd+PU4F5SqmDwGKMG+gZvHe8AGitM6znHIxfegre/b1IB9K11uut90swguDNY3ZyHrBZa51tvW/TmH1BADYAQ63IiUCMubS0m8fkKUuB663X12P87F6BUkoBLwGpWuunXVZ585jjlFJ9rNfBmDmLVIwQXGFt5jVj1lo/qLVO1FonYb63X2itr8VLxwuglApVSoU7X2P80zvw4u+F1joLOKKUGm4tOgvYhReP2YUFNLh/oK1j7u4JjC6aJDkf2IPx9/6qu8fTzBjfBI4CNZg7kpsw/t6VwF5gBRDd3eN0Ge9MjHm5DdhiPc738jGPA763xrwDeMRangJ8B+zDmNJB3T1WN2OfBXzk7eO1xrbVeux0/t68+XthjW8CsNH6brwPRPWAMYcCx4BIl2VtGrOUghAEQfBRfMEFJAiCILhBBEAQBMFHEQEQBEHwUUQABEEQfBQRAEEQBB9FBEAQBMFHEQEQBEHwUf4f/v1BFdDXCQsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(hist.history[\"accuracy\"][6:]);\n",
    "plt.plot(hist.history[\"val_accuracy\"][6:]);\n",
    "plt.legend([\"accuracy\", \"val accuracy\"]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "cbba5870",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:42:28.983105Z",
     "iopub.status.busy": "2021-09-02T19:42:28.982389Z",
     "iopub.status.idle": "2021-09-02T19:42:29.194084Z",
     "shell.execute_reply": "2021-09-02T19:42:29.193654Z",
     "shell.execute_reply.started": "2021-09-02T18:16:55.273909Z"
    },
    "papermill": {
     "duration": 2.617846,
     "end_time": "2021-09-02T19:42:29.194209",
     "exception": false,
     "start_time": "2021-09-02T19:42:26.576363",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAABFa0lEQVR4nO2dd3Rc1bWHvz3qVm+WZUuy3HuXTTfFFEPoJRgIGEJCCjxCSHkO5BFCQhpJgCQkhIQkkNC7qYbQTDOuMu62XGQVFxVbktXLeX+cGWs0GlmjOtLM/tbSunPPPXNnz2jmd/fdZ599xBiDoiiKErg4/G2AoiiK0reo0CuKogQ4KvSKoigBjgq9oihKgKNCryiKEuCE+tsAT1JSUkx2dra/zVAUZRCxrWwbABOSJ/jZEv+xZs2aUmNMqrdjA07os7OzWb16tb/NUBRlEHHav04D4IPrP/CrHf5ERPI7OqahG0VRlABHhV5RFCXAUaFXFEUJcAZcjF5RXDQ2NlJYWEhdXZ2/TRmwREZGkpGRQVhYmL9NUQYwKvTKgKWwsJDY2Fiys7MREX+bM+AwxlBWVkZhYSGjRo3ytznKAEZDN8qApa6ujuTkZBX5DhARkpOT9Y5H6RQVemVAoyJ/bPTzUXwh+IR+/dNQf8TfViiKovQbwSX0h/bAS9+ALUv9bYkySIiJifG3CYrSY4JL6Our7Lauwr92KIqi9CPBJfSNtXbrEnxF8RFjDD/4wQ+YOnUq06ZN45lnngFg3759zJ8/n5kzZzJ16lQ++ugjmpubuf7664/2vf/++/1svRLs+JReKSILgQeBEODvxphfeRyfDzwATAcWGWOe9zgeB2wGXjbG3NILdnePxhq7VaEfdPz01U1sLq7s1XNOHh7HTy6Y4lPfF198kdzcXNavX09paSlz585l/vz5PPnkk5xzzjnceeedNDc3U1NTQ25uLkVFRWzcuBGAw4cP96rditJVOvXoRSQEeAg4F5gMXCUikz267QWuB57s4DQ/A5Z338xewuXRN+hgrNI1Pv74Y6666ipCQkJIS0vj1FNPZdWqVcydO5d//vOf3H333WzYsIHY2FhGjx7Nrl27+J//+R/eeust4uLi/G2+EuT44tHPA/KMMbsARORp4CKshw6AMWaP81iL55NFZA6QBrwF5PTc5B6gHv2gxVfPu7+ZP38+y5cv5/XXX+f666/n9ttv57rrrmP9+vUsW7aMhx9+mGeffZZ//OMf/jZVCWJ8idGPAArc9gudbZ0iIg7gd8D3u25aH9DgEnr16JWuccopp/DMM8/Q3NxMSUkJy5cvZ968eeTn55OWlsbXv/51vva1r7F27VpKS0tpaWnhsssu4+c//zlr1671t/lKkNPXJRC+DbxhjCk81sQOEbkJuAkgKyur76zRwVilm1xyySV89tlnzJgxAxHhN7/5DcOGDeOxxx7jvvvuIywsjJiYGB5//HGKioq44YYbaGmxN7i//OUv/Wy9Euz4IvRFQKbbfoazzRdOAE4RkW8DMUC4iBwxxixx72SMeQR4BCAnJ8f4eO6u4wrdNKjQK75x5Ii9+xMR7rvvPu677742xxcvXszixYvbPU+9eGUg4YvQrwLGicgorMAvAq725eTGmGtcj0XkeiDHU+T7FfXoFUUJQjqN0RtjmoBbgGXAFuBZY8wmEblHRC4EEJG5IlIIXAH8VUQ29aXR3aZRY/SKogQfPsXojTFvAG94tN3l9ngVNqRzrHP8C/hXly3sTTS9UlGUICTIZsbWtG6bm/xri6IoSj8RnEIP6tUrihI0BJnQ17Y+1gFZRVGChCATevXolb6lo7LGWu5Y8SdBJvS1gHPilmbeKIoSJASf0A9Jto/re7cSohJ4LFmyhIceeujo/t13381vf/tbjhw5woIFC5g9ezbTpk3jlVde8fmcWu5Y8Qd9XQJhYNFQDTFpUFOqoZvBxptLYP+G3j3nsGlw7q86PHzllVdy2223cfPNNwPw7LPPsmzZMiIjI3nppZeIi4ujtLSU448/ngsvvNCn9Vu13LHiD4JL6BtrITEbDm7SwVilU2bNmsXBgwcpLi6mpKSExMREMjMzaWxs5I477mD58uU4HA6Kioo4cOAAw4YN6/Scxyp3/NWvfpXGxkYuvvhiZs6c2abc8Ze+9CXOPvvsfnjXSiASfEIfM9Q+1hj94OIYnndfcsUVV/D888+zf/9+rrzySgCeeOIJSkpKWLNmDWFhYWRnZ1NXV9ej19Fyx0pfEmQx+hqITrWP1aNXfODKK6/k6aef5vnnn+eKK64AoKKigqFDhxIWFsb7779Pfn6+z+fTcseKPwgej765EVoaITIBQiK0gqXiE1OmTKGqqooRI0aQnp4OwDXXXMMFF1zAtGnTyMnJYeLEiT6fT8sdK/4geITelUMfFgURsRq6UXxmw4a2g8ApKSl89tlnXvu6yhp31K7ljhV/EDyhG9es2LAoiIjR0I2iKEFDEAm906MPj4bwWE2vVBQlaAgioXf36GPVox8kGNN3C44FAvr5KL4QhEI/REM3g4TIyEjKyspUzDrAGENZWRmRkZH+NkUZ4ATvYGz5Lv/ao3RKRkYGhYWFlJSU+NuUAUtkZCQZGcdc80dRgkjoG9yEPlw9+sFAWFgYo0aN8rcZijLoCaLQjUvoh2h6paIoQUUQCb17jD4WGquhpdm/NimKovQDQST0bh59uHMRCE2xVBQlCPBJ6EVkoYhsE5E8EVni5fh8EVkrIk0icrlb+0wR+UxENonIFyJyZW8a3yU80ytB4/SKogQFnQq9iIQADwHnApOBq0Rkske3vcD1wJMe7TXAdcaYKcBC4AERSeihzd3Dc2YsaJxeUZSgwJesm3lAnjFmF4CIPA1cBGx2dTDG7HEea3F/ojFmu9vjYhE5CKQCh3tqeJdprLbFzBwhEBFn2zR0oyhKEOBL6GYEUOC2X+hs6xIiMg8IB3Z6OXaTiKwWkdV9ljPdWGu9eWiN0etygoqiBAH9MhgrIunAv4EbjDEtnseNMY8YY3KMMTmpqal9Y0Rjja1zAxq6URQlqPBF6IuATLf9DGebT4hIHPA6cKcxZkXXzOtF3D16HYxVFCWI8EXoVwHjRGSUiIQDi4Clvpzc2f8l4HFjzPPdN7MXaBO6cQq9xugVRQkCOhV6Y0wTcAuwDNgCPGuM2SQi94jIhQAiMldECoErgL+KyCbn078MzAeuF5Fc59/MvngjndJQbXPowS10ox69oiiBj0+1bowxbwBveLTd5fZ4FTak4/m8/wD/6aGNvUNjLYQ7hT40AkLCVegVRQkKgmhmbG2rRw82Tq+hG0VRgoAgEvqatkKvFSwVRQkSgkjo3QZjQStYKooSNASR0Ne0D93ohClFUYKAIBN6N48+PEZj9IqiBAXBIfTNTdDc4MWjV6FXFCXwCQ6hb3JWrgx3F3odjFUUJTgIDqF3L1HsIlzTKxVFCQ6CROjdVpdy4cqjb2lXY01RFCWgCBKh9+LRR+hygoqiBAfBIfQNHXj0oEKvKErAExxCfzR045FeCTogqyhKwBMkQu8K3US3trmWE9QUS0VRApwgEXovHn2ELieoKEpwECRC7y29UgdjFUUJDoJE6Kvt1ttgrMboFUUJcIJE6L2lV7qEXj16RVECmyATem/plerRK4oS2ASJ0NfYpQND3FZODI0AR5iGbhRFCXiCROg9Fh1xoRUsFUUJAoJE6D0WHXGhFSwVRQkCfBJ6EVkoIttEJE9Elng5Pl9E1opIk4hc7nFssYjscP4t7i3Du0RDjXePXitYKooSBHQq9CISAjwEnAtMBq4Skcke3fYC1wNPejw3CfgJcBwwD/iJiCT23Owu0ljbgUevywkqihL4+OLRzwPyjDG7jDENwNPARe4djDF7jDFfAJ41f88B3jHGlBtjDgHvAAt7we6u4bmMoIuIGI3RK4oS8Pgi9COAArf9QmebL/j0XBG5SURWi8jqkpISH0/dngOVddQ1Nrc/cCyPXkM3iqIEOANiMNYY84gxJscYk5Oamtqtc+wureaUX7/Pc2sK2x/saDA2XAdjFUUJfHwR+iIg020/w9nmCz15bpfITh7CtIx4Hv5gJ43NHhEkTa9UFCWI8UXoVwHjRGSUiIQDi4ClPp5/GXC2iCQ6B2HPdrb1OiLCLWeMpehwLS+t87iWdJheGWtnxupygoqiBDCdCr0xpgm4BSvQW4BnjTGbROQeEbkQQETmikghcAXwVxHZ5HxuOfAz7MViFXCPs61POG18KlNHxPHn9/NobjGtBzoajHVVsHQVPVMURQlAQjvvAsaYN4A3PNrucnu8ChuW8fbcfwD/6IGNPiMi3HL6OL75nzW89kUxF810jvseK3QDNnzjeqwoihJgDIjB2N7k7MlpjE+L4aH382hpMTYs01QH4dHtO2upYkVRgoCAE3qHQ7j59LFsP3CEtzcfgCYvJYpdaAVLRVGCgIATeoAvTUsnO3kIf3p/B6bBtYxgB+mVoB69oigBTUAKfWiIg2+fNpaNRZWs2Oacr9XRzFjQFEtFUQKagBR6gItnjWBEQhTPfrbdNhxzMFY9ekVRApeAFfrwUAeXz8lgV7GzpEKYl8HYcFeMXj16RVECl4AVeoCc7EQiTL3dUY9eUZQgJaCFflZWIkOkwe54G4wNjQBHqAq9oigBTUALfUxEKOMSnW/Rm0cvYjNvNHSjKEoAE9BCDzAh2U7+bQ71IvQAEXHq0SuKEtAEvNCPdXr0Ow95qVMPum6soigBT8AL/cg4AWDtvnrvHXTxEUVRApyAF/qE0CYAVhbVee+gi48oihLgBLzQS1MtzYSwcm8HYh4Rq0KvKEpAE/BCT0MNzSFRFB6q5UClF68+Mh7qKvrfLkVRlH4i8IW+sQYJtxk3a/IPtT+uQq8oSoATBEJfS2hkNJFhDu9CH5Vg69U3dhDDVxRFGeQEgdDXIGHRTM9IYHVHHj2oV68oSsASBEJvlxHMGZnIpqIK6ho98ukjE+y27nB/W6YoitIvBI3QzxmZSFOLYX3B4bbHoxLsttajXVEUJUDwSehFZKGIbBORPBFZ4uV4hIg84zz+uYhkO9vDROQxEdkgIltE5Ee9bH/nNFZD2BBmZyUCsGavR/jmqEevoRtFUQKTToVeREKAh4BzgcnAVSIy2aPbjcAhY8xY4H7g1872K4AIY8w0YA7wDddFoN9wevSJ0eGMSY1mzZ6OhP5wv5qlKIrSX/ji0c8D8owxu4wxDcDTwEUefS4CHnM+fh5YICICGCBaREKBKKABqOwVy32lsfZoieKckUms2XsIY0zrcQ3dKIoS4IT60GcEUOC2Xwgc11EfY0yTiFQAyVjRvwjYBwwBvmuMKfd8ARG5CbgJICsrq4tvwY3qMggJg8i41rbGGgi3Qj9nZCLPrC7gPyvyMUBJVT3lVdXcCxq6URQlYPFF6HvCPKAZGA4kAh+JyH+NMbvcOxljHgEeAcjJyTHtzuILh/bAgzPh/N9Dzldb252hG4DjRicB8H+vbALAIRAdEcodJoJD+4rJ6NYLK4qiDGx8EfoiINNtP8PZ5q1PoTNMEw+UAVcDbxljGoGDIvIJkAPsordJGAlxI2Dne61C39JiPXpn6GZkcjRvfucUjIHU2AiSosNpbG6h6hcxrNueT1xdI3GRYb1umqIoij/xJUa/ChgnIqNEJBxYBCz16LMUWOx8fDnwnrGB8L3AGQAiEg0cD2ztDcPbIQJjz4Bdy6HZVqykyTnb1W11qUnpcUweHkdqbAQhDiEyLISYhBTCGyu597UtfWKaoiiKP+lU6I0xTcAtwDJgC/CsMWaTiNwjIhc6uz0KJItIHnA74ErBfAiIEZFN2AvGP40xX/T2mzjKmAVQXwFFq+1+Y63delsv1o2o2GQmJjbzzOoC3t92sM/MUxRF8Qc+xeiNMW8Ab3i03eX2uA6bSun5vCPe2vuM0aeCOCDvXcg63oZtwPt6se5ExpMZVcX4tBh+9MIGln13PvFRGsJRFCUwCKyZsVGJMGKOjdODzx49UQk46ir47RUzKDlSz89e29z2+M73YPMrvW+voihKPxBYQg82fFO8FmrK3Tz6ToQ+MgHqDjM9I4FvnTqG59cU8v5WtxDOx/fDOz/pM5MVRVH6ksAT+rELwLTArg+6FLqhvhJamvmfBWMZnxbDnS9t4Ei9c1C3uhQO74Xmxj41XVEUpS8IPKEfPtsK9853fffoXbNj6yqICA3hl5dOZ19lHb95y5kgVF0CptmKvaIoyiAj8IQ+JBRGnQp570GDrx59gt06693MGZnI4hOy+feKfFbvKoGaMnu8bGefmKwoitKXBJ7Qgw3fVBXDvly732mMvv3iIz84ZwLD46P4xQuf2lAQQLkKvaIog4/AFPoxC+x2y2t2G+5j6MatsFl0RCi/uHQaVeX7W/uV9/6EXkVRlL6mr2vd+IeETEgZD6Xb7L4vg7HQrrDZqeNT2TQuDPaCQRAN3SiKMggJTI8eWr168C29ErzWpL9uRgwAux1ZNJfm9Y5tiqIo/UgAC/0ZdisOCAk/dt9j1KSPabJtK5vGQUUBpRVVvWaioihKfxC4Qp99khX4sCG24NmxCBsCjlDvNemrSwAh56SzCKGF7z/yKiVV9X1isqIoSl8QuEIfHg1ZJ3QetgF7IXDOjm1HdQkMSWbs5NkARFbu4aq/reBgVV2vmqsoitJXBK7QA5z5EzjnF771jUrwvpxgdQlEp0LSaACWHB9O0aFarnpkBeXVDb1mqqIoSl8R2EI/Yg5M97F4ZmR8B6GbUohOgSHJEBFPNvv51w1z2Vtewz2vbupdexVFUfqAwBb6rnCs0E10qg3vJI+G8p0cNzqZb502lpdzi7V+vaIoAx4VehcdefQ1To8eIGnM0TIIN58+hrFDY/jxSxtbi58piqIMQFToXXiL0Tc1WPGPTrX7SaOhogCaGogIDeHXl02juKKW3y7b1t/WKoqi+IwKvQtX6MaY1raaUrt1efTJY2zdm8P5AMwZmcR1x4/ksc/2UPLkN+GNH/SryYqiKL6gQu8iMh5amlpLG4Mzhx43j36M3bqVQvjBwomMiTXEb3+Oll3L+8lYRVEU31Ghd+Ftdmw7obcplu7FzWIiQrl/bjnhNNF4qLDPzVQURekqKvQuvNW7qXaFbpxCPyTJev4e5YqnVX0CQETzEWio7ls7FUVRuohPQi8iC0Vkm4jkicgSL8cjROQZ5/HPRSTb7dh0EflMRDaJyAYRiexF+3sPbxUsj3r0zhi9SJvMGwCam2DHMuodtkKmqdzXD8YqiqL4TqdCLyIhwEPAucBk4CoRmezR7UbgkDFmLHA/8Gvnc0OB/wDfNMZMAU4DBubCqx2FbhxhEBHX2pY0um1d+oIVUHuInennA3CkpKDPTVUURekKvnj084A8Y8wuY0wD8DRwkUefi4DHnI+fBxaIiABnA18YY9YDGGPKjDHNvWN6L+M1dFPWOlnKRfKYoymWAGx7E0LCqZp0JQDl+/f0h7WKoig+44vQjwDc3dRCZ5vXPsaYJqACSAbGA0ZElonIWhH5obcXEJGbRGS1iKwuKSnp6nvoHToK3bjCNi6SnCmWh/bYVMytr8Oo+SRlTQHgSKkOyCqKMrDo68HYUOBk4Brn9hIRWeDZyRjziDEmxxiTk5qa2scmdYBL6D1DN9Ee9rhn3pRsg0O7YcJ5ZAxL44iJpPFwUb+YqyiK4iu+CH0RkOm2n+Fs89rHGZePB8qw3v9yY0ypMaYGeAOY3VOj+wRHiI3Ft/HoS9sLfbIzl758J2x73T6ecC5R4SGUShKOKh2MVRRlYOGL0K8CxonIKBEJBxYBSz36LAUWOx9fDrxnjDHAMmCaiAxxXgBOBTb3jul9gHthM2O8h26GJNl+ZTttfH74LIgbDkBVeCoRtQf602JFUZRO6VTonTH3W7CivQV41hizSUTuEZELnd0eBZJFJA+4HVjifO4h4PfYi0UusNYY83qvv4veIiq+NXTTUA1Nte09erDhm70roHA1TDjvaHNdVBpxjaX9Y6uiKIqPhPrSyRjzBjbs4t52l9vjOsBr4XdjzH+wKZYDn8iE1tCNZw69O8ljYMNz9rGb0BMzjOTD5dQ3NhIRFtanpiqKoviKzox1JzK+NXRTU2a3Xj16Z5w+PgvSphxtDkvMIFya2VesA7KDjscvgrf/z99WKEqfoELvjnup4mN59K7Mm4nntcmxj0m1Y9Ylxbv7zkal92lphvxPIe9df1uitLRAZbG/rQg4VOjd8Rq68eLRj5gNIREw9fI2zUnpIwGoOKizYwcVlcXQ3AAlW6Gx1t/WBDebX4IHpkOFzkfpTVTo3YlMgMZqaG5sFfohXjz6lHFwRzFkzm3TnDg0C4C6ch+/pJX7YOn/QENN532VvsNVpM40wwFdB9ivFOdCS6NNdFB6DRV6d1z1buoqbA59eAyED/HeN6T9OLbEDqMFoaXCx1vPra/B2sdh72fds1fpHdxrFxWv858dCpTl2e2+3GP3c18gSOkUFXp33GfHesuh74yQMCpDEgmr3u9b/4POKQXqRfqXsp0QGglDkmHfen9bE9yU7rDb4tyO+2x8AX4zWkuCdwEVenfcC5tVl3gP23RCTcRQohtKML54HAdU6AcE5bshcRSkz+zck1T6juYmW1IE7P+ho9/QtjehttzWm1J8QoXenaOFzQ63Vq7sIs3RaaSackqO1B+7ozFwcIt9rELvX8p32Uyq9Bn2f9JY52+LgpPD+XY5zxE5UHsIDu/13m/vCmd/TXrwFRV6d9xr0ncndAM44oeTJuUUlHcywFpZDPUVEJVosz1cZY+V/qWlxXqRyaNh+EwrNAcHbpWOgMYVtpnmnHvp7e6qosiWCYfWrdIpKvTuuEI3tYegxktBM19OkZxBkhyh4OChY3d0efNTLrFZBmU7uvxaSi9QVQxNda0ePWj4xl+4fgOTLwJHqPc4fcGK1scq9D6jQu+OK3TjuoXshtDHpdoUy/IDHdx2unB5ja5cfA3f+AfXspBJoyFhpL3Y64CsfyjdYQfE49IhdZL3C+7eFRAWbWela+jGZ1To3QmLtNkXpc4Ur24IfViCrWRZ3dmSgge3QMwwyJwHIeFwYGOXX0vpBVyplUlj7Czn9BnHzvhQ+o6yPEgeZx8Pd/4fPAdk966AjDmQlK0efRdQofckMqE1l7cbMXpirdB3ugDJwU0wdBKEhEHqBPXo/UX5LjvLOc65aNrwmfZuqy/HTBqq7diA0payPEgZax+nz7SZNe5iXl9lHaLM461Hr7NnfUaF3pPI+NYUr+4IfVw6AHKsBUhamu3qVEOda6ynTYX96tH7hfJdkDQKHM6fQvoMZzmELe37VhZD/ZGevV5Tg53i/+mDPTtPoFFXCUcOQLJT6Ic71ydyv7sqXGWX8cw6HhIyoWq/JjH4iAq9J1EJNj4P3QrdEJlAoyOC6IZSahs6WAf90B47ADh0kt1PmwpH9tvZuP6kdIedqRtMuFIrXaTPtFvP8E3tYfjLifD2nT17vQMb7UD/F8/27DyBhmsg1hW6SZtiB2Td4/R7PwdxQMZciM8ADFSqV+8LKvSeuAZkwQ4MdRURGqLSGCbl7O0oxdI1EJvm8uidpY79Gb4xBpbeamvvlAZJBlBLi50s5S70iaPskpKeA7Ir/mKzsXZ90LPXLFpjtwc3t44FKa2fRYpT6MMi7YCs+wW3YIX9rUTGQbxzdVMdkPUJFXpPXCmWUYk2ft4NTGw6aXLoGELvDAukTrTbtKl260+h370c9n5qH+c+6T87+pOqfXYVMXehdzhs+Mbdk6w9BCv+DGFD7N1YRQ/WGyhaa2soAWx9tfvnCTTKdoCE2Auti+EzWmfINjdBwSobnwcbugGN0/uICr0nrklT3QnbOAlPHMEwOvHoE7MhPNrux6RC9NC+Ffr6I1ZkvGEMfPhriE2HUafCF8/YcYRA52jGzei27ekz7JhJc6Pd/+whqK+EL/3O7ud/2v3XLFoD2SfbtYa3vNb98wQapTsgcSSEhre2pc+0CwBVFNqQV2O1jc9D6+C5Zt74hAq9J67QTQ+EPixhhPXoSzsYuDu4pXUg1kXaFDiwoduv2Smf/gH+djps9bJk756PIP8TOPl2yLkBKotg94d9Z8tAodwth96d9JnQXG8HzGvKbdhm8sUw/Uob1sn/pHuvV1cBpdthxByYdAEUre7Z3UEgUbazNT7vYvgsu92X21r2wCX0oRE2PVlDNz6hQu+JK3TTnYwbJxKXTqQ0UlZ6oP3BpnrrvbgGYl2kTYGDW+0tal/gii2/9M3WSUIuPnB687Ovg/Hn2otdMIRvynfZOQzxGW3b3WfIfvpHmw552hJwhFihOZZHv+Odjmu0FOcCxi5cM/EC2+btwhtstLQ4c+jHtm1Pm2LDOcW5Nj4fl9H2f5WQCRWdTExUAB+FXkQWisg2EckTkSVejkeIyDPO45+LSLbH8SwROSIi3+8lu/sOV+imG5UrjxJrUyxrvS1AUrrDLnDRzqOfar3I8p3tn9NTGqptyGDqZVasnr2udbGT3R9B/sdw8nftAFhYpJ2tu+W11tW2ApXyXTaE5ghp25481sbRd74Hn/8Vpl7aemEeeSKUboMjJe3PV3UAnvwyvPMT76/nGogdPhtSx0PKBNiytNfezqClssiOlaR4CH1YlP3cXR591nFtj8dnaozeRzoVehEJAR4CzgUmA1eJiIdKcSNwyBgzFrgf+LXH8d8Db/bc3H6gF0I3xNlJU1Tuo6XFY2afayDWU+iHuQZk+yCfvuBzmzI642q47O92LOC177bG5mOGwezFrf1nXmN/eJte7n1bBhJlu1oXenfH4YBh02zd88YaOPV/W4+NPNlu93rx6je9aPO88/7rPb+7aI0NEw1JsvuTLrB3B9VlPX8vgxnP1Ep30mfCno/twHnWCW2PxWdYodfJZ53ii0c/D8gzxuwyxjQATwMXefS5CHjM+fh5YIGIXTVbRC4GdgODY+pnL4RuXB59kilnf6VHyduDm21+sOdtasp4297VAdmmBut9f/bnjr/wuz+yt8BZx8PYM+G0H8EXT8MrN9v4vMubdzFitrUnkMM3xrTPoXfHlU8/7Qo7c/lo+wybfeMtfLPhOVtCo77S3iV5UrTWxuddTLrA3t1tHxw+UJ/hmVrpzvCZds4JQKaHR5+QZSe3VR/sU/MCAV+EfgTgPuJR6Gzz2scY0wRUAMkiEgP8L/DTnpvaT7i88YSR3T9H7DAAhlHOh9s9bvEPbrGei3t2AdjBpZTxvs2QNcammr12O/xuPDxzDSz7ERSu9N5/z8dWvCOcaX3zfwDjzobcJ6w3P2dx2/4iMPNqGxf1jOcHCkdTK0d5Pz76VFs8y92bB/t/y5gLezwGZMt2Wo/9lO9BaJRdHMOdymJbKdNd6NNn2Kn8W4I8zbJsB4THQkxa+2OuC254bOt8ExeaS+8zfT0YezdwvzHmmPPGReQmEVktIqtLSrzEPvuT5DHwrU9h3FndP0doBGZIMlNjq/nDuzuoa3RLVTy4uf1ArIu0KZ179C3N8NgF8OiZ1uMeswCufMJ6khtfaN+//ggUr4XsU1rbHA645K82DHHOvTYW6sn0K+0sxPVPtbZVl8F7P4eXbx788XtXamWyl9ANUDf6bFZdubZ93Bhg5Ek2xFbrVop6w3OAwKyvwJgzYOsbbQtyuVJb3YVeBCadb8cC6qt69n4GM2V59v9ggwBtGTbV3o1mzm0/luIamNUUy07xReiLgEy3/Qxnm9c+IhIKxANlwHHAb0RkD3AbcIeI3OL5AsaYR4wxOcaYnNTUHsTGe4u0Kd6/dF1AYoczN7mefRV1/GdFvm2sr7IlkD3j8+6vW1nYVkA8yX3ChlvO+DF8fztc/qgVi/HnwKaX2mftFKyw8fnsk9u2D0mCG16HaZd7f5244TD6dFj/tB1kfPv/4IFpsPy3VvwfPXtwL+XWUQ490NJi+O4zuVzx9zXkHfTio2SfBBg7JR+soH/xrP2M44bDhHPt/3H/F63PKVpjQ3PDprU916QLbPhhxzu9874GI6V53sM2YJ2Q0++AE25uf+zopCkfhL6x1i7dufsjm+mU+xSseBi2L+u+3YOIUB/6rALGicgorKAvAq726LMUWAx8BlwOvGfsoqlH3UgRuRs4Yoz5Uy/YPfCJSyfpyAFOGZfCQ+/nceXcTGJLt9ljHXr0rgHZzU4x8aD+iPWoM+bBKd9vezGaejlsfsXGhkef1tq++yMrMK78464w82p44Ua4f4qNJU+9zL7ukQPw7LXwtzNg0ZPdO7e/KdsJjrDW2383/vheHm9utAu8r9pTztihMW07jJhj0zLzP4YJC6F4nc2WOvk2e3z8QkBs+MaVqlm0xl7IPe+eMo+zA/9bXrXZPcFGY60V6uSvdNxnfgfJepHxEBHvPXRzKN/OZi7dbi8kFQVAB2vQXv0cjD+7y6YPJjr16J0x91uAZcAW4FljzCYRuUdELnR2exQbk88DbgfapWAGHbHDoHIfPzxnIodqGvnbR7vb17jxpLOaN5/+wYrsOfe2v+MYd5aNY3qGb/Z8bIXJNQu3K0z8kr2oTLsCbl5lM3aGTrTx66+9Z39oj10A65/p+rn9TQeplW9t3M/9/93OpbNHkBQdzpp8L3dXYVH2M3UNyG54zgr/JOfPISbVCrgrR76lxV4M3MM2LhwhMPF82Pwy/PtSe2fQUN1rb3PAU7YTMN5DZL6QkOndo//kAVj5NzvhLes4Ow/iskdh8atw04dw6zr43jYYOsUmJQR45pMvHj3GmDeANzza7nJ7XAdc0ck57u6GfYOX2OFQXcK09CGcN20Yj360i2/N3UhUaBQkZHfwnHRbY2fbG3aGqnutncpi+OQPdunBzHntnxsWZYV581I473d20LC+ygrMyd/t3nsIi4KvdRBSSBkLX3vX5uS/dJO9CGXMtQKXOc+GRHoY/upTPIuZAVv3V3L7s7nMyEzgF5dM45Yn13kXerBx+o/vt2MVG1+wg9uuORgAE8+Dd+6y6X8NNTYTx5vQA5x5ty2g98Wz8OLX7SDwpPMh58b2ueMdYQysftQK54K7vI+7DESOlVrpC64Uyyi334oxsOO/Npx51VMdPxfg0r/CI6fDa7fBlx8f2N/ZHqAzY/uKuHTAQO6T3JP6PneYv9H4xQvWI3Z08LGL2NDIrvfhySvb1j5/714bPjnz7o5fc+plUHfYDu6BnWRimtvH53uLIUnwlRfhzJ9CzFAreC9/E/44G34/Gd66w4YsPFcJ8pXGOihcDav+bqtq/nU+/GY0fPS7ntXicaVWug3EHqpu4OuPryYmIpRHrp1DZFgIOdmJ7C6tpuxIfftzjDzRfrbLf2vvsqZ/ue3xCefZ7bY3WydKdST0UQmw4P/gO+vhhjftuMm2t+AfZ8Pfz7IX72O938ZaeOkb8Pr3bLjin+dax2Aw4Eqt7GBQvFPiM9uHbkq32xmzY8/s/PnDpsEZd9qJa1943JlW7Yenr4G/LRj0de998uiVbuBKz3z1VlKAS8JjyasfSsbU60k61vNOvAUiYu2Epn99Ca55zn7hcp+wA1KJ2R0/d/Rp9o5g4ws2drznIxuH9sw/7k1Cw21s+uTbWhdUKVwJ29+GlY/Aioes5zzlUvujSsi0P87o1PbekzH2R5r3Lux816YwNtXaY1GJNtVuSDK8e48dRLv4L90TiCMHbIEsN49+yYtfcKCinqe/cTxpcXZOwZyRiQCsyT/E2VOGtT1H5jybDbLizzZOPO6ctsdTxlkvdevrrTNtU8Yf2y6Hw15ARp4IC38J656wn9+z19qqjvO+bu/a3L8Dhwtseu2+L+D0O23478Wb4JHT7PhJRk5r35Zme4dnWuzd10DwXsvybIGy7oQWwX6f6itswoHDKWd5/7VbXzPnTrzVfp/e+IG9U4vPsHdXb/7QuRpYI6z7N8y9sXs2DgBU6PuKUafCV16AqCRIGsXh+kgu/+0HTF8fz6/GVjF2aGzHz52z2OYUP3e9zW6JTrFeX0eDUi5CnXHijS/YcMHuj+wPPXxIb76zjnGE2PGHtMkw53qbPbTlVdjwPHz8eyswLkIirGiLAGJTOZtqodqZXps81tbeGXWKFfj4DNvXGHu+N74HD58C5/wc5tzgm2i5LkSbX7H7TqE3xvBJXhlfnpvB7KzEo92njYgnLERYs9eL0EfE2sk8RWtg8gVtJ5y5mHienchWWWQLdHmmBx6L8Gg47iYrLltfszV3lt1h/1In2Qt56kRYdqfN2rnqadsG8LX/wlOL4J/nwZd+ayd47XjbCmCNMxY9dIq9cEz/8rFFtqXFlnzYv9F+xqHOMhmhUVZc6yvt6lB1FdaOjLn2z3OeCFiHZf8Guz1yAI4ctHevrnLd3cE1mN5UD+FOOdvxji0vkZDl2zkcIXDJw/CXk+xFMirBhk8zj4OLHrJ3kx/+xiYnDJaQmAcq9H2Fw9Hm1nF4FPzikmn89NVNnPPAR1xzXBa3nTmepGgvPwiwP9rFr9raKYd2w8JfW6+2M6ZeBmsfs2K/L9dO4PEXUYlWrGdfZ4XgUL6Np1YU2MJfdYediRDGCrg47MLPYxbYkrXeEIHpV1iv95Wb7Z3P8t/Zwe/oFOdF0fk5tTRbMWputKGaorXQ4MxXjxl2NCOmvLqBI/VNjE5pm10TGRbC1BHxrNnTUZz+RCv0077s/fiE8+CTB+1dyoRzu/TRHcURApMvsn9lO2H7WzYc9MkfbOgoZbz13N3TE4dOgq+/b8dPlv6PbYtKsh7uuLNtWYeVj9i49Ds/gRmL7OftCLPjQiHh9uJU8Lldvq+rcybCom3W2OjT7X7hKvvnOWgaGW/Lc89Y1L3PBjyEPtp64PmfwLybunaexGxY+CtYeou9mJ19Lxz/Lfv5L7jLhsNW/g1OurX7tvoRFfp+5PI5GZw+IZUH/ruDJz7fy0vrivjOgnF89aRROBxePNLMudY727IUcr7q24tkn2x/PO/eYz1o94lS/iQyHtKn27/eIH4EXPuSvaXe/ZFdnq+yyIYwasvtRcMRan+oEmL7z7jSepsjctpM0NlTZgu8Zae0v/OZk5XI4yvyqW9qJiLUwyPPudEpah2MgWTMtcXxako7js93heQxNnx3ws32bql4nX2NCC93h0OS7Oez+RXr2Y6Y0/aOYta1VshXPmIHcV3LZx5F7AVjyiXWs02faZ/fVGfHTppq7ecaGW9XfIqMtxfr/E+tl77rA3sXAbbqZOZcK5zDZ9v/RfRQ73dBXcWVS9/sHEfZ/ZG9sxi7oOvnmvUVe5EbMadtFtDIE63T9vHv7Z1qZFyPze5vVOj7meSYCH528VSuO2EkP399Cz9/fQuHaxr5/jkTOnjCmK5lzThC7I9z5V/tl9Zbhk6gINJ6x9AD9pbbdMaspPYhjJzsRP7+8W42FVe2CesAtnzC6T/q+MSOEHtntu4/rYtd9xZRiXYG7rEICet4QpyInf+QdbwdaGyqtXc+zQ32LzKhbRaRr0w63/6BvXsTR2tZkb4geqj9nrvq4eS9Y0NVI73MQ+kMEesMeOOMH9txj88eOvb/fICiWTd+YlxaLP+6YS6L5mbyp/fzeHFtL5ZbnXqZ3WbMHbQxxf5kT2kNIpCZ1P6zmu0akO0ofNMZp3wPzv1N+5r3A4nQcGcYJcWKcmJ290Tek/iMvhV5sCHSuBE2dAM2Pj9qvq0d1ZsMn2XDZ5/9qfOc+8ZaW0PftULZAECF3o+ICPdcNJUTRiez5IUNrNpT3jsnzphrM3B6EvsMIvaW1zA8Pqp9aAYYGhtJVtKQjvPpOyNpNBz3jYGR4RKoJGRaoW+stSVGfEmr7A6n32nHNz7+fdv25kYoXGNTbR+7AH41Eh451ebnd7R8Zz+joRs/Ex7q4OGvzOGSP3/CN/69hpe/fRJZyT3MknE44LpXesfAIGBPWTVZSR1/5nNGJvLRjlKMMYgK9sAjPgv21rfWiOpJQcJjkToBZlxlB2Wb6u0Af/kum1hgnPMc0qbabKbEbDvf4+8L7JjKaXf0X/abF9SjHwDEDwnj0evn0mIMN/xrJdsPVJFbcJj3tx7khTWFvJJbRLPnAiZuNLcYDnjWvVd8Zm9ZjdeBWBezRyZSeqSegvLafrRK8YYxhoLyGtbkH6Kx2Zmum5BpxxVqyuzchWPNNekpp/6vHRNY/5RNBR4+046hXfYofD8PvvWJLVEy7+vw7RV2/OjTP8JfTrCzdbs7ebCHqEc/QBiVEs3DX5nDtY9+ztn3L293/LUv9vGHRbOICm8bXjhc08C3n1jL57vLee6bJ7QfMFSOSVVdI2XVDV4HYl3kOOP0q/PLe363pXSZrfsreX9rCev2HmJdwWFKqmw8PjYylNMnDOWr0c602LoKmNlH3ryLxJHwv7ttRldnd3dRCXDBg7bg4Ku3whOX2Vj/ybfb+kYdzZDvA1ToBxDHj07m+W+eyPYDVSRFhx/9e2/rQe55bTOL/raCRxfnkBJjB5ryDlZx42Or2Xe4jsQhYfzgufW8fuspRIZ1YWJOkJPvSq08hoCPT4slNiKUNfmHuHT2AB5UDUBeXV/Md5/JpanFMColmlPGpjArK4Gk6Ag+2HaQd7cepKS2BpzTUT51zGJmQxNDnJOnjDFsKq7kpXVFvL15PyOTorl09gjOmTKM6Ihuyp97DSpfGHWK9e7XPwUfP2BnOiePsyGd2PTWTKemeptNNfG87tl1DMT46VaiI3Jycszq1av9bcaAY9mm/dz61DrS4iJ57Kvz2FNWza1PriMizMFfr51DdX0z1/1jJd88dQxLzu3BTMMg440N+/j2E2t5/daTmTI8vsN+1z76OSVV9bx12/x+tC64eWrlXu54aQM5IxN56JrZDI1tn3ff3GLYtDGXW1+cSwvCgbpncYRFcdbkNMYOjeH1L/ax7UAVYSHCyWNTyCs5QkF5LUPCQ1g4ZRjXnZjNzMyE/ntTLc22UulH98OBDe2Pj5gDX3+vW6cWkTXGmBxvx9SjHyScM2UYT910PF97bDUX/uljquubmDAsjr9dN4eMROuNLpqbySPLd3Lu1GHM6M8v7yBmT5nNoR+ZfOxaK3NGJvLguzuorGskLrKLHp3SZf764U5++eZWTpuQyl+umdMuZOkixCFMnzwZXgRHVAKPXzefV9YX88aGfSxdX8zsrAR+dvFUzp+WTmJ0OMYYVucf4sW1hbz2xT5e27CPz5acQXJML6djdoQjxKY/T7nUloNoabTlQELCbZprWDdr/nSCCv0gYnZWIi9+60S+8e81jE2L4TeXTW9z+3nHlybx4fYSvv/cel679WSv6YJKW/aW1ZASE05MJ7fxOSOTMAZy9x5m/vgBsApagGKM4b5l2/jzBzs5f3o6v//yTMJDO4llh0bY2HlkPMeNTua40cncfcEUKmobSY1tK+AiwtzsJOZmJ3HDSaM4+/7lvLi2iK/P72CR+L5CpPdmifuAZt0MMrJToln23fk8dPXsdjHGuMgwfnnpNHYcPMIf383zk4WDi85SK13MyIzHIfCXD3aydH3x0QFBpXd5c+N+/vzBTq6al8mDi2Z1LvIu4jMhorU0QXioo53IezI+LZbZWQk8tWovAy2E3duo0AcYp00YyhVzMvjLhzv5aIefF1ofBOwtqyG7k7ANQGxkGNefOIqNRRXc+tQ65t77X86+/0N+9/a2Y6a+Kl3js51lxESE8vOLpxHirf5TL7NoXha7SqpZ3d0JcYMEFfoA5MfnTyY9PpJrH13JFQ9/yjubD9DiFKPmFsNHO0q4/ZlcZvz0bW5+Yi0VNQNnqnZ/UtfYzL7KOp9TJu+6YDLr7jqLV24+if9dOJGUmAj++F4ej326p28NDSJyCw4zPSO+X0Qe4Pzp6cREhPLUyr398nr+QmP0AUh8VBjLbpvPM6sKePTj3Xz98dWMSY3mxDEpvL15Pwcq64mNDOWkMSks27Sf3ILDPLhoJjnZ7ZdEaWkx3itrBgCFh2owBp88ehehIQ5mZCYwIzOBb546mhsfW819y7Zx1uQ0Mn0IASkdU9fYzJZ9lf0aLx8SHsqFM4fz4tpCfnLBFOKjAnOgXT36ACU6IpSvnjyKD39wGg8umklkWAhPrdzLtBEJ/Pma2ay680wevnYOz3/rREIcwpWPrOCP7+6gucWQd7CKvy3fxTV/X8GE/3uTf36y299vp0/YU2pz6Ls7CUpE+PnFUwlxCD96cUPAx3n7mk3FlTS1mP5NdwSumptFXWMLr+QW9evr9ifq0Qc4oSEOLpo5ggtnDKepxRAW0vbaPjMzgddvPZk7X9rI797ZziMf7aKqztYmHzc0htEpMfzu7e18aXq61zzmwUx+uWuyVPdT2oYnRLHk3In8+OWNPLemkC/nZPaWeUHH+oLDAP0u9NMy4pkyPI6nVhZw7fEjA7KekU8evYgsFJFtIpInIku8HI8QkWecxz8XkWxn+1kiskZENji3nRTQVvoKEWkn8i5iI8N4cNFMfnfFDM6YOJR7L5nKJ0vO4J3bT+Xha+dQ39TMb5dt62eL+578smpiI0JJHNKz2/Wr52Uxb1QSP39tMwe15lC3yS04THp85NE1e/uTRfOy2LKvkg1FXVxNa5DQqdCLSAjwEHAuMBm4SkQme3S7EThkjBkL3A/82tleClxgjJkGLAb+3VuGK72LiHDZnAweXDSLa44byYgEW5t9VEo0N5w0iufWFLKhMLB+BPllNYxMGdJjD87hEH592XTqm1r48csbNYTTTdYXHmZGRoJfXvuimcOJCgvhqZUFnXcehPji0c8D8owxu4wxDcDTwEUefS4CHnM+fh5YICJijFlnjCl2tm8CokSkn6agKb3FLWeMJTk6nJ++uimgRGxveQ0jj1HMrCuMSonm9rPG8/bmAzy/phcXkQkSDlU3kF9Ww8ysBL+8flxkGF+ans7S3CKq6z2XVRz8+CL0IwD3y1yhs81rH2NME1ABJHv0uQxYa4xpN9NERG4SkdUisrqkRHO/BxpxkWH84JwJrM4/xKtf7GtzzBjD9gNV1DU2+8m67tHU3EJBeQ0je7Ea5Y0nj+K4UUn88IUvNOWyi+QWHgbwm0cPtoRIdUMz976xhfqmwfV97ox+yboRkSnYcM43vB03xjxijMkxxuSkpur08oHI5XMymToijl++sYXahmbqGpt5euVeznlgOWffv5xzHljOh9sHz0V6X0UdTS2mV4U+NMTBY1+dx1mT0vjJ0k38+q2tAXUH1JesLziMiB0Y9RdzRiZy/YnZPPn5Xi5+6FO2H6jymy29jS9CXwS4pxJkONu89hGRUCAeKHPuZwAvAdcZY3b21GDFP4Q4hJ9cMIV9FXV87fFVnPSr91jy4gZCHA7uOG8iISIs/sdKbn5iLfsrejYgaYxhTf4hbn8ml5ufXMvHztWdehNXMbNj1aHvDpFhIfzlK3O45rgs/vLBTr733PrWBTKctLQYvQB4kFtwmPFDYzutOdSXiAh3XziFRxfncLCyjgv++DGPfbonIP5Xvnyqq4BxIjIKK+iLgKs9+izFDrZ+BlwOvGeMMSKSALwOLDHGfNJrVit+YW52EhfNHM4rucUsmDiUG08ZxQmjkxERFp+YzV8/3MWf3s/jw+0lfO/s8Sw+IbtLk63qGptZur6Yxz/bw8aiSmIjQgkLdfD6F/sYnRLNV44fyWVzMnplUsvROvTHWFmqu4Q4bH79sLhIfvfOdjYXVxIVHsLhmkbKqxuoqG0kJiKU4QmRpMdHMTwhiknpsVwxJ7PDKo2BjDGG9QWHOWtymr9NAWDBpDTeum0+P3x+PT9Zuon3th7kV5dNIz2+/eLxgwWf6tGLyHnAA0AI8A9jzL0icg+w2hizVEQisRk1s4ByYJExZpeI/Bj4EbDD7XRnG2MOdvRaWo9+YFPf1MzhmsYOU+Dyy6q565VNfLi9hHnZSdx3xfROSwBv2VfJs6sLeGldEYdrGhmfFsN1J2RzyawRhDiENzfu4/HP8lm39zDhIQ7SEyJJjYkgNdb+ZSdHM3tkIpPT43wugnXv65t57LN8tt6zsE9n/j6/ppB/r8i3aZzR4SQNCSM+KozKuiaKD9eyr6KO4sO1lFU3MDQ2gu+cOY4v52R2mAobiOSXVXPqfR/wi0umcfVxWd06x2n/Og2AD67/oNfsMsbw7xX5/PKNrYQ6hB+fP4kv52QO2Dz7Y9Wj14VHlF7HGMMLa4v46aubaGo2/Oi8iXzluJFHBbWlxVBcUcv7Ww/y7OpCNhRVEB7i4KzJaVxzfNbRuwRPNhZV8Or6Yoor6iitqqfkSD0HK+uodE7wigh1MD0jninD42lsbqGyronK2kYq6xqZPy6V284cd/S8Nz2+mt2l1bxz+6n998Ecg5W7y/nNW1tZnX+I7OQh3H72BL40Lb1LNV82FlXQ0Nwy6JaTfCW3iO88ndvp4i/Hoi+E3kV+WTU/fP4LPt9dzinjUvjlpdOIiwpj9Z5yPt9Vzue7y2luMVw2ewSXzMogvofzMrqLCr3iF/ZV1LLkhQ18uL2EudmJDI2NZGfJEfaUVVPXaOPWE4fFcuXcTC6eOYLE6PBuvc7+ijrW7j3E2vxDrNl7iG37q4gKCyEuKow4Z5hnfcFhrpqXebQq4jn3LyczKYq/L57ba++3pxhjeH/bQX7z1ja27q9iaGwE501L54IZw5mdleD14ldR28jS3CKeXlXApuJKROBnF03lK8eP9MM76B73vLqZJ1fms/Hucwjt5p1MXwo9WOfkic/z+eWbW2lqMTQ1t9BiICxEmJGRQH1TCxuKKogIdXD+9OEsmpfJtBHx/bqsp64wpfiF9Pgo/nXDXJ5dXcBv397Owap6RqdEc9LYFMakxjAjM57J6XE9vhUeFh/JedPSOW9autfjxhh+9/Z2/vR+HrUNzdx3xQzyy6s5eVxKj163txERzpiYxmnjh/L25v28vK6YJ1fu5V+f7mFEQhTHj07GIWCAFmOorm/iw+0l1DW2MCk9jp9eOIXl20v48csbKT1Sz3cWjBuwYQZ3cgsOMW1EfLdFvj9wOIRrT8jmtAlDefjDnaTERHDc6CRmZSYeHVfZWFTBUyv38kpuMS+stXMp0uIiyEgcQkZiFGdOSuOCGcP9Yr8KvdKniAhXzs3iyrndi732lg3fP2cCUeEh3LdsGyVH6qlrbOnV1MrexOEQFk5NZ+HUdKrqGvnvlgO8un4fn+SVIgIOp3iHhgiXzc5g0dwspo6wF8yrj8viRy9u4IH/7qD0SD0/vXBqj0v+FpTX8ObGfUwdEc+87KReFeTG5hY2Fley+ITBcQeSmTSEey+Z5vXY1BHx3HvJNO44bxLvbT3IrpJqCg/VUHCohhW7yngl184d9YfYq9ArQcPNp48lMiyEn722Geh8ndiBQGxkGJfMyuCSWRk+9Q8LcXDf5dNJiYng4Q93UnakgdvOHM/o1Oh2A7xNzS3klRxhT2k1E4bFkZ3cthzE/oo6/vT+Dp5ZVUBjsw3xJg4JY8GkNM6ZMoz0+Eh2l1azu7SaPaXVlNc0cMH04VwwY7jPg+Jb91XR0NQSUGscR0eEthPz+qZmvvL3z/nec+sZnhDFnJH9O46iQq8EFTeePIro8BD++F4ek9PjOn/CIEREWHLuRFJiwvn561t4c+N+wkMcjB0aw6T0OMJDhU3FlWzdb0XWxbC4SE4ck8zxY5LZvr+Kf6/Ip7nFcOXcTG48eRTbD1Tx1sb9LNu0v12Zh+HxkYSFOvjec+u5b9k2vnpyNlfNyyI2MozSI/Ws23uYdXsPUXioltMnpnLOlGEMCQ8dEDNi+4OI0BD+em0Ol/z5E256fDUvffukbpfH7g46GKsoAczu0mrWFxxmy/5KtuyrYsu+ShqbW5icHseU4XFMGR5Pdko0m4or+GxnGZ/tLKOsugGHwCWzMrjtzHHtFlRpaGph5e5yquoaGZUazcikaKLCQzDG8OH2Eh5ZvotPd5YdTSnd6ywHHeoQEoaEUXqkgejwEBZOTWdfRS3bD1Sx6s4zezSe0NeDsb3FzpIjXPrnT0mJCefFb5/UqwudaNaNoig+YYxhx8EjRIWF9GjFrA2FFfzzk93UNDQzKyuB2SMTmTo8nohQB6v2lPPi2iJe37CPI/VNnDlpaI+znwaL0INdF/e6f3zO3Owkbj9rPOkJUQyNjejx3AkVekVRBhy1Dc18uL2ESemxPR4vGUxCD3Yi3fefW390XwRSYyI4bnQyf7xqVrfOqemViqIMOKLCQ1g4dZi/zfALl8/JYF52ErvLqtnnnCG9v6KOlNjuzSXpDBV6RVEUP5CVPKTfBmQH7gwFRVEUpVdQoVcURQlwVOgVRVECHBV6RVGUAEeFXlEUJcBRoVcURQlwVOgVRVECHBV6RVGUAGfAlUAQkRIgvwenSAFKe8mc/mCw2Qtqc38x2GwebPZCYNk80hiT6u0JA07oe4qIrO6o3sNAZLDZC2pzfzHYbB5s9kLw2KyhG0VRlABHhV5RFCXACUShf8TfBnSRwWYvqM39xWCzebDZC0Fic8DF6BVFUZS2BKJHryiKorihQq8oihLgBIzQi8hCEdkmInkissTf9nhDRP4hIgdFZKNbW5KIvCMiO5zbRH/a6ImIZIrI+yKyWUQ2ich3nO0D0m4RiRSRlSKy3mnvT53to0Tkc+f34xkR6ZulfHqAiISIyDoRec25P6BtFpE9IrJBRHJFZLWzbUB+L1yISIKIPC8iW0Vki4icMFBtFpEJzs/W9VcpIrd1x96AEHoRCQEeAs4FJgNXichk/1rllX8BCz3algDvGmPGAe869wcSTcD3jDGTgeOBm52f7UC1ux44wxgzA5gJLBSR44FfA/cbY8YCh4Ab/Wdih3wH2OK2PxhsPt0YM9Mtr3ugfi9cPAi8ZYyZCMzAft4D0mZjzDbnZzsTmAPUAC/RHXuNMYP+DzgBWOa2/yPgR/62qwNbs4GNbvvbgHTn43Rgm79t7MT+V4CzBoPdwBBgLXAcdiZhqLfvy0D4AzKcP9ozgNcAGQQ27wFSPNoG7PcCiAd240xCGQw2u9l4NvBJd+0NCI8eGAEUuO0XOtsGA2nGmH3Ox/uBNH8acyxEJBuYBXzOALbbGQLJBQ4C7wA7gcPGmCZnl4H4/XgA+CHQ4txPZuDbbIC3RWSNiNzkbBuw3wtgFFAC/NMZIvu7iEQzsG12sQh4yvm4y/YGitAHBMZeogdkvquIxAAvALcZYyrdjw00u40xzcbe7mYA84CJ/rXo2IjI+cBBY8waf9vSRU42xszGhkxvFpH57gcH2vcCCAVmA38xxswCqvEIewxAm3GOzVwIPOd5zFd7A0Xoi4BMt/0MZ9tg4ICIpAM4twf9bE87RCQMK/JPGGNedDYPeLuNMYeB97FhjwQRCXUeGmjfj5OAC0VkD/A0NnzzIAPbZowxRc7tQWzseB4D+3tRCBQaYz537j+PFf6BbDPYC+laY8wB536X7Q0UoV8FjHNmKYRjb3OW+tkmX1kKLHY+XoyNgQ8YRESAR4Etxpjfux0akHaLSKqIJDgfR2HHE7ZgBf9yZ7cBYy+AMeZHxpgMY0w29rv7njHmGgawzSISLSKxrsfYGPJGBuj3AsAYsx8oEJEJzqYFwGYGsM1OrqI1bAPdsdffgwy9OFhxHrAdG4+909/2dGDjU8A+oBHrXdyIjcW+C+wA/gsk+dtOD5tPxt4afgHkOv/OG6h2A9OBdU57NwJ3OdtHAyuBPOwtcIS/be3A/tOA1wa6zU7b1jv/Nrl+cwP1e+Fm90xgtfP78TKQOJBtBqKBMiDera3L9moJBEVRlAAnUEI3iqIoSgeo0CuKogQ4KvSKoigBjgq9oihKgKNCryiKEuCo0CuKogQ4KvSKoigBzv8DZ8PpEaZr8pgAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(hist.history[\"loss\"][6:]);\n",
    "plt.plot(hist.history[\"val_loss\"][6:]);\n",
    "plt.axvline(np.argmin(hist.history[\"val_loss\"][6:]), color=\"green\")\n",
    "plt.legend([\"loss\", \"val loss\"]);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "43f2b6da",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:42:34.009261Z",
     "iopub.status.busy": "2021-09-02T19:42:34.007821Z",
     "iopub.status.idle": "2021-09-02T19:42:35.146665Z",
     "shell.execute_reply": "2021-09-02T19:42:35.147457Z",
     "shell.execute_reply.started": "2021-09-02T18:24:39.004195Z"
    },
    "papermill": {
     "duration": 3.549226,
     "end_time": "2021-09-02T19:42:35.147623",
     "exception": false,
     "start_time": "2021-09-02T19:42:31.598397",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "149/149 [==============================] - 1s 4ms/step - loss: 0.0269 - accuracy: 0.9960\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.02685798890888691, 0.9960008263587952]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.load_weights(\"data/checkpoint.hdf5\")\n",
    "model.evaluate(x_test/255, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "c59ed85f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:42:40.175989Z",
     "iopub.status.busy": "2021-09-02T19:42:40.174972Z",
     "iopub.status.idle": "2021-09-02T19:42:46.265900Z",
     "shell.execute_reply": "2021-09-02T19:42:46.265395Z",
     "shell.execute_reply.started": "2021-08-29T15:28:45.822729Z"
    },
    "papermill": {
     "duration": 8.515312,
     "end_time": "2021-09-02T19:42:46.266038",
     "exception": false,
     "start_time": "2021-09-02T19:42:37.750726",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1337/1337 [==============================] - 6s 4ms/step - loss: 0.0113 - accuracy: 0.9974\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.011295623145997524, 0.9974274635314941]"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.evaluate(x_train/255, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "c67d59c9",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:42:51.811890Z",
     "iopub.status.busy": "2021-09-02T19:42:51.811012Z",
     "iopub.status.idle": "2021-09-02T19:42:52.462156Z",
     "shell.execute_reply": "2021-09-02T19:42:52.462623Z",
     "shell.execute_reply.started": "2021-08-29T15:28:55.929609Z"
    },
    "papermill": {
     "duration": 3.549351,
     "end_time": "2021-09-02T19:42:52.462794",
     "exception": false,
     "start_time": "2021-09-02T19:42:48.913443",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00       447\n",
      "           1       0.99      1.00      1.00       508\n",
      "           2       1.00      0.99      1.00       503\n",
      "           3       0.99      1.00      1.00       474\n",
      "           4       1.00      1.00      1.00       511\n",
      "           5       1.00      1.00      1.00       436\n",
      "           6       1.00      1.00      1.00       474\n",
      "           7       1.00      0.99      1.00       492\n",
      "           8       1.00      0.99      0.99       476\n",
      "           9       0.99      1.00      0.99       430\n",
      "\n",
      "    accuracy                           1.00      4751\n",
      "   macro avg       1.00      1.00      1.00      4751\n",
      "weighted avg       1.00      1.00      1.00      4751\n",
      "\n",
      "\n",
      "------------------------------\n",
      "\n",
      "[[447   0   0   0   0   0   0   0   0   0]\n",
      " [  1 507   0   0   0   0   0   0   0   0]\n",
      " [  0   0 500   1   0   0   0   0   1   1]\n",
      " [  0   0   1 473   0   0   0   0   0   0]\n",
      " [  0   0   0   0 510   0   0   0   0   1]\n",
      " [  0   0   0   0   0 435   0   0   0   1]\n",
      " [  1   0   0   0   0   0 472   0   1   0]\n",
      " [  0   2   0   0   0   0   0 489   0   1]\n",
      " [  0   1   1   2   0   1   0   0 470   1]\n",
      " [  0   0   0   0   1   0   0   0   0 429]]\n"
     ]
    }
   ],
   "source": [
    "y_pred = np.argmax(model.predict(x_test/255, batch_size=128, verbose=0), axis=1)\n",
    "y_true = np.argmax(y_test, axis=1)\n",
    "print(classification_report(y_true, y_pred))\n",
    "print(\"\\n------------------------------\\n\")\n",
    "print(confusion_matrix(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ac445c5d",
   "metadata": {
    "papermill": {
     "duration": 2.422496,
     "end_time": "2021-09-02T19:42:57.285545",
     "exception": false,
     "start_time": "2021-09-02T19:42:54.863049",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "Let's take a look at the labels that were mislabeled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d0f9028a",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:43:02.462796Z",
     "iopub.status.busy": "2021-09-02T19:43:02.461505Z",
     "iopub.status.idle": "2021-09-02T19:43:04.132927Z",
     "shell.execute_reply": "2021-09-02T19:43:04.133334Z",
     "shell.execute_reply.started": "2021-08-29T15:28:58.259649Z"
    },
    "papermill": {
     "duration": 4.133705,
     "end_time": "2021-09-02T19:43:04.133483",
     "exception": false,
     "start_time": "2021-09-02T19:42:59.999778",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA1gAAAJSCAYAAADec2R6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8rg+JYAAAACXBIWXMAAAsTAAALEwEAmpwYAAB3tElEQVR4nO3dd5hU5fnG8fuhC6gUEelFRMWGkSjGmtg7xsQYjWJiTexRE01+UZOYxBQ19hYVjb0l9hJ7V9BgQURsCAgoiNIUYff9/XEGs8+4O7tz5uzMOTPfz3XtxT4zZ855d+dmdt6Z88xrIQQBAAAAAErXptIDAAAAAIBqwQQLAAAAABLCBAsAAAAAEsIECwAAAAASwgQLAAAAABLCBAsAAAAAEsIEqxFmNs7Mzsx9v5WZTYm5n0vN7DfJjg5pRW4QF9lBHOQGcZAbxEFuipPZCZaZvW9mn5vZIjObk7vjuyZ9nBDCUyGEtVswnoPN7Om82x4ZQvh90mNq5Njrm9mDZjbXzFjYrABy445tZnammc00s8/M7HEzW6+1j5tVZMcdez8zm5LLzUdmdo2ZrdLax80icvO1459gZrPNbIGZXWVmHctx3KwhN+7YHc3sXDP70Mzmm9nFZta+tY+bReTGHbuiucnsBCtnjxBCV0nfkDRK0v/lb2Bm7co+qvJbJukWSYdUeiAZQW4i35f0E0lbSeoh6TlJ/6zoiNKP7ESekbRFCGFVSUMltZN0ZmWHlGrkRpKZ7STpFEnbSRqkKDu/reig0o3cRE5R9POvL2m4ot/H134X+Aq5iVQ0N1mfYEmSQggzJd2v6JcoMwtmdpSZTZU0NXfZ7mY20cw+NbNnzWzDFbc3s43N7GUzW2hmN0vq1OC6bc1sRoN6gJndYWYfm9k8M7vQzNaVdKmkzXOvGnya2/art1Nz9WFm9raZfWJmd5lZ3wbXBTM70sym5sZ4kZlZC3/+KSGEKyVNivP7q1W1nhtJQyQ9HUJ4N4RQJ+k6SSOK/DXWpFrPTghheghhboOL6iQNK+JXWJNqPTeSxkq6MoQwKYQwX9LvJR1c3G+x9pAb7SHp/BDCJyGEjyWdr+jFQRRAbiqbm6qYYJnZAEm7Svpvg4vHSNpM0ggz21jSVZKOkNRT0mWS7rLo7cMOkv6t6JX7HpJulbRPE8dpK+keSdMkDZbUT9JNIYTJko6U9FwIoWsIoVsjt/2OpD9J2ldSn9w+bsrbbHdJ35S0YW67nXK3HZgL1sCW/k7QPHKjmyStaWbDLXrbfKykB5rYFg2QHcnMtjSzzyQtzI3/701tiwi50XqSXmlQvyKpt5n1bGJ7iNysOETe9/3NbNUC29c8chMdIu/78uUmhJDJL0nvS1ok6VNFd8jFklbKXRckfafBtpdI+n3e7adI2kbS1pI+lGQNrntW0pm577eVNCP3/eaSPpbUrpHxHKzo3YCGl41rsJ8rJf2lwXVdFZ3aN7jBmLdscP0tkk4p8ncyLLpLK3//pPWL3LjjdJB0Xm4fyyW9J2lIpe+jtH6RnSZ/L/0knSFpeKXvozR+kRt3nHck7dygbp/b3+BK309p+yI37jhnKjotuZekNSS9kNtfn0rfT2n7IjfpyU3Wz8EcE0J4uInrpjf4fpCksWZ2TIPLOkjqq+iXPTPk7o2caU3sc4CkaSGE5THG2lfSyyuKEMIiM5un6MnJ+7mLZzfYfomioCF55CZymqJXhQbk9vEjSY+a2XohhCUxxloLyE6eEMJMM3tA0auO34gxzlpAbiKLJDX8MJQV3y+MMc5aQG4if5DUTdJESUslXSFpY0lzYoyzFpCbSEVzUxWnCDahYSimS/pDCKFbg6/OIYQbJc2S1C/vnM6m3m6cLmmgNd4cGBq5rKEPFYVZkmRmXRS9JTuzuR8EZVVLuRkp6eYQwowQwvIQwjhJ3UUfVly1lJ187SSt2Qr7rQW1lJtJkjZqUG8kaU4IYV4C+641NZObEMLnIYSjQwj9QghDJc2T9FIIob7UfdcgclOm3FTzBKuhKyQdaWabWaSLme1mZisr+uS05ZKONbP2ZvZdSZs2sZ8XFYXurNw+OpnZFrnr5ig6t7NDE7e9UdKPzWykRR9L+0dJL4QQ3i/1h8v9TJ0UvfKg3Lj46NvSVXVuJI2X9H0z621mbczsQEWn7LydwL5rXVVnx8wOWHHeu5kNUvRK4SOl7hfVnRtJ10o6xMxGmFk3RZ/oNS6B/da6qs6NmfUzs765n220pN9IOr3U/YLctKaamGCFECZIOkzShZLmK3oCeXDuui8lfTdXfyLpB5LuaGI/dYo+lWSYpA8kzchtL0mPKnp1braZzW3ktg8runNvVxTENSXt15LxW9TIt8iabuQbJOlz/e9TBD9XdB4tSlADufmzoibziYrO1z5B0j4hhE9bsn80rQayM0LSs2a2WNE57lNyPy9KUO25CSE8IOkvkh7LjWuaeKJcsmrPTW5fz0paLOkaRT04D7Vk32gauWld5k+vBAAAAADEVRPvYAEAAABAOTDBAgAAAICEMMECAAAAgIQwwQIAAACAhDDBKiMzO8PMrqv0OJAt5AZxkBvERXYQB7lBHNWam5qaYJnZ42Y231q4RpSZHWxmT7f2uJo49gG5j59c8bXEzIKZbVKJ8dSyLOUmd/zOZnaxmc01s8/M7MlKjaWWZTA3+5rZZDNbaGZvmNmYSo2l1mUpO2bWwcxuM7P3c3+jtq3EOJCt3OSOz2NOCmQpN2Y22sz+Y2afmNnHZnarmfWpxFiaUzMTLDMbLGkrRatK71nZ0TQvhHB9CKHrii9JP5P0rqSXKzy0mpK13ORcLqmHpHVz/55Q2eHUnqzlxsz6SbpO0s8lrSLpZEk3mNnqFR1YDcpadnKelvQjSbMrPZBalbXc8JiTDlnLjaTuip7jDFa0BuxCSVdXckBNqZkJlqSDJD2vaNX4sQ2vMLMBZnZHbjY8z8wuNLN1JV0qafPcO0if5rZ93MwObXBbN5M3s/PMbLqZLTCzl8xsq4TGP1bStYGFy8otU7kxs3UUPUgeHkL4OIRQF0J4Kc6+UJJM5UZSf0mfhhDuD5F7FS3OuGbM/SG+TGUnhPBlCOHvIYSnJdXF2QcSkanciMectMhUbnJ5uTWEsCCEsETRIslbxNlXa6u1Cdb1ua+dzKy3JJlZW0n3KFpRfrCkfpJuCiFMlnSkpOdy7yJ1a+Fxxksaqeidgxsk3WpmnRrb0MxeNbP9m9uhmQ2StLWka1s4BiQna7nZNDem31p0iuBrZrZPC8eA5GQtNxMkTTazPc2srUWn6iyV9GoLx4HkZC07SIes5YbHnHTIWm7ybS1pUgu3LauamGCZ2ZaK3kq8Jfdq/juSVtx5m0rqK+nkEMLiEMIXuVfiYgkhXBdCmBdCWB5COFtSR0lrN7HthiGEG1qw24MkPRVCeC/uuFC8jOamv6T1JX2WG9/Rkq7JveqEMshibkIIdYpewLlB0ZOcGyQdEUJYHHdsKF4Ws4PKy2JueMypvCzmJm/8G0o6TdHppalTExMsRW97PhRCmJurb9D/3godIGlaCGF5Egcys5Msatr8LPfW6aqSVitxtwdJuqbkwaFYWczN55KWSTozd+rOE5Iek7RjEuNEi2QuN2a2vaS/SNpWUgdJ20j6h5mNTGKcaLHMZQepkLnc8JiTCpnLTYP9DZN0v6TjQghPJTHGpLWr9ABam5mtJGlfSW3NbEUDbkdJ3cxsI0nTJQ00s3aNBKmxfqfFkjo3qNdocKytJP1C0naSJoUQ6s1sviQrYfxbKHoV4ba4+0DxMpybxk6voG+vTDKcm5GSngwhTMjV483sBUnbS5oYY38oUoazgwrKcG5Gisecislwbla0zTws6fchhH/G2Uc51MI7WGMUNd6OUPQfeqSiT1d7StE7Qy9KmiXpLDPrYmadcpMaSZojqb+ZdWiwv4mSvmvRR2EPk3RIg+tWlrRc0seS2pnZaYo+HacUYyXdHkJYWOJ+UJwxymZunpT0gaRTzaxdbkzflvRgzP2hOGOUzdyMl7TVilePzWxjRZ8sRT9E+YxRNrMjM+vYoJ+iQ25sTNbKY4yymRsecyprjDKYG4s+ffJRSReGEC6Ns49yqYUJ1lhJV4cQPgghzF7xpeiTRw5QNIPeQ9IwRU9MZ0j6Qe62jypqnpttZiveQj1X0peKAnaNosbAFR6U9ICktxQ1Bn6h6FWARpnZJDM7oMD1nRS9wsDpgeWXydyEEJZJ2kvSror6sK6QdFAI4c3ifnzElNXcPCHpDEm3mdlCSbdL+mMI4aHifnyUIJPZyZmi6PTkfrl9f66otwOtL5O54TGn4jKZG0mHShoq6QxrsFZscT96eVjgU78BAAAAIBG18A4WAAAAAJQFEywAAAAASAgTLAAAAABISEkTLDPb2cymmNnbZnZKUoNCdSM3iIvsIA5ygzjIDeIgN5BK+JALM2ur6BNBdlD06SLjJf0whPBGcsNDtSE3iIvsIA5ygzjIDeIgN1ihlIWGN5X0dgjhXUkys5sUfTx0kyHqYB1DJ3Up4ZCopIWaPzeE0KvE3ZCbGpNQbqQis0Nuso3cII5K5UYiO1n2hRbry7A0ibXbyE2Naeoxp5QJVj/5z7GfIWmzQjfopC7azLYr4ZCopIfDbdMS2A25qTEJ5UYqMjvkJtvIDeKoVG4kspNlL4RHktoVuakxTT3mlDLBahEzO1zS4ZLUSZ1b+3CoEuQGcZAbxEFuEBfZQRzkpvqV8iEXMyUNaFD3z13mhBAuDyGMCiGMaq+OJRwOVYLcIK5ms0Nu0Ahygzj4W4U4yA0klTbBGi9pLTMbYmYdJO0n6a5khoUqRm4QF9lBHOQGcZAbxEFuIKmEUwRDCMvN7GhJD0pqK+mqEMKkxEaGqkRuEBfZQRzkBnGQG8RBbrBCST1YIYT7JN2X0FhQI8gN4iI7iIPcIA5ygzjIDaQSFxoGAAAAAPwPEywAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABISLtKDwBAaT44/VuunnzExQW3f2hJe1efPWy9xMeE1nffzJddvfVr33O15W3/4Vu9XL3WsS+0xrCQMr2fW8XV1w56sqjbHzRta1e/95d1Xd35X+QIAPLxDhYAAAAAJIQJFgAAAAAkhAkWAAAAACSEHqwSrDm+k6sv7ve8q385Z6SrX/mm/3WH5ctbZVyoco/0d+XEdc5z9bJQ+HWTenVMfEgov3oFVz+6wc2ubpP3+ln9BvWu3uKVY13d88rnEhwdKmXJ3pu5+tpBl5W0v/yerTW3HuHqYf8qafcAapy17+Dqjw7dxNWHHHOPq8+/fXdXD/5NOv928Q4WAAAAACSECRYAAAAAJIQJFgAAAAAkhB6sIkz/jV9v6Nc9/XpDy0Kdq89c/SVXj+k3xtXLp01PbnCoGm3XHubqLW99zdXH9rjN1W3k17VqzoB2n7p65ik+1/3Oerao/aEyNj7/mILXX3Wk783buIN/PW3Ho59x9UtX8npbNfhw6/wV0JL1zg8udfVOJ4xs1eMhQZaXjc02cOW+Vz/k6jeW9HX13W/57Qf0mu/qT+7w/cGrX5LXGxN83yiqU5tO/vMJ5n9vpKsXr+H/1nxr3/+6+u5+F7p6Tt3nrh73nu/BSiv+ogIAAABAQphgAQAAAEBCmGABAAAAQELowSqgzcoru3rz3V71dUffc9WcKUf785PXPJkeLEht113L1d++9WVXH9/9rbxbFNdzlW943poT/Xec5upwVkm7R5n0+3PhXrn9NzrM1ZO2uaI1h4OUyO+RAlb4YrdvuvrSC//u6n0vPsnVO/7Ar+15y+jLXb1a22Wu7vPrzq7e5ZWfuNqemdjSoSJF2nT29+uyTddx9bvf889Jjv32g64+qttFJR1/67tPdPXwf/rPN0hrZx/vYAEAAABAQphgAQAAAEBCmGABAAAAQELowWqgXZ81XL3OPR+5+qw1xpdzOKhSbUcMd/U2N/s1IL7ec1Wct5Z96eq9nz/C1csW+R6sdU+a6uriOguRFW3yXk+78aVNXT1cE8o5HACtrN0avV192nlXuvrYsUe5erVO/m/HtD16uPrXe+7tD9DB996891fft65v+96dAX7pPaRUmy5dXD3vFr8e2jMjC/fztpFfb60+7/rHPvfrZB3z0g9dfdrIe109ZYxfc3abJ31uV77Z9wqmBe9gAQAAAEBCmGABAAAAQEKYYAEAAABAQmq6Byt8ayNXv/dz333y7zX8eaD5dnzju67+87DbXL1xB+avaL7n6uc93kz0eI8v8ccbst+rTWwZoeeqOl032vdb1OedCT/wTh6fqsHb547Ou2RiJYaBFHrnZ0Nd/Wmd761p84T/W9Q5rw998TGru7p+5hsFj/fFok1c3f2jtK5QhEI+32aEq58ZWXhtvfeWf+Hqk97fx19/t8/hgKv9c57B8/xzlD+e+gNX7/Czv7p6Tt5D3so3FxxexfAXFgAAAAAS0uwEy8yuMrOPzOz1Bpf1MLP/mNnU3L/dW3eYyCKygzjIDeIgN4iD3CAOcoPmtOQdrHGSds677BRJj4QQ1pL0SK4G8o0T2UHxxoncoHjjRG5QvHEiNyjeOJEbFNBsD1YI4UkzG5x38V6Sts19f42kxyX9MsmBtYbZx3/L1Vccd56rm+uZGvW3Y1zdfpE/v3jjMzjjsqFqyk5D+b179uwrBbefcphfS+TOhHuuLv9ssKvv2XeLvC2SPV5rq9bclNuPnj/E1ZO38T1Zne5+sZzDaXXkBnFUc26W9l7u6m1WmuXqK7tt7Orls2b7HeTXzVh99c9cPWfTbq5e7fKidpdqWcqNjVrf1aFd3nPV530PVOe3P3H1DpN8T9XMiX1cvdbVc11dN9mvrdlXPkfN9X33/9Ozrr75R74n7MfbPe7qpzuu6uqwdGkzRyiPuDOC3iGEFf9TZ0vqXWhjoAGygzjIDeIgN4iD3CAOcoOvlPyWSwghSGryo2LM7HAzm2BmE5YpHbNKpEOh7JAbNIXcIA5ygzh4joM4yA3iTrDmmFkfScr9+1FTG4YQLg8hjAohjGqvjjEPhyrSouyQG+QhN4iD3CAOnuMgDnKDr8RdB+suSWMlnZX7987ERpSg/F6Zm47/m6uHtfehfnuZfxXhu1ed5OpBF05w9Sf7+zUfmrPG8/XNb1T9MpGdhtpstK6rfzzu364+/ZU9Xd354a6ufuF7PndSp6KO/+qX/ozlI39/nKt7/neBq8Prk4raf0ZkLjetrd2A/q5+43S/hs2Noy9z9UWfrtnqY0ohcoM4qiI37eb7p3jd26zk6s9v7ebqDjv4HqpizZ3Uy9Xta2+RxXTkZtMNXHnt7X4dq/YyVz+4pJ+rf/2i780bfJZ/L2boo8+5utx3876rvuTq51b7vquXz/ywnMNpUks+pv1GSc9JWtvMZpjZIYrCs4OZTZW0fa4GHLKDOMgN4iA3iIPcIA5yg+a05FMEf9jEVdslPBZUGbKDOMgN4iA3iIPcIA5yg+bwueIAAAAAkJC4PViZ0GaZ73n67riTmtgy0u/xL1w98HH/WfxNfhxMC3W9Z2Ki+0N5vL+3X4x97y5+jYi9vzXO1Y9s3NnVq7Yprefq6F8f6+oeN/jzn8lRdWo7Yri/YIZfS+SdQwe6+q1dLnD1nLrPXf3AwVvlHeG1ksYHIN3W/I3vVfnBFju6+oERt7l6g7P835qhebcPy74seLz6Xv76VZ/t0KJxIlnLuvnPF+ie9xzk9kWruXqzTtNd/eZ3/uHqz7b1z413/p1/Lt3zCv+cpLWdPWd7V6el5yof72ABAAAAQEKYYAEAAABAQphgAQAAAEBCqroHK4z3PQYDx1doIDmf7+jX5ep094sVGgmKMeS6Wa5++2C/Xlr+emrbrbSkpOP99cOdXb3KDc+XtD9k02n33Ojqn55zjKvHHeh7rurle06/fePJrh46vrznyaN1LNl7M1e/84NLm9gStS6/Z2rR9gtdvc+Dfg3HyQde5OpR6+7v6j4H+l6XugV+DcYuk/IXzKVDuBIWDGxf8PrLj9nH1ddO9+ufvf97fz++svk1rv7PaWe7+ls7HuHqob/wuVj+3rSC48n3/pmbu/rHq57r6p2m+zVou+rdovZfLryDBQAAAAAJYYIFAAAAAAlhggUAAAAACanqHqy0WemhV1zN2cnZUPf2e65eElr3v83Emf1cPUifNLElqsm8Q/x559/s+LKrf/LTe/OuN1df/OkwVw/9JT1X1WjILyZX9PgHTds675IFjW6H9AlLff9w3c7zXf3tnY909Zl/u9bVX0zw61r97Z0dXL3rGn7t0Icv8Y9pKI/ej/q+cf3Wl9N28c9hhp0w1dUD9m3r6i1/dLSrT/j1Ta5+NW8t0Gvv889h7tjV943m92TVb7Oxq6/Z/0JXdzTfUzbvxd6upgcLAAAAAKocEywAAAAASAgTLAAAAABICD1YQJG+9/BRrn5r19LWofnVnFGuXvPYj1y9vKS9IzU23cCV736vq6sf2e+vrq7XSq4+9+FdXH3T2j4nj25ws6vP/8uxrh76C3qyqsG1g56s6PGfeX6Eq4eJdfqyqv6LL1y90r/92pwXPb2Fq98/cm1X33qoXw9pnbw1IW9d3/ferBZrlChWWOzX4nxuqe+pumiPq119xks/dvWqN/hFY7td6/92/PMp34f5p3M7u/qVTf0ajis/+LirJ3/ue7T+b7Ur5fn+4pe+rHP1gEd8btOKd7AAAAAAICFMsAAAAAAgIUywAAAAACAh9GABRer8bvvmNyrCp8v8+cvhi2ycX4zC2o4Y7uoFv1vs6jc3uMbVh073PVZ79fyvq4df52+/bOXurl53jF+rZOoBF7l61Af++tUv9GvWIJ3ePnd03iUTy3r8/HWvhp1Az1WtqJs7z9UDzvSPGXt1O8HVU354savf/d5lrt76kcNdvdKdvucLyaib4/tzT//poa6++0q/ztRTf/Z/K7Y+YF9XL3rErzuVb/EC/5ylLtS7eu8unxSs/dbSAe/t6OqFR/vjt5no/zamFe9gAQAAAEBCmGABAAAAQEKYYAEAAABAQujBAirs4v5+XZvh5/rz1If/+KVyDgcJ6XPVh67+14DHXD1+qX99a/qpa7n6iqmruDrMeM3VHVfx168zwR9/p3/58+47/3K2q+d9vrmre17JOllptMXoNyp6fNa9QpOCX69og+cOcvWLo/9RztGgCR0e9H8cdj7uOFf/9I+3ufrJDW/xO9iw2CNa85sU8O4437/cc2I2/zbxDhYAAAAAJIQJFgAAAAAkhAkWAAAAACSEHiwgZa7e+mpXn7HbIa7ueO/4cg4HLdRuQH9XXz7gLlcfOv07rv5w9EJXt9XLrl7ezPHqFiwoeH3bx/z+uvoWMJ373s2u/tW7RxS8PSrj2kFPNr8RUAF1PZf5Cyb6vtAvNqtz9bz1/FPO/ne2yrDQjC63v+Dq6x9Z39V/OmI9V7f/ll+36oi1nnL1n5/e1dWdZvi1Qnu94v+arX7Su66+ceiDrv7RCfe7+qGnRrm6bsrbygLewQIAAACAhDDBAgAAAICEMMECAAAAgITQgwWkzBad/Hntv7vgClefOX+sq+3ZV1p9TGjetP0HurpewdXP37+Bqwfq2VYfUyHHn3qMq5es719v653XswUADQ0d+JGrO5zT2dV/2nsrV9f71hykRN2nn7m6358L/236l3q5eriK6wv/yDZ19ZzzP3f1Ud3ecfVFh+zi6qG/oAcLAAAAAGpKsxMsMxtgZo+Z2RtmNsnMjstd3sPM/mNmU3P/dm/94SIryA3iIDeIi+wgDnKDOMgNmtOSd7CWSzoxhDBC0mhJR5nZCEmnSHokhLCWpEdyNbACuUEc5AZxkR3EQW4QB7lBQc32YIUQZkmalft+oZlNltRP0l6Sts1tdo2kxyX9slVGWSXmHvgNV/f8x3MVGknrIzfJ2byjX0tk0cCVXL1yZVt5EpXl3HSa63uu/vtlvat/tu+9rr5mhl87pOeVrft4MO+QzV094feXuHpZ8Dnb/YJNWnU8SctydlA55Ca+kd1nuHpy/TBXn7q6Xy9p/ES/nlGWkZv4Vvr3i67eZpuTXP3mvhe5+uQ9/IJpd57j+5mXz56T4OiSU1QPlpkNlrSxpBck9c4FTJJmS+qd7NBQLcgN4iA3iIvsIA5ygzjIDRrT4gmWmXWVdLuk40MICxpeF0IIUt5HZv3vdoeb2QQzm7BMS0saLLKH3CAOcoO44mSH3IDHHMRBbtCUFk2wzKy9ogBdH0K4I3fxHDPrk7u+j6SPGrttCOHyEMKoEMKo9uqYxJiREeQGcZAbxBU3O+SmtvGYgzjIDQpptgfLzEzSlZImhxDOaXDVXZLGSjor9++djdwcDfz0xH+5+uze33X1gD9UTzNNNeemzXJf18v32rRh9YPYspyb/B6qsQOOc/UR37/P1Xec/ldXbzf8ZFcP/WVpPVnv/sX3XD2ynz/esuB7+da+7ShXr6XnSzp+uWU5O4VsddQRrn7qosvKevy+Tzb6AnzVqNbctIY2Xbq4+tNl5uovV+9azuFUFLlJztoX+R6qh/bwOfvxKtNdfd6Px7i6/5/S2YPVkoWGt5B0oKTXzGxi7rJfKQrPLWZ2iKRpkvZtlREiq8gN4iA3iIvsIA5ygzjIDQpqyacIPi3Jmrh6u2SHg2pBbhAHuUFcZAdxkBvEQW7QHM5lAgAAAICEtOQUwZrVdrWerp591WquPmot31PVnINWmenqe3ea6urFfyhqd6iQvn/1vXIbbHKIq1/b6kpX05NVmwae4XNy/31buXrw9XNdPfyCD1yd1+rXrFn/XtfV3x/0jKvHf9HX1X/68wGuXquV1+FCPJ3/9YKrD/rF1q6+dtCTiR4vv+cr//ioXfWLF7u6S7v2rp65dSdXv/7lyq7ufN9EV1d3dx9aqu7t91x9zL0Hu3rK9/y6WPbNz1p7SIngmR8AAAAAJIQJFgAAAAAkhAkWAAAAACSEHqwCZu27tqtf3OSCkvY3/L4jXb3ur94vaX9IhyH7verqO9/yvXp7d/mkpP3ftbi7q1ed7M8/9qtwIbVefM2V5/58f1d3le/RbNamG7jynm9c4urtbvTrar1ywQBX95xBz1UWzdl8gavXPNf/Xdli9Buubq5Hi54rxPXWgtVdffXB/jnSoRMPcnW/ZZNafUzIvnX/NsPVj+3me/ue2fQfrt5/8A9cvfx9389cKbyDBQAAAAAJYYIFAAAAAAlhggUAAAAACaEHq4A+j33s6uHr/7Tg9sPX/tDV96xzp6vbfup/3XUf+/2jOlw5fIivNaSJLeOanPD+UAmd7n7R1cWue5Xf03XYwC1dPVS+x6ro/SMThp3wvKvn5F2/k0YWvH1n0XOFeKbP7+bqb65trrZnVy3jaFAtlk/3PVgnvLKvqyeOvtbV7/+wv6v7/4keLAAAAACoKkywAAAAACAhTLAAAAAAICH0YBVQN3mqq4f/rLjb765NXL2mWHcGAABk39J3V3H1+I2Dq/s//KmrWbMRcfTfx6+flv/cur+eLedwWox3sAAAAAAgIUywAAAAACAhTLAAAAAAICH0YAEAAKAoa57s+8pPP3mTvC3eKN9ggJThHSwAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASIiFEMp3MLOPJU2TtJqkuWU7cPEYX+MGhRB6lfug5CYx5CadGF/jyE1hjK9xFcmNRHYSQm7Si/E1rtHslHWC9dVBzSaEEEaV/cAtxPjSKe0/N+NLp7T/3IwvndL+czO+9Er7z57m8aV5bK0t7T874ysOpwgCAAAAQEKYYAEAAABAQio1wbq8QsdtKcaXTmn/uRlfOqX952Z86ZT2n5vxpVfaf/Y0jy/NY2ttaf/ZGV8RKtKDBQAAAADViFMEAQAAACAhZZ1gmdnOZjbFzN42s1PKeeymmNlVZvaRmb3e4LIeZvYfM5ua+7d7hcY2wMweM7M3zGySmR2XpvGVU9qyQ26ygdwUNTZyk0Nuih4f2RG5iTE+cqP05UZKd3aykpuyTbDMrK2kiyTtImmEpB+a2YhyHb+AcZJ2zrvsFEmPhBDWkvRIrq6E5ZJODCGMkDRa0lG531laxlcWKc3OOJGbVCM3RSM3Ijcx1Xx2yE0s5CaduZHSnZ1s5CaEUJYvSZtLerBBfaqkU8t1/GbGNljS6w3qKZL65L7vI2lKpceYG8udknZI6/hqLTvkJt1f5IbckBuyQ27ITZq/0pqbLGUnrbkp5ymC/SRNb1DPyF2WRr1DCLNy38+W1LuSg5EkMxssaWNJLyiF42tlWclO6u4XckNu4iA35CauGs4OuSkBuflKWnMjpfB+SXNu+JCLZoRoKlzRj1o0s66Sbpd0fAhhQcPr0jA+fF0a7hdykz1puF/ITfak5X4hO9mSlvuE3GRPGu6XtOemnBOsmZIGNKj75y5Lozlm1keScv9+VKmBmFl7RQG6PoRwR9rGVyZZyU5q7hdyI4ncFI3cSCI3sZAdchMHuclMbqQU3S9ZyE05J1jjJa1lZkPMrIOk/STdVcbjF+MuSWNz349VdH5n2ZmZSbpS0uQQwjkNrkrF+MooK9lJxf1Cbr5CbopAbr5CbopEdiSRm6KRG0nZyY2UkvslM7kpcyParpLekvSOpF9XsvmswZhulDRL0jJF574eIqmnok8gmSrpYUk9KjS2LRW9xfmqpIm5r13TMr5azg65ycYXuSE35IbskBtyk+avtOUm7dnJSm4sN1gAAAAAQIn4kAsAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABICBMsAAAAAEgIEywAAAAASAgTLAAAAABICBMsAAAAAEgIE6xGmNk4Mzsz9/1WZjYl5n4uNbPfJDs6pBW5QVxkB3GQG8RBbhAHuSlOZidYZva+mX1uZovMbE7uju+a9HFCCE+FENZuwXgONrOn8257ZAjh90mPqZFjjzWzl8xsgZnNMLO/mFm71j5uFpGbJsfxiJkFctM0suOOzWNOC5Ebd+yOZnaumX1oZvPN7GIza9/ax80icuOOvZ+ZTTGzz8zsIzO7xsxWae3jZhG5ccde38weNLO5ZhZa+3j5MjvBytkjhNBV0jckjZL0f/kb1Mgf/c6Sjpe0mqTNJG0n6aRKDijlyE0DZnaAJJ7ktAzZifCYUxxyEzlF0c+/vqThin4fX/td4CvkJvKMpC1CCKtKGiqpnaQzKzukVCM3kWWSbpF0SCUOnvUJliQphDBT0v2KHrSVeyX+KDObKmlq7rLdzWyimX1qZs+a2YYrbm9mG5vZy2a20MxultSpwXXbmtmMBvUAM7vDzD42s3lmdqGZrSvpUkmb5141+DS37Vdvp+bqw8zsbTP7xMzuMrO+Da4LZnakmU3NjfEiM7MW/vyX5F5N+DL3u7he0hYxfpU1pdZzk7v9qpJOl/SLIn99Na3Ws8NjTjy1nhtJe0g6P4TwSQjhY0nnS/pJkb/GmlPruQkhTA8hzG1wUZ2kYUX8CmsSuQlTQghXSpoU5/dXqqqYYJnZAEm7Svpvg4vHKHpldYSZbSzpKklHSOop6TJJd1l0ukIHSf+W9E9JPSTdKmmfJo7TVtI9kqZJGiypn6SbQgiTJR0p6bkQQtcQQrdGbvsdSX+StK+kPrl93JS32e6Svilpw9x2O+VuOzAXrIEt/JVsrQoFKkvIjSTpj5IukTS7wDbIQ3a+hsecFiA30SHyvu9v0Qs9aAK5kcxsSzP7TNLC3Pj/3tS2iJCbCgshZPJL0vuSFkn6VNEdcrGklXLXBUnfabDtJZJ+n3f7KZK2UfTE4ENJ1uC6ZyWdmft+W0kzct9vLuljSe0aGc/Bkp7Ou2xcg/1cKekvDa7rqujty8ENxrxlg+tvkXRKjN/LTyTNkLRape+jNH6RG3ecUZImKjrdYnBuX18bI19kp5nfC4855KaljzlnKjrdq5ekNSS9kNtfn0rfT2n7IjdN/l76STpD0vBK30dp/CI3jf5OhkkK5b4vsn4O5pgQwsNNXDe9wfeDJI01s2MaXNZBUl9Fd97MkLsXcqY1sc8BkqaFEJbHGGtfSS+vKEIIi8xsnqIHi/dzFzd8F2GJoqC1mJmNUfRKwPbBv50Or+ZzY2ZtFD3wHhdCWN7Cd9xBdhwec1qM3ET+IKmbohd2lkq6QtLGkubEGGctIDd5QggzzewBRe9yfCPGOGsBuUmBqjhFsAkNQzFd0h9CCN0afHUOIdwoaZakfnnndDb1duN0SQOt8ebA0MhlDX2oKMySJDProugt2ZnN/SAtYWY7K/pjtUcI4bUk9lmjaiU3qyh6B+tmM5staXzu8hlmtlWJ+65VtZKdFfvjMScZNZObEMLnIYSjQwj9QghDJc2T9FIIob7UfdegmslNI9pJWrMV9lsLajk3ZVXNE6yGrpB0pJltZpEuZrabma0s6TlJyyUda2btzey7kjZtYj8vKgrdWbl9dDKzFY3dcxSdS96hidveKOnHZjbSzDoq6n15IYTwfqk/XO4c1usl7RNCeLHU/eEr1ZybzxS9cjQy97Vr7vJNFJ22g9JUc3Z4zGk91Z6bfmbWN/ezjZb0G0UfsoPSVHtuDrBcn42ZDVL0Tugjpe4XVZ8bM7NOit6VU25cHUvdb0vVxAQrhDBB0mGSLpQ0X9Lbis4LVQjhS0nfzdWfSPqBpDua2E+dok9BGibpA0V9Bz/IXf2ooibv2Wb2tVNlcm/X/kbS7YqCuKak/Voyfosa+RZZ0418v5G0qqT7ctstMrP7W7JvNK2acxMis1d8KTp/WpLm5H42lKCas5PDY04rqIHcrKmoj2OxpGsU9VI81JJ9o2k1kJsRkp41s8WKevim5H5elKAGcjNI0uf63wcwfa4oO2Vh/vRKAAAAAEBcNfEOFgAAAACUAxMsAAAAAEgIEywAAAAASAgTLAAAAABICBOsMjKzM8zsukqPA9lCbhAHuUFcZAdxkBvEUa25qakJlpk9bmbzW/o5+GZ2sJk93drjKnD8fc1sspktNLM3zGxMpcZSyzKYm0PN7O3cx5c+YGZ9KzWWWpal3JjZCDObkBvvfDN72MxGVGIsyFZ28sZxmpkFM9u+0mOpRVnKjZmNNrP/mNknZvaxmd1qZn0qMZZal6Xc5I6fiefGNTPBMrPBkrZStKr0npUdTfPMrJ+k6yT9XNIqkk6WdIOZrV7RgdWYDOZmW0UL9e0lqYek9xQt5IcyylpuJH0o6XuKMrOapLsk3VTREdWoDGZHkmRma0r6vqK1bFBmGcxNd0mXSxqsaL2ihZKuruSAalHWcpOl58Y1M8GSdJCk5yWNkzS24RVmNsDM7si9ijLPzC40s3UlXSpp89w7AZ/mtn3czA5tcFs3kzez88xsupktMLOXzGyrmOPtL+nTEML9uUVh71W0OOOaMfeHeLKWm90l3RpCmJRbKPD3krbOPflB+WQqNyGET0MI74doYUSTVKdo0UiUX6ay08BFkn4piYXKKyNTuck9t7k1hLAghLBE0WK3W8TZF0qSqdwoQ8+Na22CdX3uaycz6y1JZtZW0j2Spil6JaWfpJtCCJMlHSnpuRBC1xBCtxYeZ7ykkYpeCb5B0q1m1qmxDc3sVTPbv4n9TJA02cz2NLO2Fr0FulTSqy0cB5KRtdxI0RPk/O/Xb+E4kIws5ka5P5ZfSLpA0TuhKL/MZcfMvi9paQjhvhYeG8nLXG7ybC1pUgu3RXKylpvMPDeuiQmWmW2p6C3oW0IIL0l6R9KKO29TSX0lnRxCWBxC+CKEEPvc0hDCdSGEeSGE5SGEsyV1lLR2E9tuGEK4oYnr6iRdqyiIS3P/HhFCWBx3bChOFnMj6QFJ+5rZhma2kqTTFL313znu2FCcjOZmxTbdJK0q6WhJ/407LsSTxeyY2cqKJuPHxR0LSpPF3OSNf0NFf6tOjjsuFC+LucnSc+OamGApetvzoRDC3Fx9g/73VugASdNCCMuTOJCZnWRR891nuVeDV1XU01DsfraX9BdJ20rqIGkbSf8ws5FJjBMtkrnchBAelnS6pNslvZ/7WihpRhLjRItkLjcN5f5QXSrpWkvhee1VLovZOUPSP0MI7ycxLsSSxdys2N8wSfdLOi6E8FQSY0SLZS43WXpu3K7SA2htuVfx95XU1sxm5y7uKKmbmW0kabqkgWbWrpEghUZ2uVj+3YA1GhxrK0m/kLSdpEkhhHozmy9/ylZLjZT0ZAhhQq4eb2YvSNpe0sQY+0MRMpwbhRAuUtQPITMbLun/JL0eZ18oTpZzk6dN7rj9JH2UwP7QjAxnZztJ/c3sZ7m6l6RbzOzPIYQ/x9gfipDh3MjMBkl6WNLvQwj/jLMPxJPh3IxURp4b18I7WGMUNWyPUHTHjJS0rqSnFJ17+qKiTz06y8y6mFknM1vRaDlH0R+ODg32N1HSd82sc+6Vl0MaXLeypOWSPpbUzsxOU/QpJ3GMl7TVilm5mW2s6JNeUneeaZUaowzmJjeO9S0yUNGnNJ0XQpgfZ38o2hhlMzc7mNnGuXPaV5F0jqT5kibH2R9iGaMMZkfRk6b1G4z5Q0lHKPciD1rdGGUwNxZ9Gtyjki4MIVwaZx8oyRhlMDfK0HPjWphgjZV0dQjhgxDC7BVfij6x5gBFM+g9FH1i1geKTqX6Qe62jypqupxtZiveQj1X0ackzZF0jaLGwBUeVNQD85aixsAvFL0K0Cgzm2RmBzR2XQjhCUWnXtxmZgsVnfL1xxDCQ8X9+Igpk7mR1EnR2/yLFD1APifpN0X83ChNVnPTTdHH+X+m6Dz8NSXtHEL4ouU/OkqUyezk+ioajrdO0vwQwqKifwOII5O5kXSopKGSzrDo0+gWmRmZKZ9M5iZLz40thMbe6QMAAAAAFKsW3sECAAAAgLJgggUAAAAACWGCBQAAAAAJKWmCZWY7m9kUM3vbzE5JalCobuQGcZEdxEFuEAe5QRzkBlIJH3JhZm0VfSLIDoo+XWS8pB+GEN5IbnioNuQGcZEdxEFuEAe5QRzkBiuUstDwppLeDiG8K0lmdpOkvSQ1GaIO1jF0UpcSDolKWqj5c0MIvUrcDbmpMQnlRioyO+Qm28gN4qhUbiSyk2VfaLG+DEuTWKSd3NSYph5zSplg9ZP/HPsZkjYrdINO6qLNbLsSDolKejjcNi2B3ZCbGpNQbqQis0Nuso3cII5K5UYiO1n2QngkqV2RmxrT1GNOKROsFjGzwyUdLkmd1Lm1D4cqQW4QB7lBHOQGcZEdxEFuql8pH3IxU9KABnX/3GVOCOHyEMKoEMKo9upYwuFQJcgN4mo2O+QGjSA3iIO/VYiD3EBSaROs8ZLWMrMhZtZB0n6S7kpmWKhi5AZxkR3EQW4QB7lBHOQGkko4RTCEsNzMjpb0oKS2kq4KIUxKbGSoSuQGcZEdxEFuEAe5QRzkBiuU1IMVQrhP0n0JjQU1gtwgLrKDOMgN4iA3iIPcQCpxoWEAAAAAwP8wwQIAAACAhDDBAgAAAICEMMECAAAAgIQwwQIAAACAhDDBAgAAAICEMMECAAAAgIQwwQIAAACAhDDBAgAAAICEtKv0AAAAAABUP+vY0dV9nujg6o1Wnu7q/+y2oauXv/9B6wwsYbyDBQAAAAAJYYIFAAAAAAlhggUAAAAACaEHCwCqgLX357G3HdDX1Z8PW83V7/0wuHrYVfWubv/mDFfXffxxqUNECrUbMsjV9SuvlOj+beZHrq6b90mi+weQbm179XL1sps6ufryAf8uvIN7fXn/et1KH1QZ8A4WAAAAACSECRYAAAAAJIQJFgAAAAAkhB6sKtJu0ABXz9qlv6sXDvHbD/3lc609JAAJsY3Xc/X0XVZ19cpb+l6XJza6sbgD7OjLaxb43pxbj9zJ1W2eetXfoL6uuOMhEW27d3f1gu2GF3X7kb+c6Oqz+z5d6pCcde4+ytUjzpjm6uWz5yR6PADp8v4Ra7n6lXUuKOr2R3V7x9X3a5OSx1QOvIMFAAAAAAlhggUAAAAACWGCBQAAAAAJoQerBNbO//ra5J0LH774ouDt6xcuTHQ8Xa5f4uoXhlxYcPvdf5mN81hb2/x7/fnBXS7o5uoOD4wv42hQqxbsP9rVdQfMc/UzI691db38ulVJG7uK75WZcs4rrp68/1BX1015u1XHg4h17OjqKb9Z29Vv/KC4/obW9uYeF7l6+4eOdnXnO+jBAqrZuEPOy7ukNt7bqY2fEgAAAADKgAkWAAAAACSECRYAAAAAJIQerCJM+93mrl7ab5mr39r5MlefPHszV9cHc/VD937L1Ws87/fXefy7rq6b63syluzt9//PQefkjbiTq/aeunve9bNUi5bu8k1XPzfS32/1VwZXt5G/3+rlrx/9W99TsPlhL/v9X/ENV/d+3K9XVPeWX+MBtWnD432P0/n9nszborKvh/1xjRdcfeOdM3y9Tt9yDqdmzPiV/zuxZOByV7+5R7p6rgDUtvzHrE06+OdE+c+hirVse//5Ae0ffqmk/bUW3sECAAAAgIQwwQIAAACAhDDBAgAAAICEVFUP1udjNnX1l139/PGj7X2P0zlb3FzU/ndc6UVXt7e2Bbc/ew2/ff55pxvW+V6gz4a0d/Wef33P1UM7+t6dER2ecfX4pT1d/fdpO7h60ZX9XL1KjfRg5fdcHX/+ja7Ov1+WhTpX59/P+dc/f/qFBa9vf/qzrj7usC1c/Y4fXuJY5yudpv/Gn6d+fZ+/5m3Roaj9XbdggKt/tMr0OMNqsbrA63NJ+PinvrdXu3ziyqs38GvIbFRcLL5mo2d+4urOj3Z19RknXuPqnTp/VtoBUTFtu63q6iXfGu7qL7r7v22L+/n/08tX8vtb5V3/t7LbP58rcYSoRl/08ms0zqzza7TuddYvXD3h14XXbM3X+f8+dHXdCyu7Ouk1ZuPiLyQAAAAAJKTZCZaZXWVmH5nZ6w0u62Fm/zGzqbl/u7fuMJFFZAdxkBvEQW4QB7lBHOQGzWnJO1jjJO2cd9kpkh4JIawl6ZFcDeQbJ7KD4o0TuUHxxoncoHjjRG5QvHEiNyig2R6sEMKTZjY47+K9JG2b+/4aSY9L+mWSA2uJtj17uHr/P97r6kNW/SDpI7rqmS98z9S8urxz2Sf5dac63e3Phw47+/NEX9xiXJHj8etcHX7kga7ueL/vrVlFrduTkS8t2Xn8yitcnd8jlb/OVX7PVXPXnzhrtKvP7vN8wduf19f3zrWZWXidrd37+TUf8rVde5irh17n7+fz+vpew2F7H+Hq4Q8U3H3ZpSU3ra3Tx/5+/vs8n6Pf9PJrh+T7uG6pq6/87RhX3zTLX//2fv7x6s09L2rJMJu0ZoeP8i5Zo6T9lSoruZl/sO+5+uOJV7n62ystSvR4oyf4vwtr/sL3VC1//zVXX/javq7+exf/NGHu+r4JbPyJvkcsa7KSmziWbTDU1Y9c4dd8bGvFdYksqf/S1bvP+qmr07oeUWuo5tyU6uq9LnX1KdP3cHXfO/zan+f8zPeJ79DlDVcPae97uu5cyz/X3+RgvxZp7wt833ulxO3B6h1CWPEJCbMl9U5oPKh+ZAdxkBvEQW4QB7lBHOQGXyn5Qy5CCEFqellmMzvczCaY2YRlWtrUZqhBhbJDbtAUcoM4yA3i4DkO4iA3iDvBmmNmfSQp92/+uSJfCSFcHkIYFUIY1V4dYx4OVaRF2SE3yENuEAe5QRw8x0Ec5AZfibsO1l2Sxko6K/fvnYmNqAgd7vA9BaX2XG3wzMGurn+na+Mb5gy72v/fqXvLn1e62s5dXD3vyE9d/do3rytqfJOX+XW8vnfjCa5e84lXXO3PWk2Nsmcnv+eq2HWu8q8ffrc/73zdc+a5evOtR7n6ud82s05WM8dvTn7P1V/6POXq+rzewUm7+d6bveXXj0upVDzmJKnXpX4NmZf/1cfVY/qMLXj7OZv7ns7lfX0v30qH+lw+utYNeXso7Y/6T/IeL9dS4Z6xCil/btr4/28Lv+8Xurvut39z9aB2pS1sddpHfv+v7+HXO+yzeI6rl8+fX3B/9sxEV7fPu77vEz43m7Q/ztUvHZvtnqycqni8sXr/Bkp+z9UOk31vzOIrfHaW9PLbjz/lAlcvOmGBq7s/HGuY1aQqclOsRd/fzNUj2vs+83lbFH7MueSJ7V39zPq+r/wfQ+/wN/B/6lKrJR/TfqOk5yStbWYzzOwQReHZwcymSto+VwMO2UEc5AZxkBvEQW4QB7lBc1ryKYI/bOKq7RIeC6oM2UEc5AZxkBvEQW4QB7lBc0r+kAsAAAAAQCRuD1Yq/HutB129rMnPa4kMv+dIV6/763ddPfizKa4Oy/yaD23XW9vV74xd3Y/ngH+6uldbfx7qqm38ulXNWefWo1y99m/9+IbM9z0cKe25qrhi16lqbh2st/a4xNW7H+nXqeqZ14u3xd77ufr/hvs1HHbr7Ne9yT/+3MP9ujmbH+Z7Xc7rm5+D4tbxQjrUzfE9nTMv6eXqCze60dWD2vn+h77tmuupSraRuudjNGY3ps36fk2Xx865IG+L4nqurlswwNWvLenv6smbLM+7xcyi9l+ssNR/4ln7hU1s2ITPhvjHn5W2GOnq/B4wxNf2Vf+3aJ1/+P7hwXf5O2/lCf5v48p5+9t9zF6ufnDDa1y9//o/cXX962+2dKjIsPZL/LPPzZ7w61IN038L3n6to15w9fvHfsvVq/7SP3eeV/+5q7u96z+fIC14BwsAAAAAEsIECwAAAAASwgQLAAAAABKS6R6sta715xM/uf9fXb1a25Vcfd53rnf1+ev73pj2z7/h6vpNR7r6W5f485Pv7Dkpb0S+JyG/9+WD5UtcvetVv3B17wn+PNLhj73q6rol/vZombf3H+jq4673PQDn9fW9ckmvU9V9t6muPnfnA1y9y5WXFtz/86cXXkcrv+cq6fGjddRvtbGrl/z6M1e/tME4V7fJez2svsyLU47+k1/vaPVxzzWxZW2bvkuPRPd3/XG7u7r9QxMS3X+5jf954XWy9uz3zYLXo+XqF/oeq0Gn+f+zzbStf82HD/q/pd3X7ezqz9bv5uqVXy/yAMikjveOd/Xaz/g1Gpt7xpH/+QY//9ktBbf/1i0nunrNe59vYsvK4h0sAAAAAEgIEywAAAAASAgTLAAAAABISKZ7sIae4s8n3tpOdvUbP/K9K7t09ucjf3H57a7+/aTdXP3ypleWNL5Dp2/j6tcvW9/VA69+tuDtWdcqGXV561K9k3eKf5uZxa2DlX/9fTP9ulT1eWe2j/6tXxMi3+79Nil4ff7+ix0f62DF07an76WZt6s/T/zjTf3/0JU+9L/XgRe+5ur8fogPj/fr7E3c4Na8EfjXv77eS/f1MSfpz/PWc3Xf2/26gctDKw8go14/7mJXLy3y17Tnm3u7utN035tHByUqZdHa/jFrafB94yu/T584pLrb/QpqttNiV9dv5p8LH3L1Ha4e0+VTV+82ZU9Xr3liOnuu8vEOFgAAAAAkhAkWAAAAACSECRYAAAAAJCTTPVj5ht3oz1W/fq8+rj5w5dmu3qfLfF9vel3eHn3vSlvz89GJS5f62z99pB/Pgf91dQ+xbkwa5fdMFbuOVHPXN7eO1bZ7+XWxQvC5q9dLrTo+5LTxv6dph6/j6pePLrx+T75/Hby6q0999ruuvnOTi1ydv55Zvvyeq/q8Ls0py/z92qPNclf3alvculkn9/Q9ZNtst62rV73OP54iUhdK655t2ybv9maNb1glRtx8jKuHKRv9FbXAOvrHjD9t6fvWn/6ik6unHtne1SP/0svVX3zfZ7luzkelDhEp9Osh97j6l3fv4+onNyz8+QbbvPZ9V6+8zxxXZ6X7l3ewAAAAACAhTLAAAAAAICFMsAAAAAAgIVXVg1U/8Q1X33jQTq4+4F/jStr/9j8+1NWd5vg1H4ZN9D1XyIb8daim/W5zV086xPfKJL0O1XMjb3Z1fk9YqftnHayW+fiITV1dbM9Vvr27+v6CvXe8NG+LZO+H82Zv7+oduk8qOJ7mrPtvv37b2nf6/bFOX+Pye3WXF9nz+K/hd7p6l/4/dXV7/2eu7GyUX8Nml8OeLur26//zWFev/UefKzpEK6dNly6ufvNsvxbefiu/UPD27+7oe2u2e8OvX9Rx+aISRoesenpDv8Zj/t+OrV/d19Xdf+Q/H6FusV9HKyt4BwsAAAAAEsIECwAAAAASwgQLAAAAABJSVT1Y+dq+59dpmVv3uatXa7tSUftb6T1/XqjmzW98Q2TaoNP8emWbTT/Kb5C3CMNzvy28zlWp61S19vWILN42m+d5r3DxgMdKuv3kL/2Z8X2eyFuPbeHCkvZfK771c78e4mNnX1DS/g4+/9+uvnlb3zO6fLZfI6a1LV3N/908ffWXmtiycWu86HNWt2BByWNCMmaP3cjV7+15satn5fVQ/fDNH7n6yyvXcPXKN/s1zfjLUx3aDhvi6q7j/P/hjTr459rz6v3/+c3uO97V6/7qHVfXzfukxBGmA+9gAQAAAEBCmGABAAAAQEKYYAEAAABAQqq6B6vu449dvfUNJ7v6jQN970xz7nzsFlc/+UUHV//01sNd3fMV36yzyo3+fGRkQ88rnit4/e7/2KTg9c2Zf+9arg7B97702P2tgrdfe0J7V5/dx+esuXWwlu7yTVd3vH98weNVq1Xv82vAPLzJyq7efqXq7kE69MzjXd3zlsK5R+O6ver7B/J729btUNzrmj9YeZarv/Pi7a5u7fXIjtzpx66+4fJz87bo2MojQLn0vXe6qzfq+DNXrz7B99Z0fMqv/dlR77fKuFBZ7QYNcPWmt09x9a9Wey3vFv658Stf+uccw4/wzzGqtTePd7AAAAAAICFMsAAAAAAgIUywAAAAACAhVd2DlW+Vd319ymzfe3LWGsX1nmzd6UtXT8rr6Zq1vz9f+ZSj93D1u5es7epuU/w6PGF8/nmtqEbdd5ta0u2fuXyUq5ed9oyrm1sH6+fnX+/q30/Z3dXN9YBVi+7X+J6jkwb/xNUTDz+vnMNpdaNeHOvqAXe86epqPS++tdW94f+/nHio72MZe9Gdrs7vsWpOr7Zl7nlq41+H7VHu46Nslk/zPVhrnDu9iS1RzT4+cnNX//iY+1z9s27v5d3CVMhJk7/v6u4q7TlPVvAOFgAAAAAkpNkJlpkNMLPHzOwNM5tkZsflLu9hZv8xs6m5f7u3/nCRFeQGcZAbxEV2EAe5QRzkBs1pyTtYyyWdGEIYIWm0pKPMbISkUyQ9EkJYS9IjuRpYgdwgDnKDuMgO4iA3iIPcoKBme7BCCLMkzcp9v9DMJkvqJ2kvSdvmNrtG0uOSftkqo0zIapf7Hos3H/Sf7b/H1X1cfe7QW109rH1x554PbNfV1dcMfthv8GdfX7/QH//GmZu6+sMFq7h6wK+WudoWLnH18hkzWzzWpFVTbtIuP9fb7nWAq58bebOr83uyduu8yNUP9P7A1e+UOsAipCk3Qy/zP/m3Rvnf62/W8eel79J5fmsOp2TrPHSkq9f9uW9KrZuf7vE3J03ZaajdIy+5+opf7ePqsw9a4Opfj/C52qvL3NYZWM4h03Zw9Wsf+b9D/Rf49d++/7bvJb512N2tM7AySWtukG7VlJvZJ3zL1S+f5D9PoF4hry7O0v/0yruEHqyvMbPBkjaW9IKk3rmASdJsSb2THRqqBblBHOQGcZEdxEFuEAe5QWNaPMEys66Sbpd0fAjBveQWQghS3hT3f7c73MwmmNmEZVpa0mCRPeQGcZAbxBUnO+QGPOYgDnKDprRogmVm7RUF6PoQwh25i+eYWZ/c9X0kfdTYbUMIl4cQRoUQRrUXH+9aS8gN4iA3iCtudshNbeMxB3GQGxTSbA+WmZmkKyVNDiGc0+CquySNlXRW7t87G7l5quWv+aDv+PLwvU9w9ZxN/Xx0WS/fA/XWLpe5ui4Ud6bqAXnroRywTjO/0ryWrgvmr+Wv/oFf56tu0pSixlOKas5N2oXg16TIP386fx2s/J6sFy/b2NU95Xu8WlOacrN89hxXr7aHr68Ysp2rTz6yr6ttkF/X7rrNrnT1Rh1KHWFhB763s6sH3uHv56z3XOVLU3YK6XzHC3m1v/6sI/d39SnrNvoCeGKGX+dzukbe+ovL87Zf8if/d0VXKtOykhukSzXl5oJjLnZ1W/PPdevznjM051+Le7h6jb8/G29gGdeShYa3kHSgpNfMbGLusl8pCs8tZnaIpGmS9m2VESKryA3iIDeIi+wgDnKDOMgNCmrJpwg+raaXad6uictR48gN4iA3iIvsIA5ygzjIDZpT1KcIAgAAAACa1pJTBGtW53/5c+WH/Mtfb+38r2+vbjsW3N/kM9d09cprLGxiy8g937jC1X3arlRw+2O6+7UFZl7bzdWvb1Lw5qgSC1/u6eo2G/sX2fJ7rtrkvwjX1GtycJa/N83VQ385rYktI7/tt5er52850NWzdvTdLj1ebO/qNR760O8wFO7Nqf94nqs7LX6x4PZIh16X+p7H/BVkklZsh1fnybNdvd4Th7l60jb+7xaAbCn28wNuXOg/if7v533f1b3K2MedJryDBQAAAAAJYYIFAAAAAAlhggUAAAAACaEHqwRhue+ZqJs7r4ktI8OPLHx9vh9vd5yrF/XzC+fsddKjBW+/LPhem8/3GuXqle6kJ6MaDTrNn+9cf0jhdbBOmrW1q1d/wq+LWNwKGGjK8pm+h2rlm/PrZm6f9ICAGPLXj1z56QF+g23KOBgARQubb+Tqfm2fztuic8HbbzFxP1f3/JWfSvR6pTZ7rvLxDhYAAAAAJIQJFgAAAAAkhAkWAAAAACSEHqwUa/fIS67ulnf9E9cWXhdLWuaqlUTPVS3avZ9fAG3a7zZ3dX7PlvROK48IQLXoc8MkV+/9xI8Kbt/1/dddXdyKOwBKZc+94uqfDdqyqNt3l19zlf/DjeMdLAAAAABICBMsAAAAAEgIEywAAAAASAg9WECN+XrPFQDEU/fpZ/6C/BoAahDvYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCLIRQvoOZfSxpmqTVJM0t24GLx/gaNyiE0KvcByU3iSE36cT4GkduCmN8jatIbiSykxByk16Mr3GNZqesE6yvDmo2IYQwquwHbiHGl05p/7kZXzql/edmfOmU9p+b8aVX2n/2NI8vzWNrbWn/2RlfcThFEAAAAAASwgQLAAAAABJSqQnW5RU6bksxvnRK+8/N+NIp7T8340untP/cjC+90v6zp3l8aR5ba0v7z874ilCRHiwAAAAAqEacIggAAAAACSnrBMvMdjazKWb2tpmdUs5jN8XMrjKzj8zs9QaX9TCz/5jZ1Ny/3Ss0tgFm9piZvWFmk8zsuDSNr5zSlh1ykw3kpqixkZscclP0+MiOyE2M8ZEbpS83Urqzk5XclG2CZWZtJV0kaRdJIyT90MxGlOv4BYyTtHPeZadIeiSEsJakR3J1JSyXdGIIYYSk0ZKOyv3O0jK+skhpdsaJ3KQauSkauRG5ianms0NuYiE36cyNlO7sZCM3IYSyfEnaXNKDDepTJZ1aruM3M7bBkl5vUE+R1Cf3fR9JUyo9xtxY7pS0Q1rHV2vZITfp/iI35IbckB1yQ27S/JXW3GQpO2nNTTlPEewnaXqDekbusjTqHUKYlft+tqTelRyMJJnZYEkbS3pBKRxfK8tKdlJ3v5AbchMHuSE3cdVwdshNCcjNV9KaGymF90uac8OHXDQjRFPhin7Uopl1lXS7pONDCAsaXpeG8eHr0nC/kJvsScP9Qm6yJy33C9nJlrTcJ+Qme9Jwv6Q9N+WcYM2UNKBB3T93WRrNMbM+kpT796NKDcTM2isK0PUhhDvSNr4yyUp2UnO/kBtJ5KZo5EYSuYmF7JCbOMhNZnIjpeh+yUJuyjnBGi9pLTMbYmYdJO0n6a4yHr8Yd0kam/t+rKLzO8vOzEzSlZImhxDOaXBVKsZXRlnJTiruF3LzFXJTBHLzFXJTJLIjidwUjdxIyk5upJTcL5nJTZkb0XaV9JakdyT9upLNZw3GdKOkWZKWKTr39RBJPRV9AslUSQ9L6lGhsW2p6C3OVyVNzH3tmpbx1XJ2yE02vsgNuSE3ZIfckJs0f6UtN2nPTlZyY7nBAgAAAABKxIdcAAAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgAQAAAEBCmGABAAAAQEKYYAEAAABAQphgNcLMxpnZmbnvtzKzKTH3c6mZ/SbZ0SGtyA3iIjuIg9wgDnKDOMhNcTI7wTKz983sczNbZGZzcnd816SPE0J4KoSwdgvGc7CZPZ132yNDCL9PekyNHLujmZ1rZh+a2Xwzu9jM2rf2cbOI3Hzt2HW538WKr21b+7hZRXa+dvwTzGy2mS0ws6vMrGM5jps15MYdez8zm2Jmn5nZR2Z2jZmt0trHzSJy445tZnammc3MZedxM1uvtY+bReTGHXt9M3vQzOaaWWjt4+XL7AQrZ48QQldJ35A0StL/5W9gZu3KPqryO0XRz7++pOGKfh9f+13gK+Tmf54LIXRt8PV4pQeUcmRHkpntpOhxZztJgyQNlfTbig4q3chN5BlJW4QQVlWUmXaSzqzskFKN3ES+L+knkraS1EPSc5L+WdERpRu5iSyTdIukQypx8KxPsCRJIYSZku5XNMGQmQUzO8rMpkqamrtsdzObaGafmtmzZrbhitub2cZm9rKZLTSzmyV1anDdtmY2o0E9wMzuMLOPzWyemV1oZutKulTS5rlXDT7NbfvV26m5+jAze9vMPjGzu8ysb4PrgpkdaWZTc2O8yMyshb+CPSSdH0L4JITwsaTzFT0YoQByg7jIjsZKujKEMCmEMF/S7yUdXNxvsfbUem5CCNNDCHMbXFQnaVgRv8KaVOu5kTRE0tMhhHdDCHWSrpM0oshfY82p9dyEEKaEEK6UNCnO769UVTHBMrMBknaV9N8GF4+RtJmkEWa2saSrJB0hqaekyyTdZdGpdR0k/VvRqyE9JN0qaZ8mjtNW0j2SpkkaLKmfpJtCCJMlHan/vRvQrZHbfkfSnyTtK6lPbh835W22u6RvStowt91OudsOzAVrYKFfQ973/c1s1QLb1zxyI0na2KK3z98ys99YbbyqVTKyo/UkvdKgfkVSbzPr2cT2ELnJbbOlmX0maWFu/H9valtEyI1ukrSmmQ23qP1hrKQHmtgWOeSmwkIImfyS9L6kRZI+VXSHXCxppdx1QdJ3Gmx7iaTf591+iqRtJG0t6UNJ1uC6ZyWdmft+W0kzct9vLuljSe0aGc/Bil5haXjZuAb7uVLSXxpc11XR25eDG4x5ywbX3yLplBb+Ls5UdOpFL0lrSHoht78+lb6f0vZFbtxxhip6ZbCNpA0kvSHp1ErfR2n9IjvuOO9I2rlB3T63v8GVvp/S9kVumvy99JN0hqThlb6P0vhFbtxxOkg6L7eP5ZLekzSk0vdRGr/ITaO/k2GSQrnvi6y/Wj0mhPBwE9dNb/D9IEljzeyYBpd1kNRX0Z03M+TuhZxpTexzgKRpIYTlMcbaV9LLK4oQwiIzm6foj8z7uYtnN9h+iaKgtcQfJHWTNFHSUklXSNpY0pwY46wF5Cba17sNytfM7HeSTlb0ahIaR3YiiyQ1/HCCFd8vjDHOWkBu8oQQZprZA4perf5GjHHWAnITOU3ROxgDcvv4kaRHzWy9EMKSGGOtduQmBariFMEmNAzFdEl/CCF0a/DVOYRwo6RZkvrlndPZ1NuN0yUNbOI0qtDIZQ19qCjMkiQz66LoLdmZzf0gzQkhfB5CODqE0C+EMFTSPEkvhRDqS913DaqZ3DQxFvq34qul7EyStFGDeiNJc0II8xLYd62ppdzkaydpzVbYby2opdyMlHRzCGFGCGF5CGGcpO6iDyuOWspNRVXzBKuhKyQdaWabWaSLme1mZisr+jSa5ZKONbP2ZvZdSZs2sZ8XFYXurNw+OpnZFrnr5ijqe+rQxG1vlPRjMxtp0ccZ/1HSCyGE90v94cysn5n1zf1soyX9RtLppe4XVZ+bXcysd+77dRTl5s5S9wtJVZ4dSddKOsTMRphZN0WfUjUugf3WuqrOjZkdsKJfwswGKTr74pFS94vqzo2k8ZK+b2a9zayNmR2o6LTktxPYdy2r6tzkfqZOit6VU25cZVtOpCYmWCGECZIOk3ShpPmK/lMenLvuS0nfzdWfSPqBpDua2E+dok/sGybpA0kzcttL0qOKXtWdbWZzG7ntw4qewN6uKIhrStqvJeO3qJFvkTXdyLemonNjF0u6RtH5qQ+1ZN9oWg3kZjtJr5rZYkn35cb/x5bsG4VVe3ZCCA9I+oukx3LjmiZe1ClZtedG0TsOz+Yec55R1O9xWEv2jabVQG7+rOiDdCYq6i06QdI+IYRPW7J/NK4GcjNI0uf636cIfq7oMacszJ9eCQAAAACIqybewQIAAACAcmCCBQAAAAAJYYIFAAAAAAlhggUAAAAACWGCVUZmdoaZXVfpcSBbyA3iIDeIi+wgDnKDOKo1NzU1wTKzx81sfks/B9/MDjazp1t7XE0c+4Dcx0+u+FpiZsHMNqnEeGpZlnKTO/6+ZjbZzBaa2RtmNqZSY6llGcxNZzO72MzmmtlnZvZkpcZS6zKYHR5zUiBLuTGz0Wb2HzP7xMw+NrNbzaxPJcZS6zKWm8w8N66ZCZaZDZa0laJVpfes7GiaF0K4PoTQdcWXpJ9JelfSyxUeWk3JWm7MrJ+k6yT9XNIqkk6WdIOZrV7RgdWYrOUm53JJPSStm/v3hMoOpzZlLTs85qRD1nIjqbuix5zBitYrWijp6koOqBZlLTdZem5cMxMsSQdJel7SOEljG15hZgPM7I7cqyjzzOxCM1tX0qWSNs/Nkj/Nbfu4mR3a4LZuJm9m55nZdDNbYGYvmdlWCY1/rKRrAwuXlVvWctNf0qchhPtD5F5FC1CvGXN/iCdTuTGzdRT9cT08hPBxCKEuhPBSnH2hZJnKjnjMSYtM5SaXl1tDCAtCCEsULXa7RZx9oSSZyk0jUvvcuNYmWNfnvnYys96SZGZtJd0jaZqiV1L6SbophDBZ0pGSnsvNlLu18DjjJY1U9ArwDZJuNbNOjW1oZq+a2f7N7dDMBknaWtK1LRwDkpO13EyQNNnM9jSzthadqrNU0qstHAeSkbXcbJob028tOkXwNTPbp4VjQLKylh0ec9Iha7nJt7WkSS3cFsnJbG7S/ty4JiZYZraloregb8m9KvuOpBV33qaS+ko6OYSwOITwRQgh9rmlIYTrQgjzQgjLQwhnS+ooae0mtt0whHBDC3Z7kKSnQgjvxR0XipfF3IQQ6hQ92Nyg6EnODZKOCCEsjjs2FCeLuVH0LsT6kj7Lje9oSdfkXq1EmWQxOzzmVF4Wc5M3/g0lnabo9FKUSdZzo5Q/N66JCZaitxAfCiHMzdU36H9vhQ6QNC2EsDyJA5nZSRY1+36We+t0VUmrlbjbgyRdU/LgUKzM5cbMtpf0F0nbSuogaRtJ/zCzkUmMEy2SudxI+lzSMklnhhC+DCE8IekxSTsmMU60WOayw2NOKmQuNw32N0zS/ZKOCyE8lcQY0WKZzU1Oqp8bt6v0AFqbma0kaV9Jbc1sdu7ijpK6mdlGkqZLGmhm7RoJUmPndC6W1LlBvUaDY20l6ReStpM0KYRQb2bzJVkJ499C0asIt8XdB4qX4dyMlPRkCGFCrh5vZi9I2l7SxBj7QxEynJvGTudK3Tnt1SzD2RkpHnMqJsO5WXGK18OSfh9C+GecfSCeLOcmt8/UPzeuhXewxkiqkzRC0R+CkYo+JespRbPfFyXNknSWmXUxs065O06S5kjqb2YdGuxvoqTvWvSRxsMkHdLgupUlLZf0saR2Znaaok9VKsVYSbeHEBaWuB8UZ4yymZvxkrZa8eqxmW2s6BOC6IcojzHKZm6elPSBpFPNrF1uTN+W9GDM/aF4Y5TN7PCYU1ljlMHcWPTpk49KujCEcGmcfaAkY5TB3DSQ+ufGtTDBGivp6hDCByGE2Su+FH1izQGKZtB7SBqm6AnGDEk/yN32UUVNl7PNbMVbqOdK+lJRwK5R1Bi4woOSHpD0lqLGwC8UvQrQKDObZGYHFLi+k6JXGFL7FmgVy2Rucqd2nSHpNjNbKOl2SX8MITxU3I+PmLKam2WS9pK0q6I+rCskHRRCeLO4Hx8lyGp2eMyprEzmRtKhkoZKOsMarGtU3I+OEmQ1N5l5bmwhfZ9sCAAAAACZVAvvYAEAAABAWTDBAgAAAICEMMECAAAAgISUNMEys53NbIqZvW1mpyQ1KFQ3coO4yA7iIDeIg9wgDnIDqYQPuTCztoo+EWQHRZ8uMl7SD0MIbyQ3PFQbcoO4yA7iIDeIg9wgDnKDFUpZaHhTSW+HEN6VJDO7SdHH/DYZog7WMXRSlxIOiUpaqPlzQwi9StwNuakxCeVGKjI75CbbyA3iqFRuJLKTZV9osb4MS2MvfNsAuakxTT3mlDLB6if/OfYzJG1W6Aad1EWb2XYlHBKV9HC4bVoCuyE3NSah3EhFZofcZBu5QRyVyo1EdrLshfBIUrsiNzWmqcecUiZYLWJmh0s6XJI6qXNrHw5VgtwgDnKDOMgN4iI7iIPcVL9SPuRipqQBDer+ucucEMLlIYRRIYRR7dWxhMOhSpAbxNVsdsgNGkFuEAd/qxAHuYGk0iZY4yWtZWZDzKyDpP0k3ZXMsFDFyA3iIjuIg9wgDnKDOMgNJJVwimAIYbmZHS3pQUltJV0VQpiU2MhQlcgN4iI7iIPcIA5ygzjIDVYoqQcrhHCfpPsSGgtqBLlBXGQHcZAbxEFuEAe5gVTiQsMAAAAAgP9hggUAAAAACWGCBQAAAAAJYYIFAAAAAAlhggUAAAAACSnpUwQBAJWx6PubufqZ8y5z9Yhnf+TqAd97vdXHBCC72q49zNVztunl6pfOuMTVy0Kdq0f/7mhX937sI1fXvfVOqUMEMoN3sAAAAAAgIUywAAAAACAhTLAAAAAAICH0YAFAFoze0JUn//F6V+f3Qxy/3qOu/vdq67q6bu68BAcHIOuGXjfd1bf1ucHVy0LbvNo/5jx/2oWuPu7QLVz9zjdLHSGQHbyDBQAAAAAJYYIFAAAAAAlhggUAAAAACaEHCwAyYMbJvt9ht86fFdz+x6v4forzDx3j6n5nPZvIuJBtH/3sW65e94DJrp5y7TquXu2y51p9TCiP+feu5eoL+93q6vyeq4c+7+LquuBfo9+t8yJXn9f3GVdvce9+ru6+29SWDxbIGN7BAgAAAICEMMECAAAAgIQwwQIAAACAhNCDVcimG7jyg11XdnX9uv584ylbXevqulDv6n8v7ubq067+kasHXTHF1dbVn++8/P0PCo8XVaHd0MGubnv1F66+bMhtrt7i8WNdPeAW/9+68wyf0/qJb5Q4QpTD8u9s4uqLRl5d0v4Wr7mspNujOtRt+w1XP3jqX129elv/d2fyqfe6+sRH/d+tuqnvJjg6tKalu/iFqH41/EZX569rlV//7egDC+5/lysvLXj7EKxF40S2TPvd5q5+89BLXH3shz53bx8wyNV1U95unYFVGO9gAQAAAEBCmGABAAAAQEKYYAEAAABAQujBaqDtemu7+oJb/XmkA9utVPD2dXnnF09a9qWr9+wy39dHX+Dqu37c3dU92/remaMm/tDV/feZVHA8yIY2I0e4+p1f+bVHXhnm1yaROrpq8naX+au38+WcuqWuPuTAY/zxn/hvywaKVmUd/f36h39c7upN/NXNumXR6q5e++Ilrg7F7Q5V4p2x/u9UV2vv6iH3H+rq93b5h6tnb9/b1b3owcqMx6+8wtX5PVL561zl91x1eGB8wf23kc9We/N/y8x41KlGO+46oeD15/f1uVnngFGuHnQaPVgAAAAAgAKYYAEAAABAQphgAQAAAEBCqroHq+2I4a6ue+Otgtvvduuzrn5nme+J2mP8D/z+Jvt1sVbNO410tWdmu3rRRQUPr2+v4cd32mqvubpDO3++NLJp7uF+zYhfnXS9q/N79ebm9VAdNNX34r03sZ+rV1t3rquf2uhmVy/q65t5undb1dXhS79eUv0S37uD1jHtVL/u1SYdn21iy5Y583r/eDXwv6XtDxllvi/mhyN9P8RFn67r6qHX5/XJ7OLLpd1Yyyirmlvn6ow393R1j2Z6rvINu+cIV0/azT/p+b/hfk21c3c+wNXN9XghnXbq9lrzGzWQv07WTqeNTHA06cE7WAAAAACQECZYAAAAAJAQJlgAAAAAkJCq7sF682e+h6rTrG+5evAFr7u6W9vprj7qDr8eyNBfPFfU8fM7plbaqfD2U57x64sorwfr05mruNqvcoO0aturl6tPP/kaV+/U+TNXz8rrufrOrSe5es0Tn/e1fG7z7fPEbq4ecvQUVw/4hT/+E7OGubrH/h+7uu5Tvz3imXH7eq5+dtO/5W1R3MJXe721h6ut3l9/2rsvu3pAO99bN3WZ78U74l+HubrrB/71uN7n09OVBW06d3b1tiu/6Orfve1z0ykUXquozfJkxoXWN+13vt+3vU0suH1z61S1Xdv/bZizjf/b9vbuvueqXn4drLb5D0qoCrt1/sLV9y7pVPD6fPm5qptSHeti8Q4WAAAAACSk2QmWmV1lZh+Z2esNLuthZv8xs6m5f7sX2gdqE9lBHOQGcZAbxEFuEAe5QXNa8g7WOEk75112iqRHQghrSXokVwP5xonsoHjjRG5QvHEiNyjeOJEbFG+cyA0KaLYHK4TwpJkNzrt4L0nb5r6/RtLjkn6Z5MCSENr684kv/smlrj728yNdvUHHh1w9/Nz3XJ30qedte/suqh16vFpw+4H3JzyAVpbl7CRp6nl+nar8nqt8ty3Y0NX5PVfNCVuMdHXPju+4+tIBjxa8/cMrv+nqi7vnNQ+2cg9Wtebmiz02dfVdo85xddc2K5W0/zuH3+0vGN74dv/jj9en7ZeunvxD309x/xK/7t+l//G9fXWTpzY7xtZUrbkpWRv/Omp+H8ynS3wOenYt/LSgw4LCfTpZU825GXSa7xtfdkjhdbBCKLzG2dDrfL/vbX1ucHV+z1XS62ylSTXnplT5PVfHfvhNV5/f19/vw66f5uopo1pnXOUWtwerdwhhVu772ZJ6F9oYaIDsIA5ygzjIDeIgN4iD3OArJX/IRQghSGryJS0zO9zMJpjZhGVa2tRmqEGFskNu0BRygzjIDeLgOQ7iIDeIO8GaY2Z9JCn370dNbRhCuDyEMCqEMKp9kR87jKrUouyQG+QhN4iD3CAOnuMgDnKDr8RdB+suSWMlnZX7987ERpSglXr5dV626uS7qJZutsjVDy8a4erls2a3zsBy6gf6d48PXuVDVw+70/eIDb/br1+SUZnITpL+OfpKV7fJe11ju9e/5+oZb/nevLX0gqvbDRrg6ncO6e/qmw78u6tHdvQP3suCP/4f527g6ucP2sjV9e9NVgpkPjc7/OFJVw9sV1rPVbnt0nmhq487oZur1znWr31S/0XhtU/KJPO5KbdPhxZ+WtBldv4Kj1WpKnNz4qzRrj67j+/vfX7jm1xdP9O/AdNGvkcrv+cq//r25q9vbp2tKlCVucm3dJdv5l0y0VVD7vVrKPaYkPeYcrrvwcrvydp1bf+cKKvrYrXkY9pvlPScpLXNbIaZHaIoPDuY2VRJ2+dqwCE7iIPcIA5ygzjIDeIgN2hOSz5F8IdNXLVdwmNBlSE7iIPcIA5ygzjIDeIgN2hOyR9yAQAAAACIxO3ByoS/jrzN1fV5H+iy51qvufqi/+zo6mEqbv2hYn18mv/kmPzxdXujqu+eqrVk781cPaz9M64+c65fD6nrDz5x9TrtfW/gnMM3d/XBx93n6sO73ZE3Av+6yffe2d7VH5091NWd73/F1WFpKnquMq/u299w9T6rXJi3RbYbm9/aza8ruOdFP/IbvEKO0qB+oe+dm76sp6sH9/CPP28M9eud1QW/blaXaf7xyV+LNHvmcr/A0LLT/N+m/J6p/HWsSr2+uXW2kA0nX/DPgtcPP6yZ9c1OT3AwKcY7WAAAAACQECZYAAAAAJAQJlgAAAAAkJCqbvI5+j8HufqtPS9x9Vm9X3L1a9ev5+rWXrHhkvWvz7vEz3dXv/DZVh5BbVq2oz8Pvf1DE0ran7Xv4Oo+J/o1G1Zu469ft5Nf7+yhG9dx9WXr+lys3d6f114XfDK/8fyPXb3Sfau4utctr/vrF/r11Kp+ZZJyGb2hK3e94HFXD2ufrp6ryz8b7OrbZvqesV8OecDV263k1xVEZbTp0sXVcw70uVu0jb+f1u/nH2+27/y0q7ca8q6rZw7s6uq3lvn1zNrMnudqerCyK79nqrl1rEq9vgbWwapK+ete7dZ5Ykn7O/ZDv7/8dbA+Oddv3+OEYa7OyrpYvIMFAAAAAAlhggUAAAAACWGCBQAAAAAJqeoerHYL2ja/URkt2H+0q9dv73thTpntz0vl7PbW0XHOYleX+luuH7Wuq/855B8Ft9+760e+3vAmV8+pW+7q9Z840tU9HljJ1f2vfa7w+Apei6QsW8X32h3TfWqrHu+5pf7x7YiXDnT16tf4nHT66HNXt/vQr3/UYcY0V//qrjGufmGTG+IMEyVq17+fq9e47TNX3z/Q9xYvqf/S1c8t9Tl4d7nvsdq6kz/ewnq/PuNPJvtcdft0ZuEBI7U2P+xlVye9zhXrYNWGTX77U1d/Mso/ZxmuwutgvX3AIH/BY3775/PWsNVjvtyp78hmx5gGvIMFAAAAAAlhggUAAAAACWGCBQAAAAAJqeoerDVvW+Qv+FHh7es6+V/H2xdv6uqV3/HXd5/izzvtdI/vqcr3yYjCa0Tc/eBmrh6iwr01iKf+lckl3T5sMdLVb+/bqfENW2j9fx7r6mE3furqoa9MLGn/KI+Pjvy8+Y1KsPfU3V295My+rh74sF/Xrzn1nXxu3z7H94g+t/HZebcoLeeIZ+Z3fb/CvQMvdvWQuw539boX+B6t8IFfB6tuwzVdffctV7h63xcO8/vf71VX09OZXm3X9usFDb1uuqvP6+ufU9TLPwd56HO/xtoZb+7p6s3XeM/VZ/d53tWsg1WdOt7ve6TyV3Rc7bLi9rdkaPeC1+evk/W1ni2xDhYAAAAA1BQmWAAAAACQECZYAAAAAJCQqu7BCuNfc/VxH27h6hNXf9jV+acH/2uXC1y9QYf2BY/X1vx8tS7kn63+cl6dd77yQl8v3c2fh7rSk2+6un7hwoLjQTLadlvV1ftdeY+rD1plrquXBZ+DpWGZq3c88XhXD705/7x4ZEGbjfz6Zy9tNi5/i6L2N7fO93B9++pfuHrQb19wdfv6Wa629n4drjZdfT/F279cx9Wjt5nk6rsGXpQ3osI9V0uCX29JdfRXtIblKxW+ftCd/vdeN2lKwe3t2Vdc/X9zfK/x3zbxa9Bc0mVjV9cv9usIIj3mbNPL1bf18WvX5fdc5a9Tld9z1WP3t1z9zOGb+9uf9oyrWQcLLXHyBf8seH1+z1XdlGz0XOXjHSwAAAAASAgTLAAAAABICBMsAAAAAEhIVfdg5XvlLxu5es9h33B1/2eedfXY805w9TnH+A/737pTXg9CXs9VvYrrSZh4tO/5+mC578n40aknuXqVG/waFGgdk89ey9U/WNn37uX3XNXndVG9/KXvZVn5Zu63avDBrn4tjzYlvl615R3+//ew0/3jUbt+ft2rxRv1c3Wnk/x6R3eufVfeER5WKebXf+HqHc452dVrvO7Hi2T0edb/HVhyjP+7s8opfq2j5c+s4uq6BQv8DoP/u3TnQ76v5q8H/dfXO67v6s7/8r2ASI8XT/d9lMWuc5Xfc1Xs/ptbB+v5jW9y9bDLjnD18CP8ekuoDtN+5x9jdus80dVD7vVr7w2fUh054B0sAAAAAEgIEywAAAAASAgTLAAAAABISE31YHW91Z873rWZ7df4u+8pOPuarV19xEX+s/rbtPG9N5O2urrg/v8+f7irr71mJ1cPuOdjV68ymd6dcliw/2hXv7Dj2a5+/Uu/Htr3n/TnD0/e3vfq/f69PVzdRr5nAtm0bMNk1wM6cvv/uPqKG7f0G+Qt1Ddpq0sTPX5zdv7vT1y9xrn0XJVDm6d8T9R6dx/t6vf2utzV5zw71NWX376zq/s9sdTVQ+5Y5OplB/q1i2bs7uvh/2pmwKiY/L7vYte5WrqLX3tz8VGf5e3/pYL7b24drPzrJ+3me7q+8bvjXD3oNL9GJLKh7drDXP3moZcU3H74YdXRc5WPd7AAAAAAICFMsAAAAAAgIUywAAAAACAhNdWDVaq6+fNdveb+vl66mz9/WVv58v4lK7v68d3Xc3Xf931Pgz97GeXSefYyV283wfdYDThsjquHrZV3T23vy/cm+vWK1qQHC404vrvvhzh+a1/fvmi1cg5Ha//rZ65e96wZrl5ezsHgK2sf/4qrh8//qav/9v1rXP3mYRe7+pCdfG/fjNG+Byu/N/jITZ9w9WPtVnV1WE4S0qK5dag2X+M9Vz9zuF+f6OvrXPmerub2n7/OVl3eGpG7dfZZ+1pP1iH++MN6+3WyBv3blep4f3X27mTdJ+cWvv7eJZ0Kb1AleAcLAAAAABLS7ATLzAaY2WNm9oaZTTKz43KX9zCz/5jZ1Ny/3Vt/uMgKcoM4yA3iIjuIg9wgDnKD5rTkHazlkk4MIYyQNFrSUWY2QtIpkh4JIawl6ZFcDaxAbhAHuUFcZAdxkBvEQW5QULM9WCGEWZJm5b5faGaTJfWTtJekbXObXSPpcUm/bJVRVomTbxrr6sHvV+8aD1nOTbtH/VoffR/11+f3xs393nAV0u0NK3g9/ifLuWlt+3SdW9Ltr14wwNVXvfctV89f2NnV6/7J9woun/lhScdvbbWSnbA0bx2rX/m/I5fcMsbVJ+7ne6Z6TfR9Nd37feDqNxYtdPXYXs+4+vGOvm8n6z1Y1ZSb5tbBOruPX0tz2Wn+vq1XcetY5V//t6MPLDi+Xa70a/cVu07WNv1+5OqO9xc8XKuqptyU226dv3D1+RUaR2srqgfLzAZL2ljSC5J65wImSbMl9U52aKgW5AZxkBvERXYQB7lBHOQGjWnxBMvMukq6XdLxIYQFDa8LIQQp76WT/93ucDObYGYTlmlpY5ugipEbxEFuEFec7JAb8JiDOMgNmtKiCZaZtVcUoOtDCHfkLp5jZn1y1/eR9FFjtw0hXB5CGBVCGNVeHZMYMzKC3CAOcoO44maH3NQ2HnMQB7lBIc32YJmZSbpS0uQQwjkNrrpL0lhJZ+X+vbNVRpghsw7yr0LkrxnRZ7NZqhW1lJsfHv2Qq9vkvW7R+6G8XpZWH1F2ZSk3a/1inquHn+rXI3prr0vKORzt+MZ3XW1/7eXqld7wjz+rznjb13n7y1pOs5SdQqyd/7Ns6/sez+k7d3P1sF3ecfUZA6929ciO/snbez/0axHV5x1/zfZdXb3BC/u7uu+SyV8bc5ZVS24k6cRZo12d33PV3DpWzV2/xcT9XN3lgm6u7vBA4XWpRv/2aFc/f/qFRR3/jHXucvXfdvY9X80dP0nVlJvWduyHfo3YZ64Y5erVVJ2fR9CShYa3kHSgpNfMbGLusl8pCs8tZnaIpGmS9m2VESKryA3iIDeIi+wgDnKDOMgNCmrJpwg+LeW9rPA/2yU7HFQLcoM4yA3iIjuIg9wgDnKD5hT1KYIAAAAAgKa15BRBtNDWg/258PlrUnx6d19X99b7rT0ktIL3z/TrwBzf3Z9Hfu8S380SPv+81ceE8ls+fYar1z7O9zKPnHGMq5f0911NG6//nquvG3qvq9e7y/cr5Btyh19DpuMTr7k6LJvmx1twb6iUtiN8j9V7+67m6p6jZ7v6O6v5PpM25v/OXPLxtq5evNz3YH2y1K93NmeR77laMt4ff8gFvueqLjT6oWhIgWcu970t+etcNbeOVf71w+/2faXDj3yxpPH1ftw/Rm5e7x/jnvut/1uaP74z3tzT1T3K2HOFltt0df+3Z6du/m/TM/I5rVa8gwUAAAAACWGCBQAAAAAJYYIFAAAAAAmhBytB879cqeD1HRZw7noazT7hW67uvOMcV2+ymu+1uaXPOa5eGvwHCZ36z4NcPXDus6UOERkQln3p6v5/LHy/L86r99amrh6u4vodeHTJpvqp77t60J98/0JY6tdXnNLsHvO77fJ7QD911WrK95ar6r52PdJqtcv9ekJ7X75pE1u2TLGPQc2pe8v3qffMq3f/xyYFb98jL5vIht06f+Hq8y+rznWv8vEOFgAAAAAkhAkWAAAAACSECRYAAAAAJIQerATdPPQhV3+w3J/73vOVz1xd3+ojQmPa9VnD1Ucd/m9Xj13F90Dke/jz7q7++Y0/dvXg39FzBaBl8nv3ACDLXvxokL+gb22uV8Y7WAAAAACQECZYAAAAAJAQJlgAAAAAkBB6sBJ05YL+rj739e1cPXDia+UcDpqwfNZsV5/1wi6uHrvDpa7eZ+qerl52Si9XD36+NtZ0AAAAKKTTBb5Pfch3D3P1cNVGTxbvYAEAAABAQphgAQAAAEBCmGABAAAAQELowUrQ7euu7uqBoucqC9Y6+CVX76lv5m0xq5kaAAAAHe/3PVbD76/QQCqMd7AAAAAAICFMsAAAAAAgIUywAAAAACAhTLAAAAAAICFMsAAAAAAgIUywAAAAACAhTLAAAAAAICEWQijfwcw+ljRN0mqS5pbtwMVjfI0bFELoVe6DkpvEkJt0YnyNIzeFMb7GVSQ3EtlJCLlJL8bXuEazU9YJ1lcHNZsQQhhV9gO3EONLp7T/3IwvndL+czO+dEr7z8340ivtP3uax5fmsbW2tP/sjK84nCIIAAAAAAlhggUAAAAACanUBOvyCh23pRhfOqX952Z86ZT2n5vxpVPaf27Gl15p/9nTPL40j621pf1nZ3xFqEgPFgAAAABUI04RBAAAAICElHWCZWY7m9kUM3vbzE4p57GbYmZXmdlHZvZ6g8t6mNl/zGxq7t/uFRrbADN7zMzeMLNJZnZcmsZXTmnLDrnJBnJT1NjITQ65KXp8ZEfkJsb4yI3Slxsp3dnJSm7KNsEys7aSLpK0i6QRkn5oZiPKdfwCxknaOe+yUyQ9EkJYS9IjuboSlks6MYQwQtJoSUflfmdpGV9ZpDQ740RuUo3cFI3ciNzEVPPZITexkJt05kZKd3aykZsQQlm+JG0u6cEG9amSTi3X8ZsZ22BJrzeop0jqk/u+j6QplR5jbix3StohreOrteyQm3R/kRtyQ27IDrkhN2n+SmtuspSdtOamnKcI9pM0vUE9I3dZGvUOIczKfT9bUu9KDkaSzGywpI0lvaAUjq+VZSU7qbtfyA25iYPckJu4ajg75KYE5OYrac2NlML7Jc254UMumhGiqXBFP2rRzLpKul3S8SGEBQ2vS8P48HVpuF/ITfak4X4hN9mTlvuF7GRLWu4TcpM9abhf0p6bck6wZkoa0KDun7ssjeaYWR9Jyv37UaUGYmbtFQXo+hDCHWkbX5lkJTupuV/IjSRyUzRyI4ncxEJ2yE0c5CYzuZFSdL9kITflnGCNl7SWmQ0xsw6S9pN0VxmPX4y7JI3NfT9W0fmdZWdmJulKSZNDCOc0uCoV4yujrGQnFfcLufkKuSkCufkKuSkS2ZFEbopGbiRlJzdSSu6XzOSmzI1ou0p6S9I7kn5dyeazBmO6UdIsScsUnft6iKSeij6BZKqkhyX1qNDYtlT0FuerkibmvnZNy/hqOTvkJhtf5IbckBuyQ27ITZq/0pabtGcnK7mx3GABAAAAACXiQy4AAAAAICFMsAAAAAAgIUywAAAAACAhTLAAAAAAICFMsAAAAAAgIUywAAAAACAhTLAAAAAAICFMsAAAAAAgIf8PQJei7Gm5tdYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x648 with 18 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "mislabeled = y_pred != y_true\n",
    "axes=[]\n",
    "fig=plt.figure(figsize=(12,9))\n",
    "\n",
    "for row in range(3):\n",
    "    for col in range(6):\n",
    "        axes.append(fig.add_subplot(3, 6, row * 6 + col + 1))\n",
    "        subplot_title = f\"Prediction: {y_pred[mislabeled][row * 6 + col]}\\nActual: {y_true[mislabeled][row * 6  + col]}\"\n",
    "        axes[-1].set_title(subplot_title)  \n",
    "        plt.imshow(np.array(x_test[mislabeled].reset_index(drop=True).loc[row * 6 + col, :]).reshape(28, 28))\n",
    "fig.tight_layout()    \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "792f2aca",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:43:09.241707Z",
     "iopub.status.busy": "2021-09-02T19:43:09.240766Z",
     "iopub.status.idle": "2021-09-02T19:43:09.505142Z",
     "shell.execute_reply": "2021-09-02T19:43:09.504597Z",
     "shell.execute_reply.started": "2021-08-29T15:29:16.714348Z"
    },
    "papermill": {
     "duration": 2.924966,
     "end_time": "2021-09-02T19:43:09.505272",
     "exception": false,
     "start_time": "2021-09-02T19:43:06.580306",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "fig.savefig(\"mislabeled.png\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "1acb1337",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:43:14.383717Z",
     "iopub.status.busy": "2021-09-02T19:43:14.382988Z",
     "iopub.status.idle": "2021-09-02T19:43:16.412807Z",
     "shell.execute_reply": "2021-09-02T19:43:16.413180Z",
     "shell.execute_reply.started": "2021-08-29T15:29:28.451526Z"
    },
    "papermill": {
     "duration": 4.497163,
     "end_time": "2021-09-02T19:43:16.413328",
     "exception": false,
     "start_time": "2021-09-02T19:43:11.916165",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>pixel0</th>\n",
       "      <th>pixel1</th>\n",
       "      <th>pixel2</th>\n",
       "      <th>pixel3</th>\n",
       "      <th>pixel4</th>\n",
       "      <th>pixel5</th>\n",
       "      <th>pixel6</th>\n",
       "      <th>pixel7</th>\n",
       "      <th>pixel8</th>\n",
       "      <th>pixel9</th>\n",
       "      <th>...</th>\n",
       "      <th>pixel774</th>\n",
       "      <th>pixel775</th>\n",
       "      <th>pixel776</th>\n",
       "      <th>pixel777</th>\n",
       "      <th>pixel778</th>\n",
       "      <th>pixel779</th>\n",
       "      <th>pixel780</th>\n",
       "      <th>pixel781</th>\n",
       "      <th>pixel782</th>\n",
       "      <th>pixel783</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27995</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27996</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27997</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27998</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27999</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>28000 rows × 784 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       pixel0  pixel1  pixel2  pixel3  pixel4  pixel5  pixel6  pixel7  pixel8  \\\n",
       "0           0       0       0       0       0       0       0       0       0   \n",
       "1           0       0       0       0       0       0       0       0       0   \n",
       "2           0       0       0       0       0       0       0       0       0   \n",
       "3           0       0       0       0       0       0       0       0       0   \n",
       "4           0       0       0       0       0       0       0       0       0   \n",
       "...       ...     ...     ...     ...     ...     ...     ...     ...     ...   \n",
       "27995       0       0       0       0       0       0       0       0       0   \n",
       "27996       0       0       0       0       0       0       0       0       0   \n",
       "27997       0       0       0       0       0       0       0       0       0   \n",
       "27998       0       0       0       0       0       0       0       0       0   \n",
       "27999       0       0       0       0       0       0       0       0       0   \n",
       "\n",
       "       pixel9  ...  pixel774  pixel775  pixel776  pixel777  pixel778  \\\n",
       "0           0  ...         0         0         0         0         0   \n",
       "1           0  ...         0         0         0         0         0   \n",
       "2           0  ...         0         0         0         0         0   \n",
       "3           0  ...         0         0         0         0         0   \n",
       "4           0  ...         0         0         0         0         0   \n",
       "...       ...  ...       ...       ...       ...       ...       ...   \n",
       "27995       0  ...         0         0         0         0         0   \n",
       "27996       0  ...         0         0         0         0         0   \n",
       "27997       0  ...         0         0         0         0         0   \n",
       "27998       0  ...         0         0         0         0         0   \n",
       "27999       0  ...         0         0         0         0         0   \n",
       "\n",
       "       pixel779  pixel780  pixel781  pixel782  pixel783  \n",
       "0             0         0         0         0         0  \n",
       "1             0         0         0         0         0  \n",
       "2             0         0         0         0         0  \n",
       "3             0         0         0         0         0  \n",
       "4             0         0         0         0         0  \n",
       "...         ...       ...       ...       ...       ...  \n",
       "27995         0         0         0         0         0  \n",
       "27996         0         0         0         0         0  \n",
       "27997         0         0         0         0         0  \n",
       "27998         0         0         0         0         0  \n",
       "27999         0         0         0         0         0  \n",
       "\n",
       "[28000 rows x 784 columns]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test_predict = pd.read_csv(\"../input/digit-recognizer/test.csv\")\n",
    "x_test_predict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "859bf009",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:43:21.520414Z",
     "iopub.status.busy": "2021-09-02T19:43:21.519413Z",
     "iopub.status.idle": "2021-09-02T19:43:24.493562Z",
     "shell.execute_reply": "2021-09-02T19:43:24.493108Z",
     "shell.execute_reply.started": "2021-08-29T15:29:31.531008Z"
    },
    "papermill": {
     "duration": 5.654645,
     "end_time": "2021-09-02T19:43:24.493698",
     "exception": false,
     "start_time": "2021-09-02T19:43:18.839053",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "preds = model.predict(x_test_predict/255)\n",
    "preds = np.argmax(preds, axis=1)\n",
    "preds = pd.DataFrame(preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "98b4ba42",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:43:29.352509Z",
     "iopub.status.busy": "2021-09-02T19:43:29.350808Z",
     "iopub.status.idle": "2021-09-02T19:43:29.362316Z",
     "shell.execute_reply": "2021-09-02T19:43:29.362789Z",
     "shell.execute_reply.started": "2021-08-29T15:29:40.797253Z"
    },
    "papermill": {
     "duration": 2.447986,
     "end_time": "2021-09-02T19:43:29.362943",
     "exception": false,
     "start_time": "2021-09-02T19:43:26.914957",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ImageId</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27996</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27997</th>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27998</th>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27999</th>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28000</th>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>28000 rows × 1 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Label\n",
       "ImageId       \n",
       "1            2\n",
       "2            0\n",
       "3            9\n",
       "4            0\n",
       "5            3\n",
       "...        ...\n",
       "27996        9\n",
       "27997        7\n",
       "27998        3\n",
       "27999        9\n",
       "28000        2\n",
       "\n",
       "[28000 rows x 1 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds[\"ImageId\"] = np.arange(1, 28001)\n",
    "preds[\"Label\"] = preds[0]\n",
    "preds = preds.drop(0, axis=1).set_index(\"ImageId\", drop=True)\n",
    "preds"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "9608a657",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2021-09-02T19:43:34.545631Z",
     "iopub.status.busy": "2021-09-02T19:43:34.544719Z",
     "iopub.status.idle": "2021-09-02T19:43:34.603584Z",
     "shell.execute_reply": "2021-09-02T19:43:34.603096Z",
     "shell.execute_reply.started": "2021-08-29T15:29:45.514561Z"
    },
    "papermill": {
     "duration": 2.497295,
     "end_time": "2021-09-02T19:43:34.603702",
     "exception": false,
     "start_time": "2021-09-02T19:43:32.106407",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "preds.to_csv(\"submission.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 756.921581,
   "end_time": "2021-09-02T19:43:39.986937",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2021-09-02T19:31:03.065356",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}