cnn_mnist_classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "CNN_FlippedClassroom.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xvNDbSIsTuXI"
      },
      "source": [
        "To put everything together: \n",
        "\n",
        "1. prepare the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1ac0HIYuPd7J"
      },
      "source": [
        "import tensorflow_datasets as tfds\n",
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "train_ds, test_ds = tfds.load('fashion_mnist', split=['train', 'test'], as_supervised=True)"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-o7iPNX26fYT"
      },
      "source": [
        "def prepare_fashion_mnist_data(fashion_mnist):\n",
        "  #convert data from uint8 to float32\n",
        "  fashion_mnist = fashion_mnist.map(lambda img, target: (tf.cast(img, tf.float32), target))\n",
        "  #sloppy input normalization, just bringing image values from range [0, 255] to [-1, 1]\n",
        "  #fashion_mnist = fashion_mnist.map(lambda img, target: ((img/128.)-1., target))\n",
        "  #create one-hot targets\n",
        "  fashion_mnist = fashion_mnist.map(lambda img, target: (img, tf.one_hot(target, depth=10)))\n",
        "  #cache this progress in memory, as there is no need to redo it; it is deterministic after all\n",
        "  fashion_mnist = fashion_mnist.cache()\n",
        "  #shuffle, batch, prefetch\n",
        "  fashion_mnist = fashion_mnist.shuffle(1000)\n",
        "  fashion_mnist = fashion_mnist.batch(32)\n",
        "  fashion_mnist = fashion_mnist.prefetch(20)\n",
        "  #return preprocessed dataset\n",
        "  return fashion_mnist\n",
        "\n",
        "train_dataset = train_ds.apply(prepare_fashion_mnist_data)\n",
        "test_dataset = test_ds.apply(prepare_fashion_mnist_data)"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ody3YK7qTzkt"
      },
      "source": [
        "2. prepare the model\n",
        " - This is just like the model from the section on Layers and Models, but we increase the size of hidden layers, becausee MNIST is not as tiny and simple as logical gates"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K_U90ftzUHKc"
      },
      "source": [
        "class MyModel(tf.keras.Model):\n",
        "    \n",
        "    def __init__(self):\n",
        "        super(MyModel, self).__init__()\n",
        "        self.convlayer1 = tf.keras.layers.Conv2D(filters=24, kernel_size=3, strides=(1, 1), padding=\"same\", activation=\"relu\", kernel_regularizer=tf.keras.regularizers.l1_l2)\n",
        "        self.convlayer2 = tf.keras.layers.Conv2D(filters=24, kernel_size=3, strides=(1, 1), padding=\"same\", activation=\"relu\", kernel_regularizer=tf.keras.regularizers.l1_l2)\n",
        "        self.pooling = tf.keras.layers.MaxPooling2D(pool_size=2, strides=2)\n",
        "        self.convlayer3 = tf.keras.layers.Conv2D(filters=48, kernel_size=3, strides=(1, 1), padding=\"same\", activation=\"relu\")\n",
        "        self.convlayer4 = tf.keras.layers.Conv2D(filters=48, kernel_size=3, strides=(1, 1), padding=\"same\", activation=\"relu\")\n",
        "        self.global_pool = tf.keras.layers.GlobalAvgPool2D()\n",
        "        self.out = tf.keras.layers.Dense(10, activation=\"softmax\")\n",
        "\n",
        "    @tf.function\n",
        "    def call(self, inputs):\n",
        "        x = self.convlayer1(inputs)\n",
        "        x = self.convlayer2(x)\n",
        "        x = self.pooling(x)\n",
        "        x = self.convlayer3(x)\n",
        "        x = self.convlayer4(x)\n",
        "        x = self.global_pool(x)\n",
        "        x = self.out(x)\n",
        "        return x\n",
        "    "
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fq69pQboVwoN"
      },
      "source": [
        "3. Prepare the training loop by providing functionality for training and testing:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cOyQRc0cVvqu"
      },
      "source": [
        "def train_step(model, input, target, loss_function, optimizer):\n",
        "  # loss_object and optimizer_object are instances of respective tensorflow classes\n",
        "  with tf.GradientTape() as tape:\n",
        "    prediction = model(input)\n",
        "    loss = loss_function(target, prediction)\n",
        "    gradients = tape.gradient(loss, model.trainable_variables)\n",
        "  optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
        "  return loss\n",
        "\n",
        "def test(model, test_data, loss_function):\n",
        "  # test over complete test data\n",
        "\n",
        "  test_accuracy_aggregator = []\n",
        "  test_loss_aggregator = []\n",
        "\n",
        "  for (input, target) in test_data:\n",
        "    prediction = model(input)\n",
        "    sample_test_loss = loss_function(target, prediction)\n",
        "    sample_test_accuracy =  np.argmax(target, axis=1) == np.argmax(prediction, axis=1)\n",
        "    sample_test_accuracy = np.mean(sample_test_accuracy)\n",
        "    test_loss_aggregator.append(sample_test_loss.numpy())\n",
        "    test_accuracy_aggregator.append(np.mean(sample_test_accuracy))\n",
        "\n",
        "  test_loss = tf.reduce_mean(test_loss_aggregator)\n",
        "  test_accuracy = tf.reduce_mean(test_accuracy_aggregator)\n",
        "\n",
        "  return test_loss, test_accuracy"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "hsspqW4IWTTU"
      },
      "source": [
        "Put everything together:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DjsuBWtaWVt4",
        "outputId": "d252e1d0-42e1-4f2e-ab32-010a1d956b07"
      },
      "source": [
        "tf.keras.backend.clear_session()\n",
        "\n",
        "#For showcasing we only use a subset of the training and test data (generally use all of the available data!)\n",
        "train_dataset = train_dataset.take(1000)\n",
        "test_dataset = test_dataset.take(100)\n",
        "\n",
        "### Hyperparameters\n",
        "num_epochs = 10\n",
        "learning_rate = 0.001\n",
        "\n",
        "# Initialize the model.\n",
        "model = MyModel()\n",
        "# Initialize the loss: categorical cross entropy. Check out 'tf.keras.losses'.\n",
        "cross_entropy_loss = tf.keras.losses.CategoricalCrossentropy()\n",
        "# Initialize the optimizer: Adam with default parameters. Check out 'tf.keras.optimizers'\n",
        "optimizer = tf.keras.optimizers.Adam(learning_rate)\n",
        "\n",
        "# Initialize lists for later visualization.\n",
        "train_losses = []\n",
        "test_losses = []\n",
        "test_accuracies = []\n",
        "\n",
        "#testing once before we begin\n",
        "test_loss, test_accuracy = test(model, test_dataset, cross_entropy_loss)\n",
        "test_losses.append(test_loss)\n",
        "test_accuracies.append(test_accuracy)\n",
        "\n",
        "#check how model performs on train data once before we begin\n",
        "train_loss, _ = test(model, train_dataset, cross_entropy_loss)\n",
        "train_losses.append(train_loss)\n",
        "\n",
        "# We train for num_epochs epochs.\n",
        "for epoch in range(num_epochs):\n",
        "    print(f'Epoch: {str(epoch)} starting with accuracy {test_accuracies[-1]}')\n",
        "\n",
        "    #training (and checking in with training)\n",
        "    epoch_loss_agg = []\n",
        "    for input,target in train_dataset:\n",
        "        train_loss = train_step(model, input, target, cross_entropy_loss, optimizer)\n",
        "        epoch_loss_agg.append(train_loss)\n",
        "    \n",
        "    #track training loss\n",
        "    train_losses.append(tf.reduce_mean(epoch_loss_agg))\n",
        "\n",
        "    #testing, so we can track accuracy and test loss\n",
        "    test_loss, test_accuracy = test(model, test_dataset, cross_entropy_loss)\n",
        "    test_losses.append(test_loss)\n",
        "    test_accuracies.append(test_accuracy)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Epoch: 0 starting with accuracy 0.1\n",
            "Epoch: 1 starting with accuracy 0.8\n",
            "Epoch: 2 starting with accuracy 0.83375\n",
            "Epoch: 3 starting with accuracy 0.86\n",
            "Epoch: 4 starting with accuracy 0.880625\n",
            "Epoch: 5 starting with accuracy 0.8803125\n",
            "Epoch: 6 starting with accuracy 0.8725\n",
            "Epoch: 7 starting with accuracy 0.896875\n",
            "Epoch: 8 starting with accuracy 0.8975\n",
            "Epoch: 9 starting with accuracy 0.9015625\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 279
        },
        "id": "ygQT6dcVaE2o",
        "outputId": "4e2dbee6-108e-460f-aff5-47260343127e"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Visualize accuracy and loss for training and test data.\n",
        "plt.figure()\n",
        "line1, = plt.plot(train_losses)\n",
        "line2, = plt.plot(test_losses)\n",
        "line3, = plt.plot(test_accuracies)\n",
        "plt.xlabel(\"Training steps\")\n",
        "plt.ylabel(\"Loss/Accuracy\")\n",
        "plt.legend((line1,line2, line3),(\"training\",\"test\", \"test accuracy\"))\n",
        "plt.show()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXgAAAEGCAYAAABvtY4XAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXwc9Xn48c8zs6v7sK3Dl2xkfOPbMcTglHBjIHGBEAoE0tAmbkMbcjT8AgkJMU35QSEHkEJKOFIIgQQnKflxBWhMaIAAtnHAFz7A2JIvSb50a4/n98eOxMqWpZW8u6NdPW+Y19wzz0rWM7Pf+c73K6qKMcaY7OP4HYAxxpjUsARvjDFZyhK8McZkKUvwxhiTpSzBG2NMlgr4HUC88vJyra6u9jsMY4zJGKtWrapX1Yqe1g2qBF9dXc3KlSv9DsMYYzKGiHxwtHVWRGOMMVnKErwxxmQpS/DGGJOlBlUZvDHGf6FQiJqaGtra2vwOxcTJy8ujqqqKYDCY8D6W4I0x3dTU1FBcXEx1dTUi4nc4BlBVGhoaqKmpYcKECQnvZ0U0xphu2traKCsrs+Q+iIgIZWVl/f5WZQneGHMES+6Dz0B+Jxmf4MOhEL/+7+W8+eaf/Q7FGGMGlYxP8K4jnL/mi7S//qDfoRhjkuDAgQPcc889/d7v/PPP58CBA71u853vfIcXX3xxoKFlnIxP8OIG2BUYR+GhLX6HYoxJgqMl+HA43Ot+zzzzDMOGDet1m5tvvpmzzjrrmOLLJBmf4AEOFU9iVMc2rHcqYzLf9ddfz9atW5k7dy4nnngif/VXf8WSJUs44YQTALjwwgv5yEc+wowZM7jvvvu69quurqa+vp5t27Yxffp0vvCFLzBjxgzOOeccWltbAfjc5z7H8uXLu7a/6aabmD9/PrNmzWLjxo0A1NXVcfbZZzNjxgw+//nPc9xxx1FfX5/mn0JyZEU1yWj5VEbv/z176uoZWdljmzvGmAFY9v/WsX7noaQe84QxJdz0yRlHXX/rrbeydu1a1qxZw0svvcQFF1zA2rVru6oHPvjgg4wYMYLW1lZOPPFEPvWpT1FWVtbtGJs3b+axxx7jpz/9KZdeeim//vWvufLKK484V3l5OatXr+aee+7hjjvu4P7772fZsmWcccYZ3HDDDTz33HM88MADSf386ZQVd/CFY2P/WHZuWeNzJMaYZDvppJO61f2+6667mDNnDgsXLmTHjh1s3rz5iH0mTJjA3LlzAfjIRz7Ctm3bejz2xRdffMQ2f/rTn7jssssAWLx4McOHD0/ip0mvrLiDHzlpDrwEh3asBc72OxxjskZvd9rpUlhY2DX90ksv8eKLL/Laa69RUFDAaaed1mPd8Nzc3K5p13W7imiOtp3run2W8WeirLiDHzZmCu0EoW6j36EYY45RcXExjY2NPa47ePAgw4cPp6CggI0bN/LnPye/evSiRYv41a9+BcDzzz/P/v37k36OdEnZHbyITAV+GbfoeOA7qvqjpJ/McdkVHEeR1aQxJuOVlZWxaNEiZs6cSX5+PiNHjuxat3jxYn7yk58wffp0pk6dysKFC5N+/ptuuonLL7+cRx55hJNPPplRo0ZRXFyc9POkg6Sj5omIuEAt8FFVPWrj9AsWLNCBdvjxzp2XMGLfGsZ8d7O9hWfMMdiwYQPTp0/3OwzftLe347ougUCA1157jS9+8YusWTM4nu/19LsRkVWquqCn7dNVBn8msLW35H6souVTGbv/BWr31jN2pNWkMcYMzPbt27n00kuJRqPk5OTw05/+1O+QBixdCf4y4LGeVojIUmApwPjx4wd8gsKqmbAZdm19m7EjzxzwcYwxQ9vkyZN56623/A4jKVL+kFVEcoAlwBM9rVfV+1R1gaouqKgY+J33yIlzADi0/Z0BH8MYY7JJOmrRnAesVtU9qTxJ8egphAhYTRpjjPGkI8FfzlGKZ5LKDbA7WGU1aYwxxpPSBC8ihcTePPpNKs/T6VDRJEZ1fEA0am3SGGNMShO8qjarapmqHkzlebrOVzGVKuqo2ZuZDQMZYwbeXDDAj370I1paWpIcUebKijdZOxWOnYEjSu2Wt/0OxRgzQJbgkycr2qLpVDlpDqyAppp1xKreG2MyTXxzwWeffTaVlZX86le/or29nYsuuohly5bR3NzMpZdeSk1NDZFIhG9/+9vs2bOHnTt3cvrpp1NeXs6KFSv8/ii+y6oEXzhqKmFcdK/VpDEmKZ69HnYnuerxqFlw3q1HXR3fXPDzzz/P8uXLeeONN1BVlixZwssvv0xdXR1jxozh6aefBmJt1JSWlvKDH/yAFStWUF5entyYM1RWFdHgBtkTrKLYatIYkxWef/55nn/+eebNm8f8+fPZuHEjmzdvZtasWbzwwgt84xvf4H//938pLS31O9RBKavu4AEOFU9kdMN6wpEoATe7rl/GpF0vd9rpoKrccMMN/MM//MMR61avXs0zzzzDjTfeyJlnnsl3vvMdHyIc3LIuA2r5VMaxhw/2NPgdijFmAOKbCz733HN58MEHaWpqAqC2tpa9e/eyc+dOCgoKuPLKK7nuuutYvXr1EfuaLLyDL6qaibtJ2bV1LRPHnO53OMaYfopvLvi8887jiiuu4OSTTwagqKiIn//852zZsoXrrrsOx3EIBoPce++9ACxdupTFixczZswYe8hKmpoLTtSxNBfcqa3mHfLu/xjPTvlXzrvi2iRFZszQMdSbCx7M+ttccNYV0eSNmkIYB+re9TsUY4zxVdYleAK51AXHUtxoNWmMMUNb9iV4oLF4IqND2+kIR/0OxRhjfJOVCV7Lp3Ecu9m2Z5/foRhjjG+yMsEXjZtBQKLUbl3rdyjGGOObrEzwFcfHendqrrEEb4wZurIywedUTiWCY707GZOBjqU1SbAWJeNlZYInmEd9cIy1SWNMBsqGBB8Oh309f6fsTPDEatKMCW2nLRTxOxRjTD/ENxd83XXXAXD77bdz4oknMnv2bG666SYAmpubueCCC5gzZw4zZ87kl7/8JXfddVdXk8Gnn37km+w333wzJ554IjNnzmTp0qV0vui5ZcsWzjrrLObMmcP8+fPZunUrALfddhuzZs1izpw5XH/99QCcdtppdL6QWV9fT3V1NQA/+9nPWLJkCWeccQZnnnkmTU1NnHnmmcyfP59Zs2bx5JNPdsXx8MMPM3v2bObMmcNVV11FY2MjEyZMIBQKAXDo0KFu8wOVdU0VdCmfSnXDn9i0ax8zxlf4HY0xGem2N25j477kFnVOGzGNb5z0jaOuj28uGGItSm7evDkpTQb/8z//c1ejZFdddRVPPfUUn/zkJ/nMZz7D9ddfz0UXXURbWxvRaJRnn32WJ598ktdff52CggL27eu7Vt7q1at5++23GTFiBOFwmN/+9reUlJRQX1/PwoULWbJkCevXr+d73/ser776KuXl5ezbt4/i4mJOO+00nn76aS688EIef/xxLr74YoLB4EB+xF1S3SfrMBFZLiIbRWSDiJycyvPFKxo3k6BE2PX+unSd0hiTAslsMnjFihV89KMfZdasWfzhD39g3bp1NDY2Ultby0UXXQRAXl4eBQUFvPjii1x99dUUFBQAMGLEiD6Pf/bZZ3dtp6p885vfZPbs2Zx11lnU1tayZ88e/vCHP/DpT3+66wLUuf3nP/95HnroIQAeeughrr766v7/sA6T6jv4O4HnVPUSEckBClJ8vi5lE2I1aWK9O52WrtMak1V6u9NOl2Q1GdzW1sY111zDypUrGTduHN/97ndpa2vrdzyBQIBoNNp1zHiFhYVd048++ih1dXWsWrWKYDBIdXV1r+dbtGgR27Zt46WXXiISiTBz5sx+x3a4lN3Bi0gpcCrwAICqdqjqgVSd73DByilEEcRq0hiTUQ5v8jdZTQZ3Jtfy8nKamppYvnx51/ZVVVX893//NwDt7e20tLRw9tln89BDD3U9sO0soqmurmbVqlUAXcfoycGDB6msrCQYDLJixQo++OADAM444wyeeOIJGhoauh0X4LOf/SxXXHFFUu7eIbVFNBOAOuAhEXlLRO4XkcLDNxKRpSKyUkRW1tXVJe/sOQU0BEdT1Lg1ecc0xqRcfHPB1113Heecc05Xk8GzZs3ikksuobGxkXfeeYeTTjqJuXPnsmzZMm688UbgwyaDD3/IOmzYML7whS8wc+ZMzj33XE488cSudY888gh33XUXs2fP5pRTTmH37t0sXryYJUuWsGDBAubOncsdd9wBwNe//nXuvfde5s2bR319/VE/x2c+8xlWrlzJrFmzePjhh5k2bRoAM2bM4Fvf+hYf//jHmTNnDl/72te67bN//34uv/zypPwsU9ZcsIgsAP4MLFLV10XkTuCQqn77aPsko7ngeO/d9Qk66rcx7ltrKMzN3ufJxiSTNRfsn+XLl/Pkk0/yyCOP9Li+v80FpzLr1QA1qvq6N78cuD6F5ztSxTSOb3iVDbv3M+c4q0ljjBm8vvSlL/Hss8/yzDPPJO2YKSuiUdXdwA4RmeotOhNYn6rz9aR43ExyJMKu99N6WmOM6be7776bLVu2MGXKlKQdM9XlFl8CHvVq0LwHJOfJQYJGVHs1aXasAz6ezlMbk9FUFRHxOwwTZyDF6SlN8Kq6BuixbCgd3MrYlVDqrXcnYxKVl5dHQ0MDZWVlluQHCVWloaGBvLy8fu2X3U8ecwppCI6ixHp3MiZhVVVV1NTUkNRabeaY5eXlUVVV1a99sjvBA03FE6mq386hthAlecf22q8xQ0EwGGTChAl+h2GSIGsbG+tSOZ3jZRebd6XtHStjjBkUsj7BF1XNJFdC7Nxmb7QaY4aWrE/ww4+bBUBzzTs+R2KMMemV9QneqYxVw3fqNvkciTHGpFfWJ3hyi9kfHEmxtUljjBlisj/BE6tJMy6ynf3NHX6HYowxaTMkEjwV05gktWyymjTGmCFkSCT44nEzyZMQO7dbObwxZugYEgm+1KtJ01Kz1udIjDEmfYZEgpcKr0HLOmuTxhgzdAyJBE9eKQeDFZQ2bhlQi2zGGJOJhkaCJ1aTZnx0B/VNVpPGGDM0DJkET+V0JslONu8+6HckxhiTFkMmwRePm0GBtLPzA6tJY4wZGoZOgq+aCUBTzTqfIzHGmPQYMgleKqcB4FjvTsaYISKlHX6IyDagEYgAYVX1rfs+8odzKFBGSdNW62/SGDMkpKNHp9NVtT4N5+lTc8kkqut3sOdQO6NK+9e3oTHGZJohU0QDQKXXJs3uQ35HYowxKZfqBK/A8yKySkSW9rSBiCwVkZUisjLVnfyWjJtJkbRRu31zSs9jjDGDQaoT/MdUdT5wHvBPInLq4Ruo6n2qukBVF1RUVKQ0mEKvJk2z1aQxxgwBCSV4Efm+iMzo78FVtdYb7wV+C5zU32MkVUWsJo1YTRpjzBCQ6B38BuA+EXldRP5RREr72kFECkWkuHMaOAfwtznHghE0BYYzzKtJY4wx2SyhBK+q96vqIuCzQDXwtoj8QkRO72W3kcCfROQvwBvA06r63LEGfKyaSyYxQXdQe6DV71CMMSalEi6DFxEXmOYN9cBfgK+JyOM9ba+q76nqHG+Yoar/lpSIj5FYTRpjzBCRaBn8D4GNwPnALar6EVW9TVU/CcxLZYDJVjxuFiXSSu2O9/0OxRhjUirRF53eBm5U1eYe1vn74LSf8secAEDzjrXAyf4GY4wxKZRoEc0B4i4GIjJMRC4EUNXMan/Xq0njNFhNGmNMdks0wd8Un8hV9QBwU2pCSrGiCpoDwxjWtJVI1GrSGGOyV6IJvqft0tGOTUo0l0xiAjXs2NfidyjGGJMyiSb4lSLyAxGZ6A0/AFalMrBUksqpTJEaq0ljjMlqiSb4LwEdwC+9oR34p1QFlWol42ZSKi3U7NjmdyjGGJMyCRWzeLVnrk9xLGmTOzrW6kJL7TqsJo0xJlsllOBFpAL4P8AMoKshdVU9I0VxpZZXk8Zt2OhzIMYYkzqJFtE8SuxFpwnAMmAb8GaKYkq9okpa3RKGNb1POBL1OxpjjEmJRBN8mao+AIRU9Y+q+ndAZt69A4jQXDKR46WGbQ1Wk8YYk50STfAhb7xLRC4QkXnAiBTFlBbOyOlWk8YYk9USTfDf85oI/hfg68D9wFdTFlUaFFXNZLg0UVPzgd+hGGNMSvT5kNVrRXKyqj4FHAR6ayI4Y+SMng5AS+16rCaNMSYb9XkHr6oR4PI0xJJe1iaNMSbLJdrcwCsi8mNiLzl1tSipqqtTElU6FI+mzS1iRPN7tIcj5AZcvyMyxpikSjTBz/XGN8ctUzK8Jk1L6SQm1dfwfn0z00aV+B2RMcYkVaJvsmZFufvhnMppTGp4hlf3NFmCN8ZknUTfZP1OT8tV9eaelmeKoqoZBDY+Tk3Ndpgzxu9wjDEmqRKtJtkcN0SA84h1vt0nEXFF5C0ReWpAEaZQYJTXu1PNep8jMcaY5Eu0iOb78fMicgfw+wTP8WVgAzD4ykC8mjSBfVaTxhiTfRK9gz9cAVDV10YiUgVcQOzFqMGnZCwdTgEjWt6jLRTxOxpjjEmqRMvg3yFWawbABSroXqPmaH5ErBXK4l6OvRRYCjB+/PhEwkkeEZpLJzGpvpYte5uYObY0vec3xpgUSrSa5CfipsPAHlUN97aDiHwC2Kuqq0TktKNtp6r3AfcBLFiwIO2dpDojpzN53+95ZW+jJXhjTFZJtIhmNLBPVT9Q1VogX0Q+2sc+i4AlIrINeBw4Q0R+PvBQU6OoagaVcoAPamr9DsUYY5Iq0QR/L9AUN9/sLTsqVb1BVatUtRq4DPiDql45oChTyK2MtUnTWrvO50iMMSa5Ek3woqpdxSeqGiXx4p3BrWIqAO6+TT4HYowxyZVogn9PRK4VkaA3fBl4L9GTqOpLqvqJvrf0Qek4Opx8ylvep7m918cKxhiTURJN8P8InALUAjXAR/FqvmQ8x6G1dCKTpYYte5v63t4YYzJEQgleVfeq6mWqWqmqI1X1ClXdm+rg0sWpnM5kp5Z39zT6HYoxxiRNQgleRP5LRIbFzQ8XkQdTF1Z6FVbNYJTsZ3vtTr9DMcaYpEm0iGa2qh7onFHV/cC81ISUfo5Xk6Zl5wafIzHGmORJNME7IjK8c0ZERpAttWigqyZNoGGjz4EYY0zyJJqkvw+8JiJPAAJcAtySsqjSbdh4Qk4ulW3bONQWoiQv6HdExhhzzBJ9yPowcDGwB9gNXOwtyw6OS2vJRCZLLZvtQasxJksk3Jqkqq5X1R8DzwKfEpGsevXTHTmdSU4tm/ZYVUljTHZItBbNGBH5qoi8Cazz9rsspZGlWf7YGYyVBrbt3O13KMYYkxS9JngRWSoiK4CXgDLg74FdqrpMVd9JQ3xp41TGOv9orbXenYwx2aGvh6w/Bl4DrlDVlQAikvYmfdOiq3cna5PGGJMd+krwo4FPA98XkVHAr4DsrGIyvJqwk8PI9m3sb+5geGGO3xEZY8wx6bWIRlUbVPUnqvpx4EzgALBHRDaISPZUkwRwXNpKjmeK1LDJatIYY7JAX2XwYzqnVbVGVb+vqguAvwbaUh1cujkjY23SbLJGx4wxWaCvWjT3i8ifReRWETlNRAIAqrpJVRPpkzWj5I+ZQZXUs612j9+hGGPMMeuriOZ84DRitWguAv4sIr/xatekuYfs1BOvJo21SWOMyQZ9NlWgqm3Ac96AiEwAzgN+LCKjVPWk1IaYRl5NmuC+d1FVRMTngIwxZuASaotGRAqBVq+rviCxTj8+RaxdmuwxfAIRCTImtJ36pg4qinP9jsgYYwYs0aYKXgbyRGQs8DxwFfCQqnYcbQcRyRORN0TkLyKyTkSWJSHe1HIDtJUezySpsTZpjDEZrz+dbrcQa3DsHlX9NDCrj33agTNUdQ4wF1gsIgsHHmp6uCOnMVmsdydjTOZLOMGLyMnAZ4CnE9lXYzrrGwa9YdC/BZs7+gTGOXW8v6ve71CMMeaYJJrgvwLcAPxWVdeJyPHAir52EhFXRNYAe4EXVPX1HrZZKiIrRWRlXV1df2JPCamYhoNaTRpjTMZLtD34P6rqElW9TUQcoF5Vr01gv4iqzgWqgJNEZGYP29ynqgtUdUFFRUW/P0DSed33ddakMcaYTJVoc8G/EJESrzbNWmC9iFyX6Em8/lxXAIsHFmYajTieqASoCm9nz6F2v6MxxpgBS7SI5gRVPQRcSKzDjwnEatIclYhUiMgwbzofOBsY/J2eukHaSiYwxR60GmMyXKIJPigiQWIJ/neqGqLvB6ajgRUi8jbwJrEy+KcGHmr6BEZOs6qSxpiMl2in2/8JbAP+ArwsIscBh3rbQVXfBuYdU3Q+yRk9g/GbnmbrznrgeL/DMcaYAUn0IetdqjpWVc/3qj9+AJye4tj8UzEVlygtu9/1OxJjjBmwRB+ylorIDzqrM4rI94HCFMfmH69NmtyGd4lGrSaNMSYzJVoG/yDQCFzqDYeAh1IVlO/KJhIVl3HRHdQeaPU7GmOMGZBEy+Anquqn4uaXeS8wZadALu0l1UzeV8vmvY2MG1Hgd0TGGNNvid7Bt4rIxzpnRGQRkNW3toGR05ksNby723p3MsZkpkTv4P8ReFhESr35/cDfpiakwSE4ajrHbXqG93Y1ABP9DscYY/ot0Vo0f/FahZwNzFbVecAZKY3MbxXTCFhNGmNMBku0iAYAVT3kvdEK8LUUxDN4dPXutImI1aQxxmSgfiX4w2RXb06HK5tEFIcJWsOOfS1+R2OMMf12LAk+u29rg3l0lBwXe9BqTRYYYzJQrwleRBpF5FAPQyMwJk0x+ibg9e5kbdIYYzJRr7VoVLU4XYEMRoFRJzBh8/Ns3b3f71CMMabfjqWIJvtVTCNAhOZdm/yOxBhj+s0SfG8qpgKQt38zoUjU52CMMaZ/LMH3pmwyijBBd/BBQ7Pf0RhjTL9Ygu9NTgEdJeOZ7NSyaY81WWCMySyW4PsQGDmdSVLLJqtJY4zJMJbg++BWTmOis4stVpPGGJNhUpbgRWSciKwQkfUisk5Evpyqc6VUxTSChGnetcXvSIwxpl8SbU1yIMLAv6jqahEpBlaJyAuquj6F50y+ylibNPkHNtMejpAbcH0OyBhjEpOyO3hV3aWqq73pRmADMDZV50uZ8ikAHE8N79dbTRpjTOZISxm8iFQD84DXe1i3tLOv17q6unSE0z85hXQUj2OKU2M1aYwxGSXlCV5EioBfA1+Ja2q4i6rep6oLVHVBRUVFqsMZENdrk2bTbqtJY4zJHClN8CISJJbcH1XV36TyXKnkVk73atIc8DsUY4xJWCpr0QjwALBBVX+QqvOkRcU0cgjRtGer35EYY0zCUnkHvwi4CjhDRNZ4w/kpPF/qeL07FRzcRFso4nMwxhiTmJRVk1TVP5EtvT5VxGrSTKKWLXubmDm2tI8djDHGf/YmayJyiwkVjfXapLEHrcaYzGAJPkHuyGlMsUbHjDEZxBJ8gpzK6UySnVaTxhiTMSzBJ6piKrl0cGi31aQxxmQGS/CJqpgOQHHjVprbwz4HY4wxfbMEnyivJs0UqWXzXiuHN8YMfpbgE5VXSrhwNJOcGqtJY4zJCJbg+8EdOY2pzk42W4I3xmQAS/D9IJVe9327j2gzzRhjBh1L8P1RMZU82mnc857fkRhjTJ8swfeH1yZNadN7HGwN+RyMMcb0zhJ8f1RMBWCy1LBlr5XDG2MGN0vw/ZE/nEjhyFjnH9ZkgTFmkLME309O5TSmurW8a707GWMGOUvw/SQVse77Nu+xmjTGmMHNEnx/VUwlnzYO7N7mdyTGGNMrS/D9VRlrk6a89X32N3f4HIwxxhydJfj+8qpKThZrssAYM7ilstPtB0Vkr4isTdU5fFEwgkhBuVeTxhK8MWbwSuUd/M+AxSk8vm+cymlMc62qpDFmcEtZglfVl4F9qTq+n6RiOpOcWt61NmmMMYOY72XwIrJURFaKyMq6ujq/w0lMxVQKtYWDez5AVf2OxhhjeuR7glfV+1R1gaouqKio8DucxHgPWivbt1HfZDVpjDGDU8DvADJSV02a2IPWiuJcnwPKPqpKWMOEo2Ei0QjhaJiwHltXiYIMfF8RHJzYWBwEb+zNOzgg4OB0LU+lqEYJR8OEoiFCkRBhDcfGncui3acPnw9Hw33u19u3U6X3b67HtC/atX/ndPw4/hxd/w1ge0WJ/f/hdoJ0+532+PuOWx6/Ln595/Th+x3+b6ZzOj+QzyVTLun15zIQluAHoqiCaH4Zk8KxqpKLJpX7HdERDk8A8X/oR8z3sU1PxwlHw11J9/Ak3LUuGiaika7pUDTUbT6ika5jxS/vnM90R0sK8cu7JYfO5BK3LdDt5xf/80qloBPEFbf3z3cMF7G+LradPxtBiP0v3ZbFn/vwdf3a3ht3nrMz8Uc1SlSjqCpRPpzuXHf48qhGj7pfIsryyjIrwYvIY8BpQLmI1AA3qeoDqTpfuknlVE5o3ckv46pKqiqhaIj2SDsdkQ46Ih20R9ppj7R3LT98Xee4r/Ud0Z73OTwxx9+ZpUJAAgSc2OA6brf5gBPomncdt9t8XiAvtlzcrm07k0jX8cQl6ASPemxX3AHfhfd119jXvvF3hZ1/zEf7I+8tKXRLAkdZDnTbpvNnEnSC3afdIAEJEHS7L+9p2yPWuUff3hU35d9AhpK+LgxRoil7lpeyBK+ql6fq2INBU9kkfte0hWcPfpE//kLpiMaS7rEShFw3lxw3p2vcbdrJoSSnhKAbJMfJ6frj7jbELctxc7r9AR+xfR/7x88HnEDXXaUxJjHx39TSzYpo+klVeXH7i9za+CZ1xbm4TWMoCYxj3PASJowYRkVRYVdC7ilB57q55Dg5R03gAQnY3ZMxJikswffDrqZd/Nvr/8Yfa/7ItMIq7vxgKy+P+io/234cG9fF7t7HDsvnlIllnDKpjFMmljOyJM/nqI0xQ5Ul+ASEo2Ee3fAo/7HmPwD4+oKv85mqswmsnc7ME0J88eoz2VrXzKtb63l1SwPPr9/DE6tqAJhYUciiSeWcMrGMhceXMawgx8+PYowZQizB92Fd/TqWvbaMDUi2rTsAAA/KSURBVPs2cGrVqXzro99iTNEYUIW8YVC3ERFhUmURkyqL+OzJ1USiyoZdh3hlSz2vbm3giZU1PPzaB4jAjDElLJpYzskTyzhpwggKcuxXYIxJDRlMb2IuWLBAV65c6XcYADR1NPHjNT/msY2PUZZXxg0fvYGzxp/VvXz8gXNBHPi7Z3s9Vkc4yl9qDvDqlgZe2VrPW9v3E4ooQVeYO24Yp0yM3eHPHT+M3EDvVdOMMSaeiKxS1QU9rrMEf6T/2f4/3PL6LdS11PE3U/+Ga+dfS3FO8ZEb/r8vw5pfwPiFUDwaikcdNvamA91fhGrtiPDmtn28urWBV7fW807twdgXgqDDidUjOGViOYsmlTFjTCmuYw9cjTFH11uCt/KBOLubd3PL67ewYscKpgyfwg9P+yGzK2YffYePXA2tB6BxF2x/DRp3Q6SHpgvyR3RL/PnFozi1ZDSnThgNs0dxKDCeP+91ePX9g7yypZ7bntsIQElegIXHl3HKxDIWTSpnUmWR1bAxxiTM7uCBSDTCYxsf4+637iaqUa6Zew1XnnAlQSfYvwOpQuv+WMI/tCs2btx95LhpDxzxJqJAUSUUj6I9v5Kd0eFsbili9f48NjYXsleHEy4cxQkTqzl5UgUnH1/O2OH5dodvzBBnd/C9WNewjptfu5n1Dev52NiPcePCGxlbNHZgBxOBghGxYeSMo28XjUBz3VEvALmHdjGh8S0mtNRzDkBnxZswhN912bNxGHt1OFu1kDa3kI5AMaFAMdHcYjS3BCevFKeglGDBcHKLhpFXPILC4uEUlw5nWGEupflB8oJW1m9MthuyCb4l1MLdb93NLzb+ghF5I7j947dz7nHnpqcIxHG94ppRvW8X7ojd7cclf/fQTorqduA01DCm7SCBcAM54Sby2poJtoV6PVxUhSbyqaeAJgpodQppd4voCBYTDhajOSWQV4KTPwy3oJScwmHkFQ2joLiMwtIRFJWOoLioBMe1t1mNyQRDMsGv2L6CW964hT3Ne7h06qVcO/9aSnJK/A7rSIEcGDYuNngEKPWGI4TaoO0gtB9C2w7S2riflkP7aG/aT0fzAcItB4m0HoC2gzjtjZSEGgmG9pPbsYP8tmYKtJkAvTeOFFKXQ5JPK3m0Sj7tTj5tTgEdTj4dbgEht4BwoJCIW0AkWEg0WEg0pxCCRWhuEU5OEU5eMU5uEW5+MYHcQvJyguQGHfICLnlBh9y4cW7QITeQ+tYZByVV0GjshsCYARhSCX5P8x5ufeNWXtz+IpOGTeL2825nbuVcv8NKnmBebCgeiQAF3pAwVQi10N60j6ZD+2g5uJ+Wxv10NO2jozl2cYi2HsTpaMQNtxAItxCMNFMUaSU3epDccCt50RbytI1cEm8nv0nzaCGva7yPPJo1j+aucT7tTh4hJx/HcQm6QsARAi4EHW/aEQLuh9NBB1w3btpxCDgQ6BoLAQfcrv1j69xu6xwclFibstFY0Vo03H2IHDbf0zY9rY+Eeti2cz4UG3e2ROgEIFgAgTwI5n84BPK933ki6wq8+d7WFYDbz+dOZuCiUQi3xYZoOPYMLsmGRIKPRCM8/u7j3P3W3USiEb4y/yt8dsZn+/8QNduJQE4huSMKyR0xjrJjOVYkBB3N0NEEHc1E2xoJtzXS0XKISFsj4dZGou1NRNtiYzqayO9oprCjicpQM26oGTe837uItJATaUFQiEIfXzIGJKrS1d6kei1WRhA6H4VHxCWCQxSXiLiHjQOouETF9cYB1HFRCaBObBrJjyVqx42NcwKxZOoEEMdF3CDiBhAngOMGEDeIEwjiiBDQDtxwG260DSfShhuOjZ1wK9JyECe8Bwm3IuFWCLVCuA0Jtw3sByHuhxcJN+fDeJ1g9/jdw+a7tomfD4Ab6D7vuHHH6uV44sRiESe2TJwPB8c9bJ30sG3nvNPDsY6yTpzYxTXkJd1wO4RbY+OQN+5MyJ1D6LD5o20bajtyWXyNu6JR8PV3B/zv92iyPsFv3LeRZa8uY23DWk4Zcwo3LryRccXj+t7RHBs3CPnDYgOxrsNy+PB5cb+pxv4oOmt9dRXZSPfpw9ZFVemIKO1hpT0coS0UG7dHlLZQhPZwtGscWx+lPRShLRylPfThso5IhFBYCUWidESihCJRQhFvPnzYfOf6HraPRNNTa82VKIVOhEInRKHbQaGEKHDCFEoH+dJBodNBvoTI9+bzpYN8Osijg1xvHIyECUQiBCSKqxFciRDQCAEiOEQIEMIljKuxeVcjuERw4saOhr1xbBAN42gYN0XNWaedOLFvRIFc79tQXmwcyI0tzymCgvK45XHrg3H75fVY6HrMsjbBt4RauGfNPfx8w88pzS3l30/9dxZXLx6aZbnZQCT2B9FPDpAXgLxB0ulWJKpesu/jAhGOzXdEIkSiEIlGCUfV21+7zYe9C0dsvpftokokEhu3RaM0efsevl84qoQjUSIaO2/n+SNRJaLeEPHGXeftPoQTuJAJUQJECRAmEPsu1DXvoDiiOES9703R2DJvWtBuy2LLFUei3rwetl33/V2JEhAIxI1dgYgECDs5hCWHkJNL2ImNo04uYckh4uYScWJD1M1DHZeA6+J4RX2uI7giuG5sHHDkw3UIblRww4IbiY0D3vZFHQGWpuDfW1Ym+JdrXuZ7f/4eu5p3ccmUS/jK/K9QmpuaK6Qx/eE6guu4Q6KaajTa+0Wg+4UidiEJd15IokpU1bu4fLh99LDpcNd2H+4TjsZvx1G3OzyWUPy6iNcNYFRxoorGjaNRxVElGolt2xqK9HCBixLV2OeJdn2uDy/U8TGFo0p5US5LT52Y9N9BViX4vS17ufWNW3nhgxeYWDqRh897mHmV8/wOy5ghyXEEB2EIXMuOWTRFRXdZkeAj0QhPbHqCO1ffSSga4tp51/K5GZ8jaDUCjDEZwEnRG+kpTfAishi4E3CB+1X11mSf42D7Qa558Rrern+bhaMX8u2F32Z8yfhkn8YYYzJOKjvddoH/AM4GaoA3ReR3qro+mecpySmhqriKy6dfzgUTLrCHqMYY40nlHfxJwBZVfQ9ARB4H/hpIaoIXEW479bZkHtIYY7JCKhsVGQvsiJuv8ZYZY4xJA99bjRKRpSKyUkRW1tXV+R2OMcZkjVQm+Fog/pXRKm9ZN6p6n6ouUNUFFRUVKQzHGGOGllQm+DeBySIyQURygMuA36XwfMYYY+Kk7CGrqoZF5J+B3xOrJvmgqq5L1fmMMcZ0l9J68Kr6DPBMKs9hjDGmZ74/ZDXGGJMaluCNMSZLiWp62qdOhIjUAR8McPdyoD6J4WQC+8zZb6h9XrDP3F/HqWqPVRAHVYI/FiKyUlUX+B1HOtlnzn5D7fOCfeZksiIaY4zJUpbgjTEmS2VTgr/P7wB8YJ85+w21zwv2mZMma8rgjTHGdJdNd/DGGGPiWII3xpgslfEJXkQWi8i7IrJFRK73O55UE5FxIrJCRNaLyDoR+bLfMaWLiLgi8paIPOV3LOkgIsNEZLmIbBSRDSJyst8xpZqIfNX7d71WRB4TkTy/Y0o2EXlQRPaKyNq4ZSNE5AUR2eyNhyfjXBmd4OO6BTwPOAG4XERO8DeqlAsD/6KqJwALgX8aAp+505eBDX4HkUZ3As+p6jRgDln+2UVkLHAtsEBVZxJrpPAyf6NKiZ8Biw9bdj3wP6o6Gfgfb/6YZXSCJ65bQFXtADq7BcxaqrpLVVd7043E/uizvqcsEakCLgDu9zuWdBCRUuBU4AEAVe1Q1QP+RpUWASBfRAJAAbDT53iSTlVfBvYdtvivgf/ypv8LuDAZ58r0BD+kuwUUkWpgHvC6v5GkxY+A/wNE/Q4kTSYAdcBDXrHU/SJS6HdQqaSqtcAdwHZgF3BQVZ/3N6q0Gamqu7zp3cDIZBw00xP8kCUiRcCvga+o6iG/40klEfkEsFdVV/kdSxoFgPnAvao6D2gmSV/bByuv3PmviV3cxgCFInKlv1Gln8bqriel/nqmJ/iEugXMNiISJJbcH1XV3/gdTxosApaIyDZixXBniMjP/Q0p5WqAGlXt/Ha2nFjCz2ZnAe+rap2qhoDfAKf4HFO67BGR0QDeeG8yDprpCX7IdQsoIkKsXHaDqv7A73jSQVVvUNUqVa0m9jv+g6pm9Z2dqu4GdojIVG/RmcB6H0NKh+3AQhEp8P6dn0mWP1iO8zvgb73pvwWeTMZBU9qjU6oN0W4BFwFXAe+IyBpv2Te93rNMdvkS8Kh38/IecLXP8aSUqr4uIsuB1cRqi71FFjZbICKPAacB5SJSA9wE3Ar8SkT+nliT6Zcm5VzWVIExxmSnTC+iMcYYcxSW4I0xJktZgjfGmCxlCd4YY7KUJXhjjMlSluDNoCUiZSKyxht2i0ht3HxOH/suEJG7EjjHq8mL+IhjDxORa1J1fGP6YtUkTUYQke8CTap6R9yygKqG/Yuqd15bQU95LSMak3Z2B28yioj8TER+IiKvA/8uIieJyGteg1yvdr75KSKndbYbLyLf9drgfklE3hORa+OO1xS3/Utx7a8/6r1NiYic7y1bJSJ39dQevYjMEJE3vG8Xb4vIZGIvr0z0lt3ubXediLzpbbPMW1Ydd84NXgwF3rpbvbb/3xaROw4/rzG9yeg3Wc2QVQWcoqoRESkB/sp7q/ks4BbgUz3sMw04HSgG3hWRe732TuLNA2YQa6L2FWCRiKwE/hM4VVXf995C7Mk/Aneqauebpy6xxsFmqupcABE5B5hMrJlrAX4nIqcSe0V/KvD3qvqKiDwIXCMiDwEXAdNUVUVkWP9/VGYoszt4k4meUNWIN10KPOH1jvNDYgm6J0+raruq1hNryKmn5ljfUNUaVY0Ca4BqYheG91T1fW+boyX414Bvisg3gONUtbWHbc7xhreIvY4/jVjCB9ihqq940z8HPgYcBNqAB0TkYqDlKOc2pkeW4E0mao6b/ldghVfO/UngaF28tcdNR+j522si2/RIVX8BLAFagWdE5IweNhPg/6rqXG+YpKoPdB7iyENqmNjd/nLgE8BzicZjDFiCN5mvlA+biP5cCo7/LnC898AU4G962khEjid2p38XsZYAZwONxIqEOv0e+DuvLX9EZKyIVHrrxsuHfa5eAfzJ267Ua0juq8S67TMmYZbgTab7d+D/ishbpOCZklfUcg3wnIisIpa0D/aw6aXAWq+Fz5nAw6raALwisQ6kb/d6J/oF8JqIvEPszrzzAvAusf51NwDDgXu9dU+JyNvAn4CvJfvzmexm1SSN6YOIFKlqk1er5j+Azar6wyQevxqrTmlSwO7gjenbF7w783XEioT+0+d4jEmI3cEbY0yWsjt4Y4zJUpbgjTEmS1mCN8aYLGUJ3hhjspQleGOMyVL/H/JC2t5nTZ53AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    }
  ]
}