numpy-neural-net.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Homework 3: Back-propagation Implementation\n",
    "## 2020 CS 474/574: Deep Learning\n",
    "## Graeme Holliday"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from matplotlib import pyplot as plt\n",
    "import keras\n",
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## load the digits dataset\n",
    "def load_digits(show_sample=True):\n",
    "    (x_train, y_train), (x_test, y_test) = keras.datasets.mnist.load_data()\n",
    "    \n",
    "    #show sample images\n",
    "    if show_sample == True:\n",
    "        nImg = 4\n",
    "        for i in range(nImg * nImg):\n",
    "            plt.subplot(nImg, nImg, i + 1)\n",
    "            plt.imshow(x_train[i], cmap = 'Greys_r')\n",
    "        plt.show()\n",
    "        \n",
    "    x_train_1 = np.reshape(x_train, [x_train.shape[0], x_train.shape[1] * x_train.shape[2]])\n",
    "    x_test_1 = np.reshape(x_test, [x_test.shape[0], x_test.shape[1] * x_test.shape[2]])\n",
    "    \n",
    "    return x_train_1, y_train, x_test_1, y_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Task 1: Activation functions: implement the softmax function.\n",
    "def sigm(z):\n",
    "    return 1 / (1 + np.exp(-z))\n",
    "\n",
    "def dsigm(z):\n",
    "    return sigm(z) * (1 - sigm(z))\n",
    "\n",
    "def relu(z):\n",
    "    return np.maximum(0, z)\n",
    "\n",
    "def drelu(z):\n",
    "    dz = np.copy(z)\n",
    "    dz[dz <= 0] = 0\n",
    "    dz[dz > 0] = 1\n",
    "    return dz\n",
    "\n",
    "def softmax(z):\n",
    "    ''' softmax function for the output layer. Refer to chapter 4.1 to deal with the underflow\n",
    "        overflow problems\n",
    "\n",
    "        parameters:\n",
    "            z: net input vector of the output layer\n",
    "        \n",
    "        return the result vector     \n",
    "    '''\n",
    "    top = np.exp(z - np.max(z, axis=-1, keepdims=True)) # fix numerical instability\n",
    "    return top / top.sum(axis=-1, keepdims=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Task 2: create the layer class.\n",
    "class Layer:\n",
    "    \"\"\" Regular densely-connected NN layer   \n",
    "    \"\"\"\n",
    "    def __init__(self, units, input_dim, activation='sigm'):\n",
    "        ''' initialize weights and bias. \n",
    "            \n",
    "            parameters:\n",
    "                units: the number of hidden nodes\n",
    "                input_dim: dimensionality of the layer input\n",
    "                activation: activation function\n",
    "        '''\n",
    "        self.units = units\n",
    "        self.activation = activation\n",
    "        self.input_dim = input_dim\n",
    "        \n",
    "        ## check here: https://www.deeplearning.ai/ai-notes/initialization/\n",
    "        self.W = np.random.normal(0, 1 / input_dim, (units, input_dim)) # weights\n",
    "        self.bias = np.zeros(units) # bias\n",
    "        \n",
    "        self.gW = None # gradient of weights\n",
    "        self.gb = None # gradient of bias\n",
    "        \n",
    "    def run(self, inputs):\n",
    "        ''' calculate the net input and activation output of the current layer\n",
    "        \n",
    "            parameters:\n",
    "                inputs: layer inputLast. input=(n_sample * n_features)\n",
    "          \n",
    "            return:\n",
    "                self.output: the activation output\n",
    "        '''\n",
    "        \n",
    "        self.net = inputs @ self.W.T + self.bias\n",
    "        \n",
    "        if self.activation == 'sigmoid':\n",
    "            self.output = sigm(self.net)\n",
    "        elif self.activation == 'softmax':\n",
    "            self.output = softmax(self.net)\n",
    "        elif self.activation == 'relu':\n",
    "            self.output = relu(self.net)\n",
    "        \n",
    "        return self.output\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Task 3: complete the following NN class. 60 points\n",
    "class NeuralNetwork:\n",
    "    def __init__(self):\n",
    "        self.layers = [] # list of layers\n",
    "        \n",
    "    # Task 3.1: implement the 'add' function. \n",
    "    def add(self, units, input_dim, activation='sigm'):\n",
    "        '''add one layer to neural network\n",
    "        \n",
    "            parameters:\n",
    "                units: the number of nodes of current layer\n",
    "                input_dim: input dimension (the number of nodes of the previous layer)\n",
    "                activation: the activation function\n",
    "        '''\n",
    "        \n",
    "        self.layers.append(Layer(units, input_dim, activation))\n",
    "        \n",
    "    # Task 3.2: implement the cross-entropy loss. \n",
    "    def loss(self, y_pred, y):\n",
    "        ''' loss function: 1/n_samples*sum_samples(sum_output(-y_k*log(y_pred_k)))\n",
    "            \n",
    "            parameters:\n",
    "                y_pred: predictions(n_samples * 10)\n",
    "                y: target(one-hot vectors: n_samples * 10)\n",
    "            return:\n",
    "                loss\n",
    "        '''\n",
    "        m = y.shape[0] # the number of samples\n",
    "        \n",
    "        epsilon = 1e-12\n",
    "        y_pred = np.clip(y_pred, epsilon, 1 - epsilon)\n",
    "        loss = -np.sum(y * np.log(y_pred)) / m\n",
    "        \n",
    "        return loss\n",
    "    \n",
    "    # Task 3.3: implement the forward propagation process.\n",
    "    def forward_prop(self, inputs):\n",
    "        ''' forward propagation calculates net input and output for all layers\n",
    "            \n",
    "            parameters:\n",
    "                inputs: input data(n_samples * n_features)\n",
    "            \n",
    "            return:\n",
    "                out: the output of the last layer\n",
    "            \n",
    "            Tip: call the run function layer by layer\n",
    "        '''\n",
    "        \n",
    "        out = np.copy(inputs)\n",
    "        \n",
    "        for layer in self.layers:\n",
    "            out = layer.run(out)\n",
    "        \n",
    "        return out\n",
    "    \n",
    "    # Task 3.4: implement the prediction function. 5 points\n",
    "    def predict_class(self, x):\n",
    "        '''predict class lables (0, 1, 2, 3, ..., 9) for data samples\n",
    "        \n",
    "            parameters:\n",
    "                x: input(n_samples * n_features) \n",
    "            return:\n",
    "                class labels\n",
    "        '''\n",
    "        return np.argmax(self.forward_prop(x), axis=-1) # returns index which is predicted class\n",
    "        \n",
    "    # Task 3.5: complete the following 'train' function.\n",
    "    def train(self, inputs, targets, lr=1e-3, batch_size=32, epochs=50):\n",
    "        ''' implement the SGD process and use Back-Propagation algorithm to calculate gradients \n",
    "            \n",
    "            parameters:\n",
    "                inputs: training samples\n",
    "                targets: training targets\n",
    "                lr: learning rate\n",
    "                batch_size: batch size\n",
    "                epochs: max number of epochs\n",
    "                \n",
    "            return:\n",
    "                loss_hist\n",
    "        '''\n",
    "        \n",
    "        m = len(targets)\n",
    "        loss_hist = np.zeros(epochs)\n",
    "        \n",
    "        for i in range(epochs):\n",
    "            #shuffle the data\n",
    "            idx = np.arange(m)\n",
    "            np.random.shuffle(idx)\n",
    "            inputs = inputs[idx]\n",
    "            targets = targets[idx]\n",
    "            \n",
    "            for b in range(int(m / batch_size)):\n",
    "                b_start= b * batch_size\n",
    "                b_end = min((b + 1) * batch_size, m)\n",
    "                \n",
    "                x_batch = inputs[b_start:b_end, :]\n",
    "                y_batch = targets[b_start:b_end, :]\n",
    "                \n",
    "                # 1. run forward propagation using the current batch\n",
    "                y_batch_hat = self.forward_prop(x_batch)\n",
    "                # 2. call BP to calculate all gradients\n",
    "                self.backward_prop(x_batch, y_batch, y_batch_hat)\n",
    "                # 3. update all weights and bias\n",
    "                self.update_weights(lr)\n",
    "                \n",
    "            lr *= 0.95\n",
    "            \n",
    "            # 4. calculate and record the loss of current epoch\n",
    "            predictions = keras.utils.to_categorical(self.predict_class(inputs))\n",
    "            loss_hist[i] = self.loss(predictions, targets)\n",
    "            \n",
    "            # 5. print out the loss of current epoch\n",
    "            print('loss at epoch {}: {}'.format(i, loss_hist[i]))\n",
    "            \n",
    "        return loss_hist\n",
    "   \n",
    "    # Task 3.6: implement the backpropagation algorithm.\n",
    "    def backward_prop(self, x, y, y_hat):\n",
    "        ''' Back-propagation algorithm. The implementation should be able to calculte\n",
    "            gradients for a neural network with at least 3 layers.\n",
    "            \n",
    "            parameters:\n",
    "            x: input samples (n_samples * n_features)\n",
    "            y: one-hot targets (n_samples * 10)\n",
    "            y_hat: output of forward pass\n",
    "            \n",
    "        '''\n",
    "        \n",
    "        nLayers = len(self.layers)\n",
    "        m_batch = x.shape[0]\n",
    "        \n",
    "        grad = y_hat - y # initial gradient from backwards cross entropy and softmax\n",
    "        for i in reversed(range(nLayers)):\n",
    "            self.layers[i].gb = np.sum(grad, axis=0) / m_batch\n",
    "            if i != 0: # everything except the input layer\n",
    "                self.layers[i].gW = (grad.T @ self.layers[i - 1].output) / m_batch\n",
    "                grad = (grad @ self.layers[i].W) * dsigm(self.layers[i - 1].net)\n",
    "            else: # input layer\n",
    "                self.layers[i].gW = (grad.T @ x) / m_batch\n",
    "            \n",
    "    # Task 3.7: update all weights and bias.         \n",
    "    def update_weights(self, lr):\n",
    "        for layer in self.layers:\n",
    "            layer.W -= lr * layer.gW\n",
    "            layer.bias -= lr * layer.gb\n",
    "            \n",
    "    # Task 3.8: calculate the accuracy.        \n",
    "    def Acc(self, y, y_pred):\n",
    "        ''' accuracy\n",
    "        \n",
    "            parameters:\n",
    "                y: target: categorical values (0, 1, ...9). n_samples * 1\n",
    "                y_pred: prediction: 0,1,2, ..9. n_samples *1\n",
    "        '''\n",
    "        \n",
    "        return np.mean(y == y_pred) * 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss at epoch 0: 2.0534453125\n",
      "loss at epoch 1: 1.5758890625\n",
      "loss at epoch 2: 1.2862240885416667\n",
      "loss at epoch 3: 1.1388587239583334\n",
      "loss at epoch 4: 1.0743861979166667\n",
      "loss at epoch 5: 0.9799802083333333\n",
      "loss at epoch 6: 0.8630087890625\n",
      "loss at epoch 7: 0.8201807291666666\n",
      "loss at epoch 8: 0.8169571614583333\n",
      "loss at epoch 9: 0.805444140625\n",
      "loss at epoch 10: 0.7980760416666667\n",
      "loss at epoch 11: 0.7570899739583333\n",
      "loss at epoch 12: 0.7050514973958333\n",
      "loss at epoch 13: 0.6640656901041667\n",
      "loss at epoch 14: 0.6659076822916666\n",
      "loss at epoch 15: 0.6622234375\n",
      "loss at epoch 16: 0.6028166666666667\n",
      "loss at epoch 17: 0.6124876953125\n",
      "loss at epoch 18: 0.613869140625\n",
      "loss at epoch 19: 0.611566796875\n",
      "loss at epoch 20: 0.5876197265625\n",
      "loss at epoch 21: 0.6000538411458334\n",
      "loss at epoch 22: 0.5507783203125\n",
      "loss at epoch 23: 0.576106640625\n",
      "loss at epoch 24: 0.5452521158854167\n",
      "loss at epoch 25: 0.5862381510416667\n",
      "loss at epoch 26: 0.5480151692708334\n",
      "loss at epoch 27: 0.5498572916666666\n",
      "loss at epoch 28: 0.54156796875\n",
      "loss at epoch 29: 0.554923046875\n",
      "loss at epoch 30: 0.5259103841145834\n",
      "loss at epoch 31: 0.5070292317708334\n",
      "loss at epoch 32: 0.52268681640625\n",
      "loss at epoch 33: 0.5190025065104167\n",
      "loss at epoch 34: 0.5199236328125\n",
      "loss at epoch 35: 0.5038055338541667\n",
      "loss at epoch 36: 0.5061081705729167\n",
      "loss at epoch 37: 0.5116344075520833\n",
      "loss at epoch 38: 0.49873994140625\n",
      "loss at epoch 39: 0.5038056640625\n",
      "loss at epoch 40: 0.4918321614583333\n",
      "loss at epoch 41: 0.49827939453125\n",
      "loss at epoch 42: 0.50058203125\n",
      "loss at epoch 43: 0.48400338541666665\n",
      "loss at epoch 44: 0.4913717122395833\n",
      "loss at epoch 45: 0.5038056966145833\n",
      "loss at epoch 46: 0.481240234375\n",
      "loss at epoch 47: 0.481240234375\n",
      "loss at epoch 48: 0.4830823893229167\n",
      "loss at epoch 49: 0.48768753255208336\n",
      "train accuracy: 98.23%\n",
      "test accuracy: 97.17%\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAk8klEQVR4nO3deXxddZ3/8dfn3ux72ywt3dIlLWspUHaQsqM4gowbgqgj4gKOqKOjzvxGcWYcZ1xmcAGnKtuMbAoqSBUYAasspaF0x+77kqRb9u3efH5/3NsSShLSNicnyXk/H488bu49J+d+DtzmnfM938XcHRERia5Y2AWIiEi4FAQiIhGnIBARiTgFgYhIxCkIREQiLiPsAg5XaWmpV1ZWhl2GiMiw8sorr+x297Ketg27IKisrKS6ujrsMkREhhUz29zbNjUNiYhEnIJARCTiFAQiIhGnIBARiTgFgYhIxCkIREQiTkEgIhJxkQmCv+xq4NtP/oX6ls6wSxERGVIiEwSb97Two2fXs3VfS9iliIgMKZEJgoqiHAB21beFXImIyNASmSAYmw6CmkYFgYhId5EJgtKCLGIGNboiEBF5g8CCwMwmmtmzZrbKzFaa2Wd72MfM7Ptmts7MlpnZqUHVkxGPUVqQTU1De1BvISIyLAU5+2gC+IK7LzazQuAVM3va3Vd12+ftQFX660zgzvRjICqKctjVoCsCEZHuArsicPed7r44/X0j8Bow/pDdrgLu85SXgBIzGxdUTRVFOdQoCERE3mBQ7hGYWSVwCrDwkE3jga3dnm/jzWGBmd1kZtVmVl1XV3fEdVQUZSsIREQOEXgQmFkB8Ahwq7s3HMkx3H2eu89x9zllZT0usNMvY4ty2NfSSXsiecTHEBEZaQINAjPLJBUCP3f3R3vYZTswsdvzCenXAnFgLEGtbhiLiBwUZK8hA34GvObu3+tlt8eAG9K9h84C6t19Z1A1VRSnxxKoeUhE5KAgew2dC3wIWG5mS9KvfRWYBODuPwbmA+8A1gEtwEcDrIeKomwA9RwSEekmsCBw9z8D9hb7OHBzUDUc6uDoYjUNiYgcFJmRxQDFuZlkZcTUNCQi0k2kgsDMGKuxBCIibxCpIIDUfQLNQCoi8roIBkEOtY26RyAickAkg2BXfRup+9QiIhK5IBhblENrZ5LG9kTYpYiIDAmRC4Ly9FgCrUsgIpISuSDQWAIRkTeKXBAcXLtYXUhFRIAIB4HGEoiIpEQuCHKz4hTlZCgIRETSIhcEAGOLNbpYROSASAZBau1i3SwWEYEIB0GtrghERIDIBkE2tY3tJLs0ulhEJJJBMLYoh2SXs6dZzUMiIkEuVXmXmdWa2Ypetheb2eNmttTMVppZoKuTdVd+oAtpvYJARCTIK4J7gCv62H4zsMrdTwbmAt81s6wA6zlorMYSiIgcFFgQuPsCYG9fuwCF6UXuC9L7DspMcBpdLCLyujDvEfwQOA7YASwHPuvuXT3taGY3mVm1mVXX1dUd9RuXFmQRM9RzSESEcIPgcmAJcAwwG/ihmRX1tKO7z3P3Oe4+p6ys7KjfOCMeo7QgW1cEIiKEGwQfBR71lHXARuDYwXrziqIczUAqIkK4QbAFuBjAzCqAmcCGwXrzCi1iLyICQEZQBzazB0j1Bio1s23A14BMAHf/MfDPwD1mthww4O/dfXdQ9RyqoiibVzb3dS9bRCQaAgsCd7/2LbbvAC4L6v3fytiiHPa1dNLWmSQnMx5WGSIioYvkyGJ4vQtpXaPuE4hItEU3CIo1lkBEBKIcBAcWsVcQiEjERTYIDkwzsateQSAi0RbZICjOzSQrI0at7hGISMRFNgjMjLFFOboiEJHIi2wQQOo+ge4RiEjURTwINLpYRERB0NCOu5asFJHoinQQjC3KobUzSUPboCyDICIyJEU6CMrTYwm0LoGIRFmkg2CsVioTEYl2EFQcXLtYYwlEJLoUBGiaCRGJtkgHQW5WnKKcDAWBiERapIMAYGyxRheLSLQFFgRmdpeZ1ZrZij72mWtmS8xspZn9Maha+lJRlEON5hsSkQgL8orgHuCK3jaaWQlwB/Audz8BeG+AtfSqoiiHGl0RiEiEBRYE7r4A6GtR4A8Cj7r7lvT+tUHV0peKomzqmtpJdml0sYhEU5j3CGYAo8zsOTN7xcxu6G1HM7vJzKrNrLqurm5AixhblEOyy9nTpOYhEYmmMIMgAzgNuBK4HPh/Zjajpx3dfZ67z3H3OWVlZQNaRLnGEohIxIUZBNuAJ9292d13AwuAkwe7CI0uFpGoCzMIfgOcZ2YZZpYHnAm8NthFjCtJBcG2fS2D/dYiIkNCRlAHNrMHgLlAqZltA74GZAK4+4/d/TUz+z2wDOgCfuruvXY1DUpZQTYleZmsqWka7LcWERkSAgsCd7+2H/t8G/h2UDX0h5kxo6KQNTWNYZYhIhKayI8sBphRUcCamkYtUCMikaQgAGZWFNLYltANYxGJJAUBMKOiEIDVu9Q8JCLRoyDg9SBYqxvGIhJBCgJgVH4WZYXZrNYNYxGJIAVB2kz1HBKRiFIQpFVVFLC2pokuTT4nIhGjIEibWVFIa2eSbftawy5FRGRQKQjSqg70HFLzkIhEjIIgbUZFAYDuE4hI5CgI0gpzMhlfkqsgEJHIURB0U1VRoMnnRCRyFATdzKwoZH1tE4lkV9iliIgMGgVBNzMqCulIdrFpj9YmEJHoUBB08/pUE7pPICLRoSDoZnp5AWbqQioi0RJYEJjZXWZWa2Z9rjpmZqebWcLM3hNULf2VmxVn8ug89RwSkUgJ8orgHuCKvnYwszjw78BTAdZxWKoqCtVzSEQiJbAgcPcFwN632O0zwCNAbVB1HK6ZFYVs3N1MeyIZdikiIoMitHsEZjYeeDdwZz/2vcnMqs2suq6uLtC6ZowtJNnlbKhrDvR9RESGijBvFv8X8Pfu/pad9t19nrvPcfc5ZWVlgRalqSZEJGoyQnzvOcCDZgZQCrzDzBLu/usQa2JqaQEZMVMQiEhkhBYE7j7lwPdmdg/w27BDACArI8aU0nxW79INYxGJhsCCwMweAOYCpWa2DfgakAng7j8O6n0HwoyKQlbsqA+7DBGRQRFYELj7tYex70eCquNIzKgoZP6KnbR0JMjLCrP1TEQkeBpZ3IOZYwtwh3W1ah4SkZFPQdCDA6uVaWCZiESBgqAHk0fnkZURU88hEYmEfgWBmeWbWSz9/Qwze5eZZQZbWngy4jGmlxUoCEQkEvp7RbAAyEmPBn4K+BCpuYRGrBkVBazZpSAQkZGvv0Fg7t4CXAPc4e7vBU4IrqzwzRhbyI76NhraOsMuRUQkUP0OAjM7G7gOeCL9WjyYkoaGmQcXqdENYxEZ2fobBLcCXwF+5e4rzWwq8GxgVQ0BMw72HFLzkIiMbP0aLeXufwT+CJC+abzb3f82yMLCNr4kl7ysOCs1wlhERrj+9hq638yKzCwfWAGsMrMvBltauGIx49zppfzfqlq6ujzsckREAtPfpqHj3b0BuBr4HTCFVM+hEe2ds8axq6GNxVv2hV2KiEhg+hsEmelxA1cDj7l7JzDi/0y++LgKsjJi/HbZzrBLEREJTH+D4L+BTUA+sMDMJgMNQRU1VBRkZzB3Rhnzl+9U85CIjFj9CgJ3/767j3f3d3jKZuDCgGsbEq6cNY7axnaqN6t5SERGpv7eLC42s+8dWDfYzL5L6upgxLv4uAqyM2LMX67mIREZmfrbNHQX0Ai8L/3VANwdVFFDSUF2BhfOLGf+8p0k1TwkIiNQf4Ngmrt/zd03pL9uA6b29QNmdpeZ1ZrZil62X2dmy8xsuZm9YGYnH27xg+Vg89CmvWGXIiIy4PobBK1mdt6BJ2Z2LtD6Fj9zD3BFH9s3Ahe4+0nAPwPz+lnLoLvo2HJyMmM8oeYhERmB+hsEnwR+ZGabzGwT8EPgE339gLsvAHr9E9rdX3D3A3dgXwIm9LOWQZd/sHlol5qHRGTE6W+voaXufjIwC5jl7qcAFw1gHR8jNVBtyLpy1jh2N7WzSM1DIjLCHNYKZe7ekB5hDPD5gSjAzC4kFQR/38c+Nx3osVRXVzcQb3vYDjYPaXCZiIwwR7NUpR3tm5vZLOCnwFXuvqe3/dx9nrvPcfc5ZWVlR/u2RyQvK4OLj63gdyvUe0hERpajCYKj+m1oZpOAR4EPufuaoznWYEk1D3WwcGOvmSUiMuz0OQ21mTXS8y98A3Lf4mcfAOYCpWa2DfgakAng7j8G/gkYA9xhZgAJd59zmPUPqgtnlpObGeeJZTs5Z1pp2OWIiAyIPoPA3QuP9MDufu1bbL8RuPFIjx+G3Kw4Fx1XzpMrd3Hbu04gI340F1QiIkODfpMdpneelGoeenmjeg+JyMigIDhMc2eWk5cV53H1HhKREUJBcJhys+K8c9Y4flG9lVc0I6mIjAAKgiPwD1cez7iSHG65fzF7mtrDLkdE5KgoCI5AcW4md153GnuaO7j1oSUaVyAiw5qC4AidOL6Y2951An9au5sfPLM27HJERI6YguAofOD0ifz1qRO4/Q9r+eOacKa+EBE5WgqCo2Bm/MvVJzKzopBbH3yVHfvfamZuEZGhR0FwlHKz4txx3al0Jp2b719MR6Ir7JJERA6LgmAATC0r4N//ehavbtnPN+e/FnY5IiKHRUEwQK6cNY6PnFPJPS9sYtm2/WGXIyLSbwqCAfSFy2ZQkpfJd54aFpOpiogACoIBVZiTyacumMaCNXUs3KCpqkVkeFAQDLAbzq6kvDCb7zy1GncNNBORoU9BMMBys+J85uIqFm3ax3MaWyAiw4CCIADvnzORiaNz+c6Tq+nS9BMiMsQpCAKQlRHjc5fMYOWOBn63YlfY5YiI9CmwIDCzu8ys1sxW9LLdzOz7ZrbOzJaZ2alB1RKGq2aPp6q8gO8+vZpEUoPMRGToCvKK4B7gij62vx2oSn/dBNwZYC2DLh4zvnDZTDbUNfPoq9vDLkdEpFeBBYG7LwD6Ws/xKuA+T3kJKDGzcUHVE4bLT6hg1oRibv+/tbQnkmGXIyLSozDvEYwHtnZ7vi392puY2U1mVm1m1XV1w6cnjpnxxctnsn1/Kw8s3BJ2OSIiPRoWN4vdfZ67z3H3OWVlZWGXc1jOm17KWVNH88Nn19HSkQi7HBGRNwkzCLYDE7s9n5B+bUQxS90r2N3UwSOLR9zpicgIEGYQPAbckO49dBZQ7+47Q6wnMHMmj+Kk8cXc98ImjTYWkSEnyO6jDwAvAjPNbJuZfczMPmlmn0zvMh/YAKwDfgJ8OqhawmZm3HD2ZNbWNvHies1BJCJDS0ZQB3b3a99iuwM3B/X+Q81fnXwM35z/Gve+uIlzppeGXY6IyEHD4mbxSJCTGecDZ0zi6VU1bNeSliIyhCgIBtF1Z04C4OcvbQ65EhGR1ykIBtGEUXlcclwFDy7aSlunBpiJyNCgIBhkHz6nkr3NHTyxbER2kBKRYUhBMMjOmTaGaWX53PfiprBLEREBFASDzsz48DmVLN1Wz5Kt+8MuR0REQRCGa06dQEF2Bve9sCnsUkREFARhKMjO4D2nTeC3y3ayu6k97HJEJOIUBCG5/qzJdCS7eGjR1rfeWUQkQAqCkEwvL+D8qlL+96XNWsFMREKlIAjRDWdXsrO+jf/RADMRCZGCIEQXHVvO+VWl3Pb4Kr739BrNTCoioVAQhCgeM+76yOm897QJfP8Pa/nCw0vpSKiZSEQGV2Czj0r/ZMZj/Md7ZjF5TB7feWoNO+pb+e/r51Cclxl2aSISEboiGALMjFsuquK/3j+bxZv3c82dz7N1b0vYZYlIROiKYAi5+pTxjC3O4RP/8wrvvuN5br5wOqPzsyjKzaQ4/VWUk8movEwy4spwERkYFuQNSjO7ArgdiAM/dfdvHbJ9EnAvUJLe58vuPr+vY86ZM8erq6uDKXiIWFfbxI33LmLTnp6vCnIz48yeWMLplaOYUzmaUyaVUJijpiQR6Z2ZveLuc3rcFlQQmFkcWANcCmwDFgHXuvuqbvvMA1519zvN7HhgvrtX9nXcKAQBQLLL2dPcTkNrgvrWThpaO6lPf23c3Uz15r2s2tFAl0PM4NixRZwyqYTp5QVMKc1namkB40flEo9Z2KciIkNAX0EQZNPQGcA6d9+QLuJB4CpgVbd9HChKf18M7AiwnmElHjPKC3MoL+x9n6b2BEu27GfRpr28snkfjy3dQWNb4uD2rHiMyWPymFFRyOcuncH08oJBqFxEhpsgg2A80H3+hG3AmYfs83XgKTP7DJAPXNLTgczsJuAmgEmTJg14ocNVQXYG51WVcl5Vag1kd2dPcwcbdzezoa6JDbub2VjXzJ/X7WbB2jp++MFTuWBGWchVi8hQE/bN4muBe9z9u2Z2NvA/Znaiu7+hM727zwPmQappKIQ6hwUzo7Qgm9KCbE6vHH3w9W37Wrjx3mo+evfL/OOVx/PRcysxU5ORiKQE2fVkOzCx2/MJ6de6+xjwMIC7vwjkAKUB1hRJE0bl8cinzuGS4yr4xm9X8ZVHl2vgmogcFGQQLAKqzGyKmWUBHwAeO2SfLcDFAGZ2HKkgqAuwpsjKz87gx9efxi0XTufBRVu5/mcL2dvcEXZZIjIEBBYE7p4AbgGeBF4DHnb3lWb2DTN7V3q3LwAfN7OlwAPAR1wT7gQmFjP+7vKZ3P6B2SzZup+rfvRn1tU2hV2WiIQs0HEEQYhK99GgLdm6nxvvrSZm8ItPns3kMflhlyQiAeqr+6iGp0bU7Ikl3P/xM+lMdvHBnyxkZ31r2CWJSEgUBBE2o6KQ+/7mTBpaO7nupwu1bKZIRCkIIu6kCcXc9dHT2bG/lQ/97GXqWzrDLklEBpmCQDi9cjQ/uWEO62ub+PDdL9PUnnjrHxKREUNBIACcX1XGDz94Csu313PjvYto60yGXZKIDBIFgRx02Qlj+d77Tmbhxr38zT2L1EwkEhEKAnmDq2aP57vvPZlFm/Zy9R3Pa5yBSAQoCORNrjl1Avd//CwaWjt59x3Ps2CNBnuLjGQKAunR6ZWj+fXN5zK+JJeP3P0ydz+/keE2+FBE+kdBIL2aODo1Wd3Fx1Vw2+Or+Oqv3jxZnbvTkeiitSOpoBAZpsKehlqGuPzsDP77+tP4zlOrueO59Tzzl1oyYjHaE0laO5K0JbpIdqUCICseoyQvk1F5WQcfK4qy+dTc6Ywtzgn5TESkNwoCeUuxmPGlK47lhGOKmb98J9mZMXIy4+RmxsnJjJGbGScWM+pbO9nf3Mm+lg72t3Syvq6JZ1bXsmjTPn75qbPJy9LHTWQo0r9M6bcrZ43jylnjDutnnl1dy8fuWcTnH1rKHdedSkxrKIsMObpHIIG6cGY5X33Hcfx+5S7+8//WhF2OiPRAVwQSuI+dN4W1NU384Jl1TC8v4KrZ48MuSUS60RWBBM7M+OerT+SMytF88ZfLWLJ1f9gliUg3gS5MY2ZXALcDceCn7v6tHvZ5H/B1wIGl7v7Bvo6phWmGrz1N7Vz1o+fpSHTx2C3nvaknUX1rJ6t2NFDT0EZHoov2ZBcdide/HCcvK05eVgb52enHrAxG52dx3LhCzHT/QaQ3fS1ME1gQmFkcWANcCmwjtYbxte6+qts+VaQWr7/I3feZWbm71/Z1XAXB8LZ6VyPX3PE8U8ry+bvLZrJqZwMrtzewYkc9m/e0HPFxT55QzC0XVXHJceUKBJEehBUEZwNfd/fL08+/AuDu/9Ztn/8A1rj7T/t7XAXB8PeH12q48b5qDnz0Jo7O5aTxxZxwTDEnji9mfEku2Rmx9FecrIwYWRmpVszWziQt7QmaO5I0tydo6UiyuqaReQvWs3VvK8eOLeSWi6bz9hPHEe/WQymR7GLljgZe2rCHxVv2MXdmOdeeMSmM0xcJRV9BEOTN4vHA1m7PtwFnHrLPDAAze55U89HX3f33hx7IzG4CbgKYNEn/eIe7i4+r4LGbz6OxrZMTjimmOC+z3z9bkJ1BQfYbP7ZnTBnNtadP5LGlO/jRs+u45f5XmVq2ho+fP5WmtgQvbtjDoo17aUyvszAmP4snV9bQ1Jbg42+bOqDndoC768pEho2wew1lAFXAXGACsMDMTnL3/d13cvd5wDxIXREMco0SgJMmFA/o8TLiMa45dQJXzR7P71fs4gfPrOUrjy4HYGppPn81+xjOmjqGs6aOZlReFrc+uIR/nf8ajnPT26YNaC1/XFPH5x5awiXHlfONq04kJzM+oMcXGWhBBsF2YGK35xPSr3W3DVjo7p3ARjNbQyoYFgVYl4xg8Zhx5axxvOOksSzdVs/Yopwep7e4/QOzAfjm/L8A9BoGrR1J7ntxE6trGvnUBdOoqijs9b3dnbuf38S/PLGKccW5PFy9jeXbG/jx9acyeUz+0Z+cSECCDIJFQJWZTSEVAB8ADu0R9GvgWuBuMysl1VS0IcCaJCLMjNkTS3rdnhGPpcLAUmHgDp+44PUwaOtMcv/CLdzx3Hp2N7WTnRHj8aU7+OQF07j5wulv+iu/I9HFP/1mBQ8u2splx1fwn++fzcsb93LrQ0t45w/+zPfeN5tLj68I6GxFjk5gQeDuCTO7BXiSVPv/Xe6+0sy+AVS7+2PpbZeZ2SogCXzR3fcEVZNIdxnxGLe/fzYG/NvvUlcGHz13Cg9Xb+WHz6xjV0MbZ00dzZ3Xn8qU0ny++cRr/OCZdTy+dAf/cvVJnFdVCsDe5g4++b+v8PLGvdxy4XQ+f+kMYjHjwmPL+e1nzuPTP1/Mx++r5tNzp/H5S2eQER+Y4TttnUk1O8mACHQcQRDUa0gGWiLZxeceXsrjS3dQWpDN7qZ2Tps8ii9cOoNzppe+Yd/n1+3mH361nE17Wrh69jG8//RJfOmRpdQ0tPPt98zqcdR0W2eS2x5fxQMvb+GcaWP49NzpdCSTtHV2pWdwTc3kmpMZp6q8gBkVhYzKz3rTcZraE7y0fg9/XrebBWvr2FDXzJUnjeOrVx7H+JLcwP77yMgQSvfRoCgIJAiJZBf/8KsVrK9r4paLpnPBjLJee/20dSa547n13PncOjqTTllhNj+5YU6fTVEAv6jeyj/+egXth6zp0JPSgux0KBRQlJvJSxv28OqW/SS6nNzMOGdOHc3k0Xk8VJ3qmPfpudO56W1TdYUgvVIQiARgXW0jv6jexkfOrWRccf/+It++v5Ute1rIzXp9Cu+c9Fdze4I1NY2srWlKPdY2sbamkZbOJCeNL+a86aWcV1XKaZNHkZ2R+oW/bV8L/zb/LzyxfCcTRuXyj1cez+UnVKjrqryJgkBkmHJ32jq7yM3q+y/9F9bv5rbHVrG6ppHzq0q59ZIqTp00ql+B4O4kupzMAbp3IUOTgkAkAhLJLv73pc187+k1NLQlmFaWz3vnTOSaU8dTXvjGLrTJLmfxln08uWIXT62qYcveFvKy4m9YXW5UfhaFORl0dTmdSSfZ1UVnl5NIppq25kwezRUnjmXi6LwwTlcOk4JAJEKa2hPMX7aTh6u3Ur15H/GYceHMMt47ZyJZGTGeWrmLp1fVsLupg6x4jHOmj2H2xBIa2xLsa+lgX3MH+1o62d/SQVN7gpgZmfEYGXEjI2ZkxGJ0JLvYuLsZgFkTirnixLG8/cRxTCntfbxEeyJJTX07O+pb2VXfxo76VnbubyPpzhmVozln2hjKi7SkaVAUBCIRtb6uiV9Ub+ORxduoa2wHID8rztxjy7n8hLFcOLOMwpz+T/HR3ZY9LfxuxU7mr9jF0vTU4seOLeSYktyD80A1dyRoaU89NrYl3nSM4txMutwPbqsqL+Dc6aWcM20MsyeV0NSWoKahndrGNnbVt1HT0M6e5namlhZwxpTRnDKppF83yN2d1s4kze1JWjoSBx8z4zFOHF/8hnmpDkddYzsLN+7h5Y17qSjK4YNnTOqxx9dQoCAQibhEsos/r9uNO5w9bcyA9y7avr+V36/YxdOrdtHcniQvK05+dkbqMSuD3HSz07iSHMYV5zCuOJdxxTnkZ2eQ7HJW7Wjg+fW7eWF9al6o1s5kj++TmxlndH4WO+pbcYeseIyTJxZzxpTRnF45mqyMGFv3trB1bytb97WwJf39nuZ2evtVV16YzTtOSi3DetqkUb0up+ru1DW1U71pHy+u38NLG/awtrbpYF2tnUlyM+O8b84Ebjx/6pBrMlMQiMiw0Z5IsmTLflbuaGBUfiYVhTmUF+VQUZRNQXYGZkZ9ayevbN7Lwo17eXnjXpZvqyfR9frvsnjMOKYkh4mj8pg4Ko+KomzysjPIP2Q9i30tHcxfvpNnV9fRkeiioigVChcdW87+lk427m5m4+5mNuxuZmNdEw3pK5e8rDinV47mrKljOHvaGE48pogNu5v5yYIN/HrJdpJdzttPGscn3jaVWRNK+nXe7k5HMjW2ZHdTB3WN7dQ1tace019zZ5bxVycfc0T/XRUEIjKitXQkWLJlPwATR+cxrjjnsEZwN7Un+MNrNTyxbCfPrUmFwgHjS3KZUpp/8Gv2pBJOGl/cay+rmoY27n5+Ez9fuJnGtgSVY/J63DfpTntnF22dSVo7k7R1Junq5ddxZtwoK8jmw+dUvmEqlMOhIBAR6afGtk4Wb9lPeWE2lWPy37Lrbl/HeWjRVhZv2dfjdjMjJyNOblYs/ZgaT5KbGWdMQRZlhdmUFWRTVphNcW7mUY8NCWs9AhGRYacwJ5MLZpQNyHFuPD+Y9S4GmkaQiIhEnIJARCTiFAQiIhGnIBARiTgFgYhIxCkIREQiTkEgIhJxCgIRkYgbdiOLzawO2HyEP14K7B7AcoaTqJ67zjtadN69m+zuPY6UG3ZBcDTMrLq3IdYjXVTPXecdLTrvI6OmIRGRiFMQiIhEXNSCYF7YBYQoqueu844WnfcRiNQ9AhERebOoXRGIiMghFAQiIhEXmSAwsyvMbLWZrTOzL4ddT1DM7C4zqzWzFd1eG21mT5vZ2vTjqDBrDIKZTTSzZ81slZmtNLPPpl8f0eduZjlm9rKZLU2f923p16eY2cL05/0hM8sKu9YgmFnczF41s9+mn4/48zazTWa23MyWmFl1+rWj+pxHIgjMLA78CHg7cDxwrZkdH25VgbkHuOKQ174M/MHdq4A/pJ+PNAngC+5+PHAWcHP6//FIP/d24CJ3PxmYDVxhZmcB/w78p7tPB/YBHwuvxEB9Fnit2/OonPeF7j6729iBo/qcRyIIgDOAde6+wd07gAeBq0KuKRDuvgDYe8jLVwH3pr+/F7h6MGsaDO6+090Xp79vJPXLYTwj/Nw9pSn9NDP95cBFwC/Tr4+48wYwswnAlcBP08+NCJx3L47qcx6VIBgPbO32fFv6taiocPed6e93ARVhFhM0M6sETgEWEoFzTzePLAFqgaeB9cB+d0+kdxmpn/f/Ar4EdKWfjyEa5+3AU2b2ipndlH7tqD7nWrw+YtzdzWzE9hk2swLgEeBWd29I/ZGYMlLP3d2TwGwzKwF+BRwbbkXBM7N3ArXu/oqZzQ25nMF2nrtvN7Ny4Gkz+0v3jUfyOY/KFcF2YGK35xPSr0VFjZmNA0g/1oZcTyDMLJNUCPzc3R9NvxyJcwdw9/3As8DZQImZHfhDbyR+3s8F3mVmm0g19V4E3M7IP2/cfXv6sZZU8J/BUX7OoxIEi4CqdI+CLOADwGMh1zSYHgM+nP7+w8BvQqwlEOn24Z8Br7n797ptGtHnbmZl6SsBzCwXuJTU/ZFngfekdxtx5+3uX3H3Ce5eSerf8zPufh0j/LzNLN/MCg98D1wGrOAoP+eRGVlsZu8g1aYYB+5y938Nt6JgmNkDwFxS09LWAF8Dfg08DEwiNYX3+9z90BvKw5qZnQf8CVjO623GXyV1n2DEnruZzSJ1czBO6g+7h939G2Y2ldRfyqOBV4Hr3b09vEqDk24a+jt3f+dIP+/0+f0q/TQDuN/d/9XMxnAUn/PIBIGIiPQsKk1DIiLSCwWBiEjEKQhERCJOQSAiEnEKAhGRiFMQSOSYWVP6sdLMPjjAx/7qIc9fGMjjiwRBQSBRVgkcVhB0G7XamzcEgbufc5g1iQw6BYFE2beA89Pzun8uPXnbt81skZktM7NPQGrAkpn9ycweA1alX/t1etKvlQcm/jKzbwG56eP9PP3agasPSx97RXou+fd3O/ZzZvZLM/uLmf08PUoaM/uWpdZXWGZm3xn0/zoSGZp0TqLsy6RHpAKkf6HXu/vpZpYNPG9mT6X3PRU40d03pp//jbvvTU/rsMjMHnH3L5vZLe4+u4f3uobUegEnkxr1vcjMFqS3nQKcAOwAngfONbPXgHcDx6YnESsZ2FMXeZ2uCERedxlwQ3pK54WkpjWuSm97uVsIAPytmS0FXiI1oWEVfTsPeMDdk+5eA/wROL3bsbe5exewhFSTVT3QBvzMzK4BWo7y3ER6pSAQeZ0Bn0mv/DTb3ae4+4ErguaDO6XmtrkEODu9MtirQM5RvG/3uXCSQEZ6Tv0zSC2y8k7g90dxfJE+KQgkyhqBwm7PnwQ+lZ7OGjObkZ7h8VDFwD53bzGzY0ktjXlA54GfP8SfgPen70OUAW8DXu6tsPS6CsXuPh/4HKkmJZFA6B6BRNkyIJlu4rmH1Hz2lcDi9A3bOnpe8u/3wCfT7firSTUPHTAPWGZmi9PTIh/wK1LrBCwltcLUl9x9VzpIelII/MbMckhdqXz+iM5QpB80+6iISMSpaUhEJOIUBCIiEacgEBGJOAWBiEjEKQhERCJOQSAiEnEKAhGRiPv/BwAoB2OuvRAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Task 4: Evaluation.\n",
    "   \n",
    "# 1. create a one-hidden layer NN.\n",
    "np.random.seed(22)\n",
    "nn = NeuralNetwork()\n",
    "nn.add(input_dim=784, units=128, activation='relu')\n",
    "nn.add(input_dim=128, units=10, activation='softmax')\n",
    "\n",
    "# 2. train the NN.\n",
    "x_train, y_train, x_test, y_test = load_digits(show_sample=False)\n",
    "y_train_onehot = keras.utils.to_categorical(y_train)\n",
    "\n",
    "epochs = 50\n",
    "loss_hist = nn.train(x_train, y_train_onehot, epochs=epochs)\n",
    "\n",
    "# 3. calculate and print out the test and training accuracy. 10 points\n",
    "\n",
    "train_acc = nn.Acc(nn.predict_class(x_train), y_train)\n",
    "test_acc = nn.Acc(nn.predict_class(x_test), y_test)\n",
    "print('train accuracy: {0:.2f}%'.format(train_acc))\n",
    "print('test accuracy: {0:.2f}%'.format(test_acc))\n",
    "\n",
    "plt.plot(range(epochs), loss_hist)\n",
    "plt.xlabel('Iterations')\n",
    "plt.ylabel('Loss')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}