MLP_Fmnist_Saver_optimizer.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/AnkurMali/IST597_Spring_2022/blob/main/MLP_Fmnist_Saver_optimizer.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "_iYcla4kCX67"
      },
      "outputs": [],
      "source": [
        "import os\n",
        "import numpy as np\n",
        "import time\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "\n",
        "np.random.seed(1234)\n",
        "tf.random.set_seed(1234)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Kgna3kY6CX67",
        "outputId": "db4d9f30-024f-4fcc-c284-a06acb6ad281"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[]"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "tf.config.list_physical_devices('GPU')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 383
        },
        "id": "JodgHy9nCX68",
        "outputId": "3d423aa6-4349-46d8-d995-7013c9e84450"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-labels-idx1-ubyte.gz\n",
            "32768/29515 [=================================] - 0s 0us/step\n",
            "40960/29515 [=========================================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/train-images-idx3-ubyte.gz\n",
            "26427392/26421880 [==============================] - 0s 0us/step\n",
            "26435584/26421880 [==============================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-labels-idx1-ubyte.gz\n",
            "16384/5148 [===============================================================================================] - 0s 0us/step\n",
            "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/t10k-images-idx3-ubyte.gz\n",
            "4423680/4422102 [==============================] - 0s 0us/step\n",
            "4431872/4422102 [==============================] - 0s 0us/step\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAasAAACWCAYAAABggqeqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2de5xdRZXvv4uQEEh4JAFCyIMQEghJBBSE8MYBBBwZ4OPjDowCBgedqwOo1+fMcBlHHOYqMvi5zDh4A0FR0MtTRUGSgAyYy/AQSQAhISSYkBACeUACQqDuH7v6pGp1n336pE937+78vp9Pf7rWrv2ovfc6VbvWqlplIQSEEEKIKrNNbxdACCGEaIQaKyGEEJVHjZUQQojKo8ZKCCFE5VFjJYQQovKosRJCCFF5KtFYmdklZnZ9b5ejqpjZvWb2yd4uR5WQzpQjncmRvpTTF/SlZY2VmS0xsxNadb6uYma7m9kNZvaCma0zswfM7LAk/2tm9lry97qZvWNmu7rzDDezl8zs/mTbFDN72MzWxL/ZZjYlyd/FzK4zs1Xx75IGZR0Uf0wLzWxDfJbXmNn4lj2QLcDMLjaz0F3vtWo6A7UyvZ7oxa+TPDOzb5jZ8qhT95rZ1CR/u/je1pvZSjP7vDv3Dmb2b2a2Oh5/X5L3K6ePb5rZ/JJyVkZnzGx81JO0/P/QDdepor7cE+uH9Wb2ezM7rc5+18RnNDHZNtzMbo3vb6mZneWO+Vszey6e+2EzO8rlv8fM7ovP+0Uzu7CknJXRl1ieT5rZolj2O81sz0bHVKJn1U0MBR4CDgaGA9cBd5jZUIAQwjdDCEPb/oB/Ae4NIax25/kX4Cm37QXgw/G8uwI/A25M8q8AdgDGA4cCHzezT5SU9SbgL4CzgJ2BA4FHgOObueFWYmb7AB8BVvRWGXqRUxPdeH+y/SPADOBoinc/D/hhkn8JMAnYC3gf8CUzOznJvzoet3/8/7m2jBDCKU4ffwv835IyVk5ngF2Se/inXixHT3IhMCqEsBNwPnC9mY1Kd4iNzD4dHHsV8CYwEvgr4N/bPn7ih/VlFPXMzsBM4FYzGxDzdwXuBP4DGAFMBH7tL5BQGX0xs+OAbwKnUfwOngNuaHhgCKElf8AS4ISYPhe4H/g2sCYW5pRk372B3wCvAncD/xu4PsmfTvFjXQv8Hjgubj8CWA2MjfKB8fyTO1nG9cDBHWw3YDFwjtt+BEWF9Ang/jrn3Bb4DLAx2bYaeG8ifw34zzrHnwC83nZPdfa5F/hkTO8DzAVejtf5EUUl0bbvl4Hl8dk+DRwftx8KPByfwYvAdxo8qzuBD6TvtdV/VdSZsvuNz/aniTwVeCORXwDen8j/BNwY05Pjs9+pE89lPPA2ML4v6EwsbwC27Q49qbK+uPIdCrwBHJps2xb4HXBAfEYT4/YhFA3Vvsm+PwQui+n/BvxXkjckHj8qyt8EftjJ51Y1ffk2cFUi7xnvbZ/S++hGRXoL+GtgAPA3FD9ki/nzgO8A2wHHxJu+PuaNjg/pAxQ9vxOjvFvMvzQ+yO2B+cBnO1m+g6Ii7dxB3jHAa8DQZNsA4FGKntm5dNBYRUXfBLwD/H2yfbVT2L8D1tQp12XAbxqUPVWkifGZbAfsBtwH/GvM2w/4I7BnlMe3KUB85h+P6aHA9JLrfQS43b/XVv9VUWdimV4EXqL4Uj0wyduL4mt0X2Ag8L+A22LeMIof3Mhk/w8D82P67HjtK6J+zAc+VKcMF1P08uuVsVI6w+bGajmwDLgW2HVr0Je4/y8o6pZA8ZG3TZL3ReDKmE4bq3eTfODGbf8D+HlM7xR17bB4f39L0ei13d9c4EqKBncV8HNgXB/Rl28D/5bIo+OzOa20jN2oSIuSvB1iYfYAxlFU8EOS/B8nivRl3BcDcBex10NRSTwSlejOtpfXoGw7xf2/Wid/JjDLbfsc8O/J/dTrWQ0B/jvw58m264FbgB3ji38W+FOd479P/PrujCJ1kHc68LtEyVZRfEkNdPvdB/wjDSqRWOaFxK96erax6nWdAY6kqKR2AL4KrCR+VQKDKCqIEMvzHLB3zBsbtw9OznUisCSmvxbzL4nnOZbiA2n/DsqwCDi3pIxV05mhwCEUvYiRFCanu7YGfUmOHwicAnw+2TY2vsudo5w2VkcDK905/pr4kUJh7fkaRYO8ifbWmmcoPpbfCwwGvgs80Ef05YR4PwdQ/Nb+g+KD/8yy47rTZ7WyLRFC2BiTQym6fGtCCBuSfZcm6b2Aj5jZ2rY/4ChgVDzXW8AsYBpweYh3Xw8z257iq+P/hRD+uYP8HSh6Etcl2/YELqDoEZUS7+N7wA/MbPe4+QKKbvdC4HYKe+yyOqd4ue3eOoOZjTSzG6OTfz1Fw7hrLMsi4CKKCnFV3K/NcXkeRY/gD2b2kJl9sM4lLqH4IS/pbJlaSK/rTAjhgRDC6yGEjVFf1lJULFD0eN5LUQkNpvhhzo069FrcZ6fkdDtRfNFDoQ9vAd8IIbwZQvgNcA+Q+sTa/Bt7UFT49aiUzoQQXgshPBxC2BRCeBH4LPB+M9uxs2XcQnpdX5LrvxVC+BXFff9F3PyvwNdDCOs6OOQ1cl2BXF/Oo3A/TKX4uPkY8Ivk3bwO3BpCeCiE8AaFLh5hZjt3cK2q6cts4H8CN1N8gCyJ912vjgR6Z4DFCmCYmQ1Jto1L0n+kqCx3Sf6GhBAuAzCz0RQ3ei1wuZltV+9CMe82iofwqTq7nQG8QvFl0cahFC/3STNbSfE1fWgc4TWgg3NsQ/FlNxoghPBKCOGvQgh7hBCmxvz/qnP92fHcY+rdh+ObFF9o7wqFU/djFF9hxGv/OIRwFMUPMlAMECGEsDCEcCawe9x2k3sHbRwPXBDvdSVFxfxTM/tyJ8vXHfSYznRAYPPzPQj4SQhhWayYZ1GY/6aEENbEch6YHHsg8ERMP17n3J5zgFtCCK91kNdG1XTG03ZfvTWAqzf1ZVs2D6Y4HvhW8lsCmBdH/T0DbGtmk5JjU305CPhFCOGZEMI7IYQ7430dEfMfJ9efsga1cvoSQrgqhDAphDCSotHaFlhQVqgeV6YQwlIKJ9w/xuGURwGnJrtcD5xqZieZ2QAzG2xmx5nZGDMzii+emRSt+AoKJ3Y7zGwgxdfp6xTd+3fqFOkc4Afu6+lXFLbYg+LfxRT24oNCCG+b2Ylm9u5Yvp0obONriKMGzWwfMxsR80+hGCX0jTrPYzaFA/hWMzvYzLY1sx3N7NNmNqODQ3ak+CpbF39UX0zueT8z+7P443oj3vs7Me9jZrZbfA5r4yEdPZPjKb4o2+79BYqG/qo6z6/b6UGdGWdmR8ZrDDazL1J8UT4Qd3mI4ot8pJltY2YfpzD/LIr5PwD+3syGmdlkCrPOrJh3H/A88NX4jo+kGDF4V3L97YGPJsfUex6V0hkzOyyeZxszG0Fhkrq3To+i2+lBfZlsZqeY2fZmNtDMPkbhH/tN3GVfigao7bdELMetsdd3C/B1MxsS9eE0No8ufQj4czObYAUnxvO1VejXAmeY2UGxrvsHCldFu2deQX0ZbGbT4n2Noxgle2X84KtPmY2wmT86GKnj8lN77QTgP+MD6WikzmEUL/wVCkf3HRRfRhdSjNwZFPfbM+Yf3UF5jo3X3Biv0/Z3dLLPaAp78MQG95bdD4XZ8A/xfG3lOyDJ/yhFJb8ReAw4qcH5B1F04xcBGyhMFv+H6DAld35OpbCnvxbP/QVgWcw7gKIH92p8dr9gsyP0egpb82sUX2+nN/teW/1XQZ2ZSvHFuoHCdDIHOCTJH0zRaK+gGPH0KHBykr8dcA2bR0N9voPzz4vnfxI4w+WfGd99Z3wkldGZWO7nYjlWUDTae2wF+rI/8GB8dmspGpgzSspfK1+Uh1NYfjZQfMicleQZ8PW4/VWKD+GPu/P9DcWgljUUro6y0X5V0pdd2Pw7Wwn8MzCg0ftvG1kihBBCVJb+PClYCCFEP0GNlRBCiMqjxkoIIUTl6VJjZWYnm9nTVgQk/EqrCiX6L9IZ0QzSF9HGFg+wsGK+0TMUs/SXUYyEOTOE8GTriif6E9IZ0QzSF5GybReOPZQi3MliADO7kWKeQF1FMrNKDz0splhsZvz48Zn8zjubpwwMGJDPDX777bcz+fnnn8/kPjDqcnUIYbduvkZTOlN1fdnKqZy+xH2kMxUlhGCN96pPVxqr0RQzwdtYRjF3odL4BiltRAYOHJjlXXrppZn82mubgwrsvHMe1WTDhg2Z/KlP5QEz3nrrrVp6223zx75p06ZGxe4Jljbepcv0SZ0RHSJ9ET1KVxqrTmFm51NEcBCiIdIX0SzSma2DrjRWyynixrUxJm7LCCFcTRFOQ1100VBnpC8iQXWMqNGVxuohYJKZ7U2hQH9JsQplpSnzHU2cODGTjz766Ez+05/+VEs/++yzWV5qIgS46aY8aPZpp21e7boiZr/eoE/qjOg1pC+ixhY3ViGETWb2WYpAnAOAa0IITzQ4TGzFSGdEM0hfREqXfFYhhF8Cv2xRWcRWgHRGNIP0RbTR7QMseoN0xF+jIeN77LFHLf2ud70ry3vooYcyecKECbX0qlWrsjx/7Pbbb5/JM2fOrKVvueWWLO+OO+4oLaMQon8ydOjQTJ46dWot/frrr2d548aNy+TddstnDqxdu7aW9iOOfT24cmVt3cp2LgzPK6+8Uku/+eabWd6gQYMy+Y9/3Dx4s9XTdRRuSQghROVRYyWEEKLyqLESQghRefqFz6osKoX3HZ1/fj53MB2uvmDBgizPH/v444/X0oMHD87yvDxv3rxMTu2+n/70p7O8T3ziE5l8xRVX1NIPPPAAQoi+S5kPPfWDA3zoQx+qpdPwbtB+as12222XyW+88UYt7cO/ed9Sem4/lcZfN/Wd+Xpt3bp1mbx+/fpaOvWhtQL1rIQQQlQeNVZCCCEqjxorIYQQlWeL17Paoou1KG5XmY/Kc9VVV2XysmXLMjn1JXk773ve855MHjFiRC3t/VmLFi3K5OXL8xBmO+20Uy3t52gdfPDBmTxlypRa2kdv//3vf5/Jzcwpa8AjIYRDunKCVqM4b5WmcvoC1dSZdM6T9w/tt99+mXz22WfX0q+++mqWN2TIkNLr7LjjjrV0usoDtK/b0iWO/HJHvoxpvj/PihUrMjmdZzV79uxa+tVXX2XTpk1dWiJEPSshhBCVR42VEEKIytMnh65vs03exvquaWque/nll7M8v0hiuuCiHwp69913Z/KYMWNqaT9U3Zsmhw8fnslpl/7000/P8jZu3JjJafiTCy+8MMubMWMGQoi+Q5l5/qWXXsrktD7yIZO87EM1pUPOfR3ph66nbgxfl6VD4CE3P/oQUN6MmdaDt912Wy3t6+gtQT0rIYQQlUeNlRBCiMqjxkoIIUTl6ZM+K+8f8qR2VO+H8kM6U9uu92e9733vy+Tnn3++w+Ogfbh+P3R95MiRtfS0adOyvLlz52Zyen+TJ0+mjJ6ceiCEaJ4yf433AaW//bIQSdDet5QOMffHetJVz/1Q9VGjRmVy6lebP39+lvfd7343k++666665+0q6lkJIYSoPGqshBBCVB41VkIIISpPn/RZNbKFpqH0/Twrb/f1cwxSvG9s5513rqW9Tdhfx8+dmjRpUi2dLikN7W3P6fwJf14hRLUp86l7H7P3oafzrPwS8t5PXhYWyddzZXNTvd/M+9hS2Z/nyiuvzOSLLrqI7kI9KyGEEJVHjZUQQojK02fMgM1EF99rr71q6TQKMOShjCA35/lI6mlEdsi7wN48l3bfITf7Aey99961dLqaJsDuu+9et4y77rprlrfvvvtm8jPPPIMQom/iTW6p+c67GrwJzpsQy87rSU2Gu+yyS5a3ePHiTE7rxT322CPLO/zwwzM5XUHikUceKS1Ds6hnJYQQovKosRJCCFF51FgJIYSoPP3CZ+Vtu+mKmWWhQ/yx3u/kh5TvsMMOtbT3b40dOzaT/SqfqX3Z257TVYQBnnrqqVra+7MmTJiQyfJZ9U28n2DPPffM5IULF9bSZb4JUT18/dTMUPY05Ftaj3VE2eq/zSzJ4cuQhmKCfGj7uHHjsjzvl2q1nypFPSshhBCVp2FjZWbXmNkqM1uQbBtuZneb2cL4f1j3FlP0JaQzohmkL6IzdKZnNQs42W37CjAnhDAJmBNlIdqYhXRGdJ5ZSF9EAxr6rEII95nZeLf5NOC4mL4OuBf4cgvL1RRpeCXIbcR+XkAaMglyH5b3HXnSOU/enzV69OhM9nMVUh+W90MtW7Ysk1O7tQ+5Mn369Ey+8847S8vcG/QFneltvB6my89AvoS5X+bGzxVM/ac+dE7qx/DX9X4zv0TOiy++2GHZW01/0xfvo0p91I18Sal/0r8775v3+V5OKQvN5HXGk85b9fi6d/z48bX0kiVLSs/bLFvqsxoZQlgR0yuBkWU7C4F0RjSH9EVkdHk0YAghmFndkBJmdj5wflevI/oPZTojfREe1TECtryxetHMRoUQVpjZKGBVvR1DCFcDVwOUKVwjfBThFB/aKO3W+m7qihUrMjkdIuxNht788oc//KGW3meffbK8dFg7wIgRIzJ59erVtXS6kjG0NxmmJiBvbvThlvoQndKZVulLq/A64If1llE2lNgPS/7whz+cyd/73vdqaW/2S6PyA4wZM6aWXrNmTZZ39NFHZ3Jq6vOmoVWr8lfSU2bAOvR4HdMq/FDwsrrLk/72G01ZKDMLen3z9VM6TcdP5znggAMyOZ0ec/rpp2d5fih7q01/KVtqBvwZcE5MnwPc3priiH6MdEY0g/RFZHRm6PoNwDxgPzNbZmbnAZcBJ5rZQuCEKAsBSGdEc0hfRGfozGjAM+tkHd/isoh+gnRGNIP0RXSGPhNuqQw/tDL1M/jhuUuXLs3kdGi4t+H74cVDhgyppf0w39QnBe2Hsi9fvryWbrQUSWq39mVK/ROiPs2Enikb+u39Den0Bf/OmxmyvGDBgkweOTIf7Jb6Yf30BR/K66STTqqlH3jggSzP+xCGDds8t9afx99Pir+3Rsv0iM00M3Q9fa7eP+plX7d5v1SK1+P0ffr6yP8G5s+fX0t7f/uXvvSlutdsNQq3JIQQovKosRJCCFF51FgJIYSoPP3CZ5Xa4SH3NaX+H4AXXnghk9P8wYMHZ3k+fElq5/VzHLy93y9NkoZY8vt638Hw4cNraW8/9vfa3ylbXiHF+1CaWSIhfcbTpk3L8m644YZMnjNnTi190UUXZXl+6ZfU39jI55Oe1/O5z30uk1O/GcBuu+1WS5977rlZ3gc/+MFMfvjhh+tep4yt3UfVzDIfnmbmWaVz6vy8T18feX1bu3Zt3X19fZX6Qf1vxfvQjzrqqFr6Jz/5SZb385//nJ5CPSshhBCVR42VEEKIytMnzYA+3IwfwpmaAX1X2Q//TPf13XV/bNq19uZFP5TdhzBJV4L1Q5FTsx/k5qN169ZleX4147SMzZgb+htlZrY0EjS0N9+lJtozz8yn/MyYMSOTr7322rplKFu5tWzoMLTXtfRcc+fOzfIOOeSQTP7+979fS3vTUTplohFlUbt9+bc2s2BP3e/KlStr6cmTJ2d5GzduzGT/vtLwTD5MmyfVP1+X+TBPaX17wgknZHmPPfZYJj/xxBOl1+0K6lkJIYSoPGqshBBCVB41VkIIISpPn/RZ+TBIfuXd1L7cTKh870fwttvURlyWB/Dcc89lcuqX8kOPffnXr19fS/tlGtKQT5CH6fHLn/QH0ndZ5gMq8ylMnTo1k9Oh3gCXXHJJLX3WWWeVliddouW4447L8n784x9nsl/ao4yyofY+HM7MmTPr7uvDiTVDM8P9xWb8b98/xzLd9HVO6h/y793Xe96HlR7rffNl9aC/jl8xPa2PLr/88tIydSfqWQkhhKg8aqyEEEJUHjVWQgghKk+f9Fn55Td8+KKxY8fW0mkIEmg/72TvvfeupVPbLLSf05T6ScrmOED7MElpvp8XduONN2byscceW0t7H5WX02fRH31WKVvqU7njjjtK5enTp9fSF198cZZ3xhlnZHIaruuRRx7J8g4//PBMvvvuuztdRu+7SOe+LF68uHTf1P/g5+b4Z5b6Obyvwu+bLrXj5+I8+OCDiIJGepn6pCdMmJDl+aVhUj/nb3/72yzPz8f0y8+n78vPz/T10csvv9zhcdA+/Fu6LIhf2sbrYiq3em6aelZCCCEqjxorIYQQlUeNlRBCiMrTJ31WPv6Zj6s2YsSIWtov8eD9Rakt3sfT2nHHHTM5nfNUFkcN2ttr16xZU0v72Ia/+93vMvmYY46ppVP/G7T3wfln0Z8YOHBgNict9S8+++yz2b5lfgM/b8QvzZ3qz+zZs7O8X//615mcPv+FCxdmef5dpDH8Jk6cmOWl87WgvT8i1T2/dI2/H+/HTPHHpnifrCddVmLVqlVZ3oknnlh6bG/SnX6TjvD65P1S6bv18wP9kkWpfh100EFZXlqHANx8882ZPGbMmFp6ypQpWV7ZEiHe1+3rmPRY/96b8ct2FfWshBBCVB41VkIIISpPnzQDepOJ7+KmXW1v1jnvvPMyOe3yehNh2dIjfsimD2+Smqz8sX4YsA9ZkpobfWgmH8KnzATU1xk0aFD2HK+44opa2ptO/fDb1Hzql2/x7yo1xTQahpzqnje1+LBZ6bt5+umnszyvs14nUh325mk/TSKVvZnJy+l1G91rel1vrqoyW2r6S9+BN7WmS/wA7LXXXrW0dxd4/UqHgvvfqzcZpu/Em/3233//TD711FMzOZ2Ws2zZsizPr1yeDm33vx2/b1ofHXrooVmezIBCCCFEghorIYQQlUeNlRBCiMrTJ31W3kbsl/1I7c2LFi3K8vxw3dSGX7a0uJfTpeehvZ3cXzddWt37EU455ZRMTpe29jbisiWn+xsbNmzIQs4cdthhtbS3s3vf3rBhw2ppvyRIOrUBcj+U90346zz55JO1tPebeb9O6gPx+pKGu4H2upbqqfeBeP1Pfav+3sqmWPjzelK/hp8C0hdJ/UzQPtRRqifep+hJwxf58EQ+jFXZsh/e953m+9+2L5MPeXXEEUd0WD6AV155JZPTOsjXe2Wh4/z0DP/7SHWq1VMIGvaszGysmd1jZk+a2RNmdmHcPtzM7jazhfH/sEbnEv0f6YtoFumM6AydMQNuAr4QQpgCTAc+Y2ZTgK8Ac0IIk4A5URZC+iKaRTojGtKwsQohrAghPBrTrwJPAaOB04Dr4m7XAad3VyFF30H6IppFOiM6Q1M+KzMbD7wbeBAYGUJoi9OxEhhZ57CW431UXp4zZ07dY73/IvUd+DlMZXZWb9dtVMbUlvv8889neePGjcvk2267rZY+++yzszxfRm8DrxLdqS/e3+KXfvGy6Bu0QmdSX+FJJ51US/vfrJ/fmOqUn5vnw1al/i+f55caSv2Gfo5o2RJA3r/u/Y/+t5/O5fNzRH29kd6rP4/3qaf1oC+Tlxv5QbtCpxsrMxsK3AxcFEJY75xnwcw6rNnN7Hzg/K4WVPQtpC+iWaQzooxODV03s4EUSvSjEMItcfOLZjYq5o8CVnV0bAjh6hDCISGEQzrKF/0P6YtoFumMaETDnpUVnzczgadCCN9Jsn4GnANcFv/f3i0l7AA/hNPLs2bNqnus73anQy99l9YP6Uy7w36osQ/P4iMXp0Or09VaoX1k9XRIqi9Do0jZvU0V9UVUm1bqzJAhQ5g2bVpNPuuss2ppH/LKDzlPf7NlUwkgN/35UFl++kNZ5Ht/bDrFwZeh7DyQh/vyx/pQYKlJ0ZsXvZxOpfGm1Ebh4FpJZ8yARwIfB+ab2WNx29coFOinZnYesBT4aPcUUfQxpC+iWaQzoiENG6sQwv2A1ck+vrXFEX0d6YtoFumM6AwKtySEEKLy9JlwS+nwykYh+dOQOH6oug9Dktp2va3W+4tS/LB276Pyx6bn9kNb0xU+/bn9MHe/9Eh3DhUVoq8xcODAzGd000031dKTJk3K9vW+pSOPPLKWbuTHSUcqlvmzIB8G7+sfH1LJ+7PrXRPah/BKQyH55UX8kiFpvef9UN6Xl57Ll3/q1KmZvGTJklq61as0q2clhBCi8qixEkIIUXnUWAkhhKg8fcZnlfqefJgUb0dNGT16dOm+ZWHsve029UM1ssd6O3BqX/a2Zh9mP+Wpp57K5AMPPDCT33zzzdJyCLE1MXjwYKZMmVKT09/W3Llzs339shnpb8nPf/Lh01IfuvdP+7mcqU/Lz3/yvrC0DH7fN954I5P9nKbU/+XzvE89vR9/b97nlpbRh4Yro8eXCBFCCCF6GzVWQgghKk+fMQOm3WNvNvvlL39Z9zjfbd1vv/0yOR367Yeg+sjFqWnPd/39cHq/EuyoUaNqab9CaZrnSYeCAixdujST991337rHCrG1sXLlSi699NKafO6559bSM2bMyPb19Uhq2vNmtJdeeimT0yHm3uTvj01dD77e8MPR09BxPoycL2+Zuc7n+euk+zYK4Zbejy/TvHnzSo9tJepZCSGEqDxqrIQQQlQeNVZCCCEqT5/xWV1wwQW19PTp07O8b33rW3WPu+eeezLZhx1Jh6j6FTO9vysd/tkoHIsfnp4Ovff2Yy+n+OHz3jfmQ8YIITaTLhfklw7yU2BGjBhRS6fLjEB7v3J6rPdDeR9W6j8q+61D7t/yv30/lN37j9L8sjwvNxpOn+b7VYQXLFhAPRRuSQghxFaHGishhBCVR42VEEKIytNnfFbp0sr3339/lrd8+fK6x3m776OPPtragnUz3sbt52j5ECxCiM6xbt26uvLixYt7ujiiAepZCSGEqDxqrIQQQlQea/XwwtKLmfXcxTZfs1RO77+ZfRs9t2aGp3v8UNIUv7KoD+3SBR4JIRzSqpO1gt7QF9FpKqcvIJ2pMiGEzleCHaCelRBCiMqjxkoIIUTlUWMlhBCi8vT00PXVwFJg15judrxvqY6vaVdgdSv9dy04V4fPqIU+Ks9e3XXiLtDj+tIEVStTT5enivoCxTPYQLXeDVRPX6Bny9RlfenRARa1i5o9XCXnbNXKA9UsU29RxWdRtfCGb+kAAAJKSURBVDJVrTy9SRWfhcrUdWQGFEIIUXnUWAkhhKg8vdVYXd1L161H1coD1SxTb1HFZ1G1MlWtPL1JFZ+FytRFesVnJYQQQjSDzIBCCCEqT482VmZ2spk9bWaLzOwrPXntpAzXmNkqM1uQbBtuZneb2cL4f1gPlmesmd1jZk+a2RNmdmFvl6lKSGc6LI90pg7Slw7L0y/0pccaKzMbAFwFnAJMAc40syk9df2EWcDJbttXgDkhhEnAnCj3FJuAL4QQpgDTgc/E59KbZaoE0pm6SGc6QPpSl/6hLyGEHvkDDgfuSuSvAl/tqeu7sowHFiTy08ComB4FPN0b5YrXvx04sUpl6sVnIZ2RzkhfpC+EEHrUDDga+GMiL4vbqsDIEMKKmF4JjCzbubsws/HAu4EHq1KmXkY60wDpTIb0pQF9WV80wMIRis+M3ljKZChwM3BRCGF9FcokOod0RjSD9GXL6MnGajkwNpHHxG1V4EUzGwUQ/6/qyYub2UAKJfpRCOGWKpSpIkhn6iCd6RDpSx36g770ZGP1EDDJzPY2s0HAXwI/68Hrl/Ez4JyYPofCptsjWLEq40zgqRDCd6pQpgohnekA6UxdpC8d0G/0pYcdex8AngGeBf6ul5yLNwArgLcobNrnASMoRsMsBGYDw3uwPEdRdL8fBx6Lfx/ozTJV6U86I52RvkhfQgiKYCGEEKL6aICFEEKIyqPGSgghROVRYyWEEKLyqLESQghRedRYCSGEqDxqrIQQQlQeNVZCCCEqjxorIYQQlef/A5sgn1fo4KxZAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 3 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "(X_train, y_train), (X_test, y_test) = keras.datasets.fashion_mnist.load_data() # Load MNIST or FMNIST\n",
        "assert X_train.shape == (60000, 28, 28)\n",
        "assert X_test.shape == (10000, 28, 28)\n",
        "assert y_train.shape == (60000,)\n",
        "assert y_test.shape == (10000,)\n",
        "\n",
        "\n",
        "# Display randomly selected data\n",
        "indices = list(np.random.randint(X_train.shape[0],size=3))\n",
        "for i in range(3):\n",
        "    plt.subplot(1,3,i+1)\n",
        "    plt.imshow(X_train[indices[i]].reshape(28,28), cmap='gray', interpolation='none')\n",
        "    plt.title(\"Index {} Class {}\".format(indices[i], y_train[indices[i]]))\n",
        "    plt.tight_layout()"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Important\n",
        "* Always have a validation set, the procedure to create validation or dev set is by performing random sample without replacement on train set and then only using that fraction as dev set. \n",
        "* Simple approach is to set some K samples, you can extract them from start, mid or end.\n",
        "* Imagine validation set that partially approximates test set distribution and we assume our model would produce identical results when we test it on test set.\n",
        "* Always optimize your hyperparameters by looking at performance on validation set and not test set.\n",
        "* Do not touch test set, we have this to test how our model would work on unseen data."
      ],
      "metadata": {
        "id": "PowjAHuw-wm7"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oIRI-uLoCX69",
        "outputId": "e664546e-92d4-435a-ae37-7f6836d5e783"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "size of training set is 50000 samples\n",
            "every train example is 28 by 28\n",
            "size of validation set is 10000 samples\n",
            "every validation example is 28 by 28\n",
            "size of training set is 50000 samples\n",
            "every train example has 784 features\n",
            "size of validation set is 10000 samples\n",
            "every validation example has 784 features\n"
          ]
        }
      ],
      "source": [
        "# Split train dataset into train and validation\n",
        "X_val = X_train[50000:60000]\n",
        "X_train = X_train[0:50000]\n",
        "y_val = y_train[50000:60000]\n",
        "y_train = y_train[0:50000]\n",
        "\n",
        "print(\"size of training set is\", str(X_train.shape[0]), \"samples\")\n",
        "print(\"every train example is\", str(X_train.shape[1]), \"by\", str(X_train.shape[2]))\n",
        "\n",
        "print(\"size of validation set is\", str(X_val.shape[0]), \"samples\")\n",
        "print(\"every validation example is\", str(X_val.shape[1]), \"by\", str(X_val.shape[2]))\n",
        "\n",
        "X_train = X_train.reshape(50000, 28*28)\n",
        "X_val = X_val.reshape(10000, 28*28)\n",
        "X_test = X_test.reshape(10000, 28*28)\n",
        "\n",
        "print(\"size of training set is\", str(X_train.shape[0]), \"samples\")\n",
        "print(\"every train example has\", str(X_train.shape[1]), \"features\")\n",
        "\n",
        "print(\"size of validation set is\", str(X_val.shape[0]), \"samples\")\n",
        "print(\"every validation example has\", str(X_val.shape[1]), \"features\")\n",
        "\n",
        "# Split dataset into batches\n",
        "#train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).batch(16)\n",
        "#test_ds = tf.data.Dataset.from_tensor_slices((X_test, y_test)).batch(4)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Points to remember\n",
        "* If using any type of neural network, normalize your input between 0-1.\n",
        "* One can use various procedures to achieve this, divide by largest value (for images we use 255), subtract mean from data and then normalize, one can even augment them and use other steps for normalization.\n",
        "* Normalization is important step, one could observe significant boost in performance just by having better normalization scheme.\n",
        "* For targets we always use one-hot encodings."
      ],
      "metadata": {
        "id": "VdMEIaFKAscU"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "mDyZ8bZjCX69",
        "outputId": "4f18de99-636d-4c05-b085-85e37206b037"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "1.0"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ],
      "source": [
        "#Normalize Data\n",
        "\n",
        "X_train = X_train/255\n",
        "X_val = X_val/255\n",
        "X_test = X_test/255\n",
        "# X_train[0]\n",
        "np.max(X_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3lIIy313CX69",
        "outputId": "527be8e1-561e-40c7-86ce-069f1955d162"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tf.Tensor([10000    10], shape=(2,), dtype=int32)\n"
          ]
        }
      ],
      "source": [
        "size_input = X_train.shape[1]\n",
        "size_hidden1 = 128\n",
        "size_hidden2 = 128\n",
        "size_hidden3 = 128\n",
        "size_output = 10\n",
        "\n",
        "number_of_train_examples = X_train.shape[0]\n",
        "number_of_test_examples = X_test.shape[0]\n",
        "\n",
        "y_train = tf.keras.utils.to_categorical(y_train, num_classes=10) # Other function is tf.one_hot(y_train,depth=10)\n",
        "y_val = tf.keras.utils.to_categorical(y_val, num_classes=10)\n",
        "y_test = tf.keras.utils.to_categorical(y_test, num_classes=10)\n",
        "print(tf.shape(y_val))"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Importance of weight initialization\n",
        "\n",
        "* One reason backprop based models can perform bettter lies with the weight initialization method, one important point one should remember is that, if yur weights are initialized to be too high or low, backprop would struggle.\n",
        "* Hence one should always carefully initialize weights of your model, below i have shown approach with random_normal, one can use random_uniform, truncated version of both, Xavier init and orthogonal. \n",
        "* You will find modern day NNs have achieved stable and better performance by simply switching to better init and majority of cases Xavier or Orthogonal works best.\n",
        "* Always initialize your bias using zero or some small constant (ideally 0.01 or less works better). We use bias to shift the activation and in some cases it can stabalize learning, but having large bias can cause negative results.\n",
        "\n",
        "# Loss function\n",
        "\n",
        "* We will always cross-entropy loss for classification.\n",
        "\n",
        "* tf softmax,\n",
        "loss= tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_pred_tf, labels=y_true_tf)), this function is simply saying that it will calculate softmax for you, simply provide logits to it. \n",
        "\n",
        "* In other output of your forward pass directly goes this function. Now this operator will calculate or apply softmax over prediction or logits and calculate cross-entropy between prediction and target. I am using reduce_mean since we apply this over batches.\n",
        "* Second is using keras\n",
        "Method 1 :- This function requires logits, hence same as above you will pass logits or output variable to this function. Now remember you need from_logits = True, for this to work.\n",
        "cce = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\n",
        "loss_x = cce(y_true_tf, y_pred_tf) \n",
        "\n",
        "* Method 2:- In this we will apply softmax to output function and then pass to CCE loss.\n",
        "So the approach is \n",
        "output = tf.nn.softmax(output)\n",
        "cce = tf.keras.losses.CategoricalCrossentropy(from_logits=False)\n",
        "loss_x = cce(y_true_tf, y_pred_tf) "
      ],
      "metadata": {
        "id": "U7KCVarVCVPW"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "obN7WPLpCX69"
      },
      "outputs": [],
      "source": [
        "# Define class to build mlp model\n",
        "class MLP(object):\n",
        " def __init__(self, size_input, size_hidden1, size_hidden2, size_hidden3, size_output, device=None):\n",
        "    \"\"\"\n",
        "    size_input: int, size of input layer\n",
        "    size_hidden1: int, size of the 1st hidden layer\n",
        "    size_hidden2: int, size of the 2nd hidden layer\n",
        "    size_output: int, size of output layer\n",
        "    device: str or None, either 'cpu' or 'gpu' or None. If None, the device to be used will be decided automatically during Eager Execution\n",
        "    \"\"\"\n",
        "    self.size_input, self.size_hidden1, self.size_hidden2, self.size_hidden3, self.size_output, self.device =\\\n",
        "    size_input, size_hidden1, size_hidden2, size_hidden3, size_output, device\n",
        "    \n",
        "    # Initialize weights between input mapping and a layer g(f(x)) = layer\n",
        "    self.W1 = tf.Variable(tf.random.normal([self.size_input, self.size_hidden1],stddev=0.1)) # Xavier(Fan-in fan-out) and Orthogonal\n",
        "    # Initialize biases for hidden layer\n",
        "    self.b1 = tf.Variable(tf.zeros([1, self.size_hidden1])) # 0 or constant(0.01)\n",
        "    \n",
        "    # Initialize weights between input layer and 1st hidden layer\n",
        "    self.W2 = tf.Variable(tf.random.normal([self.size_hidden1, self.size_hidden2],stddev=0.1))\n",
        "    # Initialize biases for hidden layer\n",
        "    self.b2 = tf.Variable(tf.zeros([1, self.size_hidden2]))\n",
        "    \n",
        "    # Initialize weights between 1st hidden layer and 2nd hidden layer\n",
        "    self.W3 = tf.Variable(tf.random.normal([self.size_hidden2, self.size_hidden3],stddev=0.1))\n",
        "    # Initialize biases for hidden layer\n",
        "    self.b3 = tf.Variable(tf.zeros([1, self.size_hidden3]))\n",
        "    \n",
        "     # Initialize weights between 2nd hidden layer and output layer\n",
        "    self.W4 = tf.Variable(tf.random.normal([self.size_hidden3, self.size_output],stddev=0.1))\n",
        "    # Initialize biases for output layer\n",
        "    self.b4 = tf.Variable(tf.zeros([1, self.size_output]))\n",
        "    \n",
        "    # Define variables to be updated during backpropagation\n",
        "    self.variables = [self.W1, self.W2, self.W3, self.W4, self.b1, self.b2, self.b3, self.b4]\n",
        "  \n",
        " def forward(self, X):\n",
        "    \"\"\"\n",
        "    forward pass\n",
        "    X: Tensor, inputs\n",
        "    \"\"\"\n",
        "    if self.device is not None:\n",
        "      with tf.device('gpu:0' if self.device=='gpu' else 'cpu'):\n",
        "        self.y = self.compute_output(X)\n",
        "    else:\n",
        "      self.y = self.compute_output(X)\n",
        "      \n",
        "    return self.y\n",
        "\n",
        " def loss(self, y_pred, y_true):\n",
        "    '''\n",
        "    y_pred - Tensor of shape (batch_size, size_output)\n",
        "    y_true - Tensor of shape (batch_size, size_output)\n",
        "    '''\n",
        "    #y_true_tf = tf.cast(tf.reshape(y_true, (-1, self.size_output)), dtype=tf.float32)\n",
        "    y_true_tf = tf.cast(y_true, dtype=tf.float32)\n",
        "    y_pred_tf = tf.cast(y_pred, dtype=tf.float32)\n",
        "    cce = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\n",
        "    loss_x = cce(y_true_tf, y_pred_tf)\n",
        "    # Use keras or tf_softmax, both should work for any given model\n",
        "    #loss_x = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_pred_tf, labels=y_true_tf))\n",
        "    \n",
        "    return loss_x\n",
        "\n",
        " def backward(self, X_train, y_train, opti):\n",
        "    \"\"\"\n",
        "    backward pass\n",
        "    \"\"\"\n",
        "    optimizer = opti\n",
        "    \n",
        "    with tf.GradientTape() as tape:\n",
        "        \n",
        "      predicted = self.forward(X_train)\n",
        "      current_loss = self.loss(predicted, y_train)\n",
        "        \n",
        "    grads = tape.gradient(current_loss, self.variables)\n",
        "    optimizer.apply_gradients(zip(grads, self.variables))\n",
        "           \n",
        " def compute_output(self, X):\n",
        "    \"\"\"\n",
        "    Custom method to obtain output tensor during forward pass\n",
        "    \"\"\"\n",
        "    # Cast X to float32\n",
        "    X_tf = tf.cast(X, dtype=tf.float32)\n",
        "    #X_tf = X\n",
        "    \n",
        "    # Compute values in hidden layers\n",
        "    z1 = tf.matmul(X_tf, self.W1) + self.b1\n",
        "    h1 = tf.nn.relu(z1)\n",
        "    \n",
        "    z2 = tf.matmul(h1, self.W2) + self.b2\n",
        "    h2 = tf.nn.relu(z2)\n",
        "    \n",
        "    z3 = tf.matmul(h2, self.W3) + self.b3\n",
        "    h3 = tf.nn.relu(z3)\n",
        "\n",
        "    # Compute output\n",
        "    output = tf.matmul(h3, self.W4) + self.b4\n",
        "    \n",
        "    #Now consider two things , First look at inbuild loss functions if they work with softmax or not and then change this \n",
        "    # Second add tf.Softmax(output) and then return this variable\n",
        "    return (output)\n",
        "\n",
        "#  def stderr(self,y_pred):\n",
        "#     \"\"\"\n",
        "#      Calculate standard error\n",
        "#      \"\"\"\n",
        "#     y_pred_tf = tf.cast(y_pred, dtype=tf.float32)\n",
        "#     std_dev = np.std(y_pred_tf) #Calculates standard deviation\n",
        "#     std_err = std_dev/sqrt(len(y_pred_tf))\n",
        "#     return std_err \n",
        "\n",
        "\n",
        "#  def var(self,y_pred):\n",
        "#     \"\"\"\n",
        "#      Calculate variance \n",
        "#      \"\"\"\n",
        "#     y_pred_tf = tf.cast(y_pred, dtype=tf.float32)\n",
        "#     std_dev = np.std(y_pred_tf) #Calculates standard deviation\n",
        "#     variance = (std_dev**2) # calculate variance\n",
        "#     return variance \n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "pOnhvVlUCX6-",
        "outputId": "dba83f32-51dc-412a-b36a-4e3a1868d639"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Train Accuracy: 0.8374\n",
            "Number of Epoch = 1 - Average Cross Entropy:= 0.00525556640625 \n",
            "\n",
            "Validation Accuracy: 0.8295\n",
            "\n",
            "Train Accuracy: 0.8537\n",
            "Number of Epoch = 2 - Average Cross Entropy:= 0.0035484661865234374 \n",
            "\n",
            "Validation Accuracy: 0.8444\n",
            "\n",
            "Train Accuracy: 0.8612\n",
            "Number of Epoch = 3 - Average Cross Entropy:= 0.003150406188964844 \n",
            "\n",
            "Validation Accuracy: 0.8514\n",
            "\n",
            "Train Accuracy: 0.8692\n",
            "Number of Epoch = 4 - Average Cross Entropy:= 0.0028947052001953126 \n",
            "\n",
            "Validation Accuracy: 0.8592\n",
            "\n",
            "Train Accuracy: 0.8763\n",
            "Number of Epoch = 5 - Average Cross Entropy:= 0.002747757873535156 \n",
            "\n",
            "Validation Accuracy: 0.8629\n",
            "\n",
            "Train Accuracy: 0.8813\n",
            "Number of Epoch = 6 - Average Cross Entropy:= 0.002602099914550781 \n",
            "\n",
            "Validation Accuracy: 0.8648\n",
            "\n",
            "Train Accuracy: 0.8854\n",
            "Number of Epoch = 7 - Average Cross Entropy:= 0.002496734161376953 \n",
            "\n",
            "Validation Accuracy: 0.8679\n",
            "\n",
            "Train Accuracy: 0.8922\n",
            "Number of Epoch = 8 - Average Cross Entropy:= 0.002399256286621094 \n",
            "\n",
            "Validation Accuracy: 0.8701\n",
            "\n",
            "Train Accuracy: 0.8971\n",
            "Number of Epoch = 9 - Average Cross Entropy:= 0.002292811279296875 \n",
            "\n",
            "Validation Accuracy: 0.8783\n",
            "\n",
            "Train Accuracy: 0.8977\n",
            "Number of Epoch = 10 - Average Cross Entropy:= 0.0022163417053222656 \n",
            "\n",
            "Validation Accuracy: 0.8756\n",
            "\n",
            "Total time taken (in seconds): 92.91\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAUtElEQVR4nO3df4xd5X3n8fcH25A47ZrEmUUEg8da3FYmbEg1a2WXVaXizeI0AVMJtUZOxR9Io2hhl5RVE1v+pyB5FVbdwv5BKnkDCUqmMZabKIY/oNQg7T+VYZxQjE28GQEGsyRMCbjNWjKYfPePe0zmTAZ8/fNc+75f0mjOfc7zPPd5rmR/5pzznHtSVUiSdNR5XQ9AkjRYDAZJUovBIElqMRgkSS0GgySpZX7XAzgVPv7xj9fo6GjXw5Cks8quXbv+sapGZpefE8EwOjrK5ORk18OQpLNKkv1zlXsqSZLUYjBIkloMBklSi8EgSWoxGCRJLUMbDBO7Jxi9d5Tz7jyP0XtHmdg90fWQJGkgnBPLVY/XxO4Jxh8e59A7hwDYf3A/4w+PA7DuynVdDk2SOjeURwwbd2x8LxSOOvTOITbu2NjRiCRpcAxlMLx88OXjKpekYTKUwXDZosuOq1yShslQBsOmVZtYuGBhq2zhgoVsWrWpoxFJ0uAYymBYd+U6Nl+3maWLlhLC0kVL2XzdZi88SxKQc+GZz2NjY+WX6EnS8Umyq6rGZpcP5RGDJOn9GQySpBaDQZLUYjBIkloMBklSi8EgSWoxGCRJLQaDJKnFYJAktfQVDElWJ9mXZCrJ+jn2X5DkoWb/ziSjM/ZtaMr3Jbl2RvlLSXYneSbJ5IzyjyV5PMlPmt8fPbkpSpKOxzGDIck84D7gc8AK4KYkK2ZVuwV4s6ouB+4B7m7argDWAlcAq4GvN/0d9ftVddWsW7LXAzuqajmwo3ktSTpD+jliWAlMVdULVfU2sAVYM6vOGuDBZnsbsCpJmvItVXW4ql4Eppr+PsjMvh4EbuhjjJKkU6SfYLgEeGXG6wNN2Zx1quoIcBBYfIy2Bfxtkl1JxmfUuaiqXmu2fwpcNNegkownmUwyOT093cc0JEn96PLi87+vqt+ld4rq1iS/N7tC9b76dc6vf62qzVU1VlVjIyMjp3mokjQ8+gmGV4FLZ7xe0pTNWSfJfGAR8MYHta2qo79fB77Pr04x/SzJxU1fFwOv9z8dSdLJ6icYngaWJ1mW5Hx6F5O3z6qzHbi52b4ReKL5a387sLZZtbQMWA48leQjSX4TIMlHgP8IPDdHXzcDPzixqUmSTsT8Y1WoqiNJbgMeA+YBD1TVniR3AZNVtR24H/h2king5/TCg6beVmAvcAS4tareTXIR8P3e9WnmA39dVY82b/k1YGuSW4D9wB+dwvlKko7BJ7hJ0pDyCW6SpL4YDJKkFoNBktRiMEiSWgwGSVKLwSBJajEYJEktBoMkqcVgkCS1GAySpBaDQZLUYjBIkloMBklSi8EgSWoxGCRJLQaDJKnFYJAktRgMkqQWg0GS1GIwSJJaDAZJUovBIElqMRgkSS19BUOS1Un2JZlKsn6O/RckeajZvzPJ6Ix9G5ryfUmundVuXpIfJXlkRtm3kryY5Jnm56oTn54k6XjNP1aFJPOA+4DPAgeAp5Nsr6q9M6rdArxZVZcnWQvcDfxxkhXAWuAK4BPA3yX5rap6t2l3O/A88C9mve2fVdW2k5mYJOnE9HPEsBKYqqoXquptYAuwZladNcCDzfY2YFWSNOVbqupwVb0ITDX9kWQJ8HngGyc/DUnSqdJPMFwCvDLj9YGmbM46VXUEOAgsPkbbe4GvAL+c4z03JXk2yT1JLphrUEnGk0wmmZyenu5jGpKkfnRy8TnJF4DXq2rXHLs3AL8D/BvgY8BX5+qjqjZX1VhVjY2MjJy+wUrSkOknGF4FLp3xeklTNmedJPOBRcAbH9D2auD6JC/ROzV1TZLvAFTVa9VzGPgmzaknSdKZ0U8wPA0sT7Isyfn0LiZvn1VnO3Bzs30j8ERVVVO+tlm1tAxYDjxVVRuqaklVjTb9PVFVXwRIcnHzO8ANwHMnNUNJ0nE55qqkqjqS5DbgMWAe8EBV7UlyFzBZVduB+4FvJ5kCfk7vP3uaeluBvcAR4NYZK5Lez0SSESDAM8CXTnBukqQTkN4f9me3sbGxmpyc7HoYknRWSbKrqsZml3vnsySpxWCQJLUYDJKkFoNBktRiMEiSWgwGSVKLwSBJajEYJEktBoMkqcVgkCS1GAySpBaDQZLUYjBIkloMBklSi8EgSWoxGCRJLQaDJKnFYJAktRgMkqQWg0GS1GIwSJJaDAZJUovBIElqMRgkSS19BUOS1Un2JZlKsn6O/RckeajZvzPJ6Ix9G5ryfUmundVuXpIfJXlkRtmypo+pps/zT3x6kqTjdcxgSDIPuA/4HLACuCnJilnVbgHerKrLgXuAu5u2K4C1wBXAauDrTX9H3Q48P6uvu4F7mr7ebPqWJJ0h/RwxrASmquqFqnob2AKsmVVnDfBgs70NWJUkTfmWqjpcVS8CU01/JFkCfB74xtFOmjbXNH3Q9HnDiUxMknRi+gmGS4BXZrw+0JTNWaeqjgAHgcXHaHsv8BXglzP2Lwbeavp4v/cCIMl4kskkk9PT031MQ5LUj04uPif5AvB6Ve060T6qanNVjVXV2MjIyCkcnSQNt36C4VXg0hmvlzRlc9ZJMh9YBLzxAW2vBq5P8hK9U1PXJPlO0+bCpo/3ey9J0mnUTzA8DSxvVgudT+9i8vZZdbYDNzfbNwJPVFU15WubVUvLgOXAU1W1oaqWVNVo098TVfXFps2TTR80ff7gJOYnSTpOxwyG5nz/bcBj9FYQba2qPUnuSnJ9U+1+YHGSKeAOYH3Tdg+wFdgLPArcWlXvHuMtvwrc0fS1uOlbknSGpPdH+tltbGysJicnux6GJJ1VkuyqqrHZ5d75LElqMRgkSS0GgySpxWCQJLUYDJKkFoNBktRiMEiSWgwGSVKLwSBJajEYJEktBoMkqcVgkCS1GAySpBaDQZLUYjBIkloMBklSi8EgSWoxGCRJLQaDJKnFYJAktRgMHZvYPcHovaOcd+d5jN47ysTuia6HJGnIze96AMNsYvcE4w+Pc+idQwDsP7if8YfHAVh35bouhyZpiHnE0KGNOza+FwpHHXrnEBt3bOxoRJLUZzAkWZ1kX5KpJOvn2H9Bkoea/TuTjM7Yt6Ep35fk2qbsQ0meSvIPSfYkuXNG/W8leTHJM83PVSc/zcH08sGXj6tcks6EYwZDknnAfcDngBXATUlWzKp2C/BmVV0O3APc3bRdAawFrgBWA19v+jsMXFNVnwKuAlYn+cyM/v6sqq5qfp45qRkOsMsWXXZc5ZJ0JvRzxLASmKqqF6rqbWALsGZWnTXAg832NmBVkjTlW6rqcFW9CEwBK6vnF039Bc1PneRczjqbVm1i4YKFrbKFCxayadWmjkYkSf0FwyXAKzNeH2jK5qxTVUeAg8DiD2qbZF6SZ4DXgceraueMepuSPJvkniQXzDWoJONJJpNMTk9P9zGNwbPuynVsvm4zSxctJYSli5ay+brNXniW1KnOViVV1bvAVUkuBL6f5JNV9RywAfgpcD6wGfgqcNcc7Tc3+xkbGztrjzbWXbnOIJA0UPo5YngVuHTG6yVN2Zx1kswHFgFv9NO2qt4CnqR3DYKqeq051XQY+Ca9U1mSpDOkn2B4GlieZFmS8+ldTN4+q8524OZm+0bgiaqqpnxts2ppGbAceCrJSHOkQJIPA58Ffty8vrj5HeAG4LmTmaAk6fgc81RSVR1JchvwGDAPeKCq9iS5C5isqu3A/cC3k0wBP6cXHjT1tgJ7gSPArVX1bvOf/4PNCqXzgK1V9UjzlhNJRoAAzwBfOpUTliR9sPT+sD+7jY2N1eTkZNfDkKSzSpJdVTU2u9w7nyVJLQaDJKnFYJAktRgMkqQWg0GS1GIwSJJaDAZJUovBIElqMRgkSS0GgySpxWCQJLUYDJKkFoNBktRiMEiSWgwGATCxe4LRe0c5787zGL13lIndE10PSVJHOnvmswbHxO4Jxh8e59A7hwDYf3A/4w+PA/g8amkIecQgNu7Y+F4oHHXonUNs3LGxoxFJ6pLBIF4++PJxlUs6txkM4rJFlx1XuaRzm8EgNq3axMIFC1tlCxcsZNOqTR2NSFKXDAax7sp1bL5uM0sXLSWEpYuWsvm6zV54loZUqqrrMZy0sbGxmpyc7HoYknRWSbKrqsZml3vEIElqMRgkSS19BUOS1Un2JZlKsn6O/RckeajZvzPJ6Ix9G5ryfUmubco+lOSpJP+QZE+SO2fUX9b0MdX0ef7JT1OS1K9jBkOSecB9wOeAFcBNSVbMqnYL8GZVXQ7cA9zdtF0BrAWuAFYDX2/6OwxcU1WfAq4CVif5TNPX3cA9TV9vNn1Lks6Qfo4YVgJTVfVCVb0NbAHWzKqzBniw2d4GrEqSpnxLVR2uqheBKWBl9fyiqb+g+ammzTVNHzR93nCCc5MknYB+guES4JUZrw80ZXPWqaojwEFg8Qe1TTIvyTPA68DjVbWzafNW08f7vRdN+/Ekk0kmp6en+5iGJKkfnV18rqp3q+oqYAmwMsknj7P95qoaq6qxkZGR0zNISRpC/QTDq8ClM14vacrmrJNkPrAIeKOftlX1FvAkvWsQbwAXNn2833tJkk6jfoLhaWB5s1rofHoXk7fPqrMduLnZvhF4onp3zm0H1jarlpYBy4GnkowkuRAgyYeBzwI/bto82fRB0+cPTnx6kqTjdcxgaM733wY8BjwPbK2qPUnuSnJ9U+1+YHGSKeAOYH3Tdg+wFdgLPArcWlXvAhcDTyZ5ll7wPF5VjzR9fRW4o+lrcdO3hoQPDJK651diaGDMfmAQ9L7Mz+9tkk4PvxJDA88HBkmDwWDQwPCBQdJgMBg0MHxgkDQYDAYNDB8YJA0Gg0EDwwcGSYPBVUmSNKRclSRJ6ovBIM3iTXYadvOPXUUaHrNvstt/cD/jD48DeK1DQ8MjBmkGb7KTDAapxZvsJINBavEmO8lgkFq8yU4yGKQWb7KTvMFNkoaWN7hJkvpiMEiSWgwGaUB5B7a64p3P0gDyDmx1ySMGaQB5B7a6ZDBIA8g7sNUlg0EaQN6BrS4ZDNIA8g5sdamvYEiyOsm+JFNJ1s+x/4IkDzX7dyYZnbFvQ1O+L8m1TdmlSZ5MsjfJniS3z6j/50leTfJM8/MHJz9N6eziHdjq0jHvfE4yD/g/wGeBA8DTwE1VtXdGnf8E/Ouq+lKStcAfVtUfJ1kBfBdYCXwC+Dvgt4B/CVxcVT9M8pvALuCGqtqb5M+BX1TVX/Q7Ce98lk6fid0TbNyxkZcPvsxliy5j06pNBtQ54mTufF4JTFXVC1X1NrAFWDOrzhrgwWZ7G7AqSZryLVV1uKpeBKaAlVX1WlX9EKCq/hl4HrjkRCYm6fQ5umx2/8H9FPXeslnvqTi39RMMlwCvzHh9gF//T/y9OlV1BDgILO6nbXPa6dPAzhnFtyV5NskDST4616CSjCeZTDI5PT3dxzQkHS+XzQ6nTi8+J/kN4G+AL1fVPzXFfwX8K+Aq4DXgf8zVtqo2V9VYVY2NjIyckfFKw8Zls8Opn2B4Fbh0xuslTdmcdZLMBxYBb3xQ2yQL6IXCRFV972iFqvpZVb1bVb8E/he9U1mSOuCy2eHUTzA8DSxPsizJ+cBaYPusOtuBm5vtG4EnqndVezuwtlm1tAxYDjzVXH+4H3i+qv5yZkdJLp7x8g+B5453UpJODZfNDqdjBkNzzeA24DF6F4m3VtWeJHclub6pdj+wOMkUcAewvmm7B9gK7AUeBW6tqneBq4E/Aa6ZY1nqf0+yO8mzwO8Df3qqJivp+AzSslm/VPDM8UE9kgbe7C8VhN6Ri/d2nBwf1CPprOXqqDPLYJA08FwddWYZDJIGnqujziyDQdLAc3XUmWUwSBp4g7Q6ahi4KkmSjsO59KWC77cqyWc+S1KfhuVZ3J5KkqQ+DcuyWYNBkvo0LMtmDQZJ6tOwLJs1GCSpT4O0bPZ0fneUwSBJfRqUZbOn+8l6LleVpLPM6L2j7D+4/9fKly5ayktffqnvfvwSPUk6R5zui+AGgySdZU73RXCDQZLOMqf7IrjBIElnmdN9EdyLz5I0pLz4LEnqi8EgSWoxGCRJLQaDJKnFYJAktZwTq5KSTAO/fn/42eXjwD92PYgB4ufxK34WbX4ebSfzeSytqpHZhedEMJwLkkzOtWxsWPl5/IqfRZufR9vp+Dw8lSRJajEYJEktBsPg2Nz1AAaMn8ev+Fm0+Xm0nfLPw2sMkqQWjxgkSS0GgySpxWDoWJJLkzyZZG+SPUlu73pMXUsyL8mPkjzS9Vi6luTCJNuS/DjJ80n+bddj6kqSP23+jTyX5LtJPtT1mM6kJA8keT3JczPKPpbk8SQ/aX5/9FS8l8HQvSPAf62qFcBngFuTrOh4TF27HXi+60EMiP8JPFpVvwN8iiH9XJJcAvwXYKyqPgnMA9Z2O6oz7lvA6lll64EdVbUc2NG8PmkGQ8eq6rWq+mGz/c/0/uFf0u2oupNkCfB54Btdj6VrSRYBvwfcD1BVb1fVW92OqlPzgQ8nmQ8sBP5vx+M5o6rqfwM/n1W8Bniw2X4QuOFUvJfBMECSjAKfBnZ2O5JO3Qt8Bfhl1wMZAMuAaeCbzam1byT5SNeD6kJVvQr8BfAy8BpwsKr+tttRDYSLquq1ZvunwEWnolODYUAk+Q3gb4AvV9U/dT2eLiT5AvB6Ve3qeiwDYj7wu8BfVdWngf/HKTpVcLZpzp2voReWnwA+kuSL3Y5qsFTv3oNTcv+BwTAAkiygFwoTVfW9rsfToauB65O8BGwBrknynW6H1KkDwIGqOnoEuY1eUAyj/wC8WFXTVfUO8D3g33U8pkHwsyQXAzS/Xz8VnRoMHUsSeueQn6+qv+x6PF2qqg1VtaSqRuldWHyiqob2r8Kq+inwSpLfbopWAXs7HFKXXgY+k2Rh829mFUN6IX6W7cDNzfbNwA9ORacGQ/euBv6E3l/HzzQ/f9D1oDQw/jMwkeRZ4Crgv3U8nk40R03bgB8Cu+n93zVUX42R5LvA3wO/neRAkluArwGfTfITekdVXzsl7+VXYkiSZvKIQZLUYjBIkloMBklSi8EgSWoxGCRJLQaDJKnFYJAktfx/UrRBly61hBoAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "# Set number of epochs\n",
        "NUM_EPOCHS = 10\n",
        "\n",
        "# Initialize model using CPU\n",
        "mlp_on_cpu = MLP(size_input, size_hidden1, size_hidden2, size_hidden3, size_output, device='gpu')\n",
        "\n",
        "time_start = time.time()\n",
        "opti = tf.keras.optimizers.SGD(learning_rate = 0.1)\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "    \n",
        "  loss_total = tf.zeros([1,1], dtype=tf.float32)\n",
        "  lt = 0\n",
        "    \n",
        "  train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(25, seed=epoch*(1234)).batch(128)\n",
        "  kz = 0\n",
        "  accuracy_z = 0.0\n",
        "  cur_train_acc = 0.0\n",
        "  for inputs, outputs in train_ds:\n",
        "    qw, tr = tf.shape(inputs)\n",
        "    kz = kz + 1\n",
        "    preds = mlp_on_cpu.forward(inputs) \n",
        "    loss_total = loss_total + mlp_on_cpu.loss(preds, outputs)\n",
        "    lt = lt + mlp_on_cpu.loss(preds, outputs)\n",
        "    mlp_on_cpu.backward(inputs, outputs, opti)\n",
        "\n",
        "  preds = mlp_on_cpu.forward(X_train)\n",
        "  # Get probs, remember we only have logits from our forward function, we need to apply softmax on top of it to get probs\n",
        "  preds = tf.nn.softmax(preds)\n",
        "  correct_prediction = tf.equal(tf.argmax(preds, 1), tf.argmax(y_train, 1))\n",
        "  accuracy_z = accuracy_z + tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
        "  cur_train_acc += accuracy_z.numpy()\n",
        "  ds = cur_train_acc\n",
        "  print('\\nTrain Accuracy: {:.4f}'.format(ds))\n",
        "  print('Number of Epoch = {} - Average Cross Entropy:= {} '.format(epoch + 1, np.sum(loss_total) / X_train.shape[0]))\n",
        "  preds_val = mlp_on_cpu.forward(X_val)\n",
        "  preds_val = tf.nn.softmax(preds_val)\n",
        "  correct_prediction = tf.equal(tf.argmax(preds_val, 1), tf.argmax(y_val, 1))\n",
        "\n",
        "  # Calculate accuracy\n",
        "  accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
        "  cur_val_acc = accuracy.numpy()\n",
        "\n",
        "  print('\\nValidation Accuracy: {:.4f}'.format(cur_val_acc))\n",
        "  \n",
        "  plt.plot(epoch + 1, np.sum(loss_total) / X_train.shape[0], 'go')\n",
        "\n",
        "        \n",
        "time_taken = time.time() - time_start\n",
        "    \n",
        "# Validate model\n",
        "    \n",
        "\n",
        "\n",
        "print('\\nTotal time taken (in seconds): {:.2f}'.format(time_taken))\n",
        "#For per epoch_time = Total_Time / Number_of_epochs\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Set number of epochs\n",
        "NUM_EPOCHS = 10\n",
        "\n",
        "# Initialize model using CPU\n",
        "mlp_on_cpu = MLP(size_input, size_hidden1, size_hidden2, size_hidden3, size_output, device='gpu')\n",
        "\n",
        "time_start = time.time()\n",
        "opti = tf.keras.optimizers.SGD(learning_rate = 0.1, momentum=0.9)\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "    \n",
        "  loss_total = tf.zeros([1,1], dtype=tf.float32)\n",
        "  lt = 0\n",
        "    \n",
        "  train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(25, seed=epoch*(1234)).batch(128)\n",
        "  kz = 0\n",
        "  accuracy_z = 0.0\n",
        "  cur_train_acc = 0.0\n",
        "  for inputs, outputs in train_ds:\n",
        "    qw, tr = tf.shape(inputs)\n",
        "    kz = kz + 1\n",
        "    preds = mlp_on_cpu.forward(inputs) \n",
        "    loss_total = loss_total + mlp_on_cpu.loss(preds, outputs)\n",
        "    lt = lt + mlp_on_cpu.loss(preds, outputs)\n",
        "    mlp_on_cpu.backward(inputs, outputs, opti)\n",
        "\n",
        "  preds = mlp_on_cpu.forward(X_train)\n",
        "  # Get probs, remember we only have logits from our forward function, we need to apply softmax on top of it to get probs\n",
        "  preds = tf.nn.softmax(preds)\n",
        "  correct_prediction = tf.equal(tf.argmax(preds, 1), tf.argmax(y_train, 1))\n",
        "  accuracy_z = accuracy_z + tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
        "  cur_train_acc += accuracy_z.numpy()\n",
        "  ds = cur_train_acc\n",
        "  print('\\nTrain Accuracy: {:.4f}'.format(ds))\n",
        "  print('Number of Epoch = {} - Average Cross Entropy:= {} '.format(epoch + 1, np.sum(loss_total) / X_train.shape[0]))\n",
        "  preds_val = mlp_on_cpu.forward(X_val)\n",
        "  preds_val = tf.nn.softmax(preds_val)\n",
        "  correct_prediction = tf.equal(tf.argmax(preds_val, 1), tf.argmax(y_val, 1))\n",
        "\n",
        "  # Calculate accuracy\n",
        "  accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
        "  cur_val_acc = accuracy.numpy()\n",
        "\n",
        "  print('\\nValidation Accuracy: {:.4f}'.format(cur_val_acc))\n",
        "  \n",
        "  plt.plot(epoch + 1, np.sum(loss_total) / X_train.shape[0], 'go')\n",
        "\n",
        "        \n",
        "time_taken = time.time() - time_start\n",
        "    \n",
        "# Validate model\n",
        "    \n",
        "\n",
        "\n",
        "print('\\nTotal time taken (in seconds): {:.2f}'.format(time_taken))\n",
        "#For per epoch_time = Total_Time / Number_of_epochs"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "wP34CXEreaHC",
        "outputId": "8bbb04bb-76e0-4815-d6d0-b2559b2aa59b"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Train Accuracy: 0.8300\n",
            "Number of Epoch = 1 - Average Cross Entropy:= 0.004634127197265625 \n",
            "\n",
            "Validation Accuracy: 0.8224\n",
            "\n",
            "Train Accuracy: 0.8575\n",
            "Number of Epoch = 2 - Average Cross Entropy:= 0.0033160650634765625 \n",
            "\n",
            "Validation Accuracy: 0.8470\n",
            "\n",
            "Train Accuracy: 0.8758\n",
            "Number of Epoch = 3 - Average Cross Entropy:= 0.0030181243896484377 \n",
            "\n",
            "Validation Accuracy: 0.8616\n",
            "\n",
            "Train Accuracy: 0.8779\n",
            "Number of Epoch = 4 - Average Cross Entropy:= 0.00283033447265625 \n",
            "\n",
            "Validation Accuracy: 0.8641\n",
            "\n",
            "Train Accuracy: 0.8861\n",
            "Number of Epoch = 5 - Average Cross Entropy:= 0.0026698907470703126 \n",
            "\n",
            "Validation Accuracy: 0.8694\n",
            "\n",
            "Train Accuracy: 0.8840\n",
            "Number of Epoch = 6 - Average Cross Entropy:= 0.002522196502685547 \n",
            "\n",
            "Validation Accuracy: 0.8619\n",
            "\n",
            "Train Accuracy: 0.8828\n",
            "Number of Epoch = 7 - Average Cross Entropy:= 0.0024284268188476564 \n",
            "\n",
            "Validation Accuracy: 0.8619\n",
            "\n",
            "Train Accuracy: 0.8837\n",
            "Number of Epoch = 8 - Average Cross Entropy:= 0.0023536662292480467 \n",
            "\n",
            "Validation Accuracy: 0.8650\n",
            "\n",
            "Train Accuracy: 0.8737\n",
            "Number of Epoch = 9 - Average Cross Entropy:= 0.0023053169250488283 \n",
            "\n",
            "Validation Accuracy: 0.8517\n",
            "\n",
            "Train Accuracy: 0.8945\n",
            "Number of Epoch = 10 - Average Cross Entropy:= 0.0022337586975097654 \n",
            "\n",
            "Validation Accuracy: 0.8700\n",
            "\n",
            "Total time taken (in seconds): 93.55\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAS8klEQVR4nO3df4xd5X3n8ffHPyBxu2saMqoIBo9VnEYmqGQ1tbKb1f6BF+E2IWYlpLhyKv5A8laCXbKtmuD1Pwtar8qqLd4/aCVvILGSUQ1yI2VAu2UTQFqt1BrGiRtjE6sjwGCWlik/3GYtGUy++8c9JnMmY3zHHvvc8X2/pJHPec5znvs8V/J85tzznOemqpAk6bQlXXdAkjRYDAZJUovBIElqMRgkSS0GgySpZVnXHVgIH//4x2t0dLTrbkjSorJ///6/r6qR2eWXRDCMjo4yOTnZdTckaVFJcnSucj9KkiS1GAySpBaDQZLUYjBIkloMBklSy9AGw/jBcUZ3jrLkviWM7hxl/OB4112SpIFwSUxXna/xg+NsfXwrJ947AcDR40fZ+vhWALbcsKXLrklS54byimH7U9s/CIXTTrx3gu1Pbe+oR5I0OIYyGF45/sq8yiVpmAxlMFy78tp5lUvSMBnKYNixYQcrlq9ola1YvoIdG3Z01CNJGhxDGQxbbtjCrlt3sXrlakJYvXI1u27d5Y1nSQJyKXzn89jYWLmIniTNT5L9VTU2u3worxgkSWdmMEiSWgwGSVJLX8GQZGOSI0mmktw7x/HLkzzaHN+XZHTGsW1N+ZEkt8w6b2mSHyZ5YkbZN5O8lORA83PjuQ9PkjRfZ10SI8lS4CHgZuAY8FySiao6PKPancDbVXVdks3AA8CXkqwDNgPXA58Avp/kk1X1fnPePcALwD+d9bK/X1V7z2dgkqRz088Vw3pgqqperKp3gT3Apll1NgG7m+29wIYkacr3VNXJqnoJmGraI8kq4PPA189/GJKkhdJPMFwNvDpj/1hTNmedqjoFHAeuPMu5O4GvAj+d4zV3JPlRkgeTXD5Xp5JsTTKZZHJ6erqPYUiS+tHJzeckXwDeqKr9cxzeBnwK+HXgY8DX5mqjqnZV1VhVjY2MjFy4zkrSkOknGF4Drpmxv6opm7NOkmXASuDNDzn3c8AXk7xM76Opm5J8G6CqXq+ek8A3aD56kiRdHP0Ew3PA2iRrklxG72byxKw6E8AdzfbtwNPVe6R6AtjczFpaA6wFnq2qbVW1qqpGm/aerqovAyS5qvk3wG3A8+c1QknSvJx1VlJVnUpyN/AksBR4pKoOJbkfmKyqCeBh4FtJpoC36P2yp6n3GHAYOAXcNWNG0pmMJxkBAhwAfuccxyZJOgeulSRJQ8q1kiRJfTEYJEktBoMkqcVgkCS1GAySpBaDQZLUYjBIkloMBklSi8EgSWoxGCRJLQaDJKnFYJAktRgMkqQWg0GS1GIwSJJaDAZJUovBIElqMRgkSS0GgySpxWCQJLUYDJKkFoNBktRiMEiSWgwGSVKLwSBJajEYJEktBoMkqcVgkCS1GAySpBaDQZLUYjBIkloMBklSS1/BkGRjkiNJppLcO8fxy5M82hzfl2R0xrFtTfmRJLfMOm9pkh8meWJG2ZqmjammzcvOfXiSpPk6azAkWQo8BPwGsA74rSTrZlW7E3i7qq4DHgQeaM5dB2wGrgc2An/StHfaPcALs9p6AHiwaevtpm1J0kXSzxXDemCqql6sqneBPcCmWXU2Abub7b3AhiRpyvdU1cmqegmYatojySrg88DXTzfSnHNT0wZNm7edy8AkSeemn2C4Gnh1xv6xpmzOOlV1CjgOXHmWc3cCXwV+OuP4lcA7TRtnei0AkmxNMplkcnp6uo9hSJL60cnN5yRfAN6oqv3n2kZV7aqqsaoaGxkZWcDeSdJw6ycYXgOumbG/qimbs06SZcBK4M0POfdzwBeTvEzvo6mbkny7OeeKpo0zvZYk6QLqJxieA9Y2s4Uuo3czeWJWnQngjmb7duDpqqqmfHMza2kNsBZ4tqq2VdWqqhpt2nu6qr7cnPNM0wZNm989j/FJkubprMHQfN5/N/AkvRlEj1XVoST3J/liU+1h4MokU8DvAvc25x4CHgMOA38B3FVV75/lJb8G/G7T1pVN25KkiyS9P9IXt7GxsZqcnOy6G5K0qCTZX1Vjs8t98lmS1GIwSJJaDAZJUovBIElqMRgkSS0GgySpxWCQJLUYDJKkFoNBktRiMEiSWgwGSVKLwSBJajEYJEktBoMkqcVgkCS1GAySpBaDQZLUYjBIkloMBklSi8EgSWoxGCRJLQaDJKnFYJAktRgMkqQWg0GS1GIwSJJaDAZJUovBIElqMRgkSS0GgySpxWCQJLUYDJKklr6CIcnGJEeSTCW5d47jlyd5tDm+L8nojGPbmvIjSW5pyj6S5Nkkf53kUJL7ZtT/ZpKXkhxofm48/2FKkvq17GwVkiwFHgJuBo4BzyWZqKrDM6rdCbxdVdcl2Qw8AHwpyTpgM3A98Ang+0k+CZwEbqqqnyRZDvyfJP+zqv6qae/3q2rvQg1SktS/fq4Y1gNTVfViVb0L7AE2zaqzCdjdbO8FNiRJU76nqk5W1UvAFLC+en7S1F/e/NR5jkWStAD6CYargVdn7B9ryuasU1WngOPAlR92bpKlSQ4AbwDfq6p9M+rtSPKjJA8muXwe45EknafObj5X1ftVdSOwClif5NPNoW3Ap4BfBz4GfG2u85NsTTKZZHJ6evqi9FmShkE/wfAacM2M/VVN2Zx1kiwDVgJv9nNuVb0DPANsbPZfbz5qOgl8g95HWT+nqnZV1VhVjY2MjPQxDElSP/oJhueAtUnWJLmM3s3kiVl1JoA7mu3bgaerqpryzc2spTXAWuDZJCNJrgBI8lF6N7Z/3Oxf1fwb4Dbg+fMZoCRpfs46K6mqTiW5G3gSWAo8UlWHktwPTFbVBPAw8K0kU8Bb9MKDpt5jwGHgFHBXVb3f/PLf3cx4WgI8VlVPNC85nmQECHAA+J2FHLAk6cOl94f94jY2NlaTk5Ndd0OSFpUk+6tqbHa5Tz5LkloMBklSi8EgSWoxGCRJLQaDJKnFYJAktRgMkqQWg6Fj4wfHGd05ypL7ljC6c5Txg+Ndd0nSkDvrk8+6cMYPjrP18a2ceO8EAEePH2Xr41sB2HLDli67JmmIecXQoe1Pbf8gFE478d4Jtj+1vaMeSZLB0KlXjr8yr3JJuhgMhg5du/LaeZVL0sVgMHRox4YdrFi+olW2YvkKdmzY0VGPJMlg6NSWG7aw69ZdrF65mhBWr1zNrlt3eeNZUqdcdluShpTLbkuS+mIwSJJaDAZJUovBIElqMRgkSS0GgySpxWCQJLUYDJKkFoNBktRiMEiSWgwGSVKLwSBJajEYJEktBoMkqcVgkCS1GAySpBaDQQCMHxxndOcoS+5bwujOUcYPjnfdJUkdWdZ1B9S98YPjbH18KyfeOwHA0eNH2fr4VgC/ZlQaQn1dMSTZmORIkqkk985x/PIkjzbH9yUZnXFsW1N+JMktTdlHkjyb5K+THEpy34z6a5o2ppo2Lzv/YerDbH9q+wehcNqJ906w/antHfVIUpfOGgxJlgIPAb8BrAN+K8m6WdXuBN6uquuAB4EHmnPXAZuB64GNwJ807Z0EbqqqXwNuBDYm+WzT1gPAg01bbzdt6wJ65fgr8yqXdGnr54phPTBVVS9W1bvAHmDTrDqbgN3N9l5gQ5I05Xuq6mRVvQRMAeur5ydN/eXNTzXn3NS0QdPmbec4NvXp2pXXzqtc0qWtn2C4Gnh1xv6xpmzOOlV1CjgOXPlh5yZZmuQA8Abwvara15zzTtPGmV6L5vytSSaTTE5PT/cxDJ3Jjg07WLF8RatsxfIV7Niwo6MeSepSZ7OSqur9qroRWAWsT/LpeZ6/q6rGqmpsZGTkwnRySGy5YQu7bt3F6pWrCWH1ytXsunWXN56lIdXPrKTXgGtm7K9qyuaqcyzJMmAl8GY/51bVO0meoXcP4o+AK5Isa64a5notXQBbbthiEEgC+rtieA5Y28wWuozezeSJWXUmgDua7duBp6uqmvLNzaylNcBa4NkkI0muAEjyUeBm4MfNOc80bdC0+d1zH54kab7OesVQVaeS3A08CSwFHqmqQ0nuByaragJ4GPhWkingLXrhQVPvMeAwcAq4q6reT3IVsLuZobQEeKyqnmhe8mvAniT/Gfhh07Yk6SJJ74/0xW1sbKwmJye77oYkLSpJ9lfV2Oxyl8SQJLUYDJKkFoNBktRiMEiSWgwGSVKLwaCB4vdCSN3z+xg0MPxeCGkweMWggeH3QkiDwWDQwPB7IaTBYDBoYPi9ENJgMBg0MPxeCGkwGAwaGH4vhDQYXERPkoaUi+hJkvpiMEiSWgwGSVKLwSBJajEYJEktBoM0iwv5adi5iJ40gwv5SV4xSC0u5CcZDFKLC/lJBoPU4kJ+ksEgtbiQn2QwSC0u5Ce5iJ4kDS0X0ZMk9cVgkCS1GAySpBaDQZLUYjBIA8o1m9QV10qSBpBrNqlLfV0xJNmY5EiSqST3znH88iSPNsf3JRmdcWxbU34kyS1N2TVJnklyOMmhJPfMqP+fkryW5EDz85vnP0xpcXHNJnXprFcMSZYCDwE3A8eA55JMVNXhGdXuBN6uquuSbAYeAL6UZB2wGbge+ATw/SSfBE4Bv1dVP0jyT4D9Sb43o80Hq+oPF2qQ0mLjmk3qUj9XDOuBqap6sareBfYAm2bV2QTsbrb3AhuSpCnfU1Unq+olYApYX1WvV9UPAKrqH4EXgKvPfzjSpcE1m9SlfoLhauDVGfvH+Plf4h/UqapTwHHgyn7ObT52+gywb0bx3Ul+lOSRJL80V6eSbE0ymWRyenq6j2FIi4drNqlLnc5KSvKLwJ8DX6mqf2iK/xT4FeBG4HXgj+Y6t6p2VdVYVY2NjIxclP5KF4trNqlL/cxKeg24Zsb+qqZsrjrHkiwDVgJvfti5SZbTC4XxqvrO6QpV9Xent5P8d+CJfgcjXUq23LDFIFAn+rlieA5Ym2RNksvo3UyemFVnArij2b4deLp6q/NNAJubWUtrgLXAs839h4eBF6rqj2c2lOSqGbv/Bnh+voOStHB8nmL4nPWKoapOJbkbeBJYCjxSVYeS3A9MVtUEvV/y30oyBbxFLzxo6j0GHKY3E+muqno/yb8Efhs4mORA81L/sar+B/Bfk9wIFPAy8G8XcLyS5sHnKYaTy25LOqPRnaMcPX7058pXr1zNy195+eJ3SAvKZbclzZvPUwwng0HSGfk8xXAyGCSdkc9TDCeDQdIZDdLzFM6Ouni8+Sxp4M2eHQW9Kxcf+js/3nyWtGi52uzFZTBIGnjOjrq4DAZJA2+QZkcNw70Og0HSwBuU2VGn73UcPX6Uoj54EvxSCweDQdLAG5TZUcNyr8PvfJa0KAzCarPDcq/DKwZJ6tMg3eu4kAwGSerToNzruNAMBknq06Dc64ALOzvKJ58laZFZqCfBffJZki4RF3p2lMEgSYvMhZ4dZTBI0iJzoWdHGQyStMhc6NlRBoMkLTIXenaUs5IkaUg5K0mS1BeDQZLUYjBIkloMBklSi8EgSWq5JGYlJZkGjnbdj/P0ceDvu+7EAPH9+Bnfizbfj7bzeT9WV9XI7MJLIhguBUkm55o2Nqx8P37G96LN96PtQrwffpQkSWoxGCRJLQbD4NjVdQcGjO/Hz/hetPl+tC34++E9BklSi1cMkqQWg0GS1GIwdCzJNUmeSXI4yaEk93Tdp64lWZrkh0me6LovXUtyRZK9SX6c5IUk/7zrPnUlyX9o/o88n+TPknyk6z5dTEkeSfJGkudnlH0syfeS/E3z7y8txGsZDN07BfxeVa0DPgvclWRdx33q2j3AC113YkD8N+AvqupTwK8xpO9LkquBfw+MVdWngaXA5m57ddF9E9g4q+xe4KmqWgs81eyfN4OhY1X1elX9oNn+R3r/8a/utlfdSbIK+Dzw9a770rUkK4F/BTwMUFXvVtU73faqU8uAjyZZBqwA/m/H/bmoqup/A2/NKt4E7G62dwO3LcRrGQwDJMko8BlgX7c96dRO4KvAT7vuyABYA0wD32g+Wvt6kl/oulNdqKrXgD8EXgFeB45X1f/qtlcD4Zer6vVm+2+BX16IRg2GAZHkF4E/B75SVf/QdX+6kOQLwBtVtb/rvgyIZcA/A/60qj4D/D8W6KOCxab57HwTvbD8BPALSb7cba8GS/WePViQ5w8MhgGQZDm9UBivqu903Z8OfQ74YpKXgT3ATUm+3W2XOnUMOFZVp68g99ILimH0r4GXqmq6qt4DvgP8i477NAj+LslVAM2/byxEowZDx5KE3mfIL1TVH3fdny5V1baqWlVVo/RuLD5dVUP7V2FV/S3wapJfbYo2AIc77FKXXgE+m2RF839mA0N6I36WCeCOZvsO4LsL0ajB0L3PAb9N76/jA83Pb3bdKQ2MfweMJ/kRcCPwXzruTyeaq6a9wA+Ag/R+dw3V0hhJ/gz4S+BXkxxLcifwB8DNSf6G3lXVHyzIa7kkhiRpJq8YJEktBoMkqcVgkCS1GAySpBaDQZLUYjBIkloMBklSy/8HS0yNjEY9g4cAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#RMSProp\n",
        "\n",
        "* Maintain a moving average of the square of gradients\n",
        "* Divide the gradient by the root of this moving average"
      ],
      "metadata": {
        "id": "rWaSewweLJ0p"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Set number of epochs\n",
        "NUM_EPOCHS = 10\n",
        "\n",
        "# Initialize model using CPU\n",
        "mlp_on_cpu = MLP(size_input, size_hidden1, size_hidden2, size_hidden3, size_output, device='gpu')\n",
        "\n",
        "time_start = time.time()\n",
        "opti = tf.keras.optimizers.RMSprop(learning_rate = 1e-3)\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "    \n",
        "  loss_total = tf.zeros([1,1], dtype=tf.float32)\n",
        "  lt = 0\n",
        "    \n",
        "  train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(25, seed=epoch*(1234)).batch(128)\n",
        "  kz = 0\n",
        "  accuracy_z = 0.0\n",
        "  cur_train_acc = 0.0\n",
        "  for inputs, outputs in train_ds:\n",
        "    qw, tr = tf.shape(inputs)\n",
        "    kz = kz + 1\n",
        "    preds = mlp_on_cpu.forward(inputs) \n",
        "    loss_total = loss_total + mlp_on_cpu.loss(preds, outputs)\n",
        "    lt = lt + mlp_on_cpu.loss(preds, outputs)\n",
        "    mlp_on_cpu.backward(inputs, outputs, opti)\n",
        "\n",
        "  preds = mlp_on_cpu.forward(X_train)\n",
        "  # Get probs, remember we only have logits from our forward function, we need to apply softmax on top of it to get probs\n",
        "  preds = tf.nn.softmax(preds)\n",
        "  correct_prediction = tf.equal(tf.argmax(preds, 1), tf.argmax(y_train, 1))\n",
        "  accuracy_z = accuracy_z + tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
        "  cur_train_acc += accuracy_z.numpy()\n",
        "  ds = cur_train_acc\n",
        "  print('\\nTrain Accuracy: {:.4f}'.format(ds))\n",
        "  print('Number of Epoch = {} - Average Cross Entropy:= {} '.format(epoch + 1, np.sum(loss_total) / X_train.shape[0]))\n",
        "  preds_val = mlp_on_cpu.forward(X_val)\n",
        "  preds_val = tf.nn.softmax(preds_val)\n",
        "  correct_prediction = tf.equal(tf.argmax(preds_val, 1), tf.argmax(y_val, 1))\n",
        "\n",
        "  # Calculate accuracy\n",
        "  accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
        "  cur_val_acc = accuracy.numpy()\n",
        "\n",
        "  print('\\nValidation Accuracy: {:.4f}'.format(cur_val_acc))\n",
        "  \n",
        "  plt.plot(epoch + 1, np.sum(loss_total) / X_train.shape[0], 'go')\n",
        "\n",
        "        \n",
        "time_taken = time.time() - time_start\n",
        "    \n",
        "# Validate model\n",
        "    \n",
        "\n",
        "\n",
        "print('\\nTotal time taken (in seconds): {:.2f}'.format(time_taken))\n",
        "#For per epoch_time = Total_Time / Number_of_epochs\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "LYIiTY9CduSh",
        "outputId": "85b0539c-2021-4d1b-92e2-6cc32f0b8144"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Train Accuracy: 0.8524\n",
            "Number of Epoch = 1 - Average Cross Entropy:= 0.004470570068359375 \n",
            "\n",
            "Validation Accuracy: 0.8475\n",
            "\n",
            "Train Accuracy: 0.8586\n",
            "Number of Epoch = 2 - Average Cross Entropy:= 0.0030924380493164063 \n",
            "\n",
            "Validation Accuracy: 0.8457\n",
            "\n",
            "Train Accuracy: 0.8832\n",
            "Number of Epoch = 3 - Average Cross Entropy:= 0.002732566833496094 \n",
            "\n",
            "Validation Accuracy: 0.8716\n",
            "\n",
            "Train Accuracy: 0.8878\n",
            "Number of Epoch = 4 - Average Cross Entropy:= 0.0024964035034179687 \n",
            "\n",
            "Validation Accuracy: 0.8725\n",
            "\n",
            "Train Accuracy: 0.8959\n",
            "Number of Epoch = 5 - Average Cross Entropy:= 0.002327974395751953 \n",
            "\n",
            "Validation Accuracy: 0.8769\n",
            "\n",
            "Train Accuracy: 0.8975\n",
            "Number of Epoch = 6 - Average Cross Entropy:= 0.002187594757080078 \n",
            "\n",
            "Validation Accuracy: 0.8749\n",
            "\n",
            "Train Accuracy: 0.9050\n",
            "Number of Epoch = 7 - Average Cross Entropy:= 0.0020711753845214843 \n",
            "\n",
            "Validation Accuracy: 0.8805\n",
            "\n",
            "Train Accuracy: 0.9016\n",
            "Number of Epoch = 8 - Average Cross Entropy:= 0.0019677088928222656 \n",
            "\n",
            "Validation Accuracy: 0.8729\n",
            "\n",
            "Train Accuracy: 0.9134\n",
            "Number of Epoch = 9 - Average Cross Entropy:= 0.0018597050476074218 \n",
            "\n",
            "Validation Accuracy: 0.8813\n",
            "\n",
            "Train Accuracy: 0.8949\n",
            "Number of Epoch = 10 - Average Cross Entropy:= 0.0017836886596679688 \n",
            "\n",
            "Validation Accuracy: 0.8620\n",
            "\n",
            "Total time taken (in seconds): 191.27\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAUuklEQVR4nO3df4xd5Z3f8fcH25A4aU1iRhFgYKzF2ciEllSzVlqq/oEX4c0PzEpIceRE/IHkrgotNNEmuP6jBa2rUG0D/YNN5QYSKxnFWA5SBtSGEoNUrdQ1jAmJsYmbEWBjSsIsASepJcDw7R/3EOYMY/sO/nHGvu+XdDXnPOc5z32eK8185tzznHNSVUiS9I6zuu6AJGluMRgkSS0GgySpxWCQJLUYDJKklvldd+BEOO+882p4eLjrbkjSaWXnzp1/X1VD08vPiGAYHh5mfHy8625I0mklyb6Zyv0qSZLUYjBIkloMBklSS1/BkGRVkr1JJpLcNsP2c5Lc32zfkWR4yrb1TfneJNdM229ekp8meWhK2XeTPJfkqeZ1xfsfniRpto558jnJPOAe4GrgAPBEkrGq2jOl2o3Aq1V1aZI1wJ3AF5IsB9YAlwEXAD9J8vGqeqvZ7xbgGeAfTnvbv6yqbcczMEnS+9PPEcMKYKKqnq2qN4AtwOppdVYDm5vlbcDKJGnKt1TV61X1HDDRtEeSJcBngW8f/zBmb3TXKMN3D3PW7WcxfPcwo7tGu+iGJM05/QTDhcALU9YPNGUz1qmqw8BBYPEx9r0b+Brw9gzvuTHJz5PcleScmTqVZF2S8STjk5OTfQzjXaO7Rln34Dr2HdxHUew7uI91D64zHCSJjk4+J/kc8HJV7Zxh83rgE8CfAB8Fvj5TG1W1qapGqmpkaOg912cc1YbtGzj05qFW2aE3D7Fh+4ZZtSNJZ6J+guFF4KIp60uashnrJJkPLAJeOcq+VwLXJnme3ldTVyX5PkBVvVQ9rwPfofnq6UTaf3D/rMolaZD0EwxPAMuSLE1yNr2TyWPT6owBNzTL1wOPVu8JQGPAmmbW0lJgGfB4Va2vqiVVNdy092hVfQkgyfnNzwDXAU8f1whncPGii2dVLkmD5JjB0JwzuBl4mN4Moq1VtTvJHUmubardCyxOMgF8Bbit2Xc3sBXYA/wYuGnKjKQjGU2yC9gFnAf81eyHdXQbV25k4YKFrbKFCxayceXGE/1WknTayZnwaM+RkZGa7b2SRneNsmH7BvYf3M/Fiy5m48qNrL187UnqoSTNPUl2VtXIe8oHNRgkadAdKRi8JYYkqcVgkCS1GAySpBaDQZLUYjBIkloMBklSi8EgSWoxGCRJLQaDJKnFYJAktRgMkqQWg0GS1GIwSJJaDAZJUovBIElqMRgkSS0GgySpxWCQJLUYDJKkFoNBktRiMEiSWgwGSVKLwSBJajEYJEktBoMkqcVgkCS1GAySpJa+giHJqiR7k0wkuW2G7eckub/ZviPJ8JRt65vyvUmumbbfvCQ/TfLQlLKlTRsTTZtnv//hSZJm65jBkGQecA/wZ8By4ItJlk+rdiPwalVdCtwF3NnsuxxYA1wGrAL+pmnvHbcAz0xr607grqatV5u2JUmnSD9HDCuAiap6tqreALYAq6fVWQ1sbpa3ASuTpCnfUlWvV9VzwETTHkmWAJ8Fvv1OI80+VzVt0LR53fsZmCTp/eknGC4EXpiyfqApm7FOVR0GDgKLj7Hv3cDXgLenbF8MvNa0caT3AiDJuiTjScYnJyf7GIYkqR+dnHxO8jng5ara+X7bqKpNVTVSVSNDQ0MnsHeSNNj6CYYXgYumrC9pymask2Q+sAh45Sj7Xglcm+R5el9NXZXk+80+5zZtHOm9JEknUT/B8ASwrJktdDa9k8lj0+qMATc0y9cDj1ZVNeVrmllLS4FlwONVtb6qllTVcNPeo1X1pWafx5o2aNr80XGMT5I0S8cMhub7/puBh+nNINpaVbuT3JHk2qbavcDiJBPAV4Dbmn13A1uBPcCPgZuq6q1jvOXXga80bS1u2pYknSLp/ZN+ehsZGanx8fGuuyFJp5UkO6tqZHq5Vz5LkloMBklSi8EgSWoxGCRJLQaDJKnFYJAktRgMkqQWg0GS1GIwSJJaDAZJUovBIElqMRgkSS0GgySpxWCQJLUYDJKkFoNBktRiMEiSWgwGSVKLwSBJajEYJEktBoMkqcVgkCS1GAySpBaDQZLUYjBIkloMBklSi8EgSWrpKxiSrEqyN8lEkttm2H5Okvub7TuSDE/Ztr4p35vkmqbsA0keT/KzJLuT3D6l/neTPJfkqeZ1xfEPU5LUr/nHqpBkHnAPcDVwAHgiyVhV7ZlS7Ubg1aq6NMka4E7gC0mWA2uAy4ALgJ8k+TjwOnBVVf0+yQLgb5P8j6r6u6a9v6yqbSdqkJKk/vVzxLACmKiqZ6vqDWALsHpandXA5mZ5G7AySZryLVX1elU9B0wAK6rn9039Bc2rjnMskqQToJ9guBB4Ycr6gaZsxjpVdRg4CCw+2r5J5iV5CngZeKSqdkyptzHJz5PcleScWYxHknScOjv5XFVvVdUVwBJgRZJPNpvWA58A/gT4KPD1mfZPsi7JeJLxycnJU9JnSRoE/QTDi8BFU9aXNGUz1kkyH1gEvNLPvlX1GvAYsKpZf6n5qul14Dv0vsp6j6raVFUjVTUyNDTUxzAkSf3oJxieAJYlWZrkbHonk8em1RkDbmiWrwcerapqytc0s5aWAsuAx5MMJTkXIMkH6Z3Y/kWzfn7zM8B1wNPHM0BJ0uwcc1ZSVR1OcjPwMDAPuK+qdie5AxivqjHgXuB7SSaA39ALD5p6W4E9wGHgpqp6q/njv7mZ8XQWsLWqHmrecjTJEBDgKeAvTuSAJUlHl94/9qe3kZGRGh8f77obknRaSbKzqkaml3vlsySpxWCQJLUYDJKkFoNBktRiMEiSWgwGSVKLwSBJajEYJEktBoMkqcVgkCS1GAySpBaDQZLUYjBIkloMBklSi8EgSWoxGCRJLQaDJKnFYJAktRgMkqQWg0GS1GIwSJJaDAZJUovB0LHRXaMM3z3MWbefxfDdw4zuGu26S5IG3PyuOzDIRneNsu7BdRx68xAA+w7uY92D6wBYe/naLrsmaYB5xNChDds3/CEU3nHozUNs2L6hox5JksHQqf0H98+qXJJOBYOhQxcvunhW5ZJ0KhgMHdq4ciMLFyxslS1csJCNKzd21CNJ6jMYkqxKsjfJRJLbZth+TpL7m+07kgxP2ba+Kd+b5Jqm7ANJHk/ysyS7k9w+pf7Spo2Jps2zj3+Yc9Pay9ey6fObuGTRJYRwyaJL2PT5TZ54ltSpVNXRKyTzgP8DXA0cAJ4AvlhVe6bU+VfAP6qqv0iyBvjzqvpCkuXAD4AVwAXAT4CPA28DH6qq3ydZAPwtcEtV/V2SrcADVbUlyX8FflZV3zpaH0dGRmp8fPx9fQCSNKiS7Kyqkenl/RwxrAAmqurZqnoD2AKsnlZnNbC5Wd4GrEySpnxLVb1eVc8BE8CK6vl9U39B86pmn6uaNmjavK7vUUqSjls/wXAh8MKU9QNN2Yx1quowcBBYfLR9k8xL8hTwMvBIVe1o9nmtaeNI70Wz/7ok40nGJycn+xiGJKkfnZ18rqq3quoKYAmwIsknZ7n/pqoaqaqRoaGhk9NJSRpA/QTDi8BFU9aXNGUz1kkyH1gEvNLPvlX1GvAYsKrZ59ymjSO9lyTpJOonGJ4AljWzhc4G1gBj0+qMATc0y9cDj1bvrPYYsKaZtbQUWAY8nmQoybkAST5I78T2L5p9HmvaoGnzR+9/eJKk2TrmvZKq6nCSm4GHgXnAfVW1O8kdwHhVjQH3At9LMgH8hl540NTbCuwBDgM3VdVbSc4HNjczns4CtlbVQ81bfh3YkuSvgJ82bUuSTpFjTlc9HThdVZJm73imq0qSBojBIElqMRgkSS0GgySpxWCQJLUYDJKkFoNBktRiMEiSWgwGSVKLwSBJajEYJEktBoMAGN01yvDdw5x1+1kM3z3M6K7RrrskqSPHvLuqznyju0ZZ9+A6Dr15CIB9B/ex7sF1AKy9fG2XXZPUAY8YxIbtG/4QCu849OYhNmzf0FGPJHXJYBD7D+6fVbmkM5vBIC5edPGsyiWd2QwGsXHlRhYuWNgqW7hgIRtXbuyoR5K6ZDCItZevZdPnN3HJoksI4ZJFl7Dp85s88SwNKB/tKUkDykd7SpL6YjBIkloMBklSi8EgSWoxGCRJLQaDJKnFYJAktRgMkqQWg0GS1NJXMCRZlWRvkokkt82w/Zwk9zfbdyQZnrJtfVO+N8k1TdlFSR5LsifJ7iS3TKn/H5K8mOSp5vWZ4x+mThc+MEjq3jEf1JNkHnAPcDVwAHgiyVhV7ZlS7Ubg1aq6NMka4E7gC0mWA2uAy4ALgJ8k+ThwGPhqVT2Z5B8AO5M8MqXNu6rqr0/UIHV68IFB0tzQzxHDCmCiqp6tqjeALcDqaXVWA5ub5W3AyiRpyrdU1etV9RwwAayoqpeq6kmAqvod8Axw4fEPR6czHxgkzQ39BMOFwAtT1g/w3j/if6hTVYeBg8DifvZtvnb6FLBjSvHNSX6e5L4kH5mpU0nWJRlPMj45OdnHMDTX+cAgaW7o9ORzkg8DPwRurarfNsXfAv4IuAJ4CfjPM+1bVZuqaqSqRoaGhk5Jf3Vy+cAgaW7oJxheBC6asr6kKZuxTpL5wCLglaPtm2QBvVAYraoH3qlQVb+uqreq6m3gv9H7KksDwAcGSXNDP8HwBLAsydIkZ9M7mTw2rc4YcEOzfD3waPUe9DAGrGlmLS0FlgGPN+cf7gWeqapvTm0oyflTVv8ceHq2g9LpyQcGSXPDMWclVdXhJDcDDwPzgPuqaneSO4Dxqhqj90f+e0kmgN/QCw+aeluBPfRmIt1UVW8l+efAl4FdSZ5q3urfVdV/B/5TkiuAAp4H/uUJHK/muLWXrzUIpI75BDdJGlA+wU2S1BeDQZLUYjBIkloMBklSi8EgTeON/DTojjldVRok3shP8ohBavFGfpLBILV4Iz/JYJBavJGfZDBILd7ITzIYpBZv5Cd5ryRJGljeK0mS1BeDQZLUYjBIc5RXYKsrXvkszUFega0uecQgzUFega0uGQzSHOQV2OqSwSDNQV6BrS4ZDNIc5BXY6pLBIM1BXoGtLnnlsyQNKK98liT1xWCQdFReaDd4vMBN0hF5od1g8ohB0hF5od1gMhgkHZEX2g0mg0HSEXmh3WDqKxiSrEqyN8lEkttm2H5Okvub7TuSDE/Ztr4p35vkmqbsoiSPJdmTZHeSW6bU/2iSR5L8svn5keMfpqT3wwvtBtMxgyHJPOAe4M+A5cAXkyyfVu1G4NWquhS4C7iz2Xc5sAa4DFgF/E3T3mHgq1W1HPg0cNOUNm8DtlfVMmB7sy6pA15oN5j6OWJYAUxU1bNV9QawBVg9rc5qYHOzvA1YmSRN+Zaqer2qngMmgBVV9VJVPQlQVb8DngEunKGtzcB1729okk6EtZev5flbn+ftf/82z9/6fGeh4LTZU6efYLgQeGHK+gHe/SP+njpVdRg4CCzuZ9/ma6dPATuaoo9V1UvN8q+Aj83UqSTrkownGZ+cnOxjGJJOV+9Mm913cB9F/WHarOFwcnR68jnJh4EfArdW1W+nb6/e/TpmvGdHVW2qqpGqGhkaGjrJPZXUJafNnlr9BMOLwEVT1pc0ZTPWSTIfWAS8crR9kyygFwqjVfXAlDq/TnJ+U+d84OV+ByPpzOS02VOrn2B4AliWZGmSs+mdTB6bVmcMuKFZvh54tPlvfwxY08xaWgosAx5vzj/cCzxTVd88Sls3AD+a7aAknVmcNntqHTMYmnMGNwMP0ztJvLWqdie5I8m1TbV7gcVJJoCv0MwkqqrdwFZgD/Bj4Kaqegu4EvgycFWSp5rXZ5q2vgFcneSXwJ8265IGmNNmTy1vuy3ptDC6a5QN2zew/+B+Ll50MRtXbnTa7HE60m23DQZJGlA+j0GSToBBuJ7C225LUp8G5TbkHjFIUp8G5XoKg0GS+jQo11MYDJLUp0G5nsJgkKQ+Dcr1FAaDJPVpLt2G/GTOjvI6Bkk6zUyfHQW9I5fZhpTXMUjSGeJkz44yGCTpNHOyZ0cZDJJ0mjnZs6MMBkk6zZzs2VEGgySdZk727ChnJUnSgHJWkiSpLwaDJKnFYJAktRgMkqQWg0GS1HJGzEpKMgns67ofx+k84O+77sQc4ufxLj+LNj+PtuP5PC6pqqHphWdEMJwJkozPNG1sUPl5vMvPos3Po+1kfB5+lSRJajEYJEktBsPcsanrDswxfh7v8rNo8/NoO+Gfh+cYJEktHjFIkloMBklSi8HQsSQXJXksyZ4ku5Pc0nWfupZkXpKfJnmo6750Lcm5SbYl+UWSZ5L806771JUk/7b5HXk6yQ+SfKDrPp1KSe5L8nKSp6eUfTTJI0l+2fz8yIl4L4Ohe4eBr1bVcuDTwE1Jlnfcp67dAjzTdSfmiP8C/LiqPgH8Ywb0c0lyIfBvgJGq+iQwD1jTba9Oue8Cq6aV3QZsr6plwPZm/bgZDB2rqpeq6slm+Xf0fvEv7LZX3UmyBPgs8O2u+9K1JIuAfwHcC1BVb1TVa932qlPzgQ8mmQ8sBP5vx/05parqfwG/mVa8GtjcLG8GrjsR72UwzCFJhoFPATu67Umn7ga+BrzddUfmgKXAJPCd5qu1byf5UNed6kJVvQj8NbAfeAk4WFX/s9tezQkfq6qXmuVfAR87EY0aDHNEkg8DPwRurarfdt2fLiT5HPByVe3sui9zxHzgnwDfqqpPAf+PE/RVwemm+e58Nb2wvAD4UJIvdduruaV61x6ckOsPDIY5IMkCeqEwWlUPdN2fDl0JXJvkeWALcFWS73fbpU4dAA5U1TtHkNvoBcUg+lPguaqarKo3gQeAf9Zxn+aCXyc5H6D5+fKJaNRg6FiS0PsO+Zmq+mbX/elSVa2vqiVVNUzvxOKjVTWw/xVW1a+AF5L8cVO0EtjTYZe6tB/4dJKFze/MSgb0RPw0Y8ANzfINwI9ORKMGQ/euBL5M77/jp5rXZ7rulOaMfw2MJvk5cAXwHzvuTyeao6ZtwJPALnp/uwbq1hhJfgD8b+CPkxxIciPwDeDqJL+kd1T1jRPyXt4SQ5I0lUcMkqQWg0GS1GIwSJJaDAZJUovBIElqMRgkSS0GgySp5f8DW1tyzxvfK+AAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Adam\n",
        "* Combine RMSprop with momentum\n",
        "* Default optimizer in many applications\n",
        "* In some scenarios leads to poor performance such as Continual learning, lifelong learning, metalearning"
      ],
      "metadata": {
        "id": "1wPc6gzpKLRt"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Set number of epochs\n",
        "NUM_EPOCHS = 10\n",
        "\n",
        "# Initialize model using CPU\n",
        "mlp_on_cpu = MLP(size_input, size_hidden1, size_hidden2, size_hidden3, size_output, device='gpu')\n",
        "\n",
        "time_start = time.time()\n",
        "opti = tf.keras.optimizers.Adam(learning_rate = 1e-4)\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "    \n",
        "  loss_total = tf.zeros([1,1], dtype=tf.float32)\n",
        "  lt = 0\n",
        "    \n",
        "  train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(25, seed=epoch*(1234)).batch(128)\n",
        "  kz = 0\n",
        "  accuracy_z = 0.0\n",
        "  cur_train_acc = 0.0\n",
        "  for inputs, outputs in train_ds:\n",
        "    qw, tr = tf.shape(inputs)\n",
        "    kz = kz + 1\n",
        "    preds = mlp_on_cpu.forward(inputs) \n",
        "    loss_total = loss_total + mlp_on_cpu.loss(preds, outputs)\n",
        "    lt = lt + mlp_on_cpu.loss(preds, outputs)\n",
        "    mlp_on_cpu.backward(inputs, outputs, opti)\n",
        "\n",
        "  preds = mlp_on_cpu.forward(X_train)\n",
        "  # Get probs, remember we only have logits from our forward function, we need to apply softmax on top of it to get probs\n",
        "  preds = tf.nn.softmax(preds)\n",
        "  correct_prediction = tf.equal(tf.argmax(preds, 1), tf.argmax(y_train, 1))\n",
        "  accuracy_z = accuracy_z + tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
        "  cur_train_acc += accuracy_z.numpy()\n",
        "  ds = cur_train_acc\n",
        "  print('\\nTrain Accuracy: {:.4f}'.format(ds))\n",
        "  print('Number of Epoch = {} - Average Cross Entropy:= {} '.format(epoch + 1, np.sum(loss_total) / X_train.shape[0]))\n",
        "  preds_val = mlp_on_cpu.forward(X_val)\n",
        "  preds_val = tf.nn.softmax(preds_val)\n",
        "  correct_prediction = tf.equal(tf.argmax(preds_val, 1), tf.argmax(y_val, 1))\n",
        "\n",
        "  # Calculate accuracy\n",
        "  accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
        "  cur_val_acc = accuracy.numpy()\n",
        "\n",
        "  print('\\nValidation Accuracy: {:.4f}'.format(cur_val_acc))\n",
        "  \n",
        "  plt.plot(epoch + 1, np.sum(loss_total) / X_train.shape[0], 'go')\n",
        "\n",
        "        \n",
        "time_taken = time.time() - time_start\n",
        "    \n",
        "# Validate model\n",
        "    \n",
        "\n",
        "\n",
        "print('\\nTotal time taken (in seconds): {:.2f}'.format(time_taken))\n",
        "#For per epoch_time = Total_Time / Number_of_epochs\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "NRjrGeh0dmy4",
        "outputId": "4b671ff9-596b-489e-e930-c9863ddc685a"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Train Accuracy: 0.8039\n",
            "Number of Epoch = 1 - Average Cross Entropy:= 0.007583687744140625 \n",
            "\n",
            "Validation Accuracy: 0.7970\n",
            "\n",
            "Train Accuracy: 0.8350\n",
            "Number of Epoch = 2 - Average Cross Entropy:= 0.004100390014648438 \n",
            "\n",
            "Validation Accuracy: 0.8247\n",
            "\n",
            "Train Accuracy: 0.8481\n",
            "Number of Epoch = 3 - Average Cross Entropy:= 0.0036029278564453124 \n",
            "\n",
            "Validation Accuracy: 0.8397\n",
            "\n",
            "Train Accuracy: 0.8569\n",
            "Number of Epoch = 4 - Average Cross Entropy:= 0.003350390625 \n",
            "\n",
            "Validation Accuracy: 0.8485\n",
            "\n",
            "Train Accuracy: 0.8631\n",
            "Number of Epoch = 5 - Average Cross Entropy:= 0.0031754852294921874 \n",
            "\n",
            "Validation Accuracy: 0.8533\n",
            "\n",
            "Train Accuracy: 0.8676\n",
            "Number of Epoch = 6 - Average Cross Entropy:= 0.003048553161621094 \n",
            "\n",
            "Validation Accuracy: 0.8583\n",
            "\n",
            "Train Accuracy: 0.8716\n",
            "Number of Epoch = 7 - Average Cross Entropy:= 0.0029451785278320312 \n",
            "\n",
            "Validation Accuracy: 0.8604\n",
            "\n",
            "Train Accuracy: 0.8746\n",
            "Number of Epoch = 8 - Average Cross Entropy:= 0.0028560934448242188 \n",
            "\n",
            "Validation Accuracy: 0.8632\n",
            "\n",
            "Train Accuracy: 0.8778\n",
            "Number of Epoch = 9 - Average Cross Entropy:= 0.002779222412109375 \n",
            "\n",
            "Validation Accuracy: 0.8651\n",
            "\n",
            "Train Accuracy: 0.8796\n",
            "Number of Epoch = 10 - Average Cross Entropy:= 0.002712095947265625 \n",
            "\n",
            "Validation Accuracy: 0.8679\n",
            "\n",
            "Total time taken (in seconds): 91.86\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAARQklEQVR4nO3db4xldX3H8feHXUCXNmuLE6Msu7MJq2aR+KcTorUxrVvrUv9sH5C4Zm1IQ7J9AP5pmxjIPmgk2UaSpuIDNNkAltqpK91qOvhAVDDxSQPMqnFlceMUWFiqsgKurSTA4rcP7kGHYWDussM9d+b3fiWTPfd3fufe77nJ3s895/c796SqkCS154y+C5Ak9cMAkKRGGQCS1CgDQJIaZQBIUqPW9l3AqXj1q19dk5OTfZchSSvGwYMHf15VE4utW1EBMDk5yezsbN9lSNKKkeToC63zFJAkNcoAkKRGGQCS1CgDQJIaZQBIUqNWfQBMH5pm8rpJzvjUGUxeN8n0oem+S5KksbCipoGequlD0+y+dTdPPP0EAEdPHGX3rbsB2HXRrj5Lk6TereojgD237/nNh/+znnj6CfbcvqeniiRpfKzqAHjwxIOn1C5JLVnVAbBx/cZTapeklqzqANi7bS/rzlz3nLZ1Z65j77a9PVUkSeNjVQfArot2se8D+9i0fhMhbFq/iX0f2OcAsCQBWUn3BJ6amip/DE6ShpfkYFVNLbZuVR8BSJJemAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSo4YKgCTbkxxJMpfkqkXWn53ky936O5NMzlt3ddd+JMl7u7Y3JPn+vL9fJvnEcu2UJGlpa5fqkGQNcD3wHuAYcHeSmao6PK/b5cDjVXVBkp3AtcCHkmwFdgIXAq8DvpXk9VV1BHjLvOd/GPjqMu6XJGkJwxwBXAzMVdV9VfUUsB/YsaDPDuDmbvkAsC1Juvb9VfVkVd0PzHXPN9824L+r6uhL3QlJ0qkbJgDOAx6a9/hY17Zon6o6CZwAzh1y253Al17oxZPsTjKbZPb48eNDlCtJGkavg8BJzgI+CPz7C/Wpqn1VNVVVUxMTE6MrTpJWuWEC4GHg/HmPN3Rti/ZJshZYDzw6xLaXAN+tqp+dWtmSpNM1TADcDWxJsrn7xr4TmFnQZwa4rFu+FLijqqpr39nNEtoMbAHumrfdh3mR0z+SpJfPkrOAqupkkiuB24A1wE1VdU+Sa4DZqpoBbgS+mGQOeIxBSND1uwU4DJwErqiqZwCSnMNgZtFfvwz7JUlaQgZf1FeGqampmp2d7bsMSVoxkhysqqnF1nklsCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjRoqAJJsT3IkyVySqxZZf3aSL3fr70wyOW/d1V37kSTvndf+qiQHkvwoyb1J3rEcOyRJGs6SAZBkDXA9cAmwFfhwkq0Lul0OPF5VFwCfAa7ttt0K7AQuBLYDn+ueD+CzwNer6o3Am4F7T393JEnDGuYI4GJgrqruq6qngP3AjgV9dgA3d8sHgG1J0rXvr6onq+p+YA64OMl64F3AjQBV9VRV/eL0d0eSNKxhAuA84KF5j491bYv2qaqTwAng3BfZdjNwHPhCku8luSHJOS9pDyRJL0lfg8BrgbcBn6+qtwK/Ap43tgCQZHeS2SSzx48fH2WNkrSqDRMADwPnz3u8oWtbtE+StcB64NEX2fYYcKyq7uzaDzAIhOepqn1VNVVVUxMTE0OUK0kaxjABcDewJcnmJGcxGNSdWdBnBrisW74UuKOqqmvf2c0S2gxsAe6qqp8CDyV5Q7fNNuDwae6LJOkUrF2qQ1WdTHIlcBuwBripqu5Jcg0wW1UzDAZzv5hkDniMQUjQ9buFwYf7SeCKqnqme+qPAtNdqNwH/NUy75sk6UVk8EV9ZZiamqrZ2dm+y5CkFSPJwaqaWmydVwJLUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGjVUACTZnuRIkrkkVy2y/uwkX+7W35lkct66q7v2I0neO6/9gSSHknw/yexy7IwkaXhrl+qQZA1wPfAe4Bhwd5KZqjo8r9vlwONVdUGSncC1wIeSbAV2AhcCrwO+leT1VfVMt92fVNXPl3F/JElDGuYI4GJgrqruq6qngP3AjgV9dgA3d8sHgG1J0rXvr6onq+p+YK57PklSz4YJgPOAh+Y9Pta1Ldqnqk4CJ4Bzl9i2gG8kOZhk9wu9eJLdSWaTzB4/fnyIciVJw+hzEPiPquptwCXAFUnetVinqtpXVVNVNTUxMTHaCiVpFRsmAB4Gzp/3eEPXtmifJGuB9cCjL7ZtVT377yPAV/HUkCSN1DABcDewJcnmJGcxGNSdWdBnBrisW74UuKOqqmvf2c0S2gxsAe5Kck6S3wVIcg7wZ8APT393JEnDWnIWUFWdTHIlcBuwBripqu5Jcg0wW1UzwI3AF5PMAY8xCAm6frcAh4GTwBVV9UyS1wBfHYwTsxb4t6r6+suwf5KkF5DBF/WVYWpqqmZnvWRAkoaV5GBVTS22ziuBJalRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDYESmD00zed0kZ3zqDCavm2T60HTfJUlq3Nq+C2jB9KFpdt+6myeefgKAoyeOsvvW3QDsumhXn6VJaphHACOw5/Y9v/nwf9YTTz/Bntv39FSRJA0ZAEm2JzmSZC7JVYusPzvJl7v1dyaZnLfu6q79SJL3LthuTZLvJfna6e7IOHvwxIOn1C5Jo7BkACRZA1wPXAJsBT6cZOuCbpcDj1fVBcBngGu7bbcCO4ELge3A57rne9bHgXtPdyfG3cb1G0+pXZJGYZgjgIuBuaq6r6qeAvYDOxb02QHc3C0fALYlSde+v6qerKr7gbnu+UiyAXgfcMPp78Z427ttL+vOXPectnVnrmPvtr09VSRJwwXAecBD8x4f69oW7VNVJ4ETwLlLbHsd8Eng16dc9Qqz66Jd7PvAPjat30QIm9ZvYt8H9jkALKlXvcwCSvJ+4JGqOpjkj5fouxvYDbBx48o9ZbLrol1+4EsaK8McATwMnD/v8YaubdE+SdYC64FHX2TbdwIfTPIAg1NK707yr4u9eFXtq6qpqpqamJgYolxJ0jCGCYC7gS1JNic5i8Gg7syCPjPAZd3ypcAdVVVd+85ultBmYAtwV1VdXVUbqmqye747quojy7A/kqQhLXkKqKpOJrkSuA1YA9xUVfckuQaYraoZ4Ebgi0nmgMcYfKjT9bsFOAycBK6oqmdepn2RJJ2CDL6orwxTU1M1OzvbdxmStGIkOVhVU4ut80pgSWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMgIZ4X2JJ83lP4EZ4X2JJC3kE0AjvSyxpIQOgEd6XWNJCBkAjvC+xpIUMgEZ4X2JJCxkAjfC+xJIW8n4AkrSKeT8ASdLzGACS1CgDQCPnFcnSePBKYI2UVyRL48MjAI2UVyRL48MA0Eh5RbI0PgwAjZRXJEvjwwDQSHlFsjQ+DACNlFckS+PDK4HVrOlD0+y5fQ8PnniQjes3snfbXoNIq86LXQnsNFA1yemokqeA1Cino0oGgBrldFTJAFCjnI4qGQBq1DhNR/W3kdQXA0BNGpfpqM8ORh89cZSifjMYbQhoFIaaBppkO/BZYA1wQ1V9esH6s4F/Af4AeBT4UFU90K27GrgceAb4WFXdluQVwHeAsxnMRDpQVX+/VB1OA9VqM3ndJEdPHH1e+6b1m3jgEw+MviCtOqd1Q5gka4DrgUuArcCHk2xd0O1y4PGqugD4DHBtt+1WYCdwIbAd+Fz3fE8C766qNwNvAbYneftL2TlpJXMwWn0a5hTQxcBcVd1XVU8B+4EdC/rsAG7ulg8A25Kka99fVU9W1f3AHHBxDfxf1//M7m/lXJEmLZNxGox2LKI9wwTAecBD8x4f69oW7VNVJ4ETwLkvtm2SNUm+DzwCfLOq7lzsxZPsTjKbZPb48eNDlCutHOMyGO1YRJt6GwSuqmeq6i3ABuDiJG96gX77qmqqqqYmJiZGW6T0MhuXwWgvjGvTMD8F8TBw/rzHG7q2xfocS7IWWM9gMHjJbavqF0m+zWCM4IenVL20Cuy6aFfvPz/hWESbhjkCuBvYkmRzkrMYDOrOLOgzA1zWLV8K3FGD6UUzwM4kZyfZDGwB7koykeRVAEleCbwH+NHp746kl8KxiDYtGQDdOf0rgduAe4FbquqeJNck+WDX7Ubg3CRzwN8CV3Xb3gPcAhwGvg5cUVXPAK8Fvp3kBwwC5ptV9bXl3TVJw3Isok3+HLQkYDx+HtvrIpafPwctaUmORfzWOIThKPhTEJLGxjiMRbR0GsoAkDQ2xmEsoqUpsQaApLExDtdFjMtpKHj5Z0Q5BiBprPQ9FrFx/cZFB6JHPSV2FLct9QhAkuYZh9NQMJpTUQaAJM0zDqehYDSnojwFJEkL9H0aCkZzKsojAEkaQ6M4FWUASNIYGsWpKH8KQpJWsdO6JaQkaXUyACSpUQaAJDXKAJCkRhkAktSoFTULKMlx4PlXRqwsrwZ+3ncRY8L34rl8P57L9+O3Tue92FRVE4utWFEBsBokmX2hKVmt8b14Lt+P5/L9+K2X673wFJAkNcoAkKRGGQCjt6/vAsaI78Vz+X48l+/Hb70s74VjAJLUKI8AJKlRBoAkNcoAGIEk5yf5dpLDSe5J8vG+axoHSdYk+V6Sr/VdS5+SvCrJgSQ/SnJvknf0XVOfkvxN9//kh0m+lOQVfdc0SkluSvJIkh/Oa/v9JN9M8uPu399bjtcyAEbjJPB3VbUVeDtwRZKtPdc0Dj4O3Nt3EWPgs8DXq+qNwJtp+D1Jch7wMWCqqt4ErAF29lvVyP0zsH1B21XA7VW1Bbi9e3zaDIARqKqfVNV3u+X/ZfAf/Lx+q+pXkg3A+4Ab+q6lT0nWA+8CbgSoqqeq6hf9VtW7tcArk6wF1gH/03M9I1VV3wEeW9C8A7i5W74Z+IvleC0DYMSSTAJvBe7st5LeXQd8Evh134X0bDNwHPhCdzrshiTn9F1UX6rqYeAfgQeBnwAnquob/VY1Fl5TVT/pln8KvGY5ntQAGKEkvwP8B/CJqvpl3/X0Jcn7gUeq6mDftYyBtcDbgM9X1VuBX7FMh/crUXdueweDYHwdcE6Sj/Rb1Xipwdz9ZZm/bwCMSJIzGXz4T1fVV/qup2fvBD6Y5AFgP/DuJP/ab0m9OQYcq6pnjwgPMAiEVv0pcH9VHa+qp4GvAH/Yc03j4GdJXgvQ/fvIcjypATACScLgHO+9VfVPfdfTt6q6uqo2VNUkgwG+O6qqyW95VfVT4KEkb+iatgGHeyypbw8Cb0+yrvt/s42GB8XnmQEu65YvA/5zOZ7UABiNdwJ/yeCb7ve7vz/vuyiNjY8C00l+ALwF+Iee6+lNdyR0APgucIjBZ1RTPwmR5EvAfwFvSHIsyeXAp4H3JPkxg6OkTy/La/lTEJLUJo8AJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElq1P8D7//1YzpTshoAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "#Nadam Optimizer\n",
        "* Combine Adam with Nesterov momentum\n",
        "* In somecases (image segmentation, object detection) it performs stably and leads to faster convergence "
      ],
      "metadata": {
        "id": "wFt--2EnIpoR"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Set number of epochs\n",
        "NUM_EPOCHS = 10\n",
        "\n",
        "# Initialize model using CPU\n",
        "mlp_on_cpu = MLP(size_input, size_hidden1, size_hidden2, size_hidden3, size_output, device='gpu')\n",
        "\n",
        "time_start = time.time()\n",
        "opti = tf.keras.optimizers.Nadam(learning_rate = 1e-4)\n",
        "for epoch in range(NUM_EPOCHS):\n",
        "    \n",
        "  loss_total = tf.zeros([1,1], dtype=tf.float32)\n",
        "  lt = 0\n",
        "    \n",
        "  train_ds = tf.data.Dataset.from_tensor_slices((X_train, y_train)).shuffle(25, seed=epoch*(1234)).batch(128)\n",
        "  kz = 0\n",
        "  accuracy_z = 0.0\n",
        "  cur_train_acc = 0.0\n",
        "  for inputs, outputs in train_ds:\n",
        "    qw, tr = tf.shape(inputs)\n",
        "    kz = kz + 1\n",
        "    preds = mlp_on_cpu.forward(inputs) \n",
        "    loss_total = loss_total + mlp_on_cpu.loss(preds, outputs)\n",
        "    lt = lt + mlp_on_cpu.loss(preds, outputs)\n",
        "    mlp_on_cpu.backward(inputs, outputs, opti)\n",
        "\n",
        "  preds = mlp_on_cpu.forward(X_train)\n",
        "  # Get probs, remember we only have logits from our forward function, we need to apply softmax on top of it to get probs\n",
        "  preds = tf.nn.softmax(preds)\n",
        "  correct_prediction = tf.equal(tf.argmax(preds, 1), tf.argmax(y_train, 1))\n",
        "  accuracy_z = accuracy_z + tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
        "  cur_train_acc += accuracy_z.numpy()\n",
        "  ds = cur_train_acc\n",
        "  print('\\nTrain Accuracy: {:.4f}'.format(ds))\n",
        "  print('Number of Epoch = {} - Average Cross Entropy:= {} '.format(epoch + 1, np.sum(loss_total) / X_train.shape[0]))\n",
        "  preds_val = mlp_on_cpu.forward(X_val)\n",
        "  preds_val = tf.nn.softmax(preds_val)\n",
        "  correct_prediction = tf.equal(tf.argmax(preds_val, 1), tf.argmax(y_val, 1))\n",
        "\n",
        "  # Calculate accuracy\n",
        "  accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
        "  cur_val_acc = accuracy.numpy()\n",
        "\n",
        "  print('\\nValidation Accuracy: {:.4f}'.format(cur_val_acc))\n",
        "  \n",
        "  plt.plot(epoch + 1, np.sum(loss_total) / X_train.shape[0], 'go')\n",
        "\n",
        "        \n",
        "time_taken = time.time() - time_start\n",
        "    \n",
        "# Validate model\n",
        "    \n",
        "\n",
        "\n",
        "print('\\nTotal time taken (in seconds): {:.2f}'.format(time_taken))\n",
        "#For per epoch_time = Total_Time / Number_of_epochs"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "qa4m43RFgeE1",
        "outputId": "d820a316-5156-4b0a-912d-6f757bcecb1b"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Train Accuracy: 0.8079\n",
            "Number of Epoch = 1 - Average Cross Entropy:= 0.007469556884765625 \n",
            "\n",
            "Validation Accuracy: 0.7991\n",
            "\n",
            "Train Accuracy: 0.8344\n",
            "Number of Epoch = 2 - Average Cross Entropy:= 0.004099525146484375 \n",
            "\n",
            "Validation Accuracy: 0.8271\n",
            "\n",
            "Train Accuracy: 0.8484\n",
            "Number of Epoch = 3 - Average Cross Entropy:= 0.0036003802490234375 \n",
            "\n",
            "Validation Accuracy: 0.8427\n",
            "\n",
            "Train Accuracy: 0.8568\n",
            "Number of Epoch = 4 - Average Cross Entropy:= 0.003330947265625 \n",
            "\n",
            "Validation Accuracy: 0.8523\n",
            "\n",
            "Train Accuracy: 0.8645\n",
            "Number of Epoch = 5 - Average Cross Entropy:= 0.0031479156494140623 \n",
            "\n",
            "Validation Accuracy: 0.8582\n",
            "\n",
            "Train Accuracy: 0.8686\n",
            "Number of Epoch = 6 - Average Cross Entropy:= 0.003007879943847656 \n",
            "\n",
            "Validation Accuracy: 0.8600\n",
            "\n",
            "Train Accuracy: 0.8734\n",
            "Number of Epoch = 7 - Average Cross Entropy:= 0.002898348388671875 \n",
            "\n",
            "Validation Accuracy: 0.8628\n",
            "\n",
            "Train Accuracy: 0.8766\n",
            "Number of Epoch = 8 - Average Cross Entropy:= 0.0028031634521484376 \n",
            "\n",
            "Validation Accuracy: 0.8660\n",
            "\n",
            "Train Accuracy: 0.8801\n",
            "Number of Epoch = 9 - Average Cross Entropy:= 0.002720559387207031 \n",
            "\n",
            "Validation Accuracy: 0.8682\n",
            "\n",
            "Train Accuracy: 0.8828\n",
            "Number of Epoch = 10 - Average Cross Entropy:= 0.0026505709838867187 \n",
            "\n",
            "Validation Accuracy: 0.8699\n",
            "\n",
            "Total time taken (in seconds): 177.53\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD4CAYAAADlwTGnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAARP0lEQVR4nO3df6xfdX3H8eeLFtAyUze4M5PS3iZUTZH5IzdM52IyO0fxV/cHiTV1IRtJ9wc4XZYYSP/YIOkiyTIxGZo0giN4Z2GdZheT+QtM/McBt8pWKTTeAYUylQpYN0mA4nt/3INeLpf2W3r7Pd97P89HctPz/ZzPOfd9TtLv657zOT9SVUiS2nNa3wVIkvphAEhSowwASWqUASBJjTIAJKlRK/su4EScc845NT4+3ncZkrRk7N2796dVNbbQvCUVAOPj40xPT/ddhiQtGUkOvtw8TwFJUqMMAElqlAEgSY0yACSpUQaAJDVq2QfA5L5Jxq8f57RrTmP8+nEm9032XZIkjYQldRnoiZrcN8n227fz9HNPA3DwyEG2374dgG0XbuuzNEnq3bI+Athxx45fffm/4OnnnmbHHTt6qkiSRseyDoBHjjxyQu2S1JJlHQBrV689oXZJasmyDoCdm3ay6vRVL2pbdfoqdm7a2VNFkjQ6lnUAbLtwG7s+uIt1q9cRwrrV69j1wV0OAEsSkKX0TuCJiYnyYXCSNLgke6tqYqF5y/oIQJL08gwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0aKACSbE5yIMlMkqsWmH9mklu7+XclGZ8z7+qu/UCSi7u2Nya5d87Pz5N8YrE2SpJ0fMd9JWSSFcANwHuBQ8A9Saaqav+cbpcDT1XV+Um2AtcBH06yEdgKXAC8HvhWkjdU1QHgrXPW/xjwlUXcLknScQxyBHARMFNVD1bVs8BuYMu8PluAm7vpPcCmJOnad1fVM1X1EDDTrW+uTcB/V9XBV7oRkqQTN0gAnAs8Oufzoa5twT5VdRQ4Apw94LJbgS8NXrIkaTH0Ogic5AzgQ8C/HKPP9iTTSaYPHz48vOIkaZkbJAAeA86b83lN17ZgnyQrgdXAEwMsewnwvar6ycv98qraVVUTVTUxNjY2QLmSpEEMEgD3ABuSrO/+Yt8KTM3rMwVc1k1fCtxZs68amwK2dlcJrQc2AHfPWe4jePpHknpx3KuAqupokiuBrwMrgJuq6r4k1wLTVTUF3AjckmQGeJLZkKDrdxuwHzgKXFFVzwMkOYvZK4v+4hRslyTpOHwnsCQtY74TWJL0EgaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNGigAkmxOciDJTJKrFph/ZpJbu/l3JRmfM+/qrv1AkovntL82yZ4kDyS5P8k7F2ODJEmDOW4AJFkB3ABcAmwEPpJk47xulwNPVdX5wKeB67plNwJbgQuAzcBnu/UBfAb4WlW9CXgLcP/Jb44kaVCDHAFcBMxU1YNV9SywG9gyr88W4OZueg+wKUm69t1V9UxVPQTMABclWQ28G7gRoKqeraqfnfzmSJIGNUgAnAs8Oufzoa5twT5VdRQ4Apx9jGXXA4eBLyT5fpLPJznrFW2BJOkV6WsQeCXwduBzVfU24BfAS8YWAJJsTzKdZPrw4cPDrFGSlrVBAuAx4Lw5n9d0bQv2SbISWA08cYxlDwGHququrn0Ps4HwElW1q6omqmpibGxsgHIlSYMYJADuATYkWZ/kDGYHdafm9ZkCLuumLwXurKrq2rd2VwmtBzYAd1fVj4FHk7yxW2YTsP8kt0WSdAJWHq9DVR1NciXwdWAFcFNV3ZfkWmC6qqaYHcy9JckM8CSzIUHX7zZmv9yPAldU1fPdqj8GTHah8iDwZ4u8bZKkY8jsH+pLw8TERE1PT/ddhiQtGUn2VtXEQvO8E1iSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUqIECIMnmJAeSzCS5aoH5Zya5tZt/V5LxOfOu7toPJLl4TvvDSfYluTfJ9GJsjCRpcCuP1yHJCuAG4L3AIeCeJFNVtX9Ot8uBp6rq/CRbgeuADyfZCGwFLgBeD3wryRuq6vluuT+sqp8u4vZIkgY0yBHARcBMVT1YVc8Cu4Et8/psAW7upvcAm5Kka99dVc9U1UPATLc+SVLPBgmAc4FH53w+1LUt2KeqjgJHgLOPs2wB30iyN8n2Ey9dknQyjnsK6BT6g6p6LMlvA99M8kBVfWd+py4ctgOsXbt22DVK0rI1yBHAY8B5cz6v6doW7JNkJbAaeOJYy1bVC/8+DnyFlzk1VFW7qmqiqibGxsYGKFeSNIhBAuAeYEOS9UnOYHZQd2penyngsm76UuDOqqqufWt3ldB6YANwd5KzkrwGIMlZwB8DPzj5zZEkDeq4p4Cq6miSK4GvAyuAm6rqviTXAtNVNQXcCNySZAZ4ktmQoOt3G7AfOApcUVXPJ3kd8JXZcWJWAv9cVV87BdsnSXoZmf1DfWmYmJio6WlvGZCkQSXZW1UTC83zTmBJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQCS1CgDQJIaZQBIUqMMAElqlAEgSY0yACSpUQaAJDXKAJCkRhkAktQoA0CSGmUASFKjDABJapQBIEmNMgAkqVEGgCQ1ygAYksl9k4xfP85p15zG+PXjTO6b7LskSY1b2XcBLZjcN8n227fz9HNPA3DwyEG2374dgG0XbuuzNEkN8whgCHbcseNXX/4vePq5p9lxx46eKpIkA2AoHjnyyAm1S9IwDBQASTYnOZBkJslVC8w/M8mt3fy7kozPmXd1134gycXzlluR5PtJvnqyGzLK1q5ee0LtkjQMxw2AJCuAG4BLgI3AR5JsnNftcuCpqjof+DRwXbfsRmArcAGwGfhst74XfBy4/2Q3YtTt3LSTVaevelHbqtNXsXPTzp4qkqTBjgAuAmaq6sGqehbYDWyZ12cLcHM3vQfYlCRd++6qeqaqHgJmuvWRZA3wfuDzJ78Zo23bhdvY9cFdrFu9jhDWrV7Hrg/ucgBYUq8GuQroXODROZ8PAb/3cn2q6miSI8DZXft/zFv23G76euCTwGtOvOylZ9uF2/zClzRSehkETvIB4PGq2jtA3+1JppNMHz58eAjVSVIbBgmAx4Dz5nxe07Ut2CfJSmA18MQxln0X8KEkDzN7Suk9Sb640C+vql1VNVFVE2NjYwOUK0kaxCABcA+wIcn6JGcwO6g7Na/PFHBZN30pcGdVVde+tbtKaD2wAbi7qq6uqjVVNd6t786q+ugibI8kaUDHHQPozulfCXwdWAHcVFX3JbkWmK6qKeBG4JYkM8CTzH6p0/W7DdgPHAWuqKrnT9G2SJJOQGb/UF8aJiYmanp6uu8yJGnJSLK3qiYWmuedwJLUKANAkhplAEhSowwASWqUASBJjTIAJKlRBoAkNcoAkKRGGQAN8cX0kubypfCN8MX0kubzCKARvphe0nwGQCN8Mb2k+QyARvhieknzGQCN8MX0kuYzABrhi+klzef7ACRpGfN9AJKklzAAJKlRBoAkNcoA0ND5SAppNPgoCA2Vj6SQRodHABoqH0khjQ4DQEPlIymk0WEAaKh8JIU0OgwADZWPpJBGhwGgofKRFNLo8FEQkrSM+SgIaQHej6DWeR+AmuT9CJJHAGqU9yNIBoAa5f0IkgGgRnk/gjRgACTZnORAkpkkVy0w/8wkt3bz70oyPmfe1V37gSQXd22vSnJ3kv9Mcl+SaxZrg6RBjNL9CA5Gqy/HDYAkK4AbgEuAjcBHkmyc1+1y4KmqOh/4NHBdt+xGYCtwAbAZ+Gy3vmeA91TVW4C3ApuTvGNxNkk6vlG5H+GFweiDRw5S1K8Gow0BDcMgVwFdBMxU1YMASXYDW4D9c/psAf62m94D/GOSdO27q+oZ4KEkM8BFVfVd4P+6/qd3P0vnhgQtC9su3Nb7FT/HGozuuzYtf4OcAjoXeHTO50Nd24J9quoocAQ4+1jLJlmR5F7gceCbVXXXK9kAaSlzMFp96m0QuKqer6q3AmuAi5K8eaF+SbYnmU4yffjw4eEWKZ1iDkarT4MEwGPAeXM+r+naFuyTZCWwGnhikGWr6mfAt5kdI3iJqtpVVRNVNTE2NjZAudLS4WC0+jRIANwDbEiyPskZzA7qTs3rMwVc1k1fCtxZsw8ZmgK2dlcJrQc2AHcnGUvyWoAkrwbeCzxw8psjLS0ORqtPAz0MLsn7gOuBFcBNVbUzybXAdFVNJXkVcAvwNuBJYOucQeMdwJ8DR4FPVNW/J/ld4OZufacBt1XVtcerw4fBSafG+PXjHDxy8CXt61av4+FPPDz8grRojvUwOJ8GKonTrjmNWuBCvBB++Te/7KEiLRafBirpmEZpMNqxiOExACSNzGC0YxHDZQBIGpnBaJ/SOly+D0ASMBp3Ro/KjXGT+ybZcccOHjnyCGtXr2Xnpp2975tTwSMASSNjFMYiWjoNZQBIGhmjMBbR0mkoA0DSyBiFsYhROQ0Fp/6KKMcAJI2Uvsci1q5eu+BNccO+JHYY7632CECS5hiF01AwnFNRBoAkzTEKp6FgOKeiPAUkSfP0fRoKhnMqyiMASRpBwzgVZQBI0ggaxqkonwYqScuYTwOVJL2EASBJjTIAJKlRBoAkNcoAkKRGLamrgJIcBl56Z8TScg7w076LGBHuixdzf7yY++PXTmZfrKuqsYVmLKkAWA6STL/cJVmtcV+8mPvjxdwfv3aq9oWngCSpUQaAJDXKABi+XX0XMELcFy/m/ngx98evnZJ94RiAJDXKIwBJapQBIEmNMgCGIMl5Sb6dZH+S+5J8vO+aRkGSFUm+n+SrfdfSpySvTbInyQNJ7k/yzr5r6lOSv+r+n/wgyZeSvKrvmoYpyU1JHk/ygzltv5Xkm0l+2P37m4vxuwyA4TgK/HVVbQTeAVyRZGPPNY2CjwP3913ECPgM8LWqehPwFhreJ0nOBf4SmKiqNwMrgK39VjV0/wRsntd2FXBHVW0A7ug+nzQDYAiq6kdV9b1u+n+Z/Q9+br9V9SvJGuD9wOf7rqVPSVYD7wZuBKiqZ6vqZ/1W1buVwKuTrARWAf/Tcz1DVVXfAZ6c17wFuLmbvhn4k8X4XQbAkCUZB94G3NVvJb27Hvgk8Mu+C+nZeuAw8IXudNjnk5zVd1F9qarHgL8HHgF+BBypqm/0W9VIeF1V/aib/jHwusVYqQEwREl+A/hX4BNV9fO+6+lLkg8Aj1fV3r5rGQErgbcDn6uqtwG/YJEO75ei7tz2FmaD8fXAWUk+2m9Vo6Vmr91flOv3DYAhSXI6s1/+k1X15b7r6dm7gA8leRjYDbwnyRf7Lak3h4BDVfXCEeEeZgOhVX8EPFRVh6vqOeDLwO/3XNMo+EmS3wHo/n18MVZqAAxBkjB7jvf+qvqHvuvpW1VdXVVrqmqc2QG+O6uqyb/yqurHwKNJ3tg1bQL291hS3x4B3pFkVff/ZhMND4rPMQVc1k1fBvzbYqzUABiOdwF/yuxfuvd2P+/ruyiNjI8Bk0n+C3gr8Hc919Ob7khoD/A9YB+z31FNPRIiyZeA7wJvTHIoyeXAp4D3Jvkhs0dJn1qU3+WjICSpTR4BSFKjDABJapQBIEmNMgAkqVEGgCQ1ygCQpEYZAJLUqP8H2uPcdEKFZtYAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Saving your model weights or parameters\n",
        "* It is always advisable to save your model checkpoints every k epochs. Look at Saver object in tensorflow/keras.\n",
        "* Visualize your model performance using tensorboard.\n",
        "\n",
        "# Steps to save model weights using pickle\n",
        "* Save your model(trainable variables), in our case self.variables into a pickle file.\n",
        "* Load saved file\n",
        "* Redefine model\n",
        "* Load weights\n",
        "* Re-train or test your model\n"
      ],
      "metadata": {
        "id": "2O5EAiXOHQir"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "#\n",
        "import pickle\n",
        "variables_old = mlp_on_cpu.variables\n",
        "with open('weights.pickle', 'wb') as handle:\n",
        "    pickle.dump(variables_old, handle)"
      ],
      "metadata": {
        "id": "-gtwqTXnjW30"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "with open('weights.pickle', 'rb') as handle:\n",
        "    b = pickle.load(handle)"
      ],
      "metadata": {
        "id": "l9Jc34APkghg"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Define class to build mlp model\n",
        "class MLP_check(object):\n",
        " def __init__(self, size_input, size_hidden1, size_hidden2, size_hidden3, size_output, device=None):\n",
        "    \"\"\"\n",
        "    size_input: int, size of input layer\n",
        "    size_hidden1: int, size of the 1st hidden layer\n",
        "    size_hidden2: int, size of the 2nd hidden layer\n",
        "    size_output: int, size of output layer\n",
        "    device: str or None, either 'cpu' or 'gpu' or None. If None, the device to be used will be decided automatically during Eager Execution\n",
        "    \"\"\"\n",
        "    self.size_input, self.size_hidden1, self.size_hidden2, self.size_hidden3, self.size_output, self.device =\\\n",
        "    size_input, size_hidden1, size_hidden2, size_hidden3, size_output, device\n",
        "    \n",
        "    # Initialize weights between input mapping and a layer g(f(x)) = layer\n",
        "    self.W1 = tf.Variable(tf.random.normal([self.size_input, self.size_hidden1],stddev=0.1)) # Xavier(Fan-in fan-out) and Orthogonal\n",
        "    # Initialize biases for hidden layer\n",
        "    self.b1 = tf.Variable(tf.zeros([1, self.size_hidden1])) # 0 or constant(0.01)\n",
        "    \n",
        "    # Initialize weights between input layer and 1st hidden layer\n",
        "    self.W2 = tf.Variable(tf.random.normal([self.size_hidden1, self.size_hidden2],stddev=0.1))\n",
        "    # Initialize biases for hidden layer\n",
        "    self.b2 = tf.Variable(tf.zeros([1, self.size_hidden2]))\n",
        "    \n",
        "    # Initialize weights between 1st hidden layer and 2nd hidden layer\n",
        "    self.W3 = tf.Variable(tf.random.normal([self.size_hidden2, self.size_hidden3],stddev=0.1))\n",
        "    # Initialize biases for hidden layer\n",
        "    self.b3 = tf.Variable(tf.zeros([1, self.size_hidden3]))\n",
        "    \n",
        "     # Initialize weights between 2nd hidden layer and output layer\n",
        "    self.W4 = tf.Variable(tf.random.normal([self.size_hidden3, self.size_output],stddev=0.1))\n",
        "    # Initialize biases for output layer\n",
        "    self.b4 = tf.Variable(tf.zeros([1, self.size_output]))\n",
        "    \n",
        "    # Define variables to be updated during backpropagation\n",
        "    self.variables = [self.W1, self.W2, self.W3, self.W4, self.b1, self.b2, self.b3, self.b4]\n",
        "  \n",
        " def forward(self, X):\n",
        "    \"\"\"\n",
        "    forward pass\n",
        "    X: Tensor, inputs\n",
        "    \"\"\"\n",
        "    if self.device is not None:\n",
        "      with tf.device('gpu:0' if self.device=='gpu' else 'cpu'):\n",
        "        self.y = self.compute_output(X)\n",
        "    else:\n",
        "      self.y = self.compute_output(X)\n",
        "      \n",
        "    return self.y\n",
        "\n",
        " def loss(self, y_pred, y_true):\n",
        "    '''\n",
        "    y_pred - Tensor of shape (batch_size, size_output)\n",
        "    y_true - Tensor of shape (batch_size, size_output)\n",
        "    '''\n",
        "    #y_true_tf = tf.cast(tf.reshape(y_true, (-1, self.size_output)), dtype=tf.float32)\n",
        "    y_true_tf = tf.cast(y_true, dtype=tf.float32)\n",
        "    y_pred_tf = tf.cast(y_pred, dtype=tf.float32)\n",
        "    cce = tf.keras.losses.CategoricalCrossentropy(from_logits=True)\n",
        "    loss_x = cce(y_true_tf, y_pred_tf)\n",
        "    # Use keras or tf_softmax, both should work for any given model\n",
        "    #loss_x = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=y_pred_tf, labels=y_true_tf))\n",
        "    \n",
        "    return loss_x\n",
        "\n",
        " def backward(self, X_train, y_train, opti):\n",
        "    \"\"\"\n",
        "    backward pass\n",
        "    \"\"\"\n",
        "    optimizer = opti\n",
        "    \n",
        "    with tf.GradientTape() as tape:\n",
        "        \n",
        "      predicted = self.forward(X_train)\n",
        "      current_loss = self.loss(predicted, y_train)\n",
        "        \n",
        "    grads = tape.gradient(current_loss, self.variables)\n",
        "    optimizer.apply_gradients(zip(grads, self.variables))\n",
        "           \n",
        " def compute_output(self, X):\n",
        "    \"\"\"\n",
        "    Custom method to obtain output tensor during forward pass\n",
        "    \"\"\"\n",
        "    # Cast X to float32\n",
        "    X_tf = tf.cast(X, dtype=tf.float32)\n",
        "    #X_tf = X\n",
        "    \n",
        "    # Compute values in hidden layers\n",
        "    z1 = tf.matmul(X_tf, self.W1) + self.b1\n",
        "    h1 = tf.nn.relu(z1)\n",
        "    \n",
        "    z2 = tf.matmul(h1, self.W2) + self.b2\n",
        "    h2 = tf.nn.relu(z2)\n",
        "    \n",
        "    z3 = tf.matmul(h2, self.W3) + self.b3\n",
        "    h3 = tf.nn.relu(z3)\n",
        "\n",
        "    # Compute output\n",
        "    output = tf.matmul(h3, self.W4) + self.b4\n",
        "    \n",
        "    #Now consider two things , First look at inbuild loss functions if they work with softmax or not and then change this \n",
        "    # Second add tf.Softmax(output) and then return this variable\n",
        "    return (output)\n",
        "\n"
      ],
      "metadata": {
        "id": "gXRQs7HWtgVW"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "MLP_after_check = MLP_check(size_input, size_hidden1, size_hidden2, size_hidden3, size_output, device='gpu')\n",
        "W = MLP_after_check.variables[0]\n",
        "W.assign(b[0])\n",
        "W1 = MLP_after_check.variables[1]\n",
        "W1.assign(b[1])\n",
        "W2 = MLP_after_check.variables[2]\n",
        "W2.assign(b[2])\n",
        "W3 = MLP_after_check.variables[3]\n",
        "W3.assign(b[3])\n",
        "B = MLP_after_check.variables[4]\n",
        "B.assign(b[4])\n",
        "B1 = MLP_after_check.variables[5]\n",
        "B1.assign(b[5])\n",
        "B2 = MLP_after_check.variables[6]\n",
        "B2.assign(b[6])\n",
        "B3 = MLP_after_check.variables[7]\n",
        "B3.assign(b[7])\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Qg3MXT4Stqan",
        "outputId": "15c1e16e-82a0-4e73-ae52-5d3661c0756f"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<tf.Variable 'Variable:0' shape=(784, 128) dtype=float32, numpy=\n",
            "array([[-0.16473156, -0.08003395, -0.06984133, ...,  0.05928857,\n",
            "         0.08477078, -0.08955226],\n",
            "       [-0.05746964,  0.11165438,  0.07991587, ..., -0.14035715,\n",
            "        -0.08606252,  0.05247764],\n",
            "       [-0.03853896, -0.18610518,  0.16467   , ...,  0.08612005,\n",
            "        -0.08745918,  0.01032779],\n",
            "       ...,\n",
            "       [ 0.10590637,  0.04486223,  0.07102545, ..., -0.1274214 ,\n",
            "         0.18856107, -0.1880648 ],\n",
            "       [-0.12794258, -0.09081408,  0.02673411, ..., -0.05354583,\n",
            "        -0.28470454, -0.04574722],\n",
            "       [-0.06035139, -0.14126025,  0.05768505, ..., -0.0247896 ,\n",
            "        -0.2056571 , -0.14418238]], dtype=float32)>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {
        "id": "8eNKeELWCX6_",
        "outputId": "9a61fa17-e512-4256-f424-1195fbd34658",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test loss: 0.0986\n",
            "\n",
            "Test Accuracy: 0.86\n"
          ]
        }
      ],
      "source": [
        "# Initialize\n",
        "test_loss_total = tf.Variable(0, dtype=tf.float32)\n",
        "correct_prediction = tf.Variable(0, dtype=tf.float32)\n",
        "\n",
        "\n",
        "test_ds = tf.data.Dataset.from_tensor_slices((X_test, y_test)).batch(4)\n",
        "\n",
        "\n",
        "#test_loss_total = 0.0\n",
        "for inputs, outputs in test_ds:\n",
        "  preds = mlp_on_cpu.forward(inputs)\n",
        "  test_loss_total = test_loss_total + MLP_after_check.loss(preds, outputs)\n",
        "print('Test loss: {:.4f}'.format(np.sum(test_loss_total.numpy()) / X_test.shape[0]))\n",
        "\n",
        "# Test model\n",
        "preds_test = MLP_after_check.forward(X_test)\n",
        "preds_test = tf.nn.softmax(preds_test)\n",
        "correct_prediction = tf.equal(tf.argmax(preds_test, 1), tf.argmax(y_test, 1))\n",
        "\n",
        "# Calculate accuracy\n",
        "accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
        "cur_test_acc = accuracy.numpy()\n",
        "print('\\nTest Accuracy: {:.2f}'.format(cur_test_acc))"
      ]
    },
    {
      "cell_type": "raw",
      "metadata": {
        "id": "5DRomYsUCX6_"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "id": "xYsXWqI_CX6_",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5ac8a5f6-0300-40f3-ab40-742707cc58bb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Test loss: 0.0986\n",
            "\n",
            "Test Accuracy: 0.86\n"
          ]
        }
      ],
      "source": [
        "# Initialize\n",
        "test_loss_total = tf.Variable(0, dtype=tf.float32)\n",
        "correct_prediction = tf.Variable(0, dtype=tf.float32)\n",
        "\n",
        "\n",
        "test_ds = tf.data.Dataset.from_tensor_slices((X_test, y_test)).batch(4)\n",
        "\n",
        "\n",
        "#test_loss_total = 0.0\n",
        "for inputs, outputs in test_ds:\n",
        "  preds = mlp_on_cpu.forward(inputs)\n",
        "  test_loss_total = test_loss_total + mlp_on_cpu.loss(preds, outputs)\n",
        "print('Test loss: {:.4f}'.format(np.sum(test_loss_total.numpy()) / X_test.shape[0]))\n",
        "\n",
        "# Test model\n",
        "preds_test = mlp_on_cpu.forward(X_test)\n",
        "preds_test = tf.nn.softmax(preds_test)\n",
        "correct_prediction = tf.equal(tf.argmax(preds_test, 1), tf.argmax(y_test, 1))\n",
        "\n",
        "# Calculate accuracy\n",
        "accuracy = tf.reduce_mean(tf.cast(correct_prediction, \"float\"))\n",
        "cur_test_acc = accuracy.numpy()\n",
        "print('\\nTest Accuracy: {:.2f}'.format(cur_test_acc))"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.9"
    },
    "colab": {
      "name": "MLP_Fmnist_Saver_optimizer.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}