Advanced_NN.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "# Network using Auto-differentiation\n",
    "Purpose: To derive a version of my Network module that uses AD."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from utils import *\n",
    "import NetworkAD as nnad\n",
    "from joMatAD import *\n",
    "from copy import deepcopy\n",
    "import matplotlib.pyplot as plt\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# This creates a dataset\n",
    "n_classes = 3\n",
    "in_dim = 2\n",
    "\n",
    "np.random.seed(250)\n",
    "B = np.eye(n_classes)\n",
    "A = 3.*(np.random.rand(n_classes, in_dim) - 0.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "ds = SimpleDataset(A, B, n=100, noise=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAD4CAYAAADhNOGaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAgAElEQVR4nO3deXxU9bn48c8zk2SyQCDsCRAIm4giIAFBrftCvSq4tVJrsWqpW21ra7W119uf1qVVr9W2V0VcqHXB2la4LZZWpfVaXAiKQtj3LUBCICSZLLM8vz/OSUjI7kwySeZ5+5pXZs75njPPeY3MM+e7iqpijDEmfnliHYAxxpjYskRgjDFxzhKBMcbEOUsExhgT5ywRGGNMnEuIdQBfRL9+/XT48OGxDsMYY7qUlStXFqlq/2O3d8lEMHz4cPLy8mIdhjHGdCkisqOx7VY1ZIwxcc4SgTHGxDlLBMYYE+csERhjTJyzRGCMMXHOEoExxsQ5SwTGGBPnLBEYYxpQVW79660k3p/IiCdGsPXQ1liHZNqRJQJjTAN5e/NY8NkCguEgOw7v4Id//2GsQzLtyBKBMaZFHrGviu7MPl1jTAO5WblcP+l6Ej2JjMgYwaMXPBrrkEw7kq64VGVubq7aXEPGGNM2IrJSVXOP3W53BMaYqAprmNvfup2cX+Vw019uIhgOxjok0wJLBMaYqHp19as8/+nzbC/Zzkufv8SCVQtiHZJpgSUCY0xUFfmLau8CAqEAhf7CGEdkWmKJwBgTVV8/6etk9swkNTGVAWkDuG7idbEOybQgKgvTiMgM4AnAC8xX1YeP2f84cLb7MhUYoKq93X0hYLW7b6eqXhqNmIwxR+0t3cu9y+4F4L6z7yOrZ1abz1FeXc4Lq14g0ZPIdROvw5fga7Rc39S+bLxtI3tK95DVM4skb1JEsZv2F3EiEBEv8FvgfGA3sEJEFqvq2poyqvr9OuW/A0yqc4oKVZ0YaRzGmKPKq8tZumUpmT0ymT50Ouf+7lw2HdwEwPJdy1l769p65atD1Vy/6HqWbV/GZWMv48kvP9lg7MB5vzuPVftXgcKiDYtYcs2S2n3PrHyGJz96komDJjLv4nmkJaUxvPfwdr9OEx3RuCOYCmxW1a0AIvIaMBNY20T52cB/ReF9jTGNCIQCnDL/FHaU7EBVeeDcB9hcvJmQhgDYVLypwTHP5D3Dn9b9iYpgBS+uepEzh53JVSdcBYA/4GfmqzP5cM+HteX/tvlvHKo4REZKBqv2reKOpXfgD/jZUryFQWmDeOzCxzrmYk1URKONYDCwq87r3e62BkRkGJADvFtnc7KI5InIhyIyKwrxGBPXNhVvYvvh7ZRVl1EeKGde3jy+Nv5rpCWm0SOxB9eMv6bBMcUVxQTCAcDp/llcUVy775m8Z3h/5/sNjvn1x78GoKC0AK94AagKVbGjpNFlcU0nFo07AmlkW1Oj1K4G3lB1f5o4slV1r4iMAN4VkdWquqXBm4jMBeYCZGdnRxqzMd3WkPQhJHicf9rJCclMHTyV52Y+x7UnXQvAuTnnNjhm7uS5zP90PkX+Iob1Hsbs8bPZdmgbz37yLOsK1xEmXK+81+PljbVvcKD8AP95xn8yqs8oNh7ciEc8/Pj0H7f/RZqoinhksYhMB36mqhe6r38MoKoPNVL2U+BWVV3exLleBP6iqm809542stiY5q3ev5pHP3iU7PRsfvKln5CSmNLiMcFwkAPlBxiYNpCKYAXDfzWcQ5WHSPIkkZaURnFFMRnJGZRWlaIoIQ2R6EnkvBHnsWj2IjYe3MjgnoPpldyrA67QfBFNjSyOxh3BCmC0iOQAe3B+9X+tkQCOAzKAD+psywD8qlolIv2A04BfRiEmY+La+IHjWTCrbQO5EjwJtb2Jth/eTlWoirCGqQxVMqLHCAp+UECiN5G9pXsZ+cRIgqEg1eFqPt//OQmeBMb1H9cel2I6QMRtBKoaBG4DlgLrgNdVNV9E7hORul1BZwOvaf1bkOOBPBH5DFgGPFy3t5ExJjZG9RlFn5Q+pCSkkJaYxldO+AqJ3kQAMntkMnXIVHok9SA1MZVbp94a42hNpGzSOWNMA5sObuLqN65mT+kebp1yKz8946eIHG0ODIQCPPz+wzz7ybNk9czi5ctfZmSfkTGM2LRGU1VDlgiMMQ1MeHoCq/evRlFSElIovquY5ITk2v1l1WUMeGQAFcEKBGFy1mRWfGtFDCM2rWGzjxpjWq2ovAh1O/8Fw0Eqg5X19vsD/tpxCYpSVF7U6nMXlhdy1R+u4vTnT+df2/8VvaDNF2aJwBjTwKMXPkqyNxmf18ctU26hd3LvevsHpA3gxkk34vP6SE5IbtPCNdf++VreXP8m/971by565SJKKktaPKbIX8SNi2/kitevIP9AfpuvxzTPqoaMMY0qriimMljZ7LxEe0v3kpqY2iBRAHy27zOuX3Q9/qCfh855iFnHO+NFx/x6TO3o5pSEFPJvyScnI6fZWM568SyW71pOMBwkIyWDfT/YV9t4bVrPqoaMMW3SJ6VPgyTwev7rTHx6IrPfmM2RqiNk9cxqkAQOVx7mseWPcfaCs/lk3yesL1rPZa9fxp1/vxOAe8+8t7Y30tnDz27VnETritYRCAdQlNKqUo5UHQHg0eWPMurJUXz1ja9SXl0enQuPQ3ZHYIxpla2HtnLi/5xIRbCCJG8S14y/hudnPl+vTDgcpv8j/SmuLG5wvAcP/nv8+BJ87Di8g+KKYiYMmtBgcrvGPPDeAzz4/oN4xMMZ2Wfwl6/9hQ92f8AFL11AeaAcn9fH96Z9j4fPe7jFc8Wz9hxQZoyJA/vK9uH1OHMKVYeq2XZ4W4Myb254s9EkABAmzIzfz2DR7EUM6z2MYb2Htfq97znjHmaMmsGRqiOcMewMRITC8sLaLq3VoWr2lu79AldlwKqGjDHNeGvTW0ybP42v/fFrjMoYxfgB40lLTCM1MZX/OrPhJMLVoep6v/AH9xzMyIyReNyvmuW7l/PLf3+xyQMmZ03m7Jyza5PRhaMuZEzfMaQkpNAruRd3nXbXFzqvsTsCY+JaWMPMWzmP/MJ8bph0AxMHHV0aZF/ZPq54/QoqghV8uu9Twhrm/775f6wtXMugHoPon9a/wflmjZ3F4J6D2XXEmZD4Z2f+jPTkdK5fdD3lgXLCGsYf8Ecl9uSEZD6+8WN2lOxgUI9BpCamRuW88cjuCIyJYw+9/xA/+PsP+M3Hv+H050+vV72yv2x/7a/76lA1m4s34/V4GT9wfKNJAEAQ9pXtq33902U/ZdbYWUwdPBWPeBiaPpQ7T70zavF7PV5GZIywJBAhSwTGxLFl25bV/kL3erysLTw61deJA05k4qCJpCSk4PP6uPfMe5s8j6pyuPIwqlqvasiX4KOsuow5E+bwt2v+xpbbt5DZM7P9Lsh8IZYIjIljcybOITUxlbTENJK9yeRm1e9QEggHEBFEhJKqxgd+BcNBLnrlIgY8MoBhvxrGI+c/Qt+UvgzpOYQXLn2BCU9P4Ja/3sIlr17Cox+0fuCZ6TjWRmBMHLv2pGvJ6Z3DxoMbuXjMxfXGBKwvWk/+gfzaO4ZH/v1I7eI2dS3btoz3d75PIByg0F/I8l3LKfqRM+XEB7s+oKSyBH/QOcfdb9/NWcPOYsrgKR1wdaa1LBEYE+dOzz6d07NPb7C9bhWOz+vjhP4nNHp8j6Qe1IxHSvAkkJ6czrZD25j9x9kUlBZQFayqLRvWME9+9CQvXf5SlK/CRMKqhowxjeqT0ocl1yzhvBHnMWfiHJ655JlGy00fOp3vT/s+/VL6MX3IdB4850G+/uevs2LvCnYe2UmYcO2axikJKYzpO6YjL8O0giUCY+JEQWkBZ7xwBkMfH8ozKxv/Uq/rw90fcsmrl7Bs2zKSvcmk+9KbLHv/OfdT+KNC/vXNf9E3tS9F5UWE1Vnn2CteHr3gUaYNmcbcyXO563Tr79/Z2BQTxsSJyxdezuINiwlpiJSEFNbcsoYRGSOaLD/l2Snk7XX+naUkpLD65tWtXnxmyaYlXPn6lYQ1zJwJTd9NmI5lU0wYE+cO+g/WriHgEU+L0z+nJaYhCOr+V3dhmpZcNPoi9v9wP+WBcgb1GBRR3Kb9WdWQMXHiF+f/gh5JPUjyJjFj1Ix6o4gbM//S+YzrP44+KX147PzHGJw+uE3v19PXs14S2FK8hcsWXsblCy9n26GG8xSZ2IlK1ZCIzACeALzAfFV9+Jj91wGPAHvcTb9R1fnuvjnAT93tP1fVBS29n1UNGfPFVAYr2VmykyWbltDL14trJ1xLgqdjKgZynshhZ8lOUBjZZyQbv7OxQ97XHNVuVUMi4gV+C5wP7AZWiMhiVV17TNGFqnrbMcf2Af4LyAUUWOkeeyjSuIwxDSV5k7j4lYvZWbITr8fLezve44VZL7T7+6oqu0p21TYg7yjZ0e7vaVovGlVDU4HNqrpVVauB14CZrTz2QuAfqlrsfvn/A5gRhZiMcRQXw/r1EA7HOpJOochfxM6SnVSFqvAH/Ly1+a0OeV8R4duTv107c+nNuTd3yPua1onGPeFgYFed17uBUxopd4WInAFsBL6vqruaOLbRikgRmQvMBcjOzo5C2KbbKCqC6mrIOmZJxWXL4OKLnedTpsA774DX2/HxdSJ9U/oyJH0Iu47sIkESuHDkhR323r+56DdcN/E6POLh5MyTO+x9TcuicUcgjWw7tuHhf4HhqnoS8DZQ0w7QmmOdjarzVDVXVXP792985kMTh158EYYMgREj4Ec/qr/vZz8Dv995rFzpPOKc1+Ploxs/4oFzHuBXM37FczOf67D3FhGmDJ7C5KzJtQvKmM4hGolgNzC0zushQL2lglT1oKrWjDN/Fpjc2mONadaPfgRVVc7jscfg97+Hmg4Q2dmQlOQ8D4XAfkAA0De1Lz889Yd8a/K3OqyhuC5/wM/WQ1sJhUMd/t6mcdFIBCuA0SKSIyJJwNXA4roFRKTuvLOXAuvc50uBC0QkQ0QygAvcbSbe7dkDe1vxm6Bfv6PPw2G46SZ44gnn9ZNPwn/8B5xwAjz/POTktE+sptXWFa5jyH8PYfxT48l9NpeKQEWsQzJEIRGoahC4DecLfB3wuqrmi8h9InKpW+x2EckXkc+A24Hr3GOLgftxkskK4D53m4ln998PI0c61T2PPNJ82T/9CY47Djzu/8rl5fCXvzjPMzKc/WvWwNVXt2/MplUe/eBRDlcexh/ws7l4c4c1VpvmReW+UFWXAEuO2XZvnec/Bn7cxLHPA89HIw7TDajCffdBMOi8vvdeuLOZFa3GjoX334fRo522gMREuPzyjonVtNmgtEEkeZOoClWhqvRPteq6zsCmmDCdi4jzS76w0Hndt2/Lx/TrB59+Cm+84dwdXHJJ+8ZovrB7zriHLYe2sGLvCm48+Ua+NOxLsQ7JYJPOmc5o5Ur49red6p5nn4UJE2IdUVwrLC9kxd4VTBg4oc3TTJjOxSadM13H5MnwRRJ9KASlpdCrl3NnYSK2q2QXE56eQEhDhDXM8uuXM37g+FiHZaLMJp0z3cOmTc6AsgED4MILj7YxmIgs3rCYimAFR6qOUFZdxiurX4l1SKYdWCIw3cN99zkjjAMB+OADePttKClxupCOHQt9+sDPfx7rKLuc4/sfj0ecr4nUxFS7G+imrGrIdA/p6c70EeGw0/PI54OTT4YdO5wqI4CHHoJLL4Xx451EceSIMwWFzxfb2Duxc3LO4an/eIqF+Qu5cOSFzD5xdqxDMu3AGotN91BcDFdc4YwZmDvXGTdw6qlQVna0jAicdZaTCJ57znl90klO91NrUzBxwBqLTff0r3/BV7/qTDo3f/7RMQRHjkBCgvMFX/NjRxWWL4cPP4QKd0TrihVOEmlNN1VjuilrIzCxt28f3HMP/OIXzqCwtrj6ati/Hw4dcu4IzjvP+ZJPT3e+8L/zHZgzB3r0cMpXVzvjFJKSnO6pfftC797OPr8frr/e6bX0+99H9xqN6cTsjsDEVjgM06fD7t1OHf+//w2LF7d8XI1jewctX+7MSHrzzc7gsieegMpKp8po7VpIToZFi+CPf4TDh+Guu45OTX3PPfDKK84EdnPnOm0M48ZF7VKN6awsEZjYKilxJpgLBp3H8uVtO37+fLjqKqe3UI1jF6FJToaPP4adO50pKNavh+9+FwYds6j6li1OEgCnWmn3bksEJi5Y1ZCJrd69nZHDqamQluZU74BTn3/rrU63zwsucAaKNWbmTDh40GkY9nggNxe++c2G5TwepzpowgSnHWHMGMjPr1/mrrucGHr0cGYqPeOM6F6rMZ2UJQITWyJOg+9vfwsLFsBTTznblyyB3/3Oqft/7z2n/aApGzc6iWDBAudcqamNl3vzTacd4MgRJ7G89JLTtXTDBmfbaafBtm3OOfLynDsJY+KAVQ2Z2EtNheuuq7/tvfeO9uwJBmHdOqdOv6Zht8bWrXDmmc7006mpzhf8zU2sh1t3uurERKdtYOpUp6ooIQH++U+YNMkWsDFxx+4ITMdRdfr1tzR25bPP4Ne/PjoQLBSCv//dqc7Jy6vfs+izz4429vr9zkCxppx7Lnzve04yCIXg0UedBuSau4QHH4zs+ozpoiwRmI7h98O0aU7XzTFjjk4z3Zh16xouMl9WdrQtICsLPv/c2T59uvPFnpLi3BHMbmHka3q6Uz4cdrqSVlY6271eZ2lLY+KQJQLTMRYudLpwBoPOtA+/+U3TZc87z/lST093qnBqpoAIh53eQSUl8MADzrZBg5y7gscfh5dfdnr+vPxy03cdX/qS02h87EjicBh+8pPIr9OYLsjaCEzHSE09+uXr8Ti9c5rSr59zV7B8uVOvv3Ch09C7erXzK97nq9/1c8gQp2fRKac47QhJSU6VT02yqGv6dFi6FN59F5555ui6yMnJzvTVxsQjVY34AcwANgCbgbsb2X8HsBb4HHgHGFZnXwhY5T4Wt+b9Jk+erKaLCQZV58xR7dVL9aKLVMvL236Oxx5TzclRnTVLtaTE2VZWpjppkmpioqqIqnMvoDp+fMvn+/RT1bFjVbOzVZcscbYVF6u+9prqihVtj8+YTg7I00a+UyOedE5EvMBG4HxgN84i9LNVdW2dMmcDH6mqX0RuBs5S1a+6+8pUtUdb3tMmnTO1XnrJ6SVUXu68FnHuPr7/fbj//radq7QUjj/eqXoKh53V0b72tejHbEyMNDXpXDTaCKYCm1V1q6pWA68BM+sWUNVlqlrT1eNDYEgU3teY+tU5Pp+zKM2zzzrTReTn1599tCV5eU4yKCtzGrfnz49+vMZ0QtFIBIOBXXVe73a3NeUG4K06r5NFJE9EPhSRWU0dJCJz3XJ5hc31ODHx5ZJL4IYbYOBAuOgiuPJKp7F46FCnl9KwYbB5c+vONWbM0S6rqalODyVj4kA0Gosbm8i90fomEfk6kAucWWdztqruFZERwLsislpVtzQ4oeo8YB44VUORh226BRFnYrknnnAagS+/vP44AxF4+mlnzEBLBg92GpGfftqZY+h732u/uI3pRKKRCHYDQ+u8HgLsPbaQiJwH3AOcqapVNdtVda/7d6uI/BOYBDRIBMY0av16+MEPnGqhcePqTz4HTm+goUMbP7YxU6c6D2PiSDQSwQpgtIjkAHuAq4F6LWwiMgl4BpihqgfqbM8A/KpaJSL9gNOAX0YhJhMPVJ0Vxw4ccH75r1njfPEnJzsDxbKznTaDW26JdaTGdGoRJwJVDYrIbcBSwAs8r6r5InIfTlelxcAjQA/gD+L0Jd+pqpcCxwPPiEgYp73i4bq9jYxpYPFiZ4zBFVc49f+FhTUdRqGgwLlDWLHCWVymLXcCxsQxW7PYdB3PPONUA1VVOY25GzY4vYMWLnQSwR13tL3LqDFxxNYsNl3fokVHxwsAfPqp08XzppuOrjVgjGkzm2vIdB2XXupMTZHg/n6ZNMlpG5gyxZKAMRGwOwLTddx0E2RmHm0jOHapSWPMF2KJwHQtM2c6D2NM1FjVkDHGxDlLBMYYE+csERhjTJyzRGCMMXHOEoExxsQ5SwTGGBPnLBEYY0ycs0RgjDFxzhKBMcbEOUsExhgT5ywRGGNMnLNEYIwxcc4SgTHGxDlLBMYYE+eikghEZIaIbBCRzSJydyP7fSKy0N3/kYgMr7Pvx+72DSJyYTTiMcYY03oRJwIR8QK/Bb4MjANmi8i4Y4rdABxS1VHA48Av3GPHAVcDJwAzgP9xz2eMMaaDROOOYCqwWVW3qmo18Bpw7MohM4EF7vM3gHNFRNztr6lqlapuAza75zPGGNNBopEIBgO76rze7W5rtIyqBoESoG8rjwVAROaKSJ6I5BUWFkYhbGOMMRCdRCCNbNNWlmnNsc5G1Xmqmququf37929jiMYYY5oSjUSwGxha5/UQYG9TZUQkAegFFLfyWGOMMe0oGolgBTBaRHJEJAmn8XfxMWUWA3Pc51cC76qqutuvdnsV5QCjgY+jEJMxxphWSoj0BKoaFJHbgKWAF3heVfNF5D4gT1UXA88BL4nIZpw7gavdY/NF5HVgLRAEblXVUKQxGWOMaT1xfph3Lbm5uZqXlxfrMIwxpksRkZWqmnvsdhtZbIwxcc4SgTHGxDlLBMYYE+csERhjTJyzRGCMMXHOEoExxsQ5SwTGGBPnLBEYY0ycs0RgjDFxzhKBMcbEOUsExhgT5ywRGGNMnLNEYIwxcc4SgTHGxDlLBMYYE+csERhjTJyzRGCMMXHOEoExxsS5iBKBiPQRkX+IyCb3b0YjZSaKyAciki8in4vIV+vse1FEtonIKvcxMZJ4jDHGtF2kdwR3A++o6mjgHff1sfzAN1T1BGAG8CsR6V1n/52qOtF9rIowHmOMMW0UaSKYCSxwny8AZh1bQFU3quom9/le4ADQP8L3NcYYEyWRJoKBqloA4P4d0FxhEZkKJAFb6mx+wK0yelxEfM0cO1dE8kQkr7CwMMKwjTHG1GgxEYjI2yKyppHHzLa8kYhkAi8B31TVsLv5x8BYYArQB7irqeNVdZ6q5qpqbv/+dkNhjDHRktBSAVU9r6l9IrJfRDJVtcD9oj/QRLl04K/AT1X1wzrnLnCfVonIC8AP2xS9McaYiEVaNbQYmOM+nwMsOraAiCQBfwZ+p6p/OGZfpvtXcNoX1kQYjzHGmDaKNBE8DJwvIpuA893XiEiuiMx3y3wFOAO4rpFuoi+LyGpgNdAP+HmE8RhjjGkjUdVYx9Bmubm5mpeXF+swjDGmSxGRlaqae+x2G1lsjDFxzhKBMcbEOUsExhgT5ywRGGNMnLNEYIwxcc4SgTHGxDlLBMYYE+csERhjTJyzRGCMMXHOEoExxsQ5SwTGGBPnLBEYY0ycs0RgjDFxzhKBMcbEOUsEph5V5eDBv1FQ8DyBwKFYh2OM6QCWCEw9O3c+RH7+lWza9B1WrpxMOFwV65CMMe3MEoGpZ//+lwmHywmH/QQChfj9m2IdkjGmnVkiMPX07n0WHk8KIIgkkZw8PNYhGWPaWUIkB4tIH2AhMBzYDnxFVRtULItICGddYoCdqnqpuz0HeA3oA3wCXKuq1ZHEZCIzatQTpKYeR2XlLrKyvk1CQo9Yh2SMaWeR3hHcDbyjqqOBd9zXjalQ1Ynu49I6238BPO4efwi4IcJ4TIQ8ngSGDLmdUaMeITV1VKzDMcZ0gEgTwUxggft8ATCrtQeKiADnAG98keONMcZER6SJYKCqFgC4fwc0US5ZRPJE5EMRqfmy7wscVtWg+3o3MDjCeEwzwuEgpaUrqaraG+tQjDGdSIttBCLyNjCokV33tOF9slV1r4iMAN4VkdXAkUbKaTNxzAXmAmRnZ7fhrQ1AOBzg00+/hN+fj2qYE0/8M336XBDrsIwxnUCLiUBVz2tqn4jsF5FMVS0QkUzgQBPn2Ov+3Soi/wQmAX8EeotIgntXMARo8qeqqs4D5gHk5uY2mTBM40pLV+L35xMKlQGwY8fPLREYY4DIq4YWA3Pc53OARccWEJEMEfG5z/sBpwFrVVWBZcCVzR1vosPny0Q1BICIj5QUpyG4tPQTtm+/n4MH/9bgmHA4gPMxGWO6s0gTwcPA+SKyCTjffY2I5IrIfLfM8UCeiHyG88X/sKqudffdBdwhIptx2gyeizAe4wqFKti69R7Wrr2G0tJPSU4exrhxr9Kz5xQGDPgKo0Y9QXn5ej799Ets3/4z8vOvoLDwTcCZZuLzzy/mvfeSeO+9FEpKPojx1Rhj2pN0xV98ubm5mpeXF+swOrX162/gwIFXCIcr8Xp7Mm3aThITe9crU1DwIps23UY4XA5AZua3Oe64pzl06J989tnZteUSE/tz2mmN1voZY7oQEVmpqrnHbreRxd1UaekKwuFK95VSVbWz3v5gsAyPJxVVRSQRjyeVvn0vRjWM37+5XtlQqKKDojbGxEJEI4tN55WVdQtbtvwAEQ8+XzapqcfX7gsEDrJixQRCoVJEhKys2+nb92KOHFnOmjUzgXCdMwkjRjzU4fEbYzqOJYJuavDgm+jZM5fq6j1kZJyPx5NYu6+o6H8JBg/XVgmpVpOWdgKff34hdZOASCKTJn1Aevrkjg7fGNOBLBF0Y+npuUCD6kBSUnJqn3s8qaSkjEbEA0i9ciI+0tLGtXOUxphYszaCONS795mMHPko6enTGDz4dgYPvoXExL6MHPkYHk8qHk8yaWknMWHCUrzelDadWzXM/v2vsWfPUwQCh9vpCowx0WS9hkxUbdx4G/v2vQiE8fmGMnXqWkS8sQ7LGIP1Goo7zpKTb3HgwEJCocrabQcO/IEtW+7iyJH2SaRFRW+6C9tUUFW1k+rqfe3yPsaY6LFE0E1t2XIH+flXsWHDjXz22dmoKvv2vcj69dexa9cvWbXqrAbdRKMhI+NcPJ4URBJITOxHYuLAqL+HMSa6rLG4m9q///e1vYJKSz8hGDzEoUNvEw77ARDxUFb2SdTXHDjuuPmkp59CIHCQzMxv4fHY/2LGdHb2r7Sb6tnzFA4dehvVIImJ/UhI6EX//ldSVPQmqmFASE8/FYBA4DCh0BF8vqE4y0Q0r+YtrWEAAA1MSURBVLx8LQUFz5OSMpqsrG+5PY4cHk8igwff0l6XZYxpB5YIuqlx415j165fEgweZujQHyLipX//y0hMXEp5+ef06fNlkpOHcPDg38jPvxxQ+vadxbhxryAihMMBCgqeJRAoJivrWyQlOVU81dWFfPLJNEKhUjyeVKqr95KT8/9ie7HGmIhYIuimEhJ6kJNzX4PtvXufTu/ep9e+3rr1R4TDzhQSBw8uorJyKykpI9mw4UYKC/+AaoCCgnmccspWPJ4E/P4N1Iw3CIf9HDr0jiUCY7o4ayyOc84v/aPdO73edAAOHXqHcLgC1SCBQCGBwH4AevQ4CY8nxR1vkMbAgdfEImxjTBTZHUE35PdvZPfuX5OcnM2QId/F40lqsuzYsS+ybt21VFXtISfnQZKS+gPQr99M9u1bgGoIn28oSUnOInUJCelMmfI5RUWLSEkZQUbGuR1yTcaY9mMDyrqZYLCMDz/MJhg8jMeTTN++F5OdfRc9epzcqobgGqphDhxYSDBYzIAB1zSYwtoY0/U0NaDM7gi6maqqHYTDAUAJhysoLHyDoqK/4PWm4PH4GDr0Tvr3v4Lk5ObXfRbxMHDg7I4J2hgTU9ZG0M2kpIzG5xuEx5PmblFUKwgGi6muLmDLljv46KMxbNx4W0zjNMZ0HpYIuhmPJ4nJk1cyZsxTJCePoW5DcA3VKvbufYoVK8azbt0cQiF/xwdqjOk0rGqoG0pISGfQoGvJyDiHTZtup6pqF+Xla9wVy2rahMKUl6/B799EQkIGaWnjqKjYRmbm9aSmjnZKhKvZt28B4XAVgwZdR0JCj5hdkzGm/UTUWCwifYCFwHBgO/AVVT10TJmzgcfrbBoLXK2qb4rIi8CZQIm77zpVXdXS+1pjcduFw1WUl69h06bvUlW1m0DgQO34AZ9vJIFAAeFwBV5vL6ZN20ZiYm9Wr76M4uKlQJi0tPHk5q6I7UUYYyLSXrOP3g28o6qjgXfc1/Wo6jJVnaiqE4FzAD/w9zpF7qzZ35okYL4Yj8dHz56TOfnk95k6dT0+39HG4kBgrzsHkQIhKio2EQ5Xc/DgX1GtQLWKsrI8wuHqmMVvjGk/kSaCmcAC9/kCYFYL5a8E3lJVq5RuZ9XVRWzYMJe1a69pMMuo15tMWtp4akYIq4YQ8eHxpOD19mD79p/x3ns+IFD3qGbHIxhjuq5I2wgGqmoBgKoWiMiAFspfDfz3MdseEJF7ce8oVLWqsQNFZC4wFyA7u/mujwbWrJlJaenHqIY5dOgdTj11b73J4ZKSBiKSiGo1Ignk5DyI15uM15vOxo1zG5yvZq6hLyoYLGPHjvuori4kO/su0tLGRnQ+Y0z0tJgIRORtYFAju+5pyxuJSCYwHlhaZ/OPgX1AEjAPuAtoOEEOoKrz3DLk5uZ2vVFwHczvX4dqEIBg8CChkL9eY29Ozs+prNxGefnnZGXdxtCh3wWguPhtjrYbefB4kkhM7Me4cQsjimf9+m9w8OASVKs5eHAx06fvwetNjuicxpjoaDERqOp5Te0Tkf0ikuneDWQCB5o51VeAP6tqbX1Dzd0EUCUiLwA/bGXcpgWZmXPZs+c3gJCRcXaDHj+Jib056aS/Njiud+9zSE7Owe9fg8eTSm7uJ7W9iCJRWrqSmpu9cNhPIFCI1zs04vMaYyIXaRvBYmCO+3wOsKiZsrOBV+tucJMH4sx9MAtYE2E8xjVixENMmPAPTjzxz5x44p9bfVxJyTIqK7cBzniDPXt+HZV4srK+jceThtfbg7S08fh8g6NyXmNM5CJtI3gYeF1EbgB2AlcBiEgucJOq3ui+Hg4MBf51zPEvi0h/nFbLVcBNEcZjXCJCr17T23xcOFxZOyeRarjJwWZ+/2a2br0LkSRGjnyE5OQhzZ532LCf0KvXGQSDB8nIuLBee4UxJrZs0jlTTzgcZM2aWRQXv4XPN5gxY54iFCojI+MCEhMzAFBVPvhgKNXVBYCQlnY8J5/8EYWFf8Dr7UG/fpfZF70xnZBNOmdaxeNJ4KST/kIoVElR0SLy87+CiAevtxdTp64jIaEnqiE3CYQB5+5g1aqzKC9fC8DAgUs57rh5MbwKY0xb2M820yivN5mCgnmEw35CoTJCoVJKS52RxR5PAgMHXovHk4bHk0Zm5g2Ula0iHC4nHC6nsPCPMY7eGNMWlgjiVHV1EWvWXE5eXi4HD77VaJlevU7F40kBnEFnKSljaveNHfsCEyb8nYkTlzFq1BNu428CIj7S00/tiEswxkSJVQ3FqQ0bbqS4+K+oBsnPv4Jp03bUrk5WY9iw/8LrTae8fA1ZWTfXaxB2GqOPfuFPmvQBe/Y8idfbkyFDvtth12GMiZwlgjhVWbmtdsAZCIFAUYNE4PEkkJ19JwBHjuSxYcPN9OhxIllZNzdoDPb5BjFixIMdEboxJsosEcSpnJz7WLt2NuChV6/TSU09rsmylZW7WLXqLMLhcjyeVILBEoYN+0nHBWuMaVeWCOJUv34zOeWULQQCRaSlndBsd0+/fx0izgI34bCfw4f/ZYnAmG7EEkEc8/ky8fkyWyzXs+dUPJ5kVEOAMmjQN9s/OGNMh7FEYFqUmNibKVPWUFz8Fqmpx5GefkqsQzLGRJElAtMqSUn9GTToG7EOwxjTDmwcgTHGxDlLBMYYE+csERhjTJyzRGCMMXHOEoFplfLydaxceQorVozn8OH3Yh2OMSaKLBGYVlmz5jJKS1dQXr6G1av/wx1TYIzpDiwRmFYJBIoAZxGjUKiCcDjQ/AHGmC7DEoFplREjHkbEh4iPoUN/gNebHOuQjDFRElEiEJGrRCRfRMLuOsVNlZshIhtEZLOI3F1ne46IfCQim0RkoYgkRRKPaT9ZWTcyffpupk3bysiRv4h1OMaYKIr0jmANcDnQZOuhOLOV/Rb4MjAOmC0i49zdvwAeV9XRwCHghgjjMe0oKakfPl9WrMMwxkRZRIlAVdep6oYWik0FNqvqVlWtBl4DZoqIAOcAb7jlFgCzIonHGGNM23VEG8FgYFed17vdbX2Bw3p0dZSa7Y0SkbkikicieYWFhe0WrDHGxJsWJ50TkbeBQY3sukdVF7XiPaSRbdrM9kap6jxgHkBubm6T5YwxxrRNi4lAVc+L8D12A0PrvB4C7AWKgN4ikuDeFdRsN8YY04E6ompoBTDa7SGUBFwNLFZVBZYBV7rl5gCtucMwxhgTRZF2H71MRHYD04G/ishSd3uWiCwBcH/t3wYsBdYBr6tqvnuKu4A7RGQzTpvBc5HEY4wxpu3E+WHeteTm5mpeXl6swzDGmC5FRFaqaoMxX10yEYhIIbAjCqfqh9NW0R1112uz6+p6uuu1dcXrGqaq/Y/d2CUTQbSISF5j2bE76K7XZtfV9XTXa+tO12VzDRljTJyzRGCMMXEu3hPBvFgH0I6667XZdXU93fXaus11xXUbgTHGGLsjMMaYuGeJwBhj4lxcJYI2LKSzXURWi8gqEen0I9ciXSCoMxORPiLyD3fxon+ISEYT5ULu57VKRBZ3dJyt1dJnICI+d5Gmze6iTcM7Psq2a8V1XScihXU+oxtjEWdbicjzInJARNY0sV9E5En3uj8XkZM7OsZoiKtEQCsW0qnjbFWd2EX6CUe6QFBndjfwjrt40Tvu68ZUuJ/XRFW9tOPCa71WfgY3AIdUdRTwOM7iTZ1aG/7fWljnM5rfoUF+cS8CM5rZ/2VgtPuYCzzVATFFXVwlglYupNPlRLJAUPtHF7GZOIsWQddfvKg1n0Hd630DONddxKkz66r/b7VIVd8DipspMhP4nTo+xJlRObNjooueuEoEbaDA30VkpYjMjXUwUdLUAkGd3UBVLQBw/w5oolyyu3DRhyLSWZNFaz6D2jLuhI0lOBMydmat/X/rCrf65A0RGdrI/q6oq/67qqfF9Qi6migspANwmqruFZEBwD9EZL37yyBm2nGBoJhr7tracJps9zMbAbwrIqtVdUt0Ioya1nwGnfZzakZrYv5f4FVVrRKRm3Dues5p98jaX1f8vBrodokgCgvpoKp73b8HROTPOLe+MU0E7bhAUMw1d20isl9EMlW1wL3lPtDEOWo+s60i8k9gEtDZEkFrPoOaMrtFJAHoRfNVE51Bi9elqgfrvHyWLtD20Uqd9t9VW1jV0DFEJE1EetY8By7AaYzt6hpdICjGMbXGYpxFi6CJxYtEJENEfO7zfsBpwNoOi7D1WvMZ1L3eK4F3tfOP+mzxuo6pN78UZ22S7mAx8A2399A0oKSmKrNLUdW4eQCX4WTwKmA/sNTdngUscZ+PAD5zH/k4VS8xjz3S63JfXwRsxPml3Omvy425L05voU3u3z7u9lxgvvv8VGC1+5mtBm6IddzNXE+DzwC4D7jUfZ4M/AHYDHwMjIh1zFG6rofcf0+f4axMODbWMbfyul4FCoCA+2/sBuAm4CZ3v+D0mNri/r+XG+uYv8jDppgwxpg4Z1VDxhgT5ywRGGNMnLNEYIwxcc4SgTHGxDlLBMYYE+csERhjTJyzRGCMMXHu/wNU/wrJfVd8mQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "ds.Plot()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "dl = MyDataLoader(ds, batchsize=10, shuffle=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Create Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# DenseLayer: combines dense connection weights and an activation function\n",
    "net = nnad.Network()\n",
    "cl1 = nnad.DenseLayer(in_nodes=2, out_nodes=10, act=MatReLU)\n",
    "cl2 = nnad.DenseLayer(in_nodes=10, out_nodes=3, act=MatLogistic)\n",
    "net.AddLayer(cl1)\n",
    "net.AddLayer(cl2)\n",
    "\n",
    "# Define the Loss function\n",
    "net.Loss = MatMeanCE\n",
    "\n",
    "# You can also define a loss function using composition, like this...\n",
    "#net.Loss = (lambda a,b: MatBatchMean( MatCE(a,b) ) )\n",
    "\n",
    "lrate = 1."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.00080907]]\n"
     ]
    }
   ],
   "source": [
    "# Run the network\n",
    "y = net(ds.Inputs())\n",
    "\n",
    "# Evaluate it\n",
    "loss = net.Loss(y, ds.Targets())\n",
    "print(loss())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": [
    "# Stochastic Gradient Descent\n",
    "loss = net.Learn(dl, lrate=lrate, epochs=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0.00084945]]\n"
     ]
    }
   ],
   "source": [
    "# Post-training evalutation\n",
    "loss = net.Loss( net(ds.Inputs()), ds.Targets())\n",
    "print(loss())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "Collapsed": "false"
   },
   "source": [
    "## See the output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.   , 1.   , 0.   ],\n",
       "       [0.998, 0.003, 0.   ],\n",
       "       [1.   , 0.   , 0.   ],\n",
       "       [0.   , 0.   , 1.   ],\n",
       "       [1.   , 0.   , 0.   ],\n",
       "       [0.   , 0.   , 1.   ],\n",
       "       [1.   , 0.   , 0.   ],\n",
       "       [0.   , 1.   , 0.   ],\n",
       "       [0.   , 0.   , 1.   ],\n",
       "       [1.   , 0.   , 0.   ]])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.round( net.h[-1]()[:10,:], 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 1., 0.],\n",
       "       [1., 0., 0.],\n",
       "       [1., 0., 0.],\n",
       "       [0., 0., 1.],\n",
       "       [1., 0., 0.],\n",
       "       [0., 0., 1.],\n",
       "       [1., 0., 0.],\n",
       "       [0., 1., 0.],\n",
       "       [0., 0., 1.],\n",
       "       [1., 0., 0.]])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ds.Targets()[:10,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "Collapsed": "false"
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}