data_generator.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 單元 07-2. 使用 Model Functional API 建立各種非線性堆疊網路\n",
    "\n",
    "本單元的練習是讓同學實作出著名的網路 U-net"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. 初始準備"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: KERAS_BACKEND=tensorflow\n"
     ]
    }
   ],
   "source": [
    "%env KERAS_BACKEND=tensorflow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Keras functions\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Activation\n",
    "from keras.optimizers import SGD\n",
    "\n",
    "# Keras dataset\n",
    "from keras.datasets import mnist\n",
    "\n",
    "# Keras utilis function\n",
    "from keras.utils import np_utils\n",
    "\n",
    "from keras.preprocessing.image import ImageDataGenerator"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "讀取 MNIST 手寫辨識資料"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "(x_train, y_train0), (x_test, y_test0) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find and plot first 0~9 digits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAABLCAYAAABgOHyfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJztnXd4VNXWh981k8mkQIAEEiAkBEhCIAFCr4IKivohiIA0L1i5dEGwcfXqtRcsVBGQ7gUVAQsqihdQpEgXgSTUUAIBAqGkZ2Z/f5xJSEgCIZl2c8/7POd5Zs6cOes3c/ZZZ5e11xalFDo6Ojo6//0YXC1AR0dHR8c+6A5dR0dHp4KgO3QdHR2dCoLu0HV0dHQqCLpD19HR0akg6A5dR0dHp4KgO3QdHR2dCoJbO3QRGS0i20UkS0QWuFDHEhE5LSKXRSRBRJ5wkY71IpIpIldtW7wLNFy9brOIyDQX6DCLyKcikigiV0Rkl4jc62wdNi3+IrJSRNJsegY5yW6x94eINLbtv2jb1opIYxfoCBMRdV15eckFOgZfpyHdpqulo7SUoC/Cdv8ucZQND0ed2E4kAa8D3QFvF+p4C3hcKZUlIlHAehHZpZTa4QIto5VSc11gFwClVKW81yLiCyQDX7pAigdwAugCHAfuA74QkSZKqWNO1jIDyAaCgFhgtYjsUUrtc7Ddku6PJKAvkIhWaRsFLAOaOllHHlWVUrkOsn1THUqpz4DP8t6LyCPAS8BOJ2gqyAxgmyMNuHUNXSm1Qim1CkhxsY59SqmsvLe2rYELJbkLfYGzwG/ONqyUSlNKvaKUOqaUsiqlvgOOAs6udfkCfYCXlFJXlVIbgW+Avznadkn3h1Iq1fa/KEAACxDubB3O5hZ0DAUWKSdOkxeRAUAq8Isj7bi1Q3cnRGSmiKQDccBp4HsXSXlLRM6LyO8icruLNOTh9BujJEQkCIgEHF0rvp5IwKKUSiiwbw8Q7WQdRRCRVCATmAa86UIpiSJyUkTmi0h1F+pAROoCnYFFTrTpB7wKTHC0Ld2hlxKl1EigMnAbsALIuvE3HMJzQH0gGJgNfCsiLmkpiEgoWnfHQlfYv06LCa1JvVApFedk85WAS9ftu4RWVlyKUqoqUAUYDexygYTzQGugLlrLqTIFuj5cxBDgN6XUUSfafA34VCl1wtGGdId+CyilLLYmdR1ghAvsb1VKXVFKZSmlFgK/o/Udu4IhwEYn3xhFEBEDsBitD3u0CyRcBfyu2+cHXHGBliIopdKAWcAiEQl0su2rSqntSqlcpVQy2vW521ZjdRVDcGIlRERigW7Ah86w5+6Dou6KB+7Rh57XR+oKhgBvu8g2ACIiwKdog5H3KaVyXCAjAfAQkQil1EHbvmY4v+vnRhgAH7SW3VkX6sjrmnNJmRWRjkBtYLkTzd4OhAHHteJKJcAoIo2VUi3sbcyta+gi4iEiXoAR7U/wEhGnPoREJFBEBohIJRExikh3YCDwHyfrqCoi3fP+AxEZjNYXuMaZOmxaOqA5B1dEtxTkY6ARcL9SKsMVAmw14BXAqyLia3MavdBaDQ6lpPtDRO4Skea28uoHfABcBA44WUdbEWkoIgYRCQCmAuuVUtd3UTlUR4FDhgJfKaWc2XqajVb5i7Vts4DVaJE49kcp5bYb8ArXokrytlecrKEGsAFthPoysBd40gX/RQ20kKcrNi1bgLtcdF0+ARa7uGzUtZWHTLRuj7xtsAu0+AOrgDS0EMpBTrJb7P0B9EMbvL8KnEMbwG/qAh0D0SKP0tACCRYBNZ2tw/aZl+2+6eqqMltA4xJHnV9sRnR0dHR0/stx6y4XHR0dHZ3Sozt0HR0dnQpCuRy6iNwjIvEickhEnreXKF1HxdDhTlp0HbqO/wYd5aYcnftG4DDaRBdPtNlxjV0wyKDrcEMd7qRF16Hr+G/QYY+tzIOiItIebQS5u+39C7YHxFslfcdTzMoL3zLZKwkLuWSRgY9tYl4WWvSaGW8ySSNbZRWJedV1OF7HjXAXHQBXuHheKVXDGTpudG10Ha7TcSPcWUdxlCemOxgt210eJ4G21x8kIsOAYQBe+NBWupbDZFGS1UlSOENjaQXAaZXIJS4QJc3Zqq7lwdF1OFfHjXAXHQBr1fJEZ+m40bXRdbhOx41wNx03ozx96MXN9ipS3VdKzVZKtVJKtTJhLoe58qHr0HX8L+qQltE8nnCUUQcTbn6wA3WUFV3HrVGeGvpJIKTA+zpo+YiditaVcG2SYCYZmF2QOl3XoXMznH1tDi5swbLOn9DME+7Z3xdPEl2ioyTcRUdFojw19G1AhIjUExFPYABaHmiHkXtnS04sj+G7Uzs4sTwGyx0t8KMaGVwlQ6VhVVaSOUENajlSRrHoOnRuhjOujUdYKBHbzERsMxPfbQ7NPOH9lBh8HrmW5sZdyoi76KhIlLmGrpTKFZHRaLlEjMA85aAVWqxdmgMwdd50wk0eWIFd7ecT38rCM2HtaKhi2cVvKBS1CaOSVHGEjGI5/F57DgyajkmMxDz4ELs2f+k0HcYAf6SKH8f71CazuiL8X3sgPd2h/4e0jMbq6cGp233ZN2YmADnKUuS4rn/1xbfXaayZmXazXRbS+rblnXc/5rWHhqC2/+UQG+LhgbFG4TTf8RPDsPhYqdvgLD4jhTMfeLKz1ecs+8mToRO3YTqX4ZBrk/3uZd6vvdG2x0DTBWMJ3GHF59TW/OMMYnDpPeMuOoxVqxC77iJd/fbxfs++WPY5fUVHu1OuRFdKqe9x8EIPOXe34tmZWp6jSJMnVqwcycnhktVMczNk3duawHV7qZ7p/Cf7mfEdWN//XXKUJwD+QY0IlnscatMQE8XBF7Rm6WNNNjEh4FpurkZBw4l4ZAfVpRbV7VzTUe2bcfARTz68cykmyaWb9xVylNbAs2ItcvzPMV8Qu/gx6o1IwnK+bAvZZPRqQ0aAEf95m8us+2wrA68du7/M3y8OY6MIlNlEUpeqZLRLw79KGr81+7zYY39Ir8w70+9ha5N/czQng3WNe9Dmtqb4rNxa7PHlITPQhzVRCwrt8zkl+KwoassRZaQsOFOHMbweObWqAmA6f5VT3WvwbeB05lwKgTPnnKLB0bht+lyjnx9pnaMY/+G/ucP7qm2v5kAWXOzALzPb8/srU/l57iwaLxlN/efKftOXlashVvwNnk6xJa2bcGi8kfWdplPDqA3KGDCwOr0aR7ICGVUtnsWd5/Ba66GobXvtbl+9foG4qBW39J3dHebRve1IzKvL5tCTOhvwaZAK88r0dTAYUaEZdA2M4xfpUMaTFMZyews+WDCDSNPNr3uOsvDPaY/gkaZo/+VoKp/KxXw+A5/t9nfm0jKakVO+wFCgF7XjP0YTuGCT3W3dKsdea4/VBF4NL7GzjVY5m5Van++iqznMpuoYy7HRipjga8N6g2v+Qk/fiwA0XDmS0MgkjGJg/cWGiJeXw7Rkd29F4mArI1psAGBcNW2AusncMficVqR2yKLuZwY812wvty23degnFwWzrfWMYj97NXAbP1bqwKPH7mZh2Fr8Gjt/KcOr/dryVe8pgDArNYq1D7XCN3FfMXXV8mGsUYOEKcF822Em9U0mKDDCPv9yCKv6dMJqNjHqu3hamS1kBHnjiKJ5an0IRGmvN2eaeez7J6/FOdlim9q1SGB+2E92s/mvHl/yzoG7y/x9Y4O6xHWZR+wfD1PbTg85c3wSOzJDiDQlF/lswul2ABy5Wp0FDZZzyaoImlrYoToqFV7C0Er08j1Pj7jeGIdrD5tqB51fyQHIeKAN56M98OuopV7f1XQKRi0XeP798WiVeAz7I/imcYBDNJzo6sO+ztMK7btozaT51mF80PQL4ntr3YUWJcQtiyLolGMefOeGt2faszNoZbbkP2yHHutG8yrH2fPEFECrmHXwH4i/HRJhu6VDz72zJUtjp2NAK5iPJmoxn9vXNmLv49NZl+FF4PYMDl2MwvTmOgxOTpef2aMNL781j0iTZnjhnHuoud8xBeLUwxHs6zIFMOXvW3JZCy5a9UAHLPEJSHPHL18Z+vZ2en8xEADJziHiaNFaZmr1ANZuqUw3by3d9J17++O3ruwPOZOUb6F4j7npAGQctt8CObmnzzDtnX68cU8axj8rsWek5jReP9+UQ918ALCknmZQ+5EcGwv12GM32yXRcLuJxUEfsPxqKDKxCpaDzl1bw6N+GIH/TqFvdW1B+yjTRup4mPMd2Itn2/BqYOHF7k1iJMSUAtjfoR/6sB0b+7wLeNN00yNkpmhdlI3fSCL4xD7e6/IwfvPn0tIM27IUtebtsXtFTEyeZHZrxlcvvEdtDzOPJ95F4uSGAPiu3s06n1A2rIzkqwgtjuTy7gD87WDX7Ry6tUvzAoOfVnrG9cbYNw2Aqv+naLx4NJEzTmA4sYtqv0HOGxa+ajqPx+4Yi3HdTqdoPP1wJnd4ZwJGhh7rRs0pjmvWBvc8lv96+dWafJDQlaBntXqeJV5bIOdiE8ev6KVysrHEH7rhMckPRtLE82vyWhFJSf5USj9SJnvWTrHc5rXx5gfegDBfreUWsrbooG158J+/mRrfBmBJuUB0zGPs6zyPb2Z3ITD1WjmQzXuo54QK8sVH2vN+relY8eTFX/rQKC0F+/7aG3O1X1uefmMpPXwLtpK1639/r6EAGE9foFetR0kL9WXCO59xr4/W7TE3qTNwxu6arD4WAo0+rEqrSv3nrpB7RGud5QKG2MZceu4Krc3CaUs6j899ljpp9r9/T49uxR8TpwBm+h26n9w+Ofic1ypBCkga1pKtEVoN/Yf0yoR/coLyVV803MqhS8tozj+dQaTJkx1Z8J+rjUlZFkLARe3OqLJkC1WgyA8PMppJGZdO4DrH6vOoEwzAvtvmk6MsHMiB4x9E4ov9+0TzedJM41FjCPnZgu++M1RPTChyw6YHuWoVumucG9GeqIfjCDJe6xJq9OzRMjuXxB7eBBp9yqzHIyyUvv5a7cf76EW7O7m8gd6cy1orMnrwfs59bNQ+tDrHpRqDAjnX4drdYEo1Ykk4XOS44y93IDNYC1uMHLatyOflodbYw4Wc+QVLFl1nP0vNLVmYtu8AbPfrqSSSxjXPd+bLr9bEMthoVy15hK1QTOtUn1FVD/PyZF9CH7NFzlT3J+f9K/wWtYq92bkMWPQsdd+0vzM/OK0t8Q9Owwo0+nk4UROPFQkMGD7i6/zXr78xlGon7PP0dxuHbvDxIffdy2yJWsHR3GyenjSBar8dJ9D3bKluxja1EjnmQH3G6Ia0+nfhsLf+K8bS4KstDrQKlkNHCR+vrcNc0hM8p7Vr1iM+O1obaBw64nse9ptM5QIDxK+da4HKyi7zuT3Ctd+UGVe1TN8/8ZEvHc1WPr1cB1Ivl1nHzWj0XAKPNunK/Lq/0KXfKAAqf+7YMpFPbi63NYnHJEZyFAT/WriEJL7aHpTw6sDP6O17AQBTkpH7ujyI5WDZWk4FudK/HbNCJ5NXI/86rTozxz9EyOrinWT9WufzX7/464NEnrTvwyUPr9/2M3NvZ0bddpgPmn7BP3o/AcDzL3yWPyg6aN546r5mf2d++P12xD84g0vWTPrFDaLhmAQsV7SybPDV8r+k9G1Kr0rvYcCbqC9HEb7Afk05t3HoGV2iWROlDVQ88dR4Kq/aYpcmiL1I7BnA8oBdtndGBh2+n8i3Dzu1eQtw/J8dyPWxDa0JoODBCK1AjD55O94/7nTIwJsxuiEJj1ajS6drD7XvQrT+Yy1sUXPmh3Jy6f/xBEJXJmO9UrS2eKsEbi9976axegDJfSLxf+gkGyI/Bbz4eMYDBCY7rkvMknqJlBGNOP5NBs+/vgiAFx7qjdpVhZA3NkMZk9+VhpT7GrIydCo5ysA3adUwJ6ejuDZvI7DtGX6O+QKAk7lZfJ/WiGFVjhG57DgJf4vEsv/W0gFcT81Rh6ntoTnz0Sdv58B7MfiuLtpa9agZRFKfBnwe/h7gyeiTtxO8xnFLMVjT08m5oum6wzuTTa9PB8CAYAWif32M8C9KV1G8FYxBgSzsPRMrVvrFDcLzrsT8vnlDbGNi5mlLur4eNBUw03H3ABq+csCuOtzGoTd9bTcGDDya2BXvVX+U+nt5tROjOO7GufBoe1YOf4+8gcnhJ7qQM9SM5dxxh9ksiNHPj8w2EZheSObPqGsj99pv14rDugwfTg4LReXafx1g1TGWR+avpJfv+es+KXpTjj3Un+B3NtmtkGb4GwrltbPe1hxlFE50M5NdOweDp4WfbtP+E5PAGYuZl4705oLVio/BQtDWKw6LLMnXtOcAA/71DJ+9PBmA3e0WQTuI9h1NxJzT5B45ZnebxgB/roRpXW3rMrx45odBROzakt9tCfBHzHJ2ZBn4+58PU+Mjb7KrejBsxsdEeCeTQP1yazi+KJw3x8RyOK0GFwdXwbeYgXKAhHH1+etvUwBPPrrQmKQB1Us81l54nTAVu79HXC/qT8696XhQWRAvM63MWsn3HuuJ1A3h4PA63N1tJ+MDZxPqoQ3OWgGLUsjn1bGkHrSrBn3FIh0dHZ0KglvU0FP/1p4XgyZjxZMdPzUmlNI3kXOUBStWfjzQmAjsH+VijG5oa7Jdi+7efDKMkGOOmUZeEDGbye7ShPEzF3OH9y8kW7JYl1GNfyb0AmBp9IL8Jq+XIYcjD1WlfryXQ6bbG1GFJq2A1kIAyClQBf6x0UpuGzyKKp+Vrx85K9OEFcX8SR/yzejY/P3PBczFgJChskmyWJh+7na6rR0HQNVdntT6KRlJPMm5A94EGXMcMsmqOPznbWZ0vNaH7vf2SZbWX8O+IdOJCnmChv8y2KXPuiAXu0eya7gWJTHy68eJmLAFj7BQsm3jUABHc7MZtHEMDYfHYYmNYNCbaziam8n72+8iYn/575WAuZvZMtcEpNq2wqQOaQ/A5sGTAU/SrTksWnoXdY46drKTeHhQtX0yhusSwt4X1xO6nkRx0iF2VWYWW7NMtDXn8PXaZYVmUK/NqM5B241yh/dVtmd7UnWR/cOg3MKh53pDFYMnmzPN1F+UVKq+c4OPD3GTY4AdDD5yL1FPlT2i4kYkTPIpkqsk9G3HTRDJw+DlRUr/5vz25lQAopeOoc46C+bV2wiopc2cXbqmJRMCtAdLW3MOfz4ylfYnxhK0aA/W9HS7aZHfd/PpA/fw/CMBhK7JxphR9AodfNxE3D0f281m+MO7iH5rNCGtTxXav+5sJOd+qEPAvhw8f9wG5BDJtRl2FuDUcx1obd7MsqvBdtNTGuT33QCk9w2kdf8xbH1uCnF3zGVw2N1c6mRfWylNrjmrBhO0h2e9L5ML5HHRxqIiVv1Bxr2tWTNXG5+KWj3O7pEuJbHxLa3v2mobX7lt+kTqvOP4mauXvq3Lr02/KBJbbkUc2iVhST7LyyOeYPKsmTT11OaLvL6hJ5ELMvFIvkTgUm1g+o6Q/zB03ROFyq29cAuHnkeKpVKp+hsNPj7Ev92EuF7T+SG9Ckkzwql80f6RBdYuzXm91ar893f9NQCASg5K8pSHmM3EfdCUuF6aM+8V/wCR7x3BknwWj5A6NPtG67t/JmA/l6zZtP1qArWizvJLk8/Z/NJU+g/swfmpTfBK0ULVjOvLXxuz7E+g/rMlf97oYA2wcxqbei8UX4OpxY3HLnw6a3k5XlzXh0hKPx5jLyzJZwmaepbMZ3PxEU/mhH1Hj97j7Jq/JaeKNvOw61998eYo1i7N6e2/CAMGms4ZA0Doqk2FUgI0nTOGyFeckwrg4Iy2mER7wOW14Or8ctmhFSFjeD0ODqtJXNMZWIGXzzbnq4RY/uo0H4CmVU/h6Ha155rtTKrXJv99Xvm70qsNq0O1UMUcZcD7mGNShriVQ5/4ez8i2XHDY6xdmnP26QwOtJpO17398b3nCJVxTJjYGwtmE2PSiuDE052pMlALeXJkZIt4eBD/UTPies7gZG4WPT95lrB5h8lNPktOt5bEvLOLlwO1/2j+5bos/sf9hK/YgrF6ALffNYa0/pdY2XwOdaZqXTHfpQUwO7L8A2A3I/nBcIfbuFXqfu3odlRhrJ20rqHD/byIiT2Gj2g37bQLzfH52v61MStWrOpaTT1HeWAlE6K1MLmxh+KoYdzGlxfbsOD/ulLvvH0jKkrC4OVFTExifsvWipXmM58iZKdjB0KT7qvF/sHTAKHx4tFEvBeP9+BKYGsdfX8kmlCc0wV3PbnehkL/R70Fxx0SxeceDl20fAZTOi1lBpElHpb4anu+GvIBkSZPWvwxlNq99ztUVnPPaxdh8/wWBF50fO3mxDNtiOs5haTcLPq9/Qxhq45w4c56qIcrszxmCjWMZqKXaTWwyNnn8YnXbhLL+RT8lqbgtxT6jnyWoL62FasmVAXKNhVczGZS+zWn2tf7sF4pOdb99IQOfD32XXDjlVwcibSKIWGsJ3M6LgSgs9e1+PsslcOWC/XAetquNut+q6AX/NLkc7rfO5JzsSbqmy4AnuzuoGUzM2BgR5aB395vS5WDzomNN/r5cXxEDDvCp5AXc9F62xDC5h7C4sAJV+m927L0aS0mvsPzowhf8SeE1eHpkV/kH5Nz3LnrgBak8rIt8L7j7biHQ1faU6uLdwrjFrSkwXwrpjNXSO5SA//+JxkTqq1Bea/PDr5JC2LI3nuo/oljL86J5TH5TUaAWuvPO6V28/GTWl+nl8D9w38leOxFhvp9a/vUTPS/xxL+gtYHaskt/hkfOHMTambeu1PFHnMzMu9vQ5WJx9kQPo3e2wZCfGGH7lGrJqf6ajX/z8dMzh+cTbZkYcpwbs24OIxi4GKkiZo/OM6GR726HH60Nq/0X0afSteHdMKk5FZsmNKOagvtP/hlzLKSlJtFbQ8zP8+dVWguQB5HczMZtHEMEeUcoC61pgB/zi2szo7m2mBt009sXT+vb3WoMwc41RUiTV48evx2qi7ejDKbSewdQJT5NAa0wXtziuuC+q4MaAc36X2wB+7h0G14iQcH7prFxtu8OJhVk0erHCv0+VNJt/HjplginnJsAbV2ac5HsUvIURYuWTNp/cM4ohId2xrI49erUbQ178XfaGZSde2B0iPuQY5vrkP95ZcI37cDVYIjtyfd39iQP+AaN8kPrhZe/3tAh82sClwNgNUWnz/0WHcOzW9IwArXZPkriEVZHRaU6xEWyqWWtej/6o8Mr1o0pfCE0+3YPLMV/gv+oJrVMf+Fx392MPAfE6k/Ip6FYWvz9zf7/TFkf2UAauzOJeIW5nSUF0uDYDY211oHSy6HEPqqE1P3KrCisCoDYjZzbmgL9oyYxr5sK41+1WaK1nPANP/Scqm+cx4mN3XoIhICLAJqosXEz1ZKTRGRV4AngbzM8JNsC17cMkHrz/Lc39vzTk2t8Hf2yqaT1zEAdmUZGLhhGLkpqWRMmEcWX3MeIZh6hEoEh9U+kjhK3sKt4cRQXcqXMD/T35NOXmmAkTXpoUQO25Y/Yp6p0tnHNrLIRBygY9MdtWk7+E4uNcvG45yJyFmn8DhzlrDME4VG7R2toyAHun1SwidaId2caebJrUMIf/IgAWmud+Z5pLe2X6QPaK2SC/N8GVFvAwMrF06fO/pUJ3Z+HEv21Yuc+mouWZbvHH5dqizZQsoS6EHL/H11bX3EziwfoOXrP/K01p8/91J9VvdtDyQ4TYcxIAuAhIs1iN5whm9DtAibv788jnoOCA+8VYI3pGMaXTTM196UpoaeC0xQSu0UkcrADhH52fbZh0qpyeUVYUk4zMF+YTQeM4b9D12bCRn1/UgazkwnctcOslQGWTTFT6qRq3L4g1/wV0EAhBJBXWlYXhmlQhAiHKjDknKBoKmbCLK9L6ku7mgd/xnbkUUj27CnY9HVJZZcDuF0TlXm7ewIQPgcC/V/3233FKTlwSj2qxFld29F9vgLTAr/nru90wp9lmzJoPM3E4h6MQ7/1M1kqQx8iK7w5fR6kl/MYW8rbfGKmUvup44tnbSzdHgk+EAX+D12GQaEvdm59Pn6KSJX/uUW5VJ+382Cy4EADKx8ivToWniesH88/E0dulLqNHDa9vqKiBwA7B7gm3vkGOHjj9FzfOv8fZFsyw9zMot3/orgHmLCR1Umq8CK4fbEb/cZxpy8k1khG4p85kwdN8LROozrd1LvDx9ajn2KhX//iBhP4c69/bm0viZ1Pz9F7tFEIpzQJ1gWstbWwBJrv9v42AMGEpp8mf9+RmoDpmy4G7EIUa8fJSJ5a/74yv9K+SiIat+MwEpaNsHG64YR8dNlp9+39WcdJtpzNGsffo8XT93HHz/FEP7yJrdw5nl8+ElfAAZOnEKtlw6RktoUtvxpVxu31IcuImFAc2Ar0BEYLSJDgO1otfiLxXxnGDAMwIuyp0MtSIZK4wqpVMGfVM5zgsOcVsepTDUiaYpJisZ43oqO3KOJnGxXuCnrCh2lxVE6rOnpBL+9iUlva3G1lThCJY7YNdzKEf9HzQ83cd+HLajP7psfXAodkSP+oMeIwmUhL774RkN9Fb185HFwqJm4qJWsvBpIxEc5JS7G7UgduWeSqTcpmScndQIuU/cWZpuXBntcl+DF2iLU/R/owefh39HlnwPxH1QFS+olu+ksdbtURCoBXwHjlFKXgY+BBkAsWg2+2KAcpdRspVQrpVQrkx3C2nJVLn+ymYbE4iEm6tCAjtxLW7phxosEin/i6Tocq6OsVFQd7nJdnKEjeK3Wd/7BWwNKdObu8n+UFXvosJxPwXI+hew+uTRa+3c2NFuKJaquXXWWyqGLiAnNmX+mlFoBoJRKVkpZlFJWYA7Q5kbnsAdWZeVPNlOTUAJF6/Uxixcigog24HKZC46WoevQuSHucl2cpcN3+VZ6BremWgl5vd3l/3AXLOdTiBi6k57Bre3e5SLqJvmaRUSAhcAFpdS4Avtr2frXEZHxQFul1ICbnOsKEF8OvWFordwTBfaZgBzb60CgEnAZOA/UVUrVKEbHOSDNdoyu439XB8Vp0XXoOhyso7RUv5GOYlFK3XBDmzirgD+B3bbtPmAxsNe2/xugVinOtf1mx9hDR2nslFWLrkPXoevQdZSPBY1vAAAB+UlEQVRVxy1qvmUbpYly2QgUt2hlmWLOy8qt6NAaFboOXYeuQ9fhXjocjb7AhY6Ojk4FwdkOfbYb2XGGFl3HrdvQddz6MeVF13HrNtxFRyFuOiiqo6Ojo/Pfgd7loqOjo1NB0B26jo6OTgXBaQ5dRO4RkXgROSQiz9vpnCEisk5EDojIPhF5yrb/FRE5JSK7bdt9ug5dh65D11FeLe6io0QcHUtp66M3AoeB+mhZ+PcAje1w3lpAC9vrykAC0Bh4BZio69B16Dp0HfbS4i46brQ5q4beBjiklDqilMoGlgG9yntSpdRppdRO2+srwM0yQeo6dB26Dl1HWbW4i44ScZZDD6bwdNuT2DkFrxTOBAlaJsg/RWSeiFTTdeg6dB26jnJqcRcdJeIsh17c1Cu7xUtK6TNB6jp0HboOXUdZtbiLjhJxlkM/CYQUeF8HSLLHieXWMkHqOnQdug5dR1m1uIuOkrFHh/7NNrSFNI4A9bg2mBBth/MK2nqnH123v1aB1+OBZboOXYeuQ9dRHi3uouOG57GHmFIKvg9t5PYw8A87nfOWM0HqOnQdug5dR1m1uIuOkjZ96r+Ojo5OBUGfKaqjo6NTQdAduo6Ojk4FQXfoOjo6OhUE3aHr6OjoVBB0h66jo6NTQdAduo6Ojk4FQXfoOjo6OhWE/wcu24QeX/05hgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1d8325d9d30>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "target_num = list(np.arange(10))\n",
    "target_id = []\n",
    "for i in range(y_train0.shape[0]):\n",
    "    if y_train0[i] in target_num:\n",
    "        target_num.remove(y_train0[i])\n",
    "        target_id.append(i)\n",
    "        plt.subplot(1, 10, 1+y_train0[i])\n",
    "        plt.imshow(x_train[i])\n",
    "        plt.title(i)\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = x_train[target_id]\n",
    "y_train0 = y_train0[target_id]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = x_train.reshape(10, 28, 28, 1)\n",
    "y_train = np_utils.to_categorical(y_train0, 10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [],
   "source": [
    "datagen = ImageDataGenerator(\n",
    "    featurewise_center=True,\n",
    "    featurewise_std_normalization=True,\n",
    "    rotation_range=20,\n",
    "    width_shift_range=0.2,\n",
    "    height_shift_range=0.2,\n",
    "    horizontal_flip=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "datagen.fit(x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [],
   "source": [
    "augmented_datagen = datagen.flow(x_train, y_train, batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAABLCAYAAABgOHyfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJztnXd4FVXegN9z57b03kghPYEAAemogIoCNlRUQFddFbGsuta17Lrr7vq5uzZcdUWxYC+IKIogiogLKCIQOiRAQkgggfSeW2bO98dclA5Jbm4iO+/zzAO5uXfOm5m5vznld84IKSUGBgYGBr9+TF0tYGBgYGDgHYyAbmBgYHCKYAR0AwMDg1MEI6AbGBgYnCIYAd3AwMDgFMEI6AYGBganCEZANzAwMDhF6NYBXQhxuxBitRDCIYR4ows93hFClAkh6oUQBUKIqV3oMlkIsVUI0SSE2CmEOLMLHJKFEAuEEDVCiHIhxAtCCHMXePQSQiwRQtQJIXYIIS71tYPHo/GwTRVCPN8FHuFCiE8810axEOIqXzsc5pMhhGgVQrzTReXbhBCveY5FgxAiTwgxvotclnqOxYFrJL8zyunWAR3YCzwGvN7FHv8AkqWUwcDFwGNCiIG+lhBCnAv8C7geCAJGAoW+9gBeBPYDcUB/YBRwmy8FPDeQecB8IByYBrwjhMj0pQeAlDLwwAbEAC3AR772AP4DOD0OVwMzhBA5XeBxsM9PXVi+GShBvz5DgEeA2UKI5C7yuf2gayWrMwro1gFdSjlXSvkpUNXFHpullI4DP3q2tC5Q+SvwNynlSimlJqXcI6Xc0wUeKcBsKWWrlLIc+BLwdeDIBnoA06WUqpRyCbACuMbHHodzOfrNbpkvCxVCBAATgUeklI1SyuXAZ3TR8RBCTAZqgW+6onwAKWWTlPJRKeUuz/dlPlAE+Lwy5iu6dUDvTgghXhRCNAPbgDJggY/LV4BBQJSne6HU09Xh50sPD/8GJgsh/IUQ8cB49KDuS8QxXuvjY4/DuQ54S/p+TY1MQJVSFhz02np8f6NFCBEM/A2419dlHw8hRAz6cdrcRQr/EEJUCiFWCCFGd0YBRkA/SaSUt6F3c5wJzAUcx/+E14kBLOg1wDPRuzoGAH/ysQfAd+iBoh4oBVYDn/rYYRt6Tfh+IYRFCHEeetPa38cePyOESPI4vNkFxQcCdYe9Vod+zfqavwOvSSlLuqDsoyKEsADvAm9KKbd1gcIDQCoQD8wEPhdCeL2VbwT0NuBp2i8HEoBbfVx8i+ff56WUZVLKSuAZ4HxfSgghTMAi9JtaABAJhKH37fsMKaULuAS4AChHrw3ORr/BdBXXAsullEVdUHYjEHzYa8FAgy8lhBD9gTHAdF+Wezw81+zb6OMLt3eFg5TyRyllg5TSIaV8E7170OvfXSOgtw8zPu5Dl1LWoAerrl4eMxxIBF7wXJxVwCx8fGMBkFJukFKOklJGSCnHoteAVvna4yCupWtq5wAFgFkIkXHQa7n4vnthNJAM7BZClAP3AROFEGt97AGAEEIAr6G3cCd6KgLdAcnRuw07RLcO6EIIsxDCDiiAIoSw+zo9TggR7UkVDBRCKEKIscAUYIkvPTzMAu7wOIUBd6FnefgMT8ugCLjVc35C0fuN1/vSA0AI0c9zTfgLIe5Dz7p5w9ceHpcR6M3prshuQUrZhN5q+psQIkAIcTowAb1m6ktmold2+nu2l4AvgLE+9jjADKAXcJGUsuVEb+4MhBChQoixB+KXEOJq9Ay1RV4vTErZbTfgUX7JKjmwPepjhyj0PuNa9D7jjcBNXXQ8LOgpg7Xo3QzPAfYu8OgPLAVqgEr0IBbdBR5PehwagYVAelecF4/Ly8DbXVW+xyEcfSyjCdgNXNWVPh6nR4F3uqjsnp6Y0eq5Rg5sV/vYIwo9fbPB891dCZzbGWUJT4EGBgYGBr9yunWXi4GBgYHByWMEdAMDA4NThA4FdCHEOCFEvmeiy4PekjI8Tg2P7uRieBgevwaPDtOBjn4F2ImeKmZFz3Lo3QUDH4ZHN/ToTi6Gh+Hxa/DwxtbuQVEhxHD0jJOxnp8f8twg/nGsz1iFTdoJaFd5x0LFjYMW/D0T4hye+Tc2/GilCad0HJHraXh0vsfxXCRat/Cw4UcDNZVSyijDo20eQgjUUH8CYptoKgvAVNvUJR4/+9htSIuCcLrB7UZq2knP2Ogu5+V4HM3jaHQkpzsefSWzA5QCQw9/kxBiGvpKeNjxZ6g4pwNFHsk+WUoV5fQWgwAok8XUUU22GMCP8pd1gbzpYbLbMUVGIEMCwelC3VHEPq3E5x5HoyuOR1td6qjuFh7ZYgCL5Zxib3g0TBpG+Ug9glirTKTN3I279NB103zhcTJ4w0MJC2PfFdn4XbaPpvmxRL/wfZd4ACiZaRTcFE1CbhlVTf401fmh7LcSu1Ij6NsC1Joan3h0Jgc8TkRHAvrRZjkdcU+UUs5En2xAsAjvshzJtngIsxlTciJaaAClY4J56PoPSbZUkGXR79yRSgCq1ChTm5mw/gaiHu2NyNsDqnc9OhPDo2MewmxGG5xDVT9/7JfuY1nfF3FINwBFbpVPLhnAsn72TvfoLE7kIZ1O3AGCjNAKVvvHghDQnta+omCy+6M1N7fLQ1is7B0bw0MXzuWiwJ2UqwoKklrNxroLe/L+7sE0ze8FQI95u9EqKtEcjja7dpfzciI6EtBL0aeAHyABff1yn6J3JfwyAayVFmy0cwFCk4KSlcqesVGYz6kkMbiCp+I/Ypy/A1UCnv2qUgMgTvFnUf9ZjP/r9fhNS6Z1z07veHQArx6Pbuhijo2hpU8CikPDuqsCrbYOreH4y5V0hocpKIiS+9xcmPoDd0QsR8MPi1AAyLQojAzcxtcXTcP++S8rEXSXc+MND62pCUujpG/gHpZmZqGEhp6wJnxUj0ConJhLZF4drXnb2uyhREfSkKZxWWAhz1YP4b1FI7Gm1fPZoJe5JaSYq3Ly2Zut1z3n3DSQ2R+OpudrO1D37T/UoxucF2/QkSyXn4AMIUSKEMIKTEZff9mnBBNGC420yCY0qbGPEqKIa9e+hMVMwY0RTL3pC+bnvs6rqXM51+/I2cLug6riYSY/FubOwtK3j9c8OoI3j0d3dNl3YSrN99RReV8LWx+MJ2GxxrSCQqYVFLJ/XjZl94xADMzBZP+lduxND3PPRBqvHMaOGYk8kfsxd0QsJ0bxo8jdSqPmoFHTF+EscUXQGqYc8llvepiCglAiwtv1WW952Or0iuotQ7+jeUR6uzwaZQ37+uylKdbWPg+TCSSEKf78VN2TjNcqiH/KzJiF97DZ5SRM8SfH6keO1Y+/RG1hyuQlVJ+beoRHd/nOdJR219CllG4hxO3o6xEowOtSSp+vM2wSJrJkf/JYhkTSg2QCRUi79iUdDtRAjVtCCzHhz2aXkzNXXofLaUZ1KODU738Dcwp5I2U+NmHBhKDQbcXaKskSA8iTHffoCN48Hp3m0oEGa0uU4LmsTzjHT6V0QCNLmpPZ0KI3FGf0fZd1GUn8K2c8Ga9mIVZuACm9dkzMcbEUXpfIiPM3MD/+K64vvJQt36cStAvqU+DLKU96/m4n8yr6E/H5tkN64bx5bvZf1YfaM1u5a8BOpi8eT8a7TYi8fKTLecLPessjaGcDMZY6qt2B2Pe1tPm0moSJyIsvo/nZl1hdbSeBhDZ7SKcTa53+vewfWsrq8AGYftpMr9oUphTfw4hL1nNj1H8BGGZXGB6wnTf7ncXBpXSn70xH6dBCV1LKBXj5QQ9KWBg147KoHCCw7xckLqxC3Xz8x+9FijgivXRHTX/HSW7pHSgO8NsvSVlZgWhsRja3oDXqI/nNIUHM+i6LaaE7MKFwT/4kQvL3EkkskWKcVzxOCiFQIiNx9E3CtqcONX8H4N3j0VG87RL3QytM1bu9nq86g42/yUIqepP6g0kjufWShSwa82/Guu6m964Y3GXlHfMQAob2pej3ghcGv0eieS6JZhMfNqTinBZEetlmsNloeK4HcYoVAItQ2FQeR4p/NRzWDdHR4yFsNkRWCp/88UliFBtmFDIumMXtkVeR9XAM7uKTW4LcG+dFqWrgNFsJLpuJBc7hyHb0o5tPz+SvfxzCvKlnI75vx/puDgeWev2/wwJ38G3KCIJ/cKNu20HPmVWUfp7II6E3AtCYaKc+xUTmm7twH7YbX35nxMAcii8IQfWX9Jzfgmn5Oq/t2+cP9j0RMj6G5kl1bB/yPrMbQ3gwcQpZr2QDoG3q/HXpTSvW03N9oP6Dy4Xa2nrEe6rGZ9LT+sPPP+/dHkWw+/BnC3QeSlgYALXnZVExQPD85a/zevkZ1J3hM4Uuw7ZjH/dsvJK1g9/lrojlXHDmcKJeXglAen0irzSez2k37eLHC6YzcdE9BCyoQTra/ywSJTKSHZcEMHvYs+RYzWhYGLv5ChrmxBFdnIfmuT7cFakUuPRgVi8VnMWBYPXuUuRKegrFV8QRcEYFqoTvWvzpba3hHD/Jm8Nf43cX307szP0d+nvbgrSYmVE5isdjl7Ht7gAyr29708sW2cKEwK3MCRqL1aSAdhKZBQc7tDqwerp++lr3U50j9EXhpUStqoaqakwmvesrWGoES3lEMO9UhEA7PRfNplCXqt/wp979GWMD8rl47TTEPItXi+t2U/+F00VDvT4gMSGgklFDN9OQHUJDto+aQFKiNTTo22HBvPHKYTReOQzz1fvob9uPGYUqrYWeX2io1bWdriYsVsTgvhT8MYuCP2Zx9kMr+GLKU5S7Q1i3zOfPRm4TSnAw2hn9MfXL7tAq0Fp1DdbPQvmiORCLECgXVum1Qilx79pN8gd7uG7pVGo1+OfTM6i/dECHvOtGp3LHRQvItOjSs+qSafwojpiP89FaW/WMqP69MUe1Eqm4iFRcPFMylvT3GnAXnVSm2UlTMziGUZetZU7fWdyyYzK3LLiBtY5oTAhyrU6GXpOH6wzfPYFPNDSxcPEgAk12Xh35BjueHYY2asAh4xfHQ4mKwt/uxCoE+wda2xzMATSHg5AiJ2scThLMfiQPK6Fh8jCqbhzO/ttHsP93I/T9amr7snDag0nB5O8Pw/qx86mhOB6pQ32gCnmxvo3y385VW67DsigE0+qtXi2629XQqWsg9PtoxiVcwPTUj7g5eilXX6IPuGRu74W23rsH4GRwXDCY+pvrebHPfwAYYNMw4UeBq5VHSiZgaXCBJ/Ols1CCg2k8O5t9U1r5cMhzAKgInt43hiXf9ifjX1tOJmvSN5gUlIhwhN0GLhdS1ag7K42b/jaXJTXZiJHWdu9aa24mamUVd/84iR9GvUBUQCPqsH4AiNVbcBeXkvlSMFeET2Xt4HfJumszO1uG4Devfc+9qO6lkOtXjEUo7Ha38MSSC+m1oBh3lZ5LbwoKovj8UJac/gRWoQf9DcXxZG7Y1O6/8VjUpZk4J2QLD5ZcRNGqRPqfvoOz/aoBK37CyiOxX3P2TRmk7dNbtHLbDqS78+qjsrkFxSFo1pzUqpEMHZxPVd8A8rfkEr7ORORavS/EVLoftaLiiM+LQH9qqgLZ6zbjCGtnsJUS89J1XPHFHXx+4bP8MWU+XzzQnzhrLbHmOmLNddw//nIA/F4JJWjjfmRlNWpjU7tuICdC2GzQL5Pd5wYRe3Yp32fNIMxk57RV16CtCgXguo/vJXxzI6bdO1FPYsyjLXS7gK7uryDm3RZairO5875JzM+ey6wRswCY+tC1pD6Zg1zTeWOvSkQ4TSPSKRuu4Ip2YfZ3M63fEq4L2UCYSa95mFDY5W7mgs/vJePtZpQN29A6+e4vIsIoPVswd8gr5Fj103bdrjFsfy2bjBUVqLW+6/I5HFNQEI7hWZQNt6JZJEIDZ4SGtGrYyswEF0L1eS2M9S9k+rZzQO1YkNEKCgldNpgPB/RmdsZcpj6tPyxp14uDCPt0I3LtVmyfDObFjBT+HLeQsZNTSSvKRtvQti47JSYa1V/S19JMnSa5v/hSUj924d7zS3auCA4kbWwh4SYry1v12YPhy2wIxYQ3n40jBvflzIvzONuvnMfe70Xymia0ESZUJNfvHs2o0AIuDypi65lvcNbTEwHw+0tvWLnBexKHodbXo/pJqjUn9y+aQtJCjT2jzWQMLCFqYCNbJsQAUFOVRPjKdKLWNqKUVf884UqaFXAo7HRFYavpQLNNavT8XGXB6D7cGbaNkTHraNRa+XvFMM4J2swbffSHSCX8B/JdZq6f+XuSPq9Gbi/yWveUOTGByrMSqRiikXfJs4SYDqQ9BuCQLrSfQkl+X386onvXbiQnNW2l7R6dsM+O4eny8F+RT2VMDvMfjuCSAL0747lBH/L7S24geU3nFK1ERVFyfQZZFxXwUNwyEs212IVKnGLFJg7NS/2wbiAx3wtEXj6al++yh2OOi2Xn9T34w5jPyLGaebKqNwBrlmSTtqLi58HQrkIkxlF0qYkXzn0du9CjmEuaqdfs2IWLV/eOZGr0BvJdwTTtDKGjT96SbjeReY2UOsPwE1ZeSPocgFtumcC2hFziv20g4sM8Xsg8n9wpxTw8YCEzcicSurFtg3ZabR2pQ3azVxVsdMRT9lIaIctWH5LNsW9MAn+Iew+LUPhT/iUAROYd2V3XEUxBQZQ+rDIv/r+YsRPz8mqky4nr2kT6P3IHQVutrArJIf6q1znXr4UnMuYAcF/8bQRarCeV+dJeVLuGBliiW/Db4yJzRjONfXqwZqCZiGH6gPQ/z3yLwWPqWNCcyNP5Y2jaMJyAPaDaBDnZRRQ6o/CraGeFSAjMSQkUXGoi174bk6cXebXDny/fHEHeqlyae+gVsYpcE3++cjbrbn+e7N5T6TG3P0HLdqBWVrX771eioiAqjMIrIxk2biNTo78jxOTHfrWJBU0phCrN2IULt5/eJdjZdL+A7kGtryfyix3cP+Aqhl76FABn+ancPfEzPnsh55CJAV4jNIjW/s08mfQpSWZ/wAaAIg4dalClxgMRW3l1/OmEreuJunW7910OonJMCmPGr8VucpGz7HpCv9RrgumLin7O4ugShvQFYNttVmac8Sb+Jgczy0ezYX8czvVhCA0Gj9vEB2mfYREKVxedR/J8J5WqF+omqzaiSYEblX2qfn4eSljAnqnf8+jpFxH4ej9SZ9ey/pKe/CZ4Ky+rEmG2tCm4SYeD5hcSuGj0XQTsVkj6bzHuw7owmhIFVwbWoUpBToR+LvaXB3t14M19WjqT0n7AjMJjlX1+/hvcu0vp/biCNCuIhiYeyr2Ucwa9Sx+rflNVp1airD75zJf2EFSksM0ZRm78HuqtPXAXFWMvKsZzjwVgev8r2DsqlMDx5QyMKUUbswc/xUWfgFIG+xUxvew8Ave2/YiZ7HZEahJFl0Xy1NnvEGBy8GD5YC4LW026pZHES4qoK04i+Cu9mzZ4qYW/2K+k4vyFfDfqef6ZfQ7L3hlIwqcl7TpGpqAgSq/N4J6b5vDb4APxSL8WP2rIZubMi4he3Ywz1EpQTJt33y66bUAHUCsryZ4ZzkXJUwFYPeg9bgzZzdK5WdTflIm6pcC7BVbWEPl5NGe33AWqwL/IgrUOLE0Sa6NGTaY+Wn7hld/zePRabH4upEU5wU47gEmh4KXTuO+M+SRbK7hr7vWkv1uL3PwTwBHBxde0RuutliEZO+hjreL8tTcR/HYwPXY1Ytq1jdYBKfzUpyeNCS62OOzkfZ9J2rKfvFK2KSiIjzdl8FjMKl6uHAnA4nmDsQ2q5q+9P2PMC7UUu92kWizsUzVCtjYg3W3vAwn4PI+MuZ4AetjvlMgIXJm/TFlfuUC/wSXXePfxqtVZdlJseh/0BwUDSWKj/gspcRfuAvQZtOp/w5mZkcy4AD2APZC+iMfOu4aod6uOObW+o0RucBCtNNLosqGU1xz1Rqat20LcVhumr3uyPbk3jT3MNMcIFsT0x1JvIu3tCuy7Npx0HrvSKwNHj2AaEq00X1zPn3PeZ1NLAvetmEz0SoUFKcMYMX4Djyd/wsyHR/Fd+kAAkt4vJvNfBcz5cSyv/2Y4L/R7n8Sp1bwVcC7J74k216CF1YK5WZJhLafM3cxbdQN4IGI7Duniw5JBxM1Yg3Q4sANtXwSifXTrgI6UqJvziXxKz1S47enTeSF+Oa/2XMCgSfeQ+mxFm6cbHw+1pobQT9YRtjZBf6GyGul0gaoiXW6CA/QANidmGI9fsZZAPwfS2v4BvuMiBI0TB/Gv0R9woX8F526aTOJiF3JbYacOdLUF/136oNeq7SlU9rCQHFZDfWsgpu0lNJ+ZRenVLp7p/zFrHeHctuJq0j9t9pq7KTQE2WzGjEKwWe/eSFzUgGke3H/FdVwz4VseiNjMwuYg7p99HakFG2jPyqLHq9FrdQ2YzSql7kbeqh1I5Ca95SGd3n2wfH0qJFsq9JbixqCjuzQ0kvjBLuZ9fzYz7tdvcO8MeB3nBbW4t2Z6Ndf5AMJspiXSQn+bjX2NgUSWHLuCJR0O1C0F2Lbo7d6Ig37X1vba9usi+XLKk4SbTKx1BnHnuskEfhpM7+V7UUv2EBEWxvZ1vbnszD5MPGclb982HYC3Jw9n3tIhxP4gsSwM5bmIMbyVsgDXZIVPKs8icmbbArpaVU3MO5u4Kfx2WlKdRC+18MmUXF7r/TYlJRFk9faHPN/Otex2aYsGBgYGBu2je9fQPSgr9bvc9j/1Y9KfApmZ8imLr3+CMeJ+0mYU4S7f57UcU621FY4xyCh7pgFgjddnjIb6taBZ/Do4xHd0lIhwykbDaL+9WISdsu1R9NpYjNt5aI1RyUwDTYOaOn0ihS8p1Gs0AZv7s2ZoT36f8DW3jJyGX1Yfwsft5cOM2Wx29ODRryeS+UajV7OTtOoa7OVJKMLEjiZ9mWhpNqFt2EbyBlj2iJ0fUifiSAonfVsh7qa2r9d9PITZjPv0PvSOLWGLM4y3Pj2b5M/10XpvD0KaXGARKmAiOu/oLRytqQmtqQmxt4zQ94YAMCd1ENP7zubuoTfTY6XZ6y07JTKCioGCUncj1RXBRHp178cm8+W9XFz/B2zDqmheF07Kx7Vo63/4ubtHrajAf0EdWWuj+SZ/OPtv1Fs1MxOX8vdJKym4TFLgiibJXI0iBFNCVjMre3S7/LWGBpJfzke2tCKdLiptA3krbjjTR33APa1Xk/18Omr+DsxxsT4Z7/pVBPQDXxDrtxtYN2EAe5MU+lntbJs6g9yq24h/y9HhrhclMgIRHIS6u/SoF76Sk8W2u/0BWDXsOTTshNmaqVEivB/QhYCwEM4dvIFoJYA6rYXY5eLnC8I1Ru8TtDS62DrVSkB4C60toaRdU9cpubXH4kC/rKVR0qDZGWprYtoFXxGitDApaCerHYE8+s1Esl6pa3PK4IkQVivOdH3htFVrMwDI3l54SPPdXbgLpfDIad7ewJSWzO6bVTanfclLtal6UOmkbJK0Z7bx2tiRDIhfjjPIhN/xMlekJHixfqz/e1M690b8SGOShiki3OuJBFpMODKhhUJ3IOb93p3xeDzcu3aT8Pgvk7aONgNEupy4S0qJ/qiB8lV6F+qZfX/HvtM1BuQUcUHURnbJSJY1h/PezsHEtG+aAsAhWTIxX+5mqXM4lbcF8vr4V/h9wiQaagdCo5mkBfqaQwEby1DLyjul6/RXEdAPIF1OMt5u4aUho/l3jxUA5Fy5lb0F6dgXr29XzchktyP7ZrDrvCBUf0naM/WH1HSFxYrolUr+b0P5YvQzgL7C4n61mfKmYPybXB1Za+qoKOFh7L4slnnxH9GouXi+egDBBQ1gNkP/bIqu1t8XG9vE+1mz6GfVw9iOQo2HRl3RqVkNR8PcDA7NQqDJzv3h+hLC/231Y+p315M2xwUFuzqhUDNDU/T9Wqs9A9O+HFtQVf4x8BMA3ts9iPB91UcNLF7BJFj6VX/WXLUcrqlAbEpFHucGeWBOQsnuTEw5gtvP+Ypvnsj2ulZddggTslZR4orAr6Iz2qnHoA2tcbW2Dur0sZ6QjSZC3tVoCQxkdq9zKR8ehLlFEr2uEZG31ivfY3fpHkLfKadsdRoPPX0pf+79BUPte4lR/Hh4hP4AjY83DSD2iwRC8ypQC3aeYI9t41cV0AHEmm1889VAFl65gbH+dTwYv5AJE28nqyITVrd9dp5ISWTbLTb+PfINtrbG88Gec1EcEtUmUO3QGi3pf0YBCxNfI8Wsj1VvdjmZ+P3tJLxpRuat9urfZ/L3x5GbgrN/E3VaK7Pq+vH2Z2eRtjsf95Acdlxl50/D5gEw2n8HaxzxLGxIIERp4crgDex4Ioz0e7UjnpbTKXhmRtalQ67fLwNKXzTbufOz35IxpwVlbb5Xc7J/Ltpq4doYfRU9zeL5Kiq+GRISZjP1udEUOqNYoTTSOj8GrbbzjrdaVU36a3u52v933DjmW16/9iwyHws55mQyU4Ce1hq63oJrnMZpfruYO/xcAuZ493EFLVEmUv0qKHZE4r+vc2dKd4gDNwCpV3y0hgZYtZHYg2rlXq2UaSrqlgJCH+zFY6f/htr+Li4buIbcAP07ctuoZWwZEcntS66h9+Mur+an/+oCunQ5SZtewP1chzJpFuf5NfHj2H8zuvh+EtsRW93hAfz19E+5wL+RvtY8mm+2YhEqIUoL4eZG4s019LE2YBcK7zboq7E9Nv8yMv60rnMCVUIchVcqFIycyUu1vXnntbGkz9lNy8BUSm9w8uRp7xNv1ruXphZcTcXieCI3udgzyszlkzcQ4OcAcyemUh5E8aPDAbhz4uecYa/jQHLWwppc0j9o0nPFO6Ngk4I7PoJx/g4atVZiftJL6cgEkbagxMdRdqZghP92Fjf0IXCv2inXwsG4i4rJeDuQ95IHcfO4r3mt+TzSXinBXVJ6xHu1Ft1FcUj2qgpf1fdBcXh/JrPJKSlzhvLeuiFkravtvBbKrxRt/VZi8+30CAtla2ga35w7DIApDzxPiqWV0DEv89hzV3m1zF9dQAf9i9vzi2Ye6TuBcQPfJ1oJQPZrQMnJOuFSu4djKavln5vGofX5imuCyvlz5Maff9cinSxpCWdmTQYNwxQwAAAK3klEQVSvrDmDtDf1L0Xa0pWddvE6EkJJTttHs3TybN7ZZM5YS90FuYTfWczspHksaerF/V9OASBuuSB2fyuFl1l4ZvzbvFM7EOeKCNQ93m01HI45MYGSK5J4ZPKHAFweWM7C5kgW1vTl0divCTA7UJqcnTK1+cDysYUTAqlUm7ihcCJBS/V0OV+NHkh/O736F9Pb0soN2waRVlDrk7Jl3mZsXw7HkW5h8W+fZGT4vWQ/a0HdUXToGz3jKM4gwV53CN+WZxBU7f3+/ciZP/DTTIVM2yY0V/dIpe1uaK2taGXlmGpqCd2pr+WS59QYYlPY7EjAGROA2YvL/vwqAzqAOb+Epo1ZVA1oIVoJYPqA2dxxzQ1kPBnepmwPdXcpSf/I4qkxlzP/wgKe7/kp/iaFh8rOYmFeX+IXmQjaUU+v8hLUis6vAbZGWugTsp936jMJWW7HFBXJ3pGCF3t+yqLGHN54byzZs8sAaOoVRdMDdXzT+03+VjaeTS/3IXHOxk5fimD/uYk8dusbjPfXl4c9bdV1+M8NoT5V0HDDYoqaIhB1jV4vV5jNqIN7sXOKjT+e/Ql/Lj+Hfa+mEFrzw4k/7EU0uxVVVch32bD9GIjcXXTiD3mJmOVVvJE1mrjza3lp3OvcFvYbwhYPJ/q7Mty7Sg4ZFG9MUTGhUb47nLDKmk676fhqud5fK+bkJHZNSWDgxXrkjlUcqFJhXWMSqt3k1SB8wn0JIRKBt4BY9AHlmVLKfwshHgVuAg4so/aw54EXnUKrbGYzP+GgFYEgviqNpK960nd9Jualy4iKUKhw/IPg0POIbEPclW43rNlM4mY7Vev7cs7AP6A4IGKLi147qtGKS/WHyh7LgxSSRAY75Wb2UoTFs1xAOn2IFO1YMF9KAhUHzZoVe41E+tlQoloJEm7y6pJojdIovDYWV10NlYtewHZrNb2c/iTb99OzaCXbtU3e8TgGLROG8OjDeldX31fvwFVXQ93Lb+F01qPGmpgnXKxO7YlS/iF7ZL5XPZSoSAom27hz9CLWNyWy4t3TiF+Yj4oPzstBVOcGMyCokI9qBhNW4D7kmaad7aFuKSDjEX8e87+YO85czJIzn+fD3Fxe7n0e6e8FgCLYNywYZ0MNAW//lbuebaW89hn8m1NJIL5Tjsfx8OV56XYeJgWRm83WGwOZe8F0+tv0MipVwaTC86j6vxRsX3pn5vQBTubm4AbulVKuFUIEAWuEEF97fjddSvmUV42OgUCQQT+CRRhu6WKV/Ibwpd8RkGpm3A3hvHhnKP+qyuDrO/rAzrVt3r/W2or1y59I/PKX145WoznCg28Il/pCDUlk0FNktfMv1AkqbKTCGcjjsT/ySu55OIOiGZ6ymVCTiRtj/8u0i78j1NRCxX6V/8s6A8e3QwhatJkf9swhguFe8zgqJoWScTDOr5nf7x1B6rNbcWjNtLp60TJ1DI/f+h/umlCCvLAG6XJ63UO2OrBWKTz/1TjS5rTSI28dqid1srPPy8FELd3DyrABOEIh9cfth1wnvvDQmpvJums9H008j7L7Qng4agXTpmxi3kWJbGvpQbbfXmr3OxnSFEhU70Cu+OMdbH/7H4QS5FWPk8GX56W7eZjj4yiYEsyT571LX6uFIpfeap206XqCnwzCutS7wRxOIqBLKcuAMs//G4QQW4F4r5ucAJvw+/lJ3GZhwV8G4aAFpVngkgouqdKg2jE3Or2eRngyHt7C1OigpDGMfaqDjdc9R75LJdzkxt9kY6TdSaPmYGZtLvhDfd5QYr8uQK1uwJ9Ar3oc1S3AHywaLdJJfn00Zmc1Vsw0Th7DtDs+I9xPpaFHPLHfVNAZi/mqNTX0/Mv3P/988DhGZ5+Xg3Hv2k3sdD0z4fCbvq88pMNB2Ffb+SZ4GBuv7MFzabO5Omg/jsC92IQZgqBGE/y5/BwCGyz403nH43j48rx0Jw9zXCzbb03k+cv0FTAVYeLWnZMAsM8Mw7JyQ6eMw7Wp+0YIkQwMAH4ETgduF0JcC6xGr8UfMbtHCDENmAZgx7+DujotsokGagkhnPqGPbw3q4XZs1VEaCmDC4OOup5BZ3vUUkkJOymTuwkijEz6YRFHrvNyIg81fwfiyUGcf8vNLBk8k14WO2Dh9j1nsL4ynqraQPxW659LmLMdtbKqUzyOhik4iPCYemo1N80uC67ZPQC4J/0Dcm17GP/S9dSsfoHs1hrqoNM8ToSvjkdXe6iVVcS8u4mmsl78Ztpv+aH/h9iEmTfq9fPy0s6ROL6NJGLZmv+J49EdPITZjJKUwNa7YvhmwpOkWAJZ53Bx+fe3kP6MPrbll7cGrZMmAIqTXbBICBEIfAf8n5RyrhAiBqhET+H8OxAnpbzhePsIFuFyqDinQ8Ju6WYNS0mhF9EiHodsxerp/9rJZhy0kiP0BP4f5TfUy+ojZjz8WjyE2YwpJQlHYhgmVUO4NMy1LajbPJMRNNWnx8OcEM+WR3qw48KXqdJacHqunU3OCG5eOIn6e/9DsiP1V3NeABbLOWuklIMO3sev1UMJDqb5jCycwXraasi2etz52/mpeeH/5PHoCg8xMIcdU4L404Vz+W3wfna7G7lgzTTi/yaQHVio62geR+OkauhCCAvwMfCulHIugJRy30G/fwWY307Xk0aTGhv4gViSiBZ6r49N/LIwZbxMYR0rOlvDZx7S7UbdXoj5oOXWD76v+/p4qOX7yJwVQa+Q3/L+0Ff5157xAKxbnEbLEzOIccb9T5yX7uqh1tdjW/CTJzyBW2qsY8X/7PHwpYcwm1FP70vcP3fydNwicqx6987DpRcS8k4Qcn3nphL/7HGiGroQQgBvAtVSyrsOej3O07+OEOJuYKiUcvIJ9tUAtC1R/FCS0WPawXPbLcCB9UqjgUCgHr310FNKGXUUjwqgyfMew+N/14OjuRgehkcne5wskcfzOCpSyuNuwBno3SobgHWe7XzgbWCj5/XP0LtcTrSv1Sd6jzc8Tqac9roYHoaH4WF4tNejjc5tLuNkslyWc/SHQHZaznlHPYTovIWCDA/Dw/AwPLorxgMuDAwMDE4RfB3QZ3ajcnzhYni0vQzDo+3v6SiGR9vL6C4eh3DSaYsGBgYGBt0bo8vFwMDA4BTBCOgGBgYGpwg+C+hCiHFCiHwhxA4hxINe2meiEOJbIcRWIcRmIcTvPa8/KoTYI4RY59nONzwMD8PD8OioS3fxOCadnUvp6aNXgJ1AKmAF1gO9vbDfOOA0z/+DgAKgN/AocJ/hYXgYHoaHt1y6i8fxNl/V0IcAO6SUhVJKJ/ABMKGjO5VSlkkp13r+3wCcaCVIw8PwMDwMj/a6dBePY+KrgB7PodNtS/HyErzi0JUgQV8JcoMQ4nUhRJjhYXgYHoZHB126i8cx8VVAP9rUK6/lSwp9JciPgbuklPXADCAN6I++lvvThofhYXgYHh106S4ex8RXAb0USDzo5wRgrzd2LI6xEqSUUpVSasAr6E0lw8PwMDwMj464dBePY+ONDv0TbejL9BYCKfwymJDjhf0K9OedPnvY63EH/f9u4APDw/AwPAyPjrh0F4/j7scbMicpfD76yO1O4I9e2mebV4I0PAwPw8PwaK9Ld/E41mZM/TcwMDA4RTBmihoYGBicIhgB3cDAwOAUwQjoBgYGBqcIRkA3MDAwOEUwArqBgYHBKYIR0A0MDAxOEYyAbmBgYHCK8P9kuYvsfI2L3AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1d832f2c908>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "batch_data, batch_label = augmented_datagen.next()\n",
    "for i in range(batch_data.shape[0]):\n",
    "    plt.subplot(1, 10, i+1)\n",
    "    plt.imshow(batch_data[i].squeeze())\n",
    "    plt.title(batch_label[i].argmax())\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., 0., 0., 0., 0., 1., 0., 0.],\n",
       "       [0., 0., 0., 1., 0., 0., 0., 0., 0., 0.],\n",
       "       [1., 0., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 0., 1.],\n",
       "       [0., 0., 0., 0., 0., 0., 1., 0., 0., 0.],\n",
       "       [0., 1., 0., 0., 0., 0., 0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 0., 0., 0., 0., 1., 0.],\n",
       "       [0., 0., 0., 0., 0., 1., 0., 0., 0., 0.],\n",
       "       [0., 0., 0., 0., 1., 0., 0., 0., 0., 0.],\n",
       "       [0., 0., 1., 0., 0., 0., 0., 0., 0., 0.]], dtype=float32)"
      ]
     },
     "execution_count": 193,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "batch_label"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Model Functional API"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在此之前，我們使用 Sequential 便足以建構大多數的神經網路，那是因為我們接觸的神經網路多為線性堆疊 (linear stack)。\n",
    "\n",
    "除了輸入層需指定 `input_dim` 外，其餘隱藏層只需宣告，那是因為 Sequential 會認定上一層的輸出這一層的輸入。\n",
    "\n",
    "因此，再建構線性堆疊的神經網路時，Sequential 便足以處理。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.1 Functional API 的使用時機"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "當神經網路模型為非線性的複雜網路結構，如：\n",
    "\n",
    "* 多重輸出-多重輸入模型 (Multi-input and multi-output models)\n",
    "  + 分歧 (branch)\n",
    "  + 合併 (merge)\n",
    "* 具重複/循環結構的模型，如: CycleGAN\n",
    "\n",
    "Sequential 便不足以建構這類複雜結構的神經網路，我們以下介紹 `Model` Fnuctional API 的使用。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先，我們先將 `Model` 引入。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.models import Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在 `Model` 的世界中，所有的神經網路層 (fully-connected, convolution, MaxPooling, LSTM, etc) 都被視作函數來操作，因此，我們只需關心函數的輸入和輸出即可。\n",
    "\n",
    "此外，為了讓神經網路的第一層從不需要輸入 `input_dim`，我們還需引進下面這個函數來代替 `input_dim`。 (此寫法亦可用在 `Sequential`)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Input"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.2 Functional API 的函數概念"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "回顧一下，我們想學習的手寫辨識模型是一個長得像這樣的函數\n",
    "\n",
    "$$\\hat{f} \\colon \\mathbb{R}^{784} \\to \\mathbb{R}^{10}$$\n",
    "\n",
    "我們希望建立一個具有兩個隱藏層的神經網路來學習這個函數，攤開來看的話，如下：\n",
    "\n",
    "$$\\mathbb{R}^{784} \\overset{f_1}{\\to} \\mathbb{R}^{500} \\overset{f_2}{\\to} \\mathbb{R}^{500} \\overset{f_3}{\\to} \\mathbb{R}^{10}$$\n",
    "\n",
    "$$x \\overset{f_1}{\\mapsto} h_1 \\overset{f_2}{\\mapsto} h_2 \\overset{f_3}{\\mapsto} y$$\n",
    "\n",
    "\n",
    "或是以簡易的圖來表示這個全連接神經網路\n",
    "\n",
    "<img src=\"plain_model.png\" alt=\"drawing\" style=\"width: 400px;\"/>\n",
    "\n",
    "其中，$f_1, f_2, f_3$ 代表的是全連結層所代表的函數，其他變數說明如下：\n",
    "\n",
    "* $x$: 代表的是輸入模型的圖片向量，為 784 維的向量。\n",
    "* $h_1$: $x$ 經過第一層隱藏層運算後得結果，即為 $f_1(x)$，為 500 維的向量。\n",
    "* $h_2$: $h_1$ 經過第二層隱藏層運算後得結果，即為 $f_2(h_1)$，為 500 維的向量。\n",
    "* $y$: $h_2$ 經過最後一層運算後得結果，即為 $f_3(h_2)$，為 10 維的向量，代表的是 $x$ 為哪個數字的機率。\n",
    "\n",
    "注意: 為了方便，我們將 `Dense(500)`, `Activation('sigmoid')` 兩個合併用 `Dense(500, activation='sigmoid')` 表示"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.3 Functional API 的操作方式"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "我們沿用上圖的變數名稱來定義"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_1 = Dense(500, activation='sigmoid')\n",
    "f_2 = Dense(500, activation='sigmoid')\n",
    "f_3 = Dense(10, activation='softmax')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接著，定義層前後變數之間的關係；首先，第一個變數必定以 `Input` 函數來定義"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = Input(shape=(784,))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "剩下的部分，就如變數說明，**幾乎**可以照著數學式輸入 $$h_1 = f_1(x), h_2 = f_2(h_1), y = f_3(h_2)$$"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "h_1 = f_1(x)\n",
    "h_2 = f_2(h_1)\n",
    "y = f_3(h_2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "在這裡，變數 $h_1, h_2, y$ 是以張量 (tensor) 類別來表示，我們可以嘗試 `print` 看看。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"dense_1/Sigmoid:0\", shape=(?, 500), dtype=float32)\n",
      "Tensor(\"dense_2/Sigmoid:0\", shape=(?, 500), dtype=float32)\n",
      "Tensor(\"dense_3/Softmax:0\", shape=(?, 10), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "print(h_1)\n",
    "print(h_2)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接著，透過 `Model` 將一個模型的輸入/輸出包裝起來，建立模型的過程就完成了！"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 500)               392500    \n",
      "_________________________________________________________________\n",
      "dense_2 (Dense)              (None, 500)               250500    \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 10)                5010      \n",
      "=================================================================\n",
      "Total params: 648,010\n",
      "Trainable params: 648,010\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Model(x, y)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "一樣的，當模型 compile 之後，便可以進行資料的訓練、預測等等，請有興趣的同學讀入 MNIST 手寫辨識之料後，自行完成這個模型的訓練。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(loss='mse', optimizer=SGD(lr=0.1), metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.fit(x_train, y_train, batch_size=100, epochs=5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "雖然 summary 少了很多東西，但模型架構和之前做的沒有差異，所以可以安心讀入之前訓練好的權重。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.load_weights('handwriting_model_weights.h5')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2.4 小結論\n",
    "Functional API 的操作流程如下：\n",
    "1. 將層定義成明確的函數\n",
    "2. 透過層函數將變數連接\n",
    "3. 定義神經網路的輸入與輸出"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3 非線性堆疊模型"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.1 如果建立具分歧及合併結構的神經網路模型呢？"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import concatenate, add"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先，假設我們希望在模型之間增加一個分歧，且這個分歧在模型的輸出會合併，則神經網路的結構會變成：\n",
    "\n",
    "<img src=\"branch-and-merge.png\" alt=\"drawing\" style=\"width: 400px;\"/>\n",
    "\n",
    "此模型為單一輸入、多重輸出的模型，是分歧模型最容易處理的一種。\n",
    "\n",
    "其中，$f_1, f_2$ 同之前，$f_4:\\mathbb{R}^{784}\\to\\mathbb{R}^{500}$ 的全連接層，但 `Activation` 改用 `ReLu`。\n",
    "\n",
    "需注意的是，由於 $f_3$ 的定義域改變，為 $\\mathbb{R}^{500}\\times\\mathbb{R}^{500}\\to\\mathbb{R}^{10}$ 函數，所以需要重新定義。\n",
    "\n",
    "* $x$: 代表的是輸入模型的圖片向量，為 784 維的向量。\n",
    "* $h_1$: $x$ 經過 $f_1$ 隱藏層運算後得結果，即為 $f_1(x)$，為 500 維的向量。\n",
    "* $h_2$: $h_1$ 經過 $f_2$ 隱藏層運算後得結果，即為 $f_2(h_1)$，為 500 維的向量。\n",
    "\n",
    "* $z$: $x$ 經過 $f_4$ 運算後得結果，即為 $f_4(x)$，為 250 維的向量。\n",
    "* $y$: $h_2$ 和 $z$ 經過新的 $f_3$ 運算後得結果，即為 $f_3(h_1, z)$，為 10 維的向量，代表的是 $x$ 為哪個數字的機率。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "因為上面已將 $f_4$ 及 $z$ 以外的變數定義好，我們只需定義 $f_3$, $f_4$ 及 $z$ 即可"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_4 = Dense(500, activation='relu')\n",
    "z = f_4(h_1)\n",
    "\n",
    "# new f_3\n",
    "f_3 = Dense(10, activation='softmax')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接著，再將 $y = f_3(h_1, z)$ 定義好，就會發現......"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "__call__() takes 2 positional arguments but 3 were given",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-19-8debaf5d981f>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0my\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mf_3\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mh_2\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mz\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mTypeError\u001b[0m: __call__() takes 2 positional arguments but 3 were given"
     ]
    }
   ],
   "source": [
    "y = f_3(h_2, z)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "發現沒辦法這麼做，那是因為我們先將 $h_2$ 與 $z$ 透過 `concatenate` 接在一起，叫做 $u$，再送進新的 $f_3$ 裡。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "u = concatenate([h_2, z])\n",
    "y = f_3(u)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "換句話說，模型其實是這樣畫的\n",
    "\n",
    "<img src=\"branch-and-merge_final.png\" alt=\"drawing\" style=\"width: 400px;\"/>\n",
    "\n",
    "其中，`concatenate` 是將不同的變數合併在一起，並沒有進行任何運算。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "再透過 `Model` 將模型的輸入和輸出包裝起來，即可將模型建構完成。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 784)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 500)          392500      input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_2 (Dense)                 (None, 500)          250500      dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_4 (Dense)                 (None, 500)          250500      dense_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 1000)         0           dense_2[0][0]                    \n",
      "                                                                 dense_4[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dense_5 (Dense)                 (None, 10)           10010       concatenate_1[0][0]              \n",
      "==================================================================================================\n",
      "Total params: 903,510\n",
      "Trainable params: 903,510\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model = Model(x, y)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.2 小結論\n",
    "Branch-and-Merge 的注意要點如下：\n",
    "1. 每一層分別定義成函數\n",
    "2. 分歧結構: 實就是透過新的函數來定義新的變數，無特別注意事項。\n",
    "3. 合併結構: 要合併前，將所有要進入的變數都合併起來，才能進行之後的運算。\n",
    "\n",
    "常見應用:\n",
    "1. 多重輸入-多重輸出模型。\n",
    "2. 當層函數為 convolution 時，這樣的技巧可以實現 U-net 上的重要結構 multi-resolution fusion (多解析度融合，又稱 MRF)。\n",
    "3. ResNet 上的重要結構 skip connection (跳躍式傳遞)，亦可透過分歧-合併來實現，只是 ResNet 使用的是 `add` 而非 `concatenate`。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3.3 自定義的不具可訓練權重之神經網路層\n",
    "這裡，我們將進行這單元最後一個重要的神經網路建構技巧 - 自定義神經網路層 (不具可訓練權重)\n",
    "\n",
    "** 具有可訓練重的自定義層牽扯到 TensorFlow 及 Python 類別的撰寫，若有興趣可參考: https://keras.io/layers/writing-your-own-keras-layers/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先，我們需要引入 `Lambda` 這個函數，透過 `Lambda` 函數，我們可以將 Python 上的 function，包裝成 Keras 上的 layer。\n",
    "\n",
    "此外，我們需要引進後端所使用的套件 (此處為 TensorFlow)，並使用裡面的運算進行 function 的撰寫。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers.core import Lambda\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "首先，我們透過 backend 來定義一個簡單的 function，作用是對輸入取平均，程式碼如下："
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def average_function(inputs):\n",
    "    return K.mean(inputs, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'numpy.dtype' object has no attribute 'base_dtype'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-25-fb01d882e927>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[1;32m----> 1\u001b[1;33m \u001b[0maverage_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0marray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m3\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m<ipython-input-23-a509c6a873b8>\u001b[0m in \u001b[0;36maverage_function\u001b[1;34m(inputs)\u001b[0m\n\u001b[0;32m      1\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0maverage_function\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 2\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mK\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmean\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;32m~\\Anaconda3\\lib\\site-packages\\keras\\backend\\tensorflow_backend.py\u001b[0m in \u001b[0;36mmean\u001b[1;34m(x, axis, keepdims)\u001b[0m\n\u001b[0;32m   1367\u001b[0m         \u001b[0mA\u001b[0m \u001b[0mtensor\u001b[0m \u001b[1;32mwith\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mmean\u001b[0m \u001b[0mof\u001b[0m \u001b[0melements\u001b[0m \u001b[0mof\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0mx\u001b[0m\u001b[0;31m`\u001b[0m\u001b[1;33m.\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1368\u001b[0m     \"\"\"\n\u001b[1;32m-> 1369\u001b[1;33m     \u001b[1;32mif\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdtype\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbase_dtype\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbool\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1370\u001b[0m         \u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcast\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfloatx\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1371\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mtf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreduce_mean\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mkeepdims\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mAttributeError\u001b[0m: 'numpy.dtype' object has no attribute 'base_dtype'"
     ]
    }
   ],
   "source": [
    "average_function(np.array([1, 3]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接著，使用 `Lambda` 將函數包裝成神經網路層，並指定輸出的形狀。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "average_layer = Lambda(average_function, output_shape=(1,))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "此時，一個具有將輸入取平均的神經網路可由下述方式建構。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "x = Input(shape=(4,))\n",
    "y = average_layer(x)\n",
    "average_model = Model(x, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可以發現，這樣的神經網路是不具有訓練權重的。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_2 (InputLayer)         (None, 4)                 0         \n",
      "_________________________________________________________________\n",
      "lambda_1 (Lambda)            (None, 1)                 0         \n",
      "=================================================================\n",
      "Total params: 0\n",
      "Trainable params: 0\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "average_model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "記得，在使用前記得先 compile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "average_model.compile(loss='mse', optimizer='sgd')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "接著，我們將 `[1, 2, 3, 4]` 送進這個神經網路，看看神經網路的輸出是否為這個向量的**平均。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2.5], dtype=float32)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "average_model.predict(np.array([[1, 2, 3, 4]]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "可以發現，將 `[1, 2, 3, 4]` 轉成 Numpy array 送進神經網路**預測**後，答案是 `2.5`，即為 (1+2+3+4)/4，確實是平均。\n",
    "\n",
    "我們也可以一次送進多筆資料進行平均的計算。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([2.5, 1. ], dtype=float32)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "average_model.predict(np.array([[1, 2, 3, 4],\n",
    "                                [1, 1, 1, 1]]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 加碼: 具抽樣功能的神經網路層\n",
    "\n",
    "輸入為 $(\\mu, s)$ ，$\\mu=(\\mu_1,\\cdots,\\mu_n)^T$ 和 $s=(s_1,\\cdots,s_n)$ 各自為 $n$ 維向量。\n",
    "\n",
    "我們希望神經網路層輸出為服從 $N(\\mu, e^{s}I_n)$ 的 $n$ 維向量，換言之，我們希望建構的神經網路其實是一個抽樣函數。\n",
    "\n",
    "** 由於神經網路的輸入輸出經常沒有限制，為了讓 $s$ 具有變異數的非負特性，我們考慮 $e^{s}$ 作為變異數；換言之，$s$ 為 log-variance。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "假設我們想進行抽樣的維度為 `sampling_dim`，則一個具抽樣函數功能的神經網路可由下述方式建構。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampling_dim = 2\n",
    "def sampling(args):\n",
    "    z_mean, z_log_var = args\n",
    "    epsilon = K.random_normal(shape=(sampling_dim,), mean=0., stddev=1)\n",
    "    return z_mean + K.exp(z_log_var / 2) * epsilon"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "這裡使用到常態分配的線性特性來定義函數，亦即\n",
    "\n",
    "$$X\\sim N(0, 1)\\Rightarrow \\mu+\\sigma X\\sim N(\\mu, \\sigma^2)$$\n",
    "\n",
    "\n",
    "若不熟機率論的同學，可以詢問助教。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample_layer = Lambda(sampling, output_shape=(sampling_dim,))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = Input(shape=(sampling_dim,))\n",
    "s = Input(shape=(sampling_dim,))\n",
    "z = sample_layer([m, s])\n",
    "\n",
    "sample_model = Model([m, s], z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_3 (InputLayer)            (None, 2)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_4 (InputLayer)            (None, 2)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lambda_2 (Lambda)               (None, 2)            0           input_3[0][0]                    \n",
      "                                                                 input_4[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 0\n",
      "Trainable params: 0\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "sample_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "平均為 (5, 4)\n"
     ]
    }
   ],
   "source": [
    "test_mean = np.random.randint(10, size=sampling_dim).reshape(1, 2)\n",
    "test_log_var = np.array([[0, 0]])\n",
    "\n",
    "print(\"平均為 (%d, %d)\" %(test_mean[0][0], test_mean[0][1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "透過下面的指令，我們每次可以抽樣出一服從上述要求常態分配之隨機向量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[4.587086 , 5.1148167]], dtype=float32)"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_model.predict([test_mean, test_log_var])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "來和 Numpy 上的抽樣函數進行比較吧~"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_of_samples = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "np_sample = np.random.multivariate_normal(test_mean[0], np.identity(2), size=num_of_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "kears_sample = np.zeros((num_of_samples, 2))\n",
    "for i in range(num_of_samples):\n",
    "    kears_sample[i] = sample_model.predict([test_mean, test_log_var])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x18cd1817be0>]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAW4AAAD8CAYAAABXe05zAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJztnX9wXeV557+PZYElJ7aMbdhIrCz3j3WBZoODk0mrLk6ggRhSRpvJkGRESMLOOGyLl+x0Ra1hty3ddU1G6bY07JZqUlMnqEloiu9QCIEUErN4lyYGmRBiPLMtloO0jX+AcIzkIItn/7j3SvdenV/vue855z33fD8znmude+65zznnnud93+enqCoIIYTkh2VZC0AIIcQMKm5CCMkZVNyEEJIzqLgJISRnUHETQkjOoOImhJCcQcVNCCE5g4qbEEJyBhU3IYTkjOVJHHTdunXa19eXxKEJIaQlee65506q6voo+yaiuPv6+nDw4MEkDk0IIS2JiExE3ZemEkIIyRmhiltENonIoZp/p0XkC2kIRwghZCmhphJVPQLgcgAQkTYAkwD2JSwXIYQQH0xNJVcD+EdVjWyLIYQQYhdTxf1JAF/3ekNEtovIQRE5eOLEieYlI4QQ4olEbaQgIucBmAJwmar+LGjfLVu2KKNKCMmG0vgkRh4/gqnpWXR3dWDo2k0Y2NyTtVgkBBF5TlW3RNnXJBxwG4Dnw5Q2ISQ7SuOTGH7oRczOzQMAJqdnMfzQiwBA5d1CmJhKPgUfMwkhxA1GHj+yoLSrzM7NY+TxIxlJRJIgkuIWkU4AHwbwULLiEEKaYWp61mg7ySeRFLeqzqjqWlV9I2mBCCHx6e7qMNpO8gkzJwlpIYau3YSO9ra6bR3tbRi6dlNGEpEkSKRWCSEkG6oOSEaVtDZU3IS0GAObe6ioWxyaSgghJGdQcRNCSM6g4iaEkJxBxU0IITmDipsQE8bGgL4+YNmy8uvYWNYSkQLCqBJCojI2BmzfDszMlP+emCj/DQCDg9nJRQoHZ9yEROXOOxeVdpWZmfJ2QlKEipuQqBw7ZradkISg4iYkKr29ZtsJSQgqbkKismsX0NlZv62zs7ydkBSh4iYkKoODwOgosGEDIFJ+HR2lY5KkDqNKCDFhcJCKmmQOZ9yEEJIzqLgJISRnUHETQkjOoOImhJCcQcVNCCE5g4qbEEJyBhU3IYTkDCpuQgjJGZEUt4h0ici3RORlETksIr+atGCEEEK8iZo5eQ+A76jqx0XkPACdYR8ghBCSDKGKW0RWAbgSwGcBQFXfAvBWsmIRQgjxI4qp5JcAnABwv4iMi8hXRGRlwnIR4hZsWUYcIoriXg7gvQD+XFU3A3gTwM7GnURku4gcFJGDJ06csCwmIRlSbVk2MQGoLrYso/ImGSGqGryDyL8A8Kyq9lX+/jcAdqrq9X6f2bJlix48eNCmnIRkR19fWVk3smEDcPRo2tKQFkVEnlPVLVH2DZ1xq+o/A/ipiGyqbLoawE+akI+QfMGWZcQxokaV7AAwVoko+ScAn0tOJEIco7fXe8bNlmUkIyLFcavqIVXdoqr/WlUHVPX1pAUrKqXxSfTf/RQ27nwU/Xc/hdL4ZNYikTgty+jMJAnCDjgOURqfxPBDL2J2bh4AMDk9i+GHXgQADGzuyVK0YlPteHPnnWXzSG9vWWn7dcKpOjNnZsp/V52ZtccipAmY8u4QI48fWVDaVWbn5jHy+JGMJKrA2WNZ4R49Crz9dvk1SAHfeeei0q4yM1PeTogFOON2iKnpWaPtqcDZozl0ZpKE4YzbIbq7Ooy2pwJnj+b4OS3pzCSWoOJ2iKFrN6Gjva1uW0d7G4au3eTziRScmZw9mhPHmUmIAVTcDjGwuQe7P/Zu9HR1QAD0dHVg98fe7euYrDozJ6dnoVh0ZlpV3nmbPYbZ49Ow1w8OAqOj5QQdkfLr6ChNS8QaoZmTcWDmZDr03/0UJj3s3z1dHTiw8yo7X9Jo4wbKs0cXFVGYrHk6F1I4rGZOEndJxZmZp9ljmD3elr3ehSgbF2QgmUHFnWNSc2aahMJlSZg93oa93oWCU2EyUKk3j+PXkIo7x8RxZuaSqA9RmD3ehr3ehSibIBlcGFjyTh6uoapa/3fFFVcoSYd9z7+qv7b7Se373Uf013Y/qfuef7W5Az7wgOqGDaoi5dcHHrAhZnPydHaqlh+h8r/OTm+5wvY1OZYfIvWfr/4Taf48o173IBk2bPB+b8OG5uQrEhldQwAHNaKOpeImi9hQbLYxfYjCFGCzA1MSD7XpdQ+SIamBpUhkdA2puEk8klJKzShK1xRREoNbnMHJTwbOuJsnBzNu2rgdIvPKgJaSbRbP4xH0H5hDqbMvvq3QtTjyJKJsTK97kAxM/mmeHFxDxnE7QmNlQKDsaAxKwLGOhU4vnucxdxa7H/syBg7vNz5eIWKvbXfYGRuLXsmQeJPBNTSJ46bidoRUkmkqlMYnMfL4EUxNz6K7qwND124qDw4WlKTvebxxHAfuu6X8h0g5tDAqra6ImrjuvveS5A4Txc3qgI6QVmXAwJrfpnWnDeSdWrVu8Q9TM8fgYGsp6kZiXnfWby8utHE7QlrJNKE1v5tMtvE9j9Mny/9xzFboDDGuu7P120niUHE7QlrJNEnP7D3P49wvMPT0V91Ol08RW05oJ+u3k1SgqcQRqkvbpO2V3V0dnjZoWzN77/O4HANf+piV4+cdm+aNpO8lcRfOuB1iYHMPDuy8Cq/cfT0O7LzK/0Fuoo5CGjN73/NwvP5DGtg0bxSm5AFZAhW3RVKJw26yjoJpzW9X5LYmQ8YDh03zRmb3kmQOwwEtkVoctu2YX1uEhexlLbcj8eCxwj5dC4d0TZ4WgfW4MyA1D7+LrcSizKazlrvJqn62VlPG5o0sVypeKxQXVk4k2oxbRI4C+DmAeQDnwkaFIs64N+58FF5XUgC8cvf19r4o65mrF1FkylruZcvKiqaRgGSganLL5PQsBKi7v82spoySZrK6bn4rlI4O4NSp9OUpAEkl4HxIVU/GlKnlSc3Dv2uX9wOVZWx0lNl0xnKXfv1jGLnsekytWofu0ycxtH9vOQXfJxmo0fTVqPKrq6k4intgc0/0z2W1UvFboTRuS0seUgdNJZZI1cPfUTMYrF2bfWx0lEJQcYszGTgU/cwZpfFJDPd/FpOrL4TKMkyuvhDD23ag9J4P+w4cXqavRhKPlx4bA5YtQ+mSrei/dQ823vEw+m/dg9IlWyNlnzZl3jFVxK42j25Ros64FcATIqIA/kJVRxOUKZekEofttXyddSDZIups2jR1vfF8q/bU6rFqCIqPHnn8CGZV6vafbV+Bkd+8DQODN3h+dRSlnGi8dOXcS//q1zG8bQdm21cAwMKgg43zGAj4eNPx4r293iaatWvLvzmXVnwFJKqNu1tVp0TkQgDfBbBDVZ9u2Gc7gO0A0Nvbe8WE100nzZG1nTiIJCINDM43KFpjanrW2P/gd7wqiVdurJx7/617MLn6wiVvhxUfa6po2dgYcPvtS23Z1SgcgFElCWDdxq2qU5XX4yKyD8D7ATzdsM8ogFGg7Jw0kphEI+vIjCCSKARlcL5B8dFx/A9D125aEt5ZdVD2pFGFr3KOdcW5aghbEcSOF/da1QHlmfY99yzeYyrqTAm1cYvIShF5Z/X/AK4B8OOkBSMe+NkRL7ggncSSGntz6cqPo/+/PJxsspFBE4WgIl1x/A9eyS1/8onLcTQsq9UP0+SfyjkuFOdqIMxME7tomZdTEgDe8Q4qa4eI4py8CMAzIvICgB8AeFRVv5OsWMQTr84c550HnD6dfFxtTfxu6ZevxPD7PoXJuTYoFu2nsZR3VaGJAMuXl1+ris2gE0mQco6bYRi5BEGUczSNfa6c+9D+veiYO+t5XkHEdpa7vKojCzBzMm802pLPnDGOq41VfL/G3hzX7up5Ll7LciCWPdXZpgJxfROVe11auREjV30OUysvQHdXZ+TzavY+e8pLe3ZisAOOQySuTAwTS2Kn5td8z8Y7HobK0sWacbJRkJIA3HC62iBG8k9mBA2mQOu1jXMIprw7QlVJTlaiGpoyKfhh2Ew3LDXfN/a35nhx7a5LCFt+t8ry3LWGx0HUxtt7YVAmgCQHFXeCpFK/xLAjtV9UweT0LP5z6UX/gabme6LaXUMTQMIUV7OKLcwhmFa1wBx0Da+j2o1HxPv9VhlQcwwVd4Kk0qHEMCMxaFY89uwx/4Gm5nsGXn4au3/4dfS0z/s6+yKtNrwUWpVmFVuYQzCiw9BKcam4WaNZk6eVQsGgjTtB0uzcHhUvG3cYcQplRT73qrN1YgJoawPm5+04wcIcghEchqmV6nUVR0rh5gILCWjFsnE7UBzfDxc7lFRD40yIk9odebVRXZarAufOlV9rKwrGva9hYW0Rwt6K0ozXd1WR15VC2mRQ6jbfitvx2sCudigZ2NyDHh9l3GjVjDvQNNW13sZ9DVvmRzADFKEZb6hJK0b3+cLRZK33OORbcWdwwUyxlsRhiersqlpjupaO9jYMfqDXykDT1GrDxn0NcwhGcBg2NfjkhKKsKhIlg6SlfHd5Z5aXEV41ppOqv9FUtUQb97U6M/SzO4a9D+96JVmbumxThFVF4vhVUkzQiZtv52TG1fKczdTzwc9h2CaCt1XR3dWBD/3yenzv5RPZnpNDVRDzdo9NcdGBnjssOXGL45zMMD42leQay/jNouZVF87hgWePZX9ODsU9u2bqsk3TDnTLwQG2enumSgZO3Hwr7gy93n62wS9881DyP7iYD0sc22wS9s7Qh5PRDKnRlAPdcnBAHidDC6TsxM23qSRD/JoDV0ks3reJZVmcGG7AbsPjwsdGB5FEM4oY1JqHujrboQq8MTu31FRkYNKKYnIqutmmOKaSDAmbvSbmmW8i4qJxdtXml9LcgM0oCkYx+OBIaGvjrPf1mTlMz855z4AjOJFL45PY/IdP4AvfPBQ6k6ajNDpU3DHxsg02ksgPrsmIi1qb7R/f+J7Qc7AdRcGH0we/Afn221MVI6xJct0gGxILXx0EXp+ZCz5OhSKEX9qCijsmtbNXPyInm8TojBJ5ewBe9s2bLMVx+2H14XQ4a9YYv4H31KlUzyvKALqwT4gTOWwQaPwuFzONXSXfcdwZM7C5BwObe3zttqE/OIMu5gtE7ahueA5pYRIbHWgXjXPtXMYvFhgoz8ZTOie//pyN+wAIjYUPGwQaB+umYv8LBp2TlrDabSRiZ5SsnVhxiXKtQp2YDsV6W2FsDLjpJu/3Umy4EObANnEk+zkbTY9TFNgBJy/E7YySVEU9hwh66Hu6OjD0tf+KgZ98f+mbLnaVicq6dcZt6JIgclRJyDHu+ruXPO3bXR3t+IMbLqPSbsBEcdNUkiVxUmUbTQTzlZlR2qYCC7P+oJl30DJ7cnoWw9t2AKoYOLy//s0814q+5x6rZrC4NGs+85u1U2Hbg87JLImTIegVfVAlrQJbMUPXahNvNv/hExj6mxd8Q8RCwy2Xn4+RD362fqMlJZdZ9p6txKOMnbZ+TsmV5y+n0rYETSVZYzpz9TOvVEnDVBDDvhw1+WdNZzvGf++aSPsLFK984zartv60EoQSq4HiQPMDv+Q0m4lcrQhNJXlicNDsgQqKPqi+nzQxYsnDQsOqvD4zh9L4ZF2EgZ+tu7urs65bzcjjRzC181Gs7miHCDA9Y2aX9ZOzGnNsS3E3Dg7V1QaA5r8jKEEr48gUxmPbg6aSvJFkn8aoxIglN0mwqSZmVJOF/vQTlwfG9zZm+03PzuH1GZ9svxDSSBBKNHvUgVLHjMdOnsiKW0TaRGRcRB5JUiASQq0dFChHlVRfqzOrpG2aXoOHCHDddb4fMZltNSrJsEJIkbL99jwZyeabRvZeooODAw1+Xe381EqYmEpuB3AYwKqEZCE1BNpAa80rlhJRjGyug4PAgQPAffct2ttVgb17gf5+z+/1Srzxw0tJBkU6RMr261xT70itnkcEOW3PFkNNCc1E7FhO0KpiapNPO7GraESacYvIxQCuB/CVZMUhgGF5SwttvmKV0/z2t5c6SQO+d2BzD3ZfdBo9b56C6NvoefMUblpz1sqSOspsuPv0yehyJjxbDDQlxIjYqYuC+em7UPri/VZL4ua63GqLEimqRES+BWA3gHcC+E+q+lGPfbYD2A4Avb29V0wEOdBIIEblLeMm8cT9vrjf6xPtUPri/Rg5s66p6IrQbL+5s9j92JfrY74zTtTxncEaRuw0HQUTIZmr6OVW08JqVImIfBTAcVV9TkQ+6Lefqo4CGAXK4YARZW1NmkxOMbKBWuh3F8vmavq9PiuDgS/dgYEmswIba1zURZW8eQpDT+5xLlHH15Rg6FxsKgomYjIXKzq6RxRTST+AG0TkKIBvALhKRB5IVKo8Y6GuspGDzEKbr1gOOdPvTTjaobZc7aHfvwbjv3dNud3YB9owMPHD6HJWySqJxdC5GEWp+iYURUzmYrlV9whV3Ko6rKoXq2ofgE8CeEpVfarhNEGrlOi0YHM2CqeKm21Xc72Hvv0/0SH1i6RQW7Pp99qOdoj6e4lzfbJsamA4IPoq1TeOA+vWoXTvg/726bBBs/I+w/vcw404btMHxWUlb2FmaewgM+1313C9B/7X32L3Y3+GnvZ5M4ecyffabABs+nuJIGfdrPTZeZQ2vG/xvUu2ov/me7HxR6uTT4H3G2gAz9+8p1KdO4uh/XuBU6cw8pM3/WPGwwbNyvsM73MPN1LeTRwyDqT0BtJkudFmU6Ejfd5SSVRjWW2Vo7Vc0tXTwVdxaALA8LYdmG1fsfiezRT4KNck5DdfGp/EyJ4nMdW5Bt2nT2Jo/94Fm/7GOx6GytL5mQB45d3TS4/rcXySDvkr62oSoeB6HeYYA0tVAU5Oz0KAujoPJkoicoSBhUgUr+8SAIMf6MV/G3h3pGPExoL8tfhGTbxxHAAwufrCpe/ZiKiI+luJ8pv3uSb9t+4Jlr8AJYLzQv6aBZvYPx1I6Q3E0KZaGyMLYElxHpNU6Mip1BbszV7fpQAeePYYLr/riWQr60WQ36TCn6+Db9U6TK1aZ/QZI6L6Q6L85n2uydD+veg494u6bXX26aoZSRU4d678GsXcRjLFDcVtYv90IKU3FAPbb5TiS1GVROSwLQv25iCZfLuC2yJEftOEEV8H38zr9Yk7ET4TRt2Asu33Ubpk69KdGhV1lN/8rl3Aeect2WXg//5v7O47l1/7tMv+rAxxQ3GbzFJtOrkcIIpSjqokIodtWaj7HFUma8WTagmR37SIk2/UxC1XY2j7NdYiKpYMKKsvxPC2HUuV9wUX1Cur664L/80PDgJ79gBr1y5uW7sWuP9+DNx240Ko5IGdV+VLaWcV3eM4bti4Tcl5z8Vaglp0AQnZuC1QGp/Ef/zmIc+6y42kXYc5Tj3oIEerrdrZQbb0A/fdUv6jvb08GL311uIOnZ3AZz5TLjNg6TefWD1wm7juz7JM69fjNq1h7TBeRY2qDsoewwfKZpfssAd7YHMPDk68hrFnj4Uq77QTNeLUgw4qihSnYJLX9fO3pa8vK+veXuDMmaV9J2dmykrbJ+Xd9H4nWg/cJq77szIknzPuFsPr4QPsKOC48kSduTc2lj1z9hzm3tbQz7kif5xjx+1Qf/7yZZienVtyzLoIFYOImbjnmZvaI5xx+5LPGXeL0Tijy3pGZFL/wkv2UMV274MYOXK2HHc88zqGNq3AwG03WpPf5sqjlqj3xe/6rWhfho72tuCSsQY1YOLWKbFZeyRRk0tCJWpbASpuB0mjfVYQzTzYYWaF0r0PYviVNsyuLDvRJleuxfArZ4F7H7SuvG1fq6j3xe86Tc/M4U8+cXmwojNQVv73aaY8W/Wxh9tqLZb4BKMqc4v4s2xCxe0gWVdjS7Jn4MiRswtKu8ps+wqMHDmFgaaPnixR70vQ9QsdUAyUle/3nD65OGv3aBxhq1lEKhOMAH9WLhysCeFGOCCpw7gam+VYV6tFhRpkm+pc47nbVOca5+N0o96Xpq9fxDwAz+859wsMff+v6ndsSOpptvZINRbdLxoqjQmGM80dMoozz6/ibuHAfKMHP4FYV2tFhTxk801oqc4SHY7TjXpf0irK5Pk9jQ0jqjREYtSWwTWJ7W7M9PUijSiiRBsuRyXDOPN8RpW4XmjKApGXgS573j1kK12ydWnRpsYONS7I7kPYfcl8+Z7w78Fm3kEzxInVt47la52/IlOmuKys0sZywSWr+MhWumQrRq6+xbOa3QIbNuTOIRUWnpeKUo8wqWlGDj+FCVX0nD6BoZcexcDn/23i98uJkEbLz17rhwMyMH+RGK3LrCqQoCxWH9kGZo5i4Ms3+w7ApUs/iJErb8bUqnVlxT6yt+y4dFx5hy3fUwnxDHFuNhsJ4ucQ7Tl9YjH787nH6mVJAFsO1qaw0DYwLvm0ceeh0FRaGNZuserUCbPxhcnm8X7p0g9i+CO3YXL1hVBZVq7ncfXnUfqLfebypUxQ1EmqNtkA52azcgQ2bqhi2PEpDk40d8iwblI+FXeLFZpqCsOCUcYPbpATOKwsaZhsHu+PXHlznf0bqIQLXpZerZO4BEWdZB3iGfZ9UeVYojDfOO7tEE1h9RvXwWoNC8Xa4pJPxZ3hBXON0vgk+n/6Lmz85P9A/x/9PUr7DgReB6MHN2xGHcVkFRba1vD+1Kr13vI1bncwqigo6sSVhrs25KhTmI/d5R3FUpTVr2nbQEvkU3EDiV4wkyL8WRLH7GH04IbNqBMwWXWf5+3UqdseMqCY3D+b9zpo+e5Kw13rcnD1mwn5jCpJkDRLozZLHM+60fk1eM1Ll2zFyNbPlJ2Ga1Zi6B0nMfC7n7Mallkan8Twg+OYVVmUTxS7b9y8KF9AVFFp3wGjAllp3uvMQwWTkqOFyixnSeuHAyZEaXwSv/PgC5j3uCbOVU5D/FjWODHinvHX7W3YfdFpDHzpjvJDe8EF5Tdee62pBzhUvoAwrP4/+vvIg5kTIWWEVGj9cMAEqM6+vJQ2YOhESmkGEremSOQCTDUFj0a2fmap03BuHiNn1mHg6NGl8cMeNTKiEipfQBiWiQ3fFYchIabk18ZtmbDej5GdNymmwSZuN61xAoc2zY3a+NYGAXZVExt+Zg5DBx2rJF+EKm4RWSEiPxCRF0TkJRG5Kw3B0iZolmWkDFNUYKnEslacwN1rVnq+vaDk0kyKCogqMhnMMnEYJjWwczAoFKE2bhERACtV9YyItAN4BsDtqvqs32fyaOP2s3e2ieCPb3xPdGXocgp6TErjk7jr717C6zP13VvqHHkOlSEwcb6l7jBM4joVoHZPEUjMOSkinSgr7n+vqv/gt18eFbe1CAOHFJgNvK4LAHR1tOMPbrhs8dpkqDxcidaIRBIDe4v95oqKieKOZOMWkTYROQTgOIDvBintvGLN7NBica1+tv+V5y+vvzYZJUXZSuH3jee2bYJIolwDa/cUDtMZdxeAfQB2qOqPG97bDmA7APT29l4x4TUDKAotFNcaJeQwtap3HtfURkif72rrotPW49SrK5PShvctxsT//BSGLu2M37qNM+6WwPqMu4qqTgP4PoCPeLw3qqpbVHXL+vXeacuFIW5Wp4MOprDIi1Q6kXg59D79aeC3fstKSJ9v/ZYjZ+07mgcHUfri/Ri+7j8sFtJatR7DP1sV/5q12CqPhBMlqmR9ZaYNEekA8BsAXk5asMKRYTeNIMIiL1KpeucVqaMK3Hcfutu9QzhNQvp8lb9PmzVMTDQ1uI6cWYfZ5efXbWvqmrF2T+GIMuN+F4DviciPAPwQZRv3I8mKVUDSjIM2IMz2HzrjtbGK8LPVqmJo/1ebDunzXVXMvO7/oSYG10QSfzIqdkSyITRzUlV/BGBzCrIUG4cdTEGZjIHZm7ayKf0yJQEMPPMQcM89TdnYfYvyb1pRNjk0Dqi1VAdXg/OJm/FKSBVmTlbIvCJgTptDBJpSLKwiSuOT6P/0n2HjHQ+j/9Y9KF2ytX6H3t6m6zL7ripuu7HeBOHHsWNGKwvjxB8HfR8kW1hkCo5UBMxxEoVvVIlfzDLgv73huEvuS21j4bSvj1/0xtq1wOys0b2rvWarO9ohAkzPzC1dMeT4d0HMYHVAQ5ypElcNeZuYANragPn58mwvr+GEfopOBPja10LPyfe+vHEcBx67K/3r4qdEOzqAU6eW7h8hHC900lCQUL9cJVElRGLhgK2KM1XiBgcXQ7vmKw+yI9Elsdi1y9vEoBrJXOJ7X1atz2Yw84veeO017/2D/BMV88fI6BPBUTkO+z5skUpIaYtBxQ2LVeJs2CIdjS6JxeCgv0kkguLxvS+nT2Q3mHlFb5j6J2pCP0OrLubU92FCqo2UWwQqbliqEmcrDrtVZljVQcyPCIonsKO4S4OZaQJMzeDcffqk5y4Lg1YBkmucWfHmCCpuWKpTYmum3AozrNpBzIuIimfhvrxxHKJvL+0o7spgZpoAUyP30P696Jg7W/d23aShAMk1rjRSzhN0TtrCVtW3Vogi6OtDqbNvsRbH6ZMY2r+3rHDjOFtbzUHXcD6LvTzXo3tNZ+Ecc05EdTkAnZNZYGum3AIzrNLKjRjetmOxFsfqCzG8bQdKl34wXlZfq5kLvM5HpFy5q4Ck0hCkxeCM2xatMFO2RP+Or2Jy5dol23vePIUDX7453kFbqOIigIXzqQ5ytbVLijjbJIzjzo5WUy4x2bjzEajH9FGgeOXujwJg3G4VZ3IISOawy3tWDA4WUlE30t3V6VOLo2weaLRpVuN2ASxR3q2u4BlRQeJAGzexjq1SsEVIzGBEBYkDFTexTtOlYCsUITEjk07zJPfQVEISwa8UbGl8EstEMO/hW2mcZRbBjFC9Rq1sDiL2oeImqVE1fXgpba9ZZlHqVgfVOyfEC5pKWoUc1Gz26xjfJuIZ/kYzAiHecMbdCtjqNJMwfiaOt1U9Z5w0IxDiDRV3KxBUJ8UhxR3H9EEzAiFLoamkFchJRUGaPgixA2fcrYBfM13HKgo2a/pwORnHZdlI60HF3QTOPKy7dnnXSYlYhCnN84hr+jDawt7qAAAGlklEQVTJtkwbl2UjrQlNJTFxKquviYqCTp1HAC4n47gsG2lNqLhj4tzD6tVSKwLOnYcPLifjuCwbaU1CFbeI/EsR+Z6IHBaRl0Tk9jQEc51WeVjzch4u1/RwWTbSmkSZcZ8D8DuqegmADwD4bRG5NFmx3CfvD2tpfBL9dz8Fv6K+rp2HyxEpLstGWpNQxa2q/09Vn6/8/+cADgMovMclzw9rrV3bi8zPwyML1OUuKS7LRloTo0YKItIH4GkAv6Kqp/32K0ojBWeiSgzxK94PlJVOpudhoZNQ7u4LG3AQJNQBR0TeAWA/gF2q+pDH+9sBbAeA3t7eKyb8OnyTzNm481FPE4kAeOXu69MWp54mGwPnrvEsW96RCtabBYtIO4C/BTDmpbQBQFVHVXWLqm5Zv359dGlJ6jhtn4+TBVpjWhnZ82QuomQWCCpXQIgPUaJKBMBfAjisqv89eZFI0jhtn/fL9vTbXp2xTkwAqpjqXOO5m2tRMgvkpFwBcYsoM+5+AJ8GcJWIHKr8uy5huUiCOO1M27WrbCqoJSgLtGHG2n36pOduTqwmvDAdqAhBhJR3VX0G8GjZTXKNs1X3qnbdqM66hpnp0P69GN62A7PtKxa2ObOa8KLJcgWkmLBWCXGPwcHojrmGAlsDh/cDAEauvgVTK9e6H1ViOlARAsNwwKgUJRyQOACjMkiLYD2qhBBnaaLAFiF5haYSkn9MTCuEtACccRNCSM6g4iaEkJxBxU0IITmDipsQQnIGFTchhOQMKm5CCMkZVNyEEJIzqLgJISRnUHETQkjOoOImhJCcQcVNCCE5g4qbEEJyBhU3IYTkDCpuQgjJGVTchBCSM6i4CSEkZ1BxE0JIzqDiJoSQnEHFTQghOSNUcYvIHhE5LiI/TkMgQgghwUSZcf8VgI8kLAchhJCIhCpuVX0awGspyEIIISQCtHETQkjOsKa4RWS7iBwUkYMnTpywdVhCCCENWFPcqjqqqltUdcv69ettHZYQQkgDNJUQQkjOiBIO+HUA/wfAJhF5VUT+XfJiERKDsTGgrw9Ytqz8OjaWtUSEJMLysB1U9VNpCEJIU4yNAdu3AzMz5b8nJsp/A8DgYHZyEZIANJWQ1uDOOxeVdpWZmfJ2QloMKm7SGhw7ZradkBxDxU1ag95es+2E5BgqbtIa7NoFdHbWb+vsLG8npMWg4iatweAgMDoKbNgAiJRfR0fpmCQtSWhUCSG5YXCQipoUAs64CSEkZ1BxE0JIzqDiJoSQnEHFTQghOYOKmxBCcoaoqv2DipwAMGH9wPZYB+Bk1kJkQBHPu4jnDPC888gGVY1UEzsRxe06InJQVbdkLUfaFPG8i3jOAM87azmShqYSQgjJGVTchBCSM4qquEezFiAjinjeRTxngOfd0hTSxk0IIXmmqDNuQgjJLYVS3CKyQkR+ICIviMhLInJX1jKlhYi0ici4iDyStSxpISJHReRFETkkIgezlictRKRLRL4lIi+LyGER+dWsZUoSEdlUucfVf6dF5AtZy5UkRasO+AsAV6nqGRFpB/CMiDymqs9mLVgK3A7gMIBVWQuSMh9S1bzG9cblHgDfUdWPi8h5ADrDPpBnVPUIgMuB8gQFwCSAfZkKlTCFmnFrmTOVP9sr/1reyC8iFwO4HsBXspaFJIuIrAJwJYC/BABVfUtVp7OVKlWuBvCPqupyAmDTFEpxAwsmg0MAjgP4rqr+Q9YypcCfArgDwNtZC5IyCuAJEXlORLZnLUxK/BKAEwDur5jGviIiK7MWKkU+CeDrWQuRNIVT3Ko6r6qXA7gYwPtF5FeylilJROSjAI6r6nNZy5IB/ar6XgDbAPy2iFyZtUApsBzAewH8uapuBvAmgJ3ZipQOFbPQDQD+JmtZkqZwirtKZfn4fQAfyViUpOkHcIOIHAXwDQBXicgD2YqUDqo6VXk9jrLN8/3ZSpQKrwJ4tWYl+S2UFXkR2AbgeVX9WdaCJE2hFLeIrBeRrsr/OwD8BoCXs5UqWVR1WFUvVtU+lJeRT6nqTRmLlTgislJE3ln9P4BrAPw4W6mSR1X/GcBPRWRTZdPVAH6SoUhp8ikUwEwCFC+q5F0A9lY8z8sAPKiqhQmPKxgXAdgnIkD5d/7XqvqdbEVKjR0Axiqmg38C8LmM5UkcEekE8GEAn89aljRg5iQhhOSMQplKCCGkFaDiJoSQnEHFTQghOYOKmxBCcgYVNyGE5AwqbkIIyRlU3IQQkjOouAkhJGf8f42HZw3skbGhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x18cd181cd30>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(kears_sample[:, 0], kears_sample[:, 1], 'ro')\n",
    "plt.plot(np_sample[:, 0], np_sample[:, 1], 'o')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. 恭喜你，完成學習所有建立 Variational Autoencoder 所需的重要技巧。"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Variational Autoencoder (VAE) 是一個重要的非監督式學習模型，具體應用的場合為特徵抽取/資料壓縮及還原，為影像處理中常見的模型之一。\n",
    "\n",
    "在建立 VAE中，需要的兩個重要技巧:\n",
    "* 分歧-合併\n",
    "* 自定義函數 (抽樣函數)\n",
    "\n",
    "已經完成了學習，下一節，我們將告訴各位如何建立並訓練 VAE 模型。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}