tutorials/02.  Intermediate/04. CVAE.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Original Source: https://www.tensorflow.org/tutorials/generative/cvae"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import os\n",
    "import time\n",
    "import numpy as np\n",
    "import glob\n",
    "import matplotlib.pyplot as plt\n",
    "import PIL\n",
    "import imageio\n",
    "\n",
    "from IPython import display\n",
    "\n",
    "(train_images, _), (test_images, _) = tf.keras.datasets.mnist.load_data()\n",
    "\n",
    "train_images = train_images.reshape(train_images.shape[0], 28, 28, 1).astype('float32')\n",
    "test_images = test_images.reshape(test_images.shape[0], 28, 28, 1).astype('float32')\n",
    "\n",
    "# Normalizing the images to the range of [0., 1.]\n",
    "train_images /= 255.\n",
    "test_images /= 255.\n",
    "\n",
    "# Binarization\n",
    "train_images[train_images >= .5] = 1.\n",
    "train_images[train_images < .5] = 0.\n",
    "test_images[test_images >= .5] = 1.\n",
    "test_images[test_images < .5] = 0.\n",
    "\n",
    "TRAIN_BUF = 60000\n",
    "BATCH_SIZE = 100\n",
    "\n",
    "TEST_BUF = 10000\n",
    "\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices(train_images).shuffle(TRAIN_BUF).batch(BATCH_SIZE)\n",
    "test_dataset = tf.data.Dataset.from_tensor_slices(test_images).shuffle(TEST_BUF).batch(BATCH_SIZE)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 100, Test set ELBO: -78.32476806640625, time elapse for current epoch 2.797369956970215\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAOwAAADnCAYAAAAdFLrXAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO2dZ5xV1dWHnzszDGVE+oAgHUEEQQRfVNSogIAoYoNYExEUseUXJbEEKxosUYmiISrGKD8VQiJFsYCRIk0RkSKgIEWadKQNDHPfDyf/de4Uhim3nWE/Xwbunbl373PO3qvutULhcBiHwxEMUhI9AIfDUXTcgnU4AoRbsA5HgHAL1uEIEG7BOhwBIq2wN0OhUKBdyOFwOFSU3ztW5gnHzlyjMc9QKKTvLO1HFZsjzdNJWIcjQBQqYR2OY5lkzFFwEtbhCBBuwTocAcItWIcjQLgF63AEiIQ6nU477TQAxo8fT/Xq1QGYPHkyAHfeeScAmzdvTszgYkC5cuVIT08HYN++fUByOjYcyYuTsA5HgAgVtsPHIsiemprKF198AUDHjh3zva/x7N27F4CBAwcyduxYAA4ePFis70pk4kR6ejrdu3cH4JFHHgGgRYsWlC9fHoAtW7YA0LhxYwAOHDhQ4u9KdOJEhw4dWLlyJQA7duwo8ecoUSElxZMj5cqVAyA7O5vs7GzAJcPEbcHqZsybN48OHTrkei87O9sWqH7q9w8cOMArr7wCwHPPPQfA4cOHi/SdiciKadmyJQATJkywxagHsCC2bdsGQJ06deyhLC6JWrDafHbs2GGLS6bMqFGjirTBVqpUCYBLLrmEa6+9FoAqVaoAMGfOHACGDx9uplFOTk7SLdiSZkTp7/QTICcnR5/lMp0cjqATNwn7z3/+E4AbbrjBdqKNGzcC0L9/f5YvXw74O8wFF1wAeOqkHDSdOnUCYPv27UX6znhK2OOPPx6ADz74APDGGrlz/m88+V4Tixcv5tRTTy3RdydKwkrVf/jhh/O9t2vXLmbPng3ADz/8AMDPP/8MeM5GaR3nn38+4F0/XZv9+/cD8NlnnwHQt2/fSCddzO9pcSRmKBQiLc3z3R46dKhIn6+5S0OpWLEiALt37z6q6u8krMMRIGIuYc8++2wAczQBzJo1C4ArrrgCgK1bt9pupt2tfv36AMyePZsKFSoAfhhozZo1RfrueEpY7ZYPPPAAAIMGDbJxS2Js3ryZ4447DoAGDRoAue1baRC6PkUl3hJWY5Y0iNQadB8PHDhgjrVNmzYBvk06Z84c06Tuv/9+AJo3b84vv/wCwIsvvgjAyy+/DOTWqGJ9T1NTU82ullQvzGdSo0YNu6eFPZe6RqmpqaSmpgK+/S77f8eOHSaljzTPmMVhNcBJkyblev3nn3+mW7duAOzZsyff32kyV111FQCZmZns3LkT8D2ryUhWVhYAL730EgA//fSTLdgFCxbYayeeeCIA//rXvwCoXbu2fYZeq1u3bnwGXUI2bNgAFOwsmTBhAuCZQAsXLgT8BatrlJOTY9dGC7xr1668++67AMyYMQMouooZTUKhEL///e8B+PbbbwGYOnVqvvHIoThx4kSmT58OeKYd5F7geT3fKSkp+VR/fa6uYWE4ldjhCBAxk7DKXKpatWqu16+99toCJato2LAhAE899RTg7Ujz588H/B0pmdm9ezfgqfJSlX788UfAiyPrtYJ208zMTMDfjYuy48YTxZUjtQKhkNsTTzwBeOE5SZqC5iFp+9FHHwHw+eefm+pb0vBWNKhcuTJ//OMfAX+MDz30EOA5BqW2X3rppYB3z5RPcCSHIlDgtdDvK/zlJKzDUcaImdNJzibp97J7mjRpUuAOKsmjTBm5yiPtneLaNIlInKhVqxYAp59+ukkiSdiMjAxLDrj66qsB31n1v/ECWL5xUSVNPJxOFStWNCdMXhYsWMB5550H+FpQOBzOFxaJdCzK0RIZEokIaRxxHLG+p6tXrzYtT9lbnTt3BjxbXGOTn2HKlCkWllFYTs/wkeZxJEkc+fsurONwlAFiZsNqB1VyxL333gsU7CJPS0szD7B2XNGwYcOEeAuLinZL2epK+DjnnHM466yzcv1OxYoVqVat2hE/qzCbL1HofiiFMhLlP99zzz02x5o1awKeF7Vy5cq5PkOSZ+3atSat9TPR91h2aMOGDc2mvPDCCwHPuw/efdE89Vx/9913nHHGGYCvXe3atQvw7mdBSRh5Q5gFhcWORMwWrL5YLnH9jETOlXnz5pnaK84991zAv1jJyimnnAL4IRmFbSpVqpRP9Tl8+LAtSoWvIpE6mUwLVhut1D7w7+2UKVMAzwmlUMjdd98N5M5c0u/LYTNs2DBGjBgBFP9AR7TRM/j555/bawovaVEWtIgkkGrUqGHOKW1Qeq9cuXK5TDvwzBx9Z97PLcq1cCqxwxEgYiZhperoQLp9YVqaZXgMGTIEgHbt2tn7klQzZ86M1dCiRqNGjSzIL5W4MNd+KBQyCZtX9QffMZcMaB49e/YEcqt3q1evBnynzF133WX3UE60gpIH5Fhs2bJlqY4TRpMuXboA5ErkGDduHJA/6SEcDptmJKdhZP53jx49AN9ZeNxxx9m/db83bdpkTkWFsRQKDIVCR1WJnYR1OAJETCRsKBSyHXTu3LmAr683bNiQwYMHA9CvXz/7G0nkm2++ORZDiiqSIvfff7+d0ilIskrK6CznmjVrzOnUokWLfL8/bdq0mIy3JEgKTJw4EfBs7q1bt+Z67YQTTgA8203vfffddwCMGDHC7Lbnn38e8DQS8KV2MtCnTx/Afz7nz59vOc9CNunhw4dNk3rssccATzLr2ZUNqv9nZmZaApHue6VKlfKF77RWiqJ1xGTBpqWlmVdRHjNNunr16hajlXoRDoft5irPWDmpMuiTAalGelC3b99uG5LU2RdeeAGA77//3jK6tJjbtWvHhx9+mOu1SB5//PEYjr54aHxyFP3444+28SxduhTwj82NHz/eHDTyBGdnZ9tntG7dGoChQ4cC3gH1wg71x4vIvF4VTnjttdfsmdN7kY7Ca665BvCfgcOHD/Pll18CmCqt3OktW7bYZ2RkZADeotbijUj0t/EczeGY+KvmcDiKTEwkbLly5WzH0i4io7558+amRmrnOnTokBnlo0aNArBdq3v37gl3/QuFNk466SQARo8ebXmmhcURlSM8adIkc/3nZe3atQXGOhOF1LWuXbsCXuaanE3jx48H4JtvvgE8LaogySApqrIxkc6bRMddNQ45ORUzz87ONgkoNV+Oo2rVqjFgwADAV3+nTZtmztP169cDvlYY6UCS5hEKhSznIG/Yy+USOxxljJhI2JycnHy7jEI5LVq0sPOtOm84ZswYmjdvDmAB+F/96leAV3JFu3wiSUlJMQeCpMPy5csLlRQqNjdv3jygYLtVh9UHDRqUFFJHRNqu4PkWpFmcfPLJACxatAg4smR48MEHAd/eE/v27UuKeszhcNgcfe+88w4AzZo1M+mp51PX4vrrr6devXoAdoLs2Weftec5ooBarp95v7M0OAnrcASImJzWqVSpkklU2bJnnnkm4O1S2tVUsGznzp3mRVbIQCck9u7da2Uvi5uyF82THRUrVuTGG28E/BTCMWPG5HPFyxafMmWKaQmRaA533HEHAG+//bZ9pu5FUcu4ilie1pHdOX78eNN0pAm88cYbgJe+mNfPUL9+fdauXVvgZ3bp0qXAVNWiEKvTOvLidu7cmVatWgG+t1fvXXTRRRbZuP322wHPvs2rcajYXGmkaVxLxKSmppoRLxXioosuAjyDXKVDlOERDocttqlYndTHrKyspFCfqlSpwjnnnAP44agtW7ZY3LFv374APPnkk0DBtYg3btxoucZ5N59QKGR/k8jO33nROHv16mUL9IYbbgDgtttuA2DFihW28ej+ff311/k+S4u6pIs1lkiwfPDBB2amtG3bFoDevXsDXjz99ddfB/xFWatWLRMuck7JqRSL++dUYocjQMREJQ6FQiZRVbVfjocdO3bwzDPPAL76C36lPP2dpMzf//53Bg4cWJJhRFV9qlChggXNf/Ob3wDQtGlTO6Qulb4glG+6ePHiwsaQ72RHtDsc/O97Sr3t6wTPfffdB3jSSeqvjqkVdD0UKiqNcy0RrTqk/VWuXDlfVlOlSpVMm8xbdK40uAPsDkcZIGYlYrSbysEU2fhKkkMOm8g2jEJB+Y4dO5Z4R472biznw6OPPgp4joe853jF1q1bLWGiqLZMQZ0CikKiKv/LGfjkk09ajek6derk+z056956661Sf2cim2Ed6SSWXo/mOeaENcOSB03qQt6FKWSwq67vPffcA5TuIkT75sopJO/v5MmTTV3SdVRSuNpYxINEd6+rUqUKq1atAvxqmeAfClcVjmhwrHevcyqxwxEg4tYMS9KpcuXKljGj7/7uu+9yVduLFrHajZVTPGTIEFOT1aJD4YF4kmgJGwqFrOCATmJt377dahzFQ1UsYExOwjocjsQS9w7s8STWu3FRSnrEg0RLWIBbb70V8HPBe/bsaedlo4mTsA6HIzA4CcuxM0+I3VyVrqnzvrt3745JudZj/Z66BcuxM084duZaVufpVGKHI0AUKmEdDkdy4SSswxEg3IJ1OAKEW7AOR4BwC9bhCBBuwTocAcItWIcjQLgF63AECLdgHY4AUWiZ07Ka3pWXY2WecOzMtazO00lYhyNAuAXrcAQIt2AdjgARk1YdjmMPFQ6P7KmkSphBQKVKCyplqgMyoVAoX1f2eOMkrMMRIJyEdZQYVcI86aSTGDt2LIA1+1q/fr01xPrzn/8MeE2zILpVFEuL5hDZvEz/ltbQsGFDwOttfNVVVwF+18GpU6fm69wX0/HG7ZscDkepSaiEjbQXyuJBerXx6NmzJwB33nknkydPBvzmX2quFESuvPJKwOterppOkp6hUMjqEletWhXw+wGPHj06IfWbC0M2aTgctrmo/rT6xd57772ccsopud7bsmULy5YtA/ya1LF8lhNa00kLtly5ctY/JwiFxItClSpVeOeddwC/OXVkR7cdO3YA0K9fP8BrmFxS4p04ofYkP/30EwA1a9a0Pkk33XQTANOnT7d7qvYdaupdoUIF5s6dC/hNkKPdqa8084x8LgErfD9ixAgaNGgAwMMPP6zxmHNNBeiWL18OwMqVK9m4cSMQvWbkTiV2OAJEQiWsVMbMzEzrxq6WHdqRatasaZ2/tRs/+uijRVI/4ilhtSvXrFkT8HbjLl265HovFApZ2COvCvn444/z+OOPa9zF+u54S1j1+tX9yMnJoX379gCsW7fuiH8nZ86JJ57IzTffDPiaxsiRI+3eF0Y876nGqy6EI0eONPX+9ttvBzzt4dJLLwXg17/+NeB38EtNTWX16tUAtGzZEoCsrKwifbeTsA5HGSAhTidJHNkDu3fvNhtGu5rspK+++oq6desCcOGFFwIwdOjQpHFSqWv66aefDsDrr78OeG01J0yYAMBf//pXwGsQdd555wHw1FNPAf5u/MgjjzB9+nTAb9OYrNSrVw+AzZs3AzBgwIBCJauQNrF27VpWrlwJwBNPPAHA999/bw65ZAn7aBx79uyx17Zu3QpgGuG6detYsmQJAD///DPgh7FCoRCNGzcG/PajDzzwQKme3birxKFQiE6dOgG+02LdunX5nA7qCrd7925bxEuXLgV8r93RiJX6pA3nuOOOo2/fvgA8++yzgK/mf/DBB9x9990AbNiwQeMxVbh3794AjBkzxj5TC1b9VIv64MZLJZZTZc6cOYAXawXo1q1bsR/CFi1a5Pqs4cOH8+STTwIUGtdMhCNRwuPBBx9k6tSpAHavIuet3sdauGp4DdgG1aJFiyI52JxK7HCUAeKuErdt25bWrVsDMGvWLKBgSaKMmcgMlDPOOCMOIzw62nHPP/98kwqSPprLN998w5YtW3K9Bn6n+R9//DHf56pbveacLKoheBrAl19+CUDTpk0BuPrqq4GSheIuueQSwNekevfuzV/+8heAmIT4SoPGsXjxYmbPnp3rtUikGUhdjpSwunalvadOwjocASJuElY76bvvvmv2XkG7lILV06ZNs9fktEl0VpBsV0nAunXrmk0qyandddWqVQXaKprftddem+892fTJhOY8a9YsszvlhFGGT3FJT0/nlltuAfxrmZGRYRJJ4Z1kOe2jrKZVq1aZ9C8IOSDllAM/IWTAgAFA6bUGJ2EdjgARNwn7pz/9CYBGjRqZLTp69GgAS2sDP9xRu3ZtwNtl5W1NNNodJelHjx5ttqjsWoWqFi5caH8nKZWens4VV1wBQJ8+fQB/B87JybEwUDLx5ptvAn5aIfjSorj2mJJG+vTpYwkmmv+hQ4fsVMy2bduA5JGwYufOnYVKyJdffhnI7Xfp2LEjkDs0VBritmA1mdatW9sD3KZNGwCWLFliquKdd96Z6+/0gCcje/bs4dNPP831muZx/PHHmyNKmTIDBw607CeFqJR32rhxY1Op9PPw4cMJc7w0adIEgOuvv95e27VrFwAffvjhUf8+FArZg9uoUSPAD2V1797dQkLKHFqwYIFlAWnz00aeaOeTMru6devGSy+9BOTOfVYo77e//S3gj/e6665j/vz5UR2LU4kdjgARNwmrTJgrrrjCVCNJo3A4bKGCvE6cSZMmxWuIJSLv7i818YILLuChhx4C/NMeoVCItWvXAp7zDfwc09atW3PxxRcDvpPt4MGDCZEuoVCITz75xP4tNGZJPknQtLQ0O5HTrFkzwAv56Fronh533HGAl+mkjCFlsc2YMYOdO3cCfr5toiWrxv2Pf/wD8DSFKVOmAL6GlJqaytChQwH/mR08eDCAndaKJk7COhwBIu6JE4cOHTJbSKSkpFhOrXZ0JR0EBUkb2X4jRowwx4rmlJWVxcSJEwH/pMtvfvMbwLPlJJGU4pYoCVOzZk1zAIl9+/YxbNgwwNciFO5o3Lix2aey1xs0aGAOuRkzZgD+dcjIyDBbXqeuZs6caWGtop5oiRUap8KPHTp0sPc0F93HChUqmGNp5syZgF+cIBYkRU2ncDjMrbfemuu1ESNG2HvJSnp6uiV333///YB/xEqOE8id9K74srK9dLPLlStnD7g2q0TN/aqrrjLHl8bw6aefsn37diB/hcSDBw+aCq08WvCdZ1rEl19+OeBlOakaxccffwzAxo0bLVMo0fdcjs7rrrsO8OPCu3fvNrVetZ3S0tJss9bcde+LclywuDiV2OEIEEkhYWvWrGkOF+2uTz/9dCKHVChSdT/99FOTlJImIhwOW9aTQj8jR45k06ZNgHfMCvzdOCcnp0inVeJBpDqssQwbNixfNUFpDuvXrzdnmsIdaWlpFrv929/+BvjmQk5Ojjlt/vCHPwDe0cPI+r8QX0mruV1zzTV2PE6Hz1UOZsmSJSZ1f//73wPeAXb9rVRnqcZdunQxR5oIh8Olmp+TsA5HgEiohNVO/dFHH9muI+dFYTmbiUJjVEmQVq1a2e4qyaJD3RMnTjQnTaRNqmQRfYbYs2eP7cyJZs2aNXb9VUt4xYoVFs7RXOUwisx4Uiikffv2FpKTrSuJMn/+fC666CLAz72OJJ6SVePt378/AM8884zZnkOGDAHgiy++ALznVVlu0ozC4XCuSpHgJwS99NJL5ouRRrF///584a7ITL+j4SSswxEgoiphi1JnOBQK2VlXub9btmzJPffcA5CU+bRCc3r//fcBuOyyy2yXVFKBvKW7du3KpyWUL1/eguzyNuozhw4dyi+//BLjGRSNefPmWeqgTuTs27fP5iMJEXmPFeKR72HQoEGmfSiPVpVCZO8mAzpFJjs1IyPDNAhJUZ1TvuOOO7jssssAf7579uxhwYIFgJ9mqus0cuRI86wrVHXo0CG7biXpz1PiEjGRR83kcIlsFJR3MJp8+/btee211wA/R/Oxxx6zw8vRJNblRDIyMjj11FMBX7XTg75nzx57sKX6X3fddVbzSddKMekGDRoUqB4WhWiXiKlUqZJtnFpkgwcPtiwfOVL07Jx88slWh0pH5LKzs+nVqxfgmTzRItr3VPnea9asAbx4uJ5dVXTU8628Z/AXZf/+/S2jKZqHFVyJGIejDFBsCSsVULvN7373OwsDKPC/f/9+KzwmlUPVAlu2bGm7mbJ88rq+o0WsJWxKSgpnn3024JWLAUwKLVu2zCSrwjU33XSTXT/t0M2bNwf8EEJJiLaEDYVCdOvWDYBRo0YBnoak+7Rq1SrAzxtu0KCBqb9KhOjZs2dMWjJG+55K09EpnIEDB+Y6HpcXhbl0faZNmxYTJ5mTsA5HGaDYElZn/7p27Qp4B9PltpfUqFChgrm/dSpDNs4bb7xhKVyxdt/HoySmUux0okMOtezsbDvMXqNGDX2PXaN27doBWE3b0hCLMqeSPEq9vPXWW02b0H3TCausrCzTMGLtUIrVPZWd+tZbb1mTL0lahXnGjh1rxRRi7SA80jyLvWA1MR2natKkiS1KOZZq1apFtWrVAPj2228B/0bGM74ajwUrFVceb8Xz0tLS8nXz/uWXX8zjKJMhGsSrLnHeA/aRD3S8YqfxuKd6jnUIQ3HSZHh2nUrscASIUlf+zytFwAtj5D2xkIj6PPGsEq8aVF999RXghazkdJk3bx7gOSpiUfkx3s2wEkkiW4jGEydhHY4yQEx660SeXElk5btE7MbSOFJTUy1xItYV/J2EzU9ZnaeTsA5HgIjJaZ3IEwzHGtJYkq2mrqNsEJMFG4sMF4fD4VRihyNQFOp0cjgcyYWTsA5HgHAL1uEIEG7BOhwBwi1YhyNAuAXrcAQIt2AdjgDhFqzDESDcgnU4AoRbsA5HgCg0l7isHlHKy7EyTzh25lpW5+kkrMMRINyCdTgChFuwDkeAcAs2wYRCoQIL2TkcBeEWrMMRIGLe0FnFptWeLyUlxdoPHitncSVB09PTrc2kGhrfddddAPzwww8MGjQIiH1VeUdwcRLW4QgQUZGwkqI1a9YEoEePHtbKQ+0NJFEyMzOtuW3v3r0Bvw9nWSIlJYUmTZoAcO+99wJQp04d6352zjnnAN71AGjUqJF1u3M4jkSpF2x6ejodOnQA4OqrrwZgwYIF1hBYDY7feustAP7v//6PMWPGAPDTTz8B3kJXh4Cgo0X3yiuv0KdPH8Dv0bJ582a++OILwO/XooJ1mzdvLnFD50RTvXp1fvjhBwDrqRQOh1m5ciXgb06bN29OzACjiMybRJlzTiV2OAJEiSv/Sw0++eSTrTHzZ599BsDUqVMLrct7wgknAH7z44MHD3L66acD0d254pnGpi52Tz/9NAB33nmnvffdd98B8Nhjj1m95ldeeQWA448/HoBBgwZZy8riEu/URN37LVu2AH4nwyMhbaJXr16A93yUtG51PO+p+kOp5eZtt93GaaedBsD06dMBePPNN82kk7NQZmA4HDaJrG4YmZmZtGrVCvA1jhUrVgDk0jJdaqLDUQYosQ0rSbhnzx4+/fRTAObPnw8cver9xo0bAc+eBVi2bBmTJ08GoHv37iUdUkKpWrUqAC1btgQ86TNt2jQA/vCHPwDeDnzNNdcAfiNo9dZ999134zrekiApIaehJO2RkBSVxJEWNXfu3KS118uXL2/P4FNPPQVgjbnLly9vErNTp06Ap0mpI6H8EYsXLwZg7969tGjRAoCGDRsCsGbNGvv9v//97wAsX768yOMr9YJdv3693ZDiOo727t0LwL/+9S+LRz744IMAPPHEEyUdWkKQOjRkyBAA6tata53m9V6lSpV44IEHcv1d3759AV9tTGaee+45IP9C3bRpE3/9618BfwPKyMhg165dAOzcuRPwTaD9+/fbZyRLS5fKlSsDMGHCBHOiVqhQAfAdTZFZaXr+d+zYwapVqwB/Y9L97tSpk0UBxPz583n11VcBzzkLxbv3TiV2OAJEqcM6hw8fth20pA2gFi5caDvu0KFDAUxteP7550s7xLig+OrXX38NwNKlSy0uLafFiy++SP369QFYvXo1gKnNQUBxc7Fs2TIAWrVqZZIyUhpJhZZDTr9TrVo1tm3bFpcxHw1lnr399tuAp7ZLS1DYUVl6zZo1s+d03LhxgBdjl3ovqZuRkQHAFVdcQceOHQHMofj111/bOimJduEkrMMRIKKS6SRjO3J3PUq4CPAdL2eeeab9vt6TQypoaB779++3sMcll1wCQMeOHW1X7dy5c67fDwJ//OMfAT8JRk6ZSEmh+YTDYZMk0jSkLVWrVi0pnIvp6elcfPHFgD/GYcOGmRNVGkK7du0AaN26tYUkhw8fDsD27dvzPbtyyn3wwQfmTJSdWtr77SSswxEgoiphpetnZGTkCh6DbytkZmaaF+62224DPO+pdmPtTvK2BhnZtV26dAG8tEUF2X/88ceEjaukvPfeewC8/PLLgJ8IcaSEDyWFLFmyBPBDX99++21SeIcrV65skv6rr74C4NVXX7VnUWmmslG3bdtmOfGNGjUCYNGiRTaXSpUqAVhixA8//BA1ySqismDzxtvKlStnKq0W55VXXgl4sSw5qeT+njVrFjVq1AC8mwmwdu3aaAwtoShTRjcZ/HkFSRUWus/K5vrd734HeNluytaJfHjzLlTNWc9EounRowfnn38+ALfccgsAu3btMieZFpuESGZmJm3btgV8NblBgwZ2ny+44ALAz2x75JFHCvxeqc5yYBWnAbpTiR2OABEVCaudUztRVlaWhStmzpwJwMiRIwH/5Ar4akWPHj1s11G4ozi7TrKizB5x6NAhSyYIMsOGDQP801nDhw83tfj7778HPAdT3bp1c/2dkmOkiSUKPWtPPvmkSX+NKTU1NZ/kk0OqY8eOdmRS+dOStOAnDn388ceAp2nqsyKlqr5ToUsVdCgKTsI6HAEiZiViZMvktW/BD6TLSVW9enULVitdK4g2XiTlypXjjjvuAHybpmLFiuZ0SrbUvOKge3PmmWcCXrjn8ssvB/yTWGeddVa+33/ppZfiOcwjImlXuXJlc4aOGjUK8MaopI5Zs2YBviQcN26cSVSllEaGMBHWz1EAABDUSURBVBctWgTAv//9b8DzYUiLbN68OQBt2rSxlEddj+JI2JjXdCoIXbA2bdrYT6nKmrz+H4Qc24K47LLLaN26NeBlOIF305RIXhaQF/y5557jxhtvBHxHlDZl8DesZEGb5ODBg+3eKAPtrrvusjizijDIjAmHw9x3332Av2DBz59WTFebcmTusQ68LF++3J7tkphHTiV2OAJE3CVsKBSy+Jwk7M6dO83A1+4kV3l2dra9F081ObLSIXgqbF7nWkHjkUPhoYceYsOGDQB2dHDp0qWmIpUl9u3bx6RJkwA/31gqIEC/fv0SMq6j8dprr1kOsRxL4J0+goJz45X9FMnDDz8M+M9uZLaXkFotpyq4XGKHo8wTNwkbWSpDElbG9pYtWyyJQofg5SJPS0szI12SNisrK+bOGu2OcpalpaXZLqxxRGoG0ggeffRRAJo2bWq2mzJmmjZtarZMWar2Hw6H7f4VVFNZJ5iSEflIdDLnaAwYMCDX/8PhMLNnz7Z/H43SPrdOwjocASLmElaSRBKoQoUKlnMpvT4nJ8cC7rIfIiWbgtSyiw4cOGBpb5LEsbJztSMePHjQpKMkfsWKFalTpw7gF127+eabbdxKwTvjjDPss2TDBD1sFUlqaqpVVoi0XUWiEyWiSdOmTXP9/8CBA5ZqGw/irhLXrFnTHmrl2B46dIjRo0cD+Q39cDhspWR0MPiGG26wTCqVHdFCj+VCiDw6p3ErnnrppZcC/sYUDoctP7pr166Al/UlJ5Y2ISWWK0QSJDSXunXr2vxVlxiKF18MCrpPeha2bdtmsdx4xNadSuxwBIiYS1jtRDLu09LSrOKc1Ml58+ZZN4C8x/Kys7NNCulg8bJly0ySKficiIyh7OxsO6WiQlwnnngi4I1fY1TFvBkzZli5GGUGrVu3DoCPPvrIirYFJftJ4ZoLL7zQ5qhiZjk5Ofz3v/9N2Nhihe6vntOxY8eao81JWIfDkYu42bCSmD///LOFdWQLPv3009aDpyAbVK/Jfli5cmXS5OLK5lbe8Pvvvw94h7dVImbw4MGAd5JFKXvalVWk6+qrr2bGjBmAV7sWPPs2GU8tyXbVvKpXr26OOLF7927Lzy2L6FmsWrWqFSiYOHEigGlKsSDumU4HDx60h1xJ0rNnzy72wkv0Qs2LYq6//e1vAe9GSl1Wo6jIMWtxLly4EPCS5nVdtKGFQiEzB0pakTIWaLOUU03jBd/0+fDDD/nkk0/iP7gYIxNMsfUzzjiDZs2aAX4Wk1rWxMIB6lRihyNAxF3CZmdn8+abbwJ+jaCyEKeT6qoD+0X9feWfRvbILWr1yUShMUWq63pNTrQ///nPZeK+5kW1mHUyp3nz5haHVXgnljgJ63AEiLhL2KysLF544QWgbJSBiQUFnfZIJnTfdDa0V69ezJs3D8C0pw0bNiSdnyEaKGwVeY/ko1BTq1jeNydhHY4AUeKGzkEgns1/E0m8GzonkkTe01AoxLnnngv4BehWr15tZ4HlJdZ56dJwpHm6BcuxM084duYa63kWdDwymqqw68DucJQBCpWwDocjuXAS1uEIEG7BOhwBwi1YhyNAuAXrcAQIt2AdjgDhFqzDESDcgnU4AoRbsA5HgHAL1uEIEIUer3N5p8HA5RLnp6zO00lYhyNAuAXrcAQIt2AdjgAR9xIxjmOPvGdH3QmxkuMkrMMRIBIqYVWQOhwOu123DKJi26eeeiqANcRetmyZlUN19714OAnrcASIuElYdXK77777uPfeewGssTNgPWCnTp0KQN++fYFg9k2NFslaSLwopKSkWEe7Rx99FPAaYAMsWrSIP/3pT4B/3x1FIyZF2EKhEFWqVAG8CvDgdyaXmnQ0VNP2hRdesKZLxa1zG88guzafe+65B4Dt27fz2muvAUWropeSkmI9atq1awd49X+/+OIL+/eRSKbECZk5p556KieccAIAnTt3BrBWm40aNeJXv/oVUPwFm+jECc0vsnG3+h7pHkVjk3WJEw5HGSCqEla7T5s2baz1Xr169fRZJR0jP/30EwA33HAD4PWvKUo3t3jsxpqXOpadf/759t6ePXsAuOmmmwCvw51abGZmZgIwcOBAANq2bWtOmapVqwIwd+5ca/ys9pQFkQwSVhJnwoQJgNeS8pVXXgEwbUu1fCM7vm3btq1Y35NICVu7dm3mzJkDQEZGBgBvvPEGY8eO1dgA2Lp1q/2USSftsKjdLpyEdTjKAFFxOkmyqp38e++9V2zJqh0o789wOEz16tUBeOuttwDPifH2228Dfj/SRHH66acDuSWrkF374osvAp6UlNTVTqudetmyZVY5vkWLFgBMmjTJJHIy06FDB7O109K8R+rDDz9k1qxZgGezgidZwesnK4dUcSVsIqhbty7gVfaXD2bfvn2A5xSVZJUPQv9v0qSJfYYaeX/11Vf5noHi2LylXrChUMgaBEnlqVWrVq4FB/4C/OWXX6xJsVSlFStW8NhjjwF+Y2R1+c7MzKRbt24A9OzZE4B+/fqxcuVKAKZNm5br8+NJKBTi888/L/C97Oxsa5KkmGP58uVZu3YtAEuXLgXgm2++Abx5S02UmjxnzpykbhgmVfCqq66y1zTXfv362WLUxqXnJCUlhTZt2gDw9ddfx228xUWLbNGiRYDnMI10hgL85S9/MXMlbxOzUChkG7IciR06dGDJkiUAbNmyBSjewnUqscMRIKKiEmuHkIt+xowZJkWlKq1atQqAzZs320575ZVXAl5rwsWLFwNeOAT8UMjq1atZsGABAOPHjwfg3HPPtc+QihKNBkTFpXv37vma+C5cuBCATp06mTorsyAtLc3GKzNCmkSzZs1Msuq95cuXJ2Uc9t133wVyS1bdt1atWgGeJqV5q4GUJBb4anIy88QTTwCYSQYwffp0wJOsALt37z6idhcOh036zp49G/DUZj0XJWkr6iSswxEgSh3WCYVCZqM0bdoUgJ07d5prO68NlpGRwVlnnQXAM888A3hhjPfffx+ATz75BMDc57t377bPkOQJhULm3JBUL8jWi1UIQOPYvn272eFyQlSrVg04eoaWpM15550HwL///W+zd1599VUA7rrrriLZsPEK6+g+5012WLdundnfmncoFLI5ys9w9tln299cfPHFAEyePLlYY4hHWEcZWXqGNe9ly5bRo0cPANavXw94z11x/SfSPApbey6s43CUAaLiJdZOITv14MGD+SSDJGLbtm3NNjj55JPtM/r16wf4u3D//v0BzyZUkkTkjlSaRIzSovBVhQoVbHcdPnw4UPzc59dffx3wtAyFqIYMGQIUPcgeL+bPn5/r/7ovjRs3LnCsFSpUsPfzIqmbjCiNVpJW2tN7771nvyNNavv27SVJmS3x2Eq9YHNycuxB0yJKT083dUiT1kP+9ttvW1wrEjljli1bBngOF4BDhw4V+v1FUS+ihb5LToOHHnqINWvWAL5DrKjoGkQ+zHI6yYGTTKSmplp8WNx4441A7o1F1yglJcX+rWdAhMNhWwTJRqVKlXj44YdzvaZMu6ysLHr16gX4YcR//vOf5mCNB04ldjgCRFTCOtpttKOGQiHbVZXxNHLkSAA7wZEXSWkF44t6iiNeYY9I1V8B72eeeabE3z937txc/8/JybHsrWTk8ssvz6fNyClTvnz5fNchHA6b1hR5jBISE4IrKu3atbOQoZ5raTy1a9e2bKaTTjoJgMWLF5t6H49n0UlYhyNARPUAu3aY7Oxs252Ugta6dWugYGfR4cOHLeFAaVuFOZUiQwZ5UyCjRVEKh5XkO5VYkFfTeO+995LOyQR+CGvo0KH2msapVNGqVataquiKFSsAT4pKy5LDUcTT5isqut+9e/e2+ypN4D//+Q/gnRLTnBSabN26NV9++SXgO6eOEipNrNOpIMLhsB23knohlTfSASGP6r59++x1LXBliGzZsiXfg1yuXDm7wNGuSBHpNIHo5iinp6db1pbQ599xxx1R+55oojjz4cOH2bx5MwBPPfUU4B0tA+8YYd582JSUFHM06lqKjz/+OPYDLybaVCI9//KCKz9606ZNuX4P4JRTTqF9+/aAnxcdeSBF5oCuwd69ewuMehQVpxI7HAEiqhJW0qlSpUq2oyhM89FHHwFeNpRUB6m1VatWtR1ah9QjM2EU/9POtGPHDtsFjxb2KS4aU40aNey7IDqSfMaMGflK5FxyySVAcoZywNeIbr/9djsuV5RrkZqaauVyhCTKI488Et1BRgE9fyNGjODMM88EfMeSjgWuWLHC4q+6Lhs2bDDHqjQQaZVt27Y1U1AZfNOmTStS8YUj4SSswxEgoloiRtIp0s0vnb958+YAtG/f3k6oKHmgevXq1K9fH8ByUlVCBbBzpdql/vOf/5iTQw6MaOQSR4ajOnbsCGA2586dO4vyUQWiki+S1pH/jjwJUlJimUus+1dUqRB5QkdlgnSiSeGw+vXrl1gzinUucUpKitVRfvnllwE/fPX555/bKSNJ2nnz5tlc9CxKa2ratKlpI88++yzglRJSyNLlEjscZZyoloiRR6xGjRrmKVMan/T7JUuWmPSUrp+ammolQ/SevJOhUMhyjkeNGgXkLmYWTS9uZAcCnZxRmY8FCxYU26un6yIvo74DoiNZY4kkZXFDTX369AE86aRrKJ/FddddBxRdWieCnJycXGeawdcy6tWrZ9Lz17/+NeA968qh13xr1aoFeDavivMpUSYrKyvxYR0NQKpu586d7aidxL/e27dvn6nCMupbtmxJ7dq1Ad9dHhkHVRmWf/zjHwAxrXMkFWbjxo2Arxpv377dckqL+sDJWRZ5yD0IB7eh+A2sunbtCmDZWikpKRbHVOH4mTNnFumzkg3d7zVr1ljGnlTi/v37Wz64VGPllb/66qumTutalFbAOJXY4QgQMalLXK9ePQYMGAD4NXnlzDl8+LD9nvIyIxMh8rJnzx5zAqiqYFEpjYNCDjSpNzVq1DDnkQqn7du3L5+00Dz+9re/ccstt+R677PPPrMq+NEkFk4nzV8aD+SXEjJpZs2axSmnnJLr77dt28Y555wDwPfffw9E57hgoiv/C4X9xo0bZ+aenGzKh4+sqFhcnNPJ4SgDxKS3Dvg7sw5oq+BaWlparlIvR0K7ee/evS2Vrbi7VTR341AoZHa47NyCxnPaaacBXv1ZzVN2z6BBg2Jiv8UyrKM5t2nTxiTm9ddfD/hzTU1NNamrZInhw4cndK6xlrB6duvUqWOag+oN5y2yVhKONM+Yda+Tl3jQoEGAr/527drVHoKC2LVrFwBdunQBPMdNMjgpwuFwocfClMH0/PPPA56jQubAO++8E/sBxghtTt98840tWHm4le89ZswYbr/9diC5PcDRRM/k7t27zWzSa7F0ijqV2OEIEDFTifMiqXrWWWdZlX/FOA8dOsS4ceMAePzxx4HoHMGKp/qkcI3U9ylTpnD33XcDsZc68aqaGFmgABLTbSFZVGLRoEEDq8ElrVLtUUvTRsY5nRyOMkDcOrDLFpo2bZo18y0LRLbYBK8JFHjlY8qaPVeSSvVllciCgwrxqF1o3gP70cRJWIcjQMTNhk0E8bB3IvNMwU/FLEm92pKSDA2d40Wy2LCSsDVr1rSTZm3btgUwf0xp/DBHmqdbsBw784RjZ67xmmdkLkHeskKxiMM6ldjhCBCFSliHw5FcOAnrcAQIt2AdjgDhFqzDESDcgnU4AoRbsA5HgHAL1uEIEP8PwIvREM5k3dwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 288x288 with 16 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "class CVAE(tf.keras.Model):\n",
    "    def __init__(self, latent_dim):\n",
    "        super(CVAE, self).__init__()\n",
    "        self.latent_dim = latent_dim\n",
    "        self.inference_net = tf.keras.Sequential(\n",
    "            [\n",
    "                tf.keras.layers.InputLayer(input_shape=(28, 28, 1)),\n",
    "                tf.keras.layers.Conv2D(\n",
    "                    filters=32, kernel_size=3, strides=(2, 2), activation='relu'),\n",
    "                tf.keras.layers.Conv2D(\n",
    "                    filters=64, kernel_size=3, strides=(2, 2), activation='relu'),\n",
    "                tf.keras.layers.Flatten(),                \n",
    "                tf.keras.layers.Dense(latent_dim + latent_dim),\n",
    "            ]        \n",
    "        )\n",
    "        \n",
    "        self.generative_net = tf.keras.Sequential(\n",
    "            [\n",
    "                tf.keras.layers.InputLayer(input_shape=(latent_dim,)),\n",
    "                tf.keras.layers.Dense(units=7*7*32, activation=tf.nn.relu),\n",
    "                tf.keras.layers.Reshape(target_shape=(7, 7, 32)),\n",
    "                tf.keras.layers.Conv2DTranspose(\n",
    "                    filters=64,\n",
    "                    kernel_size=3,\n",
    "                    strides=(2, 2),\n",
    "                    padding=\"SAME\",\n",
    "                    activation='relu'),\n",
    "                tf.keras.layers.Conv2DTranspose(\n",
    "                    filters=32,\n",
    "                    kernel_size=3,\n",
    "                    strides=(2, 2),\n",
    "                    padding=\"SAME\",\n",
    "                    activation='relu'),\n",
    "\n",
    "                # No activation\n",
    "                tf.keras.layers.Conv2DTranspose(\n",
    "                    filters=1, kernel_size=3, strides=(1, 1), padding=\"SAME\"),\n",
    "\n",
    "            ]                \n",
    "        )\n",
    "        \n",
    "    @tf.function\n",
    "    def sample(self, eps=None):\n",
    "        if eps is None:\n",
    "            eps = tf.random.normal(shape=(100, self.latent_dim))\n",
    "        return self.decode(eps, apply_sigmoid=True)\n",
    "    \n",
    "\n",
    "    def encode(self, x):\n",
    "        mean, logvar = tf.split(self.inference_net(x), num_or_size_splits=2, axis=1)\n",
    "        return mean, logvar\n",
    "    \n",
    "    def reparameterize(self, mean, logvar):\n",
    "        eps = tf.random.normal(shape=mean.shape)\n",
    "        return eps * tf.exp(logvar * .5) + mean\n",
    "    \n",
    "    def decode(self, z, apply_sigmoid=False):\n",
    "        logits = self.generative_net(z)\n",
    "        if apply_sigmoid:\n",
    "            probs = tf.sigmoid(logits)\n",
    "            return probs        \n",
    "        return logits\n",
    "    \n",
    "optimizer = tf.keras.optimizers.Adam(1e-4)\n",
    "\n",
    "def log_normal_pdf(sample, mean, logvar, raxis=1):\n",
    "    log2pi = tf.math.log(2. * np.pi)\n",
    "    return tf.reduce_sum(\n",
    "        -.5 * ((sample - mean) ** 2. * tf.exp(-logvar) + logvar + log2pi),\n",
    "        axis=raxis\n",
    "    )\n",
    "\n",
    "@tf.function\n",
    "def compute_loss(model, x):\n",
    "    mean, logvar = model.encode(x)\n",
    "    z = model.reparameterize(mean, logvar)\n",
    "    x_logit = model.decode(z)\n",
    "    \n",
    "    cross_ent = tf.nn.sigmoid_cross_entropy_with_logits(logits=x_logit, labels=x)\n",
    "    logpx_z = -tf.reduce_sum(cross_ent, axis=[1, 2, 3])\n",
    "    logpz = log_normal_pdf(z, 0., 0.)\n",
    "    logqz_x = log_normal_pdf(z, mean, logvar)\n",
    "\n",
    "    return -tf.reduce_mean(logpx_z + logpz - logqz_x)    \n",
    "\n",
    "\n",
    "@tf.function\n",
    "def compute_apply_gradients(model, x, optimizer):\n",
    "    with tf.GradientTape() as tape:\n",
    "        loss = compute_loss(model, x)\n",
    "        \n",
    "    gradients = tape.gradient(loss, model.trainable_variables)\n",
    "    optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
    "                \n",
    "epochs = 100\n",
    "latent_dim = 50\n",
    "num_examples_to_generate = 16\n",
    "\n",
    "# keeping the random vector constant for generation (prediction) so\n",
    "# it will be easier to see the improvement.\n",
    "random_vector_for_generation = tf.random.normal(\n",
    "    shape=[num_examples_to_generate, latent_dim])\n",
    "model = CVAE(latent_dim)        \n",
    "\n",
    "def generate_and_save_images(model, epoch, test_input):\n",
    "    predictions = model.sample(test_input)\n",
    "    fig = plt.figure(figsize=(4,4))\n",
    "\n",
    "    for i in range(predictions.shape[0]):\n",
    "        plt.subplot(4, 4, i+1)\n",
    "        plt.imshow(predictions[i, :, :, 0], cmap='gray')\n",
    "        plt.axis('off')\n",
    "\n",
    "    # tight_layout minimizes the overlap between 2 sub-plots\n",
    "    plt.savefig('image_at_epoch_{:04d}.png'.format(epoch))\n",
    "    plt.show()    \n",
    "    \n",
    "generate_and_save_images(model, 0, random_vector_for_generation)\n",
    "\n",
    "for epoch in range(1, epochs + 1):\n",
    "    start_time = time.time()\n",
    "    for train_x in train_dataset:\n",
    "        compute_apply_gradients(model, train_x, optimizer)\n",
    "\n",
    "    end_time = time.time()\n",
    "\n",
    "    loss = tf.keras.metrics.Mean()\n",
    "    for test_x in test_dataset:\n",
    "        loss(compute_loss(model, test_x))\n",
    "\n",
    "    elbo = -loss.result()\n",
    "    display.clear_output(wait=False)\n",
    "    print('Epoch: {}, Test set ELBO: {}, '\n",
    "      'time elapse for current epoch {}'.format(epoch,\n",
    "                                                elbo,\n",
    "                                                end_time - start_time))\n",
    "    generate_and_save_images(\n",
    "        model, epoch, random_vector_for_generation)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}