004_Modeling_Local.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Local Modeling"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Libraries to Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "plt.style.use('fivethirtyeight')\n",
    "\n",
    "from PIL import Image\n",
    "from tensorflow.keras.preprocessing import image, image_dataset_from_directory\n",
    "from tensorflow.keras import models, layers, optimizers\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from tensorflow.keras.layers import BatchNormalization\n",
    "\n",
    "import os\n",
    "import gc\n",
    "from timeit import default_timer as timer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Image Directories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dir = '/Users/arencarpenter/Desktop/Detecting_ALL_with_CNN/Data/training/'\n",
    "test_dir = '/Users/arencarpenter/Desktop/Detecting_ALL_with_CNN/Data/testing/'\n",
    "val_dir = '/Users/arencarpenter/Desktop/Detecting_ALL_with_CNN/Data/validation/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create flow_from_directory objects for Keras model. Using a downsized dataset with class balance for local model running."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 1346 images belonging to 2 classes.\n",
      "Found 593 images belonging to 2 classes.\n",
      "Found 947 images belonging to 2 classes.\n"
     ]
    }
   ],
   "source": [
    "train_g = image.ImageDataGenerator(rescale = 1/255).flow_from_directory(train_dir,\n",
    "                                                                  target_size = (256,256), \n",
    "                                                                  color_mode='rgb',\n",
    "                                                                  class_mode='binary')\n",
    "val_g = image.ImageDataGenerator(rescale = 1/255).flow_from_directory(val_dir,\n",
    "                                                                target_size = (256,256), \n",
    "                                                                color_mode='rgb',\n",
    "                                                                class_mode='binary')\n",
    "test_g = image.ImageDataGenerator(rescale = 1/255).flow_from_directory(test_dir,\n",
    "                                                                target_size = (256,256), \n",
    "                                                                color_mode='rgb',\n",
    "                                                                class_mode='binary')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vanilla Model - 1C1D"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu', input_shape=(256,256,3)))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(512, activation = 'relu'))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# compiling models\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.RMSprop(lr=1e-4),\n",
    "              metrics=['accuracy', 'Recall'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 410s 10s/step - loss: 1.0253 - accuracy: 0.6478 - recall: 0.6419 - val_loss: 0.6186 - val_accuracy: 0.7386 - val_recall: 0.5612\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 378s 9s/step - loss: 0.6211 - accuracy: 0.7259 - recall: 0.7073 - val_loss: 0.5100 - val_accuracy: 0.7639 - val_recall: 0.8265\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 308s 7s/step - loss: 0.4892 - accuracy: 0.7749 - recall: 0.7816 - val_loss: 0.6214 - val_accuracy: 0.7150 - val_recall: 0.8980\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 306s 7s/step - loss: 0.4327 - accuracy: 0.8061 - recall: 0.8009 - val_loss: 0.7740 - val_accuracy: 0.6358 - val_recall: 0.2721\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 346s 8s/step - loss: 0.4165 - accuracy: 0.8068 - recall: 0.7875 - val_loss: 0.5517 - val_accuracy: 0.7808 - val_recall: 0.7007\n",
      "Total Time Elapsed:  29  minutes  56  seconds\n"
     ]
    }
   ],
   "source": [
    "# training\n",
    "start = timer()\n",
    "\n",
    "history = model.fit(train_g, \n",
    "                    epochs=5, \n",
    "                    validation_data=val_g, \n",
    "                    workers = 7\n",
    "                   )\n",
    "\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 14s 474ms/step - loss: 1.2477 - accuracy: 0.5164 - recall: 0.3272\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2C1D\n",
    "\n",
    "Add another layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 100s 2s/step - loss: 0.5483 - accuracy: 0.7385 - recall: 0.6909 - val_loss: 0.5160 - val_accuracy: 0.7622 - val_recall: 0.7823\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 103s 2s/step - loss: 0.5017 - accuracy: 0.7697 - recall: 0.7311 - val_loss: 0.5093 - val_accuracy: 0.7656 - val_recall: 0.8061\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 100s 2s/step - loss: 0.4607 - accuracy: 0.7838 - recall: 0.7340 - val_loss: 0.4995 - val_accuracy: 0.7707 - val_recall: 0.7823\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 86s 2s/step - loss: 0.4479 - accuracy: 0.7987 - recall: 0.7756 - val_loss: 0.5273 - val_accuracy: 0.7589 - val_recall: 0.8435\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 92s 2s/step - loss: 0.4196 - accuracy: 0.8039 - recall: 0.7860 - val_loss: 0.5227 - val_accuracy: 0.7673 - val_recall: 0.8265\n",
      "Total Time Elapsed:  8  minutes  15  seconds\n"
     ]
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu', input_shape=(256,256,3)))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(256, activation = 'relu'))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.RMSprop(lr=1e-4),\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "\n",
    "start = timer()\n",
    "\n",
    "history = model.fit(train_g, \n",
    "                    epochs=5, \n",
    "                    validation_data=val_g, \n",
    "                    workers = 7\n",
    "                   )\n",
    "\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 12s 393ms/step - loss: 0.8072 - accuracy: 0.5744 - recall: 0.4676\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3C1D\n",
    "\n",
    "Add another layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 76s 2s/step - loss: 0.5603 - accuracy: 0.7229 - recall: 0.7103 - val_loss: 0.5338 - val_accuracy: 0.7487 - val_recall: 0.8673\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 79s 2s/step - loss: 0.5138 - accuracy: 0.7593 - recall: 0.7207 - val_loss: 0.4887 - val_accuracy: 0.7808 - val_recall: 0.7585\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 78s 2s/step - loss: 0.4982 - accuracy: 0.7727 - recall: 0.7028 - val_loss: 0.5109 - val_accuracy: 0.7572 - val_recall: 0.8333\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 78s 2s/step - loss: 0.4798 - accuracy: 0.7719 - recall: 0.7251 - val_loss: 0.4772 - val_accuracy: 0.7875 - val_recall: 0.7755\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 78s 2s/step - loss: 0.4652 - accuracy: 0.7912 - recall: 0.7623 - val_loss: 0.4864 - val_accuracy: 0.7791 - val_recall: 0.7211\n",
      "Total Time Elapsed:  6  minutes  41  seconds\n"
     ]
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu', input_shape=(256,256,3)))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(256, activation = 'relu'))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.RMSprop(lr=1e-4),\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "\n",
    "start = timer()\n",
    "\n",
    "history = model.fit(train_g, \n",
    "                    epochs=5, \n",
    "                    validation_data=val_g, \n",
    "                    workers = 7\n",
    "                   )\n",
    "\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 14s 475ms/step - loss: 0.9313 - accuracy: 0.4879 - recall: 0.2855\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2x2x1C1D\n",
    "\n",
    "Try deeper structure."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 283s 7s/step - loss: 0.5827 - accuracy: 0.7325 - recall: 0.6612 - val_loss: 0.5210 - val_accuracy: 0.7487 - val_recall: 0.8061\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 266s 6s/step - loss: 0.5241 - accuracy: 0.7578 - recall: 0.7177 - val_loss: 0.5490 - val_accuracy: 0.7521 - val_recall: 0.8810\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 266s 6s/step - loss: 0.5061 - accuracy: 0.7682 - recall: 0.7311 - val_loss: 0.5380 - val_accuracy: 0.7622 - val_recall: 0.8605\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 265s 6s/step - loss: 0.4951 - accuracy: 0.7697 - recall: 0.7281 - val_loss: 0.5025 - val_accuracy: 0.7605 - val_recall: 0.7925\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 265s 6s/step - loss: 0.4921 - accuracy: 0.7741 - recall: 0.7251 - val_loss: 0.5079 - val_accuracy: 0.7757 - val_recall: 0.7687\n",
      "Total Time Elapsed:  22  minutes  55  seconds\n"
     ]
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu', input_shape=(256,256,3)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(256, activation = 'relu'))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.RMSprop(lr=1e-4),\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "\n",
    "start = timer()\n",
    "\n",
    "history = model.fit(train_g, \n",
    "                    epochs=5, \n",
    "                    validation_data=val_g, \n",
    "                    workers = 7\n",
    "                   )\n",
    "\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 37s 1s/step - loss: 0.8040 - accuracy: 0.4857 - recall: 0.2901\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2x2x1C2D\n",
    "\n",
    "Add Dense layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 254s 6s/step - loss: 0.5606 - accuracy: 0.7281 - recall: 0.7132 - val_loss: 0.5072 - val_accuracy: 0.7673 - val_recall: 0.7211\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 240s 6s/step - loss: 0.5149 - accuracy: 0.7608 - recall: 0.7132 - val_loss: 0.5200 - val_accuracy: 0.7622 - val_recall: 0.8401\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 240s 6s/step - loss: 0.4988 - accuracy: 0.7808 - recall: 0.7355 - val_loss: 0.5376 - val_accuracy: 0.7622 - val_recall: 0.8741\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 243s 6s/step - loss: 0.4946 - accuracy: 0.7741 - recall: 0.7400 - val_loss: 0.5496 - val_accuracy: 0.7521 - val_recall: 0.8673\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 265s 6s/step - loss: 0.4794 - accuracy: 0.7838 - recall: 0.7444 - val_loss: 0.5064 - val_accuracy: 0.7605 - val_recall: 0.7857\n",
      "Total Time Elapsed:  21  minutes  11  seconds\n"
     ]
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu', input_shape=(256,256,3)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(256, activation = 'relu'))\n",
    "model.add(layers.Dense(256, activation = 'relu'))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.RMSprop(lr=1e-4),\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "\n",
    "start = timer()\n",
    "\n",
    "history = model.fit(train_g, \n",
    "                    epochs=5, \n",
    "                    validation_data=val_g, \n",
    "                    workers = 7\n",
    "                   )\n",
    "\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 37s 1s/step - loss: 0.8977 - accuracy: 0.5259 - recall: 0.3688\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2x2x1x1C1D\n",
    "\n",
    "Try deeper structure with another layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 270s 6s/step - loss: 0.5692 - accuracy: 0.7244 - recall: 0.6924 - val_loss: 0.6684 - val_accuracy: 0.6796 - val_recall: 0.9320\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 292s 7s/step - loss: 0.5274 - accuracy: 0.7563 - recall: 0.7355 - val_loss: 0.4963 - val_accuracy: 0.7605 - val_recall: 0.7925\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 293s 7s/step - loss: 0.5142 - accuracy: 0.7652 - recall: 0.7444 - val_loss: 0.4906 - val_accuracy: 0.7605 - val_recall: 0.7789\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 288s 7s/step - loss: 0.5025 - accuracy: 0.7712 - recall: 0.7296 - val_loss: 0.4999 - val_accuracy: 0.7723 - val_recall: 0.8129\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 286s 7s/step - loss: 0.4995 - accuracy: 0.7734 - recall: 0.7400 - val_loss: 0.4851 - val_accuracy: 0.7555 - val_recall: 0.7721\n",
      "Total Time Elapsed:  24  minutes  22  seconds\n"
     ]
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu', input_shape=(256,256,3)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(256, activation = 'relu'))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.RMSprop(lr=1e-4),\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "\n",
    "start = timer()\n",
    "\n",
    "history = model.fit(train_g, \n",
    "                    epochs=5, \n",
    "                    validation_data=val_g, \n",
    "                    workers = 7\n",
    "                   )\n",
    "\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 42s 1s/step - loss: 0.9306 - accuracy: 0.5048 - recall: 0.3256\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2x2x1x1C1D Adam\n",
    "\n",
    "Try Adam optimizer instead of RMSprop for faster convergence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 261s 6s/step - loss: 0.5660 - accuracy: 0.7214 - recall: 0.7088 - val_loss: 0.6083 - val_accuracy: 0.7369 - val_recall: 0.8537\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 268s 6s/step - loss: 0.5142 - accuracy: 0.7719 - recall: 0.7013 - val_loss: 0.5094 - val_accuracy: 0.7639 - val_recall: 0.7993\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 279s 6s/step - loss: 0.5071 - accuracy: 0.7645 - recall: 0.7088 - val_loss: 0.4937 - val_accuracy: 0.7757 - val_recall: 0.7551\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 266s 6s/step - loss: 0.4985 - accuracy: 0.7779 - recall: 0.7444 - val_loss: 0.4926 - val_accuracy: 0.7757 - val_recall: 0.7619\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 280s 7s/step - loss: 0.5133 - accuracy: 0.7608 - recall: 0.6909 - val_loss: 0.5149 - val_accuracy: 0.7572 - val_recall: 0.8265\n",
      "Total Time Elapsed:  22  minutes  60  seconds\n"
     ]
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu', input_shape=(256,256,3)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(256, activation = 'relu'))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.Adam(),\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "\n",
    "start = timer()\n",
    "\n",
    "history = model.fit(train_g, \n",
    "                    epochs=5, \n",
    "                    validation_data=val_g, \n",
    "                    workers = 7\n",
    "                   )\n",
    "\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 37s 1s/step - loss: 0.8501 - accuracy: 0.5248 - recall: 0.3981\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2x2x1x1C1D Adam / BatchNorm\n",
    "\n",
    "Add BatchNormalization for improved performance."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 489s 11s/step - loss: 0.9327 - accuracy: 0.7043 - val_loss: 0.6597 - val_accuracy: 0.7116\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 535s 12s/step - loss: 0.5530 - accuracy: 0.7318 - val_loss: 0.6387 - val_accuracy: 0.6779\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 469s 11s/step - loss: 0.4851 - accuracy: 0.7801 - val_loss: 0.6176 - val_accuracy: 0.7791\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 450s 10s/step - loss: 0.4551 - accuracy: 0.7853 - val_loss: 0.6077 - val_accuracy: 0.6476\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 431s 10s/step - loss: 0.4457 - accuracy: 0.7912 - val_loss: 0.6177 - val_accuracy: 0.7740\n",
      "Total Time Elapsed:  40  minutes  29  seconds\n"
     ]
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(96, 3, activation = 'relu', input_shape=(256,256,3)))\n",
    "model.add(layers.Conv2D(96, 3, activation = 'relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(layers.Conv2D(32, 3, activation = 'relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(256, activation = 'relu'))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.Adam(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "start = timer()\n",
    "\n",
    "history = model.fit(train_g, \n",
    "                    epochs=5, \n",
    "                    validation_data=val_g, \n",
    "                    workers = 7\n",
    "                   )\n",
    "\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 63s 2s/step - loss: 0.7235 - accuracy: 0.4678\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1x1C1D Adam/BatchNorm\n",
    "\n",
    "Try simpler model with Adam optimizer and Batch Normalization layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 181s 4s/step - loss: 16.3543 - accuracy: 0.6419 - recall: 0.6597 - val_loss: 2.5317 - val_accuracy: 0.5042 - val_recall: 0.0000e+00\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 187s 4s/step - loss: 3.0000 - accuracy: 0.7080 - recall: 0.6880 - val_loss: 1.9641 - val_accuracy: 0.5042 - val_recall: 0.0000e+00\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 184s 4s/step - loss: 2.2222 - accuracy: 0.7415 - recall: 0.7429 - val_loss: 0.9555 - val_accuracy: 0.5042 - val_recall: 0.0000e+00\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 184s 4s/step - loss: 1.2783 - accuracy: 0.7719 - recall: 0.7786 - val_loss: 0.6944 - val_accuracy: 0.5025 - val_recall: 0.0340\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 172s 4s/step - loss: 2.1308 - accuracy: 0.7615 - recall: 0.7756 - val_loss: 0.8881 - val_accuracy: 0.5042 - val_recall: 0.0000e+00\n",
      "Total Time Elapsed:  15  minutes  34  seconds\n"
     ]
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(96, 3, activation = 'relu', input_shape=(256,256,3)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(256, activation = 'relu'))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.Adam(),\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "\n",
    "start = timer()\n",
    "\n",
    "history = model.fit(train_g, \n",
    "                    epochs=5, \n",
    "                    validation_data=val_g, \n",
    "                    workers = 7\n",
    "                   )\n",
    "\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 25s 843ms/step - loss: 1.1091 - accuracy: 0.3200 - recall: 0.0077\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1x1C1D BatchNorm\n",
    "\n",
    "Try Batch Normalization with RMSprop optimizer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 170s 4s/step - loss: 28.2369 - accuracy: 0.6508 - recall: 0.4889 - val_loss: 0.5945 - val_accuracy: 0.6442 - val_recall: 0.9082\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 182s 4s/step - loss: 1.0080 - accuracy: 0.6263 - recall: 0.3730 - val_loss: 0.6534 - val_accuracy: 0.5312 - val_recall: 0.9286\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 173s 4s/step - loss: 2.7178 - accuracy: 0.7288 - recall: 0.6969 - val_loss: 0.8279 - val_accuracy: 0.4992 - val_recall: 0.0204\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 174s 4s/step - loss: 0.5921 - accuracy: 0.7429 - recall: 0.6152 - val_loss: 2.9566 - val_accuracy: 0.5042 - val_recall: 0.0000e+00\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 172s 4s/step - loss: 0.7711 - accuracy: 0.7511 - recall: 0.6478 - val_loss: 12.3006 - val_accuracy: 0.5042 - val_recall: 0.0000e+00\n",
      "Total Time Elapsed:  14  minutes  53  seconds\n"
     ]
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(96, 3, activation = 'relu', input_shape=(256,256,3)))\n",
    "model.add(BatchNormalization())\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(BatchNormalization())\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(256, activation = 'relu'))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.RMSprop(),\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "\n",
    "start = timer()\n",
    "\n",
    "history = model.fit(train_g, \n",
    "                    epochs=5, \n",
    "                    validation_data=val_g, \n",
    "                    workers = 7\n",
    "                   )\n",
    "\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 22s 718ms/step - loss: 17.4486 - accuracy: 0.3157 - recall: 0.0000e+00\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4x1C1D\n",
    "\n",
    "Try even deeper structure with 4 individual layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 108s 3s/step - loss: 0.6059 - accuracy: 0.7021 - val_loss: 0.5018 - val_accuracy: 0.7622\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 109s 3s/step - loss: 0.5281 - accuracy: 0.7637 - val_loss: 0.5465 - val_accuracy: 0.7622\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 111s 3s/step - loss: 0.5134 - accuracy: 0.7652 - val_loss: 0.4828 - val_accuracy: 0.7622\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 110s 3s/step - loss: 0.5010 - accuracy: 0.7741 - val_loss: 0.4880 - val_accuracy: 0.7825\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 111s 3s/step - loss: 0.4877 - accuracy: 0.7838 - val_loss: 0.5114 - val_accuracy: 0.7673\n",
      "Total Time Elapsed:  9  minutes  23  seconds\n"
     ]
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(96, 3, activation = 'relu', input_shape=(256,256,3)))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(256, activation = 'relu'))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.RMSprop(),\n",
    "              metrics=['accuracy'])\n",
    "\n",
    "start = timer()\n",
    "\n",
    "history = model.fit(train_g, \n",
    "                    epochs=5, \n",
    "                    validation_data=val_g, \n",
    "                    workers = 7\n",
    "                   )\n",
    "\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 20s 679ms/step - loss: 0.7397 - accuracy: 0.4995\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3x1C3D\n",
    "\n",
    "Try deeper Dense layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 123s 3s/step - loss: 0.6837 - accuracy: 0.7021 - recall: 0.6538 - val_loss: 0.5049 - val_accuracy: 0.7572 - val_recall: 0.8401\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 119s 3s/step - loss: 0.5155 - accuracy: 0.7615 - recall: 0.7221 - val_loss: 0.6927 - val_accuracy: 0.7066 - val_recall: 0.9320\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 124s 3s/step - loss: 0.5002 - accuracy: 0.7712 - recall: 0.7311 - val_loss: 0.5026 - val_accuracy: 0.7622 - val_recall: 0.7721\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 123s 3s/step - loss: 0.4865 - accuracy: 0.7845 - recall: 0.7563 - val_loss: 0.4912 - val_accuracy: 0.7926 - val_recall: 0.7551\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 122s 3s/step - loss: 0.4648 - accuracy: 0.7905 - recall: 0.7637 - val_loss: 0.5063 - val_accuracy: 0.7943 - val_recall: 0.7483\n",
      "Total Time Elapsed:  10  minutes  27  seconds\n"
     ]
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(96, 3, activation = 'relu', input_shape=(256,256,3)))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(128, activation = 'relu'))\n",
    "model.add(layers.Dense(64, activation = 'relu'))\n",
    "model.add(layers.Dense(32, activation = 'relu'))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.RMSprop(),\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "\n",
    "start = timer()\n",
    "\n",
    "history = model.fit(train_g, \n",
    "                    epochs=5, \n",
    "                    validation_data=val_g, \n",
    "                    workers = 7\n",
    "                   )\n",
    "\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 16s 549ms/step - loss: 1.3665 - accuracy: 0.4678 - recall: 0.2500\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2C3D\n",
    "\n",
    "Try deeper Dense layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 106s 2s/step - loss: 0.6466 - accuracy: 0.7266 - recall: 0.6508 - val_loss: 0.5084 - val_accuracy: 0.7589 - val_recall: 0.7449\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 107s 2s/step - loss: 0.5236 - accuracy: 0.7645 - recall: 0.7251 - val_loss: 0.4848 - val_accuracy: 0.7723 - val_recall: 0.7653\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 109s 3s/step - loss: 0.5049 - accuracy: 0.7808 - recall: 0.7533 - val_loss: 0.5145 - val_accuracy: 0.7774 - val_recall: 0.8061\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 107s 2s/step - loss: 0.4295 - accuracy: 0.8143 - recall: 0.7920 - val_loss: 0.5047 - val_accuracy: 0.7791 - val_recall: 0.7789\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 107s 2s/step - loss: 0.3768 - accuracy: 0.8284 - recall: 0.8172 - val_loss: 0.5594 - val_accuracy: 0.7336 - val_recall: 0.8707\n",
      "Total Time Elapsed:  9  minutes  10  seconds\n"
     ]
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(96, 3, activation = 'relu', input_shape=(256,256,3)))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(128, activation = 'relu'))\n",
    "model.add(layers.Dense(64, activation = 'relu'))\n",
    "model.add(layers.Dense(32, activation = 'relu'))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.RMSprop(),\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "\n",
    "start = timer()\n",
    "\n",
    "history = model.fit(train_g, \n",
    "                    epochs=5, \n",
    "                    validation_data=val_g, \n",
    "                    workers = 7\n",
    "                   )\n",
    "\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 18s 593ms/step - loss: 0.9085 - accuracy: 0.5438 - recall: 0.4738\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2x2C1D Dropout\n",
    "\n",
    "Try adding Dropout to combat overfitting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/8\n",
      "43/43 [==============================] - 237s 6s/step - loss: 0.8407 - accuracy: 0.6538 - recall: 0.5513 - val_loss: 0.5106 - val_accuracy: 0.7538 - val_recall: 0.7789\n",
      "Epoch 2/8\n",
      "43/43 [==============================] - 235s 5s/step - loss: 0.5510 - accuracy: 0.7511 - recall: 0.7028 - val_loss: 0.4977 - val_accuracy: 0.7656 - val_recall: 0.7755\n",
      "Epoch 3/8\n",
      "43/43 [==============================] - 236s 5s/step - loss: 0.5303 - accuracy: 0.7519 - recall: 0.7177 - val_loss: 1.6961 - val_accuracy: 0.4975 - val_recall: 1.0000\n",
      "Epoch 4/8\n",
      "43/43 [==============================] - 236s 5s/step - loss: 0.5350 - accuracy: 0.7697 - recall: 0.7325 - val_loss: 0.5011 - val_accuracy: 0.7707 - val_recall: 0.7347\n",
      "Epoch 5/8\n",
      "43/43 [==============================] - 236s 5s/step - loss: 0.4736 - accuracy: 0.7675 - recall: 0.7236 - val_loss: 0.5086 - val_accuracy: 0.7740 - val_recall: 0.7823\n",
      "Epoch 6/8\n",
      "43/43 [==============================] - 236s 5s/step - loss: 0.4471 - accuracy: 0.8068 - recall: 0.7712 - val_loss: 0.5346 - val_accuracy: 0.7572 - val_recall: 0.8265\n",
      "Epoch 7/8\n",
      "43/43 [==============================] - 236s 5s/step - loss: 0.3991 - accuracy: 0.8120 - recall: 0.7890 - val_loss: 0.5512 - val_accuracy: 0.7589 - val_recall: 0.6871\n",
      "Epoch 8/8\n",
      "43/43 [==============================] - 237s 6s/step - loss: 0.3693 - accuracy: 0.8276 - recall: 0.7979 - val_loss: 0.6107 - val_accuracy: 0.7167 - val_recall: 0.8571\n",
      "Total Time Elapsed:  32  minutes  14  seconds\n"
     ]
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu', input_shape=(256,256,3)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Dropout(0.25))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Dropout(0.25))\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(128, activation = 'relu'))\n",
    "model.add(layers.Dropout(0.25))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.RMSprop(),\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "\n",
    "start = timer()\n",
    "\n",
    "history = model.fit(train_g, \n",
    "                    epochs=8, \n",
    "                    validation_data=val_g, \n",
    "                    workers = 7\n",
    "                   )\n",
    "\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19/19 [==============================] - 18s 974ms/step - loss: 0.7373 - accuracy: 0.5767 - recall: 0.9592\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2x1x1C1D Dropout\n",
    "\n",
    "Add Dropout to model to combat overfitting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "43/43 [==============================] - 235s 5s/step - loss: 0.6728 - accuracy: 0.6768 - recall: 0.6062 - val_loss: 0.5184 - val_accuracy: 0.7555 - val_recall: 0.8027\n",
      "Epoch 2/10\n",
      "43/43 [==============================] - 240s 6s/step - loss: 0.5272 - accuracy: 0.7533 - recall: 0.6954 - val_loss: 0.5191 - val_accuracy: 0.7555 - val_recall: 0.8469\n",
      "Epoch 3/10\n",
      "43/43 [==============================] - 236s 5s/step - loss: 0.5182 - accuracy: 0.7667 - recall: 0.7177 - val_loss: 0.5123 - val_accuracy: 0.7639 - val_recall: 0.8163\n",
      "Epoch 4/10\n",
      "43/43 [==============================] - 234s 5s/step - loss: 0.4931 - accuracy: 0.7630 - recall: 0.7385 - val_loss: 0.5189 - val_accuracy: 0.7656 - val_recall: 0.7891\n",
      "Epoch 5/10\n",
      "43/43 [==============================] - 234s 5s/step - loss: 0.4833 - accuracy: 0.7749 - recall: 0.7340 - val_loss: 0.5025 - val_accuracy: 0.7589 - val_recall: 0.7823\n",
      "Epoch 6/10\n",
      "43/43 [==============================] - 230s 5s/step - loss: 0.4695 - accuracy: 0.7853 - recall: 0.7489 - val_loss: 0.4947 - val_accuracy: 0.7723 - val_recall: 0.7619\n",
      "Epoch 7/10\n",
      "43/43 [==============================] - 230s 5s/step - loss: 0.4577 - accuracy: 0.7957 - recall: 0.7578 - val_loss: 0.5084 - val_accuracy: 0.7673 - val_recall: 0.7823\n",
      "Epoch 8/10\n",
      "43/43 [==============================] - 229s 5s/step - loss: 0.4448 - accuracy: 0.8061 - recall: 0.7801 - val_loss: 0.5269 - val_accuracy: 0.7487 - val_recall: 0.8435\n",
      "Epoch 9/10\n",
      "43/43 [==============================] - 235s 5s/step - loss: 0.4258 - accuracy: 0.7935 - recall: 0.7712 - val_loss: 0.5514 - val_accuracy: 0.7622 - val_recall: 0.8231\n",
      "Epoch 10/10\n",
      "43/43 [==============================] - 240s 6s/step - loss: 0.4076 - accuracy: 0.8128 - recall: 0.7994 - val_loss: 0.5407 - val_accuracy: 0.7184 - val_recall: 0.8265\n",
      "Total Time Elapsed:  40  minutes  1  seconds\n"
     ]
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu', input_shape=(256,256,3)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Dropout(0.25))\n",
    "\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Dropout(0.25))\n",
    "\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Dropout(0.25))\n",
    "\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(128, activation = 'relu'))\n",
    "model.add(layers.Dropout(0.25))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.RMSprop(),\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "\n",
    "start = timer()\n",
    "\n",
    "history = model.fit(train_g, \n",
    "                    epochs=10, \n",
    "                    validation_data=val_g, \n",
    "                    workers = 7\n",
    "                   )\n",
    "\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 35s 1s/step - loss: 0.7906 - accuracy: 0.5544 - recall: 0.4969\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2x1x1C1D Dropout Adam\n",
    "\n",
    "Try Dropout and Adam optimizer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "43/43 [==============================] - 259s 6s/step - loss: 0.5778 - accuracy: 0.7288 - recall: 0.6597 - val_loss: 0.5040 - val_accuracy: 0.7555 - val_recall: 0.7721\n",
      "Epoch 2/10\n",
      "43/43 [==============================] - 231s 5s/step - loss: 0.5176 - accuracy: 0.7779 - recall: 0.7355 - val_loss: 0.5605 - val_accuracy: 0.7622 - val_recall: 0.8095\n",
      "Epoch 3/10\n",
      "43/43 [==============================] - 225s 5s/step - loss: 0.4981 - accuracy: 0.7697 - recall: 0.7415 - val_loss: 0.5237 - val_accuracy: 0.7622 - val_recall: 0.7823\n",
      "Epoch 4/10\n",
      "43/43 [==============================] - 223s 5s/step - loss: 0.4900 - accuracy: 0.7823 - recall: 0.7474 - val_loss: 0.5216 - val_accuracy: 0.7639 - val_recall: 0.7823\n",
      "Epoch 5/10\n",
      "43/43 [==============================] - 226s 5s/step - loss: 0.4560 - accuracy: 0.7927 - recall: 0.7533 - val_loss: 0.5130 - val_accuracy: 0.7555 - val_recall: 0.8435\n",
      "Epoch 6/10\n",
      "43/43 [==============================] - 224s 5s/step - loss: 0.4495 - accuracy: 0.8031 - recall: 0.7756 - val_loss: 0.4966 - val_accuracy: 0.7740 - val_recall: 0.7959\n",
      "Epoch 7/10\n",
      "43/43 [==============================] - 226s 5s/step - loss: 0.4172 - accuracy: 0.8076 - recall: 0.7697 - val_loss: 0.5090 - val_accuracy: 0.7521 - val_recall: 0.7857\n",
      "Epoch 8/10\n",
      "43/43 [==============================] - 224s 5s/step - loss: 0.4141 - accuracy: 0.8046 - recall: 0.7756 - val_loss: 0.5412 - val_accuracy: 0.7420 - val_recall: 0.8095\n",
      "Epoch 9/10\n",
      "43/43 [==============================] - 226s 5s/step - loss: 0.3973 - accuracy: 0.8210 - recall: 0.7860 - val_loss: 0.5106 - val_accuracy: 0.7622 - val_recall: 0.7347\n",
      "Epoch 10/10\n",
      "43/43 [==============================] - 226s 5s/step - loss: 0.3425 - accuracy: 0.8499 - recall: 0.8321 - val_loss: 0.5257 - val_accuracy: 0.7555 - val_recall: 0.7245\n",
      "Total Time Elapsed:  39  minutes  5  seconds\n"
     ]
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu', input_shape=(256,256,3)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Dropout(0.25))\n",
    "\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Dropout(0.25))\n",
    "\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Dropout(0.25))\n",
    "\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(128, activation = 'relu'))\n",
    "model.add(layers.Dropout(0.25))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.Adam(),\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "\n",
    "start = timer()\n",
    "\n",
    "history = model.fit(train_g, \n",
    "                    epochs=10, \n",
    "                    validation_data=val_g, \n",
    "                    workers = 7\n",
    "                   )\n",
    "\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 35s 1s/step - loss: 1.1240 - accuracy: 0.4625 - recall: 0.2809\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3C1D -- Fewer parameters\n",
    "\n",
    "Tried fewer numbers of filters for Conv2D layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "43/43 [==============================] - 132s 3s/step - loss: 0.8183 - accuracy: 0.7028 - recall: 0.6776 - val_loss: 0.7067 - val_accuracy: 0.7201 - val_recall: 0.9252\n",
      "Epoch 2/5\n",
      "43/43 [==============================] - 131s 3s/step - loss: 0.5054 - accuracy: 0.7741 - recall: 0.7013 - val_loss: 0.5106 - val_accuracy: 0.7521 - val_recall: 0.8673\n",
      "Epoch 3/5\n",
      "43/43 [==============================] - 133s 3s/step - loss: 0.4546 - accuracy: 0.7890 - recall: 0.7593 - val_loss: 0.5212 - val_accuracy: 0.7605 - val_recall: 0.8095\n",
      "Epoch 4/5\n",
      "43/43 [==============================] - 128s 3s/step - loss: 0.3817 - accuracy: 0.8180 - recall: 0.7771 - val_loss: 0.6056 - val_accuracy: 0.7555 - val_recall: 0.8265\n",
      "Epoch 5/5\n",
      "43/43 [==============================] - 130s 3s/step - loss: 0.3137 - accuracy: 0.8700 - recall: 0.8455 - val_loss: 0.6016 - val_accuracy: 0.7504 - val_recall: 0.7755\n",
      "Total Time Elapsed:  11  minutes  9  seconds\n"
     ]
    }
   ],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(32, 3, activation = 'relu', input_shape=(256,256,3)))\n",
    "model.add(layers.Conv2D(32, 3, activation = 'relu'))\n",
    "model.add(layers.Conv2D(32, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(128, activation = 'relu'))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.RMSprop(),\n",
    "              metrics=['accuracy', 'Recall'])\n",
    "\n",
    "start = timer()\n",
    "\n",
    "history = model.fit(train_g, \n",
    "                    epochs=5, \n",
    "                    validation_data=val_g, \n",
    "                    workers = 7\n",
    "                   )\n",
    "\n",
    "end = timer()\n",
    "elapsed = end - start\n",
    "print('Total Time Elapsed: ', int(elapsed//60), ' minutes ', (round(elapsed%60)), ' seconds')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "30/30 [==============================] - 17s 570ms/step - loss: 1.3274 - accuracy: 0.4931 - recall: 0.3441\n"
     ]
    }
   ],
   "source": [
    "scores = model.evaluate(test_g, verbose=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Final Model Summary - 2x2C1D Dropout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = models.Sequential()\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu', input_shape=(256,256,3)))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Dropout(0.25))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.Conv2D(64, 3, activation = 'relu'))\n",
    "model.add(layers.MaxPooling2D((2, 2)))\n",
    "model.add(layers.Dropout(0.25))\n",
    "model.add(layers.Flatten())\n",
    "model.add(layers.Dense(128, activation = 'relu'))\n",
    "model.add(layers.Dropout(0.25))\n",
    "model.add(layers.Dense(1, activation='sigmoid'))\n",
    "\n",
    "model.compile(loss='binary_crossentropy',\n",
    "              optimizer=optimizers.RMSprop(),\n",
    "              metrics=['accuracy', 'Recall'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d (Conv2D)              (None, 254, 254, 64)      1792      \n",
      "_________________________________________________________________\n",
      "conv2d_1 (Conv2D)            (None, 252, 252, 64)      36928     \n",
      "_________________________________________________________________\n",
      "max_pooling2d (MaxPooling2D) (None, 126, 126, 64)      0         \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 126, 126, 64)      0         \n",
      "_________________________________________________________________\n",
      "conv2d_2 (Conv2D)            (None, 124, 124, 64)      36928     \n",
      "_________________________________________________________________\n",
      "conv2d_3 (Conv2D)            (None, 122, 122, 64)      36928     \n",
      "_________________________________________________________________\n",
      "max_pooling2d_1 (MaxPooling2 (None, 61, 61, 64)        0         \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 61, 61, 64)        0         \n",
      "_________________________________________________________________\n",
      "flatten (Flatten)            (None, 238144)            0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 128)               30482560  \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 1)                 129       \n",
      "=================================================================\n",
      "Total params: 30,595,265\n",
      "Trainable params: 30,595,265\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}