ENEL_610_Project_Code.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "ENEL_610_Project_Code.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GBpuTDOFjH7h"
      },
      "source": [
        "# ENEL 610- Biometric Technologies and Systems\n",
        "## Ocular Disease Intelligent Recognition\n",
        "### Normal v/s Cataract v/s Glaucoma v/s Myopia Prediction "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BP62Zzt0jAVE"
      },
      "source": [
        "## Getting the dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RydKVFr7l40E"
      },
      "source": [
        "# Downloading the dataset\n",
        "\n",
        "import os\n",
        "os.environ[\"KAGGLE_CONFIG_DIR\"] = '/content'          # Setting the Environment variable\n",
        "!chmod 600 /content/kaggle.json                     \n",
        "!kaggle datasets download -d andrewmvd/ocular-disease-recognition-odir5k        # API command to directly import the dataset to colab.\n",
        "!unzip *.zip && rm *.zip                              # Unzipping the dataset and simultaneously deleting the zip.  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uRtpgGspjHAP"
      },
      "source": [
        "## Importing necessary libraries, functions "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_y_xfV4PMBUU"
      },
      "source": [
        "import tensorflow as tf\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "sns.set_style(\"darkgrid\")\n",
        "import cv2, random\n",
        "import matplotlib.pyplot as plt\n",
        "import itertools"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bRqfzOXKozHM"
      },
      "source": [
        "from tensorflow.keras.preprocessing.image import ImageDataGenerator, load_img, img_to_array\n",
        "from tensorflow.keras.layers import Flatten, Dense, Input, Conv2D, MaxPool2D, AveragePooling2D, BatchNormalization, Dropout\n",
        "from tensorflow.keras.models import Model, load_model\n",
        "from tensorflow.keras.optimizers import SGD, Adam \n",
        "from tensorflow.keras.callbacks import ModelCheckpoint, EarlyStopping, LearningRateScheduler\n",
        "from tensorflow.keras.utils import to_categorical\n",
        "from tensorflow.keras.metrics import AUC, Precision, Recall\n",
        "from sklearn.metrics import f1_score, confusion_matrix\n",
        "from tensorflow.keras.preprocessing.image import img_to_array\n",
        "from tqdm import tqdm"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mWWwNcx13Tfz"
      },
      "source": [
        "# Importing pre-trained models for transfer learning\n",
        "\n",
        "from tensorflow.keras.applications import VGG19, InceptionV3, Xception"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1lX5NgcZautJ"
      },
      "source": [
        "AUC_value = AUC(curve = 'ROC', name = 'auc_value', multi_label = False) \n",
        "# This metric creates four local variables, true_positives, true_negatives, false_positives and false_negatives that are used to compute the AUC. \n",
        "# Here, receiver operating characteristics at default threshold is being used.\n",
        "# It tells us how much area at the given threshold is the ROC curve covering. Higher the area, better the classifier."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e8gHgV7Izm9t"
      },
      "source": [
        "# Global Variables\n",
        "\n",
        "join = True                                                                     # If true, concatenate the data\n",
        "image_size = 224                                                                # Desired Image Size\n",
        "i_shape = (image_size, image_size, 3)                                           # Input image shape     \n",
        "\n",
        "lr_cnn = 1e-5                                                                   # Learning rate for training\n",
        "lr_ft = 1e-6                                                                    # Learning rate for fine-tuning\n",
        "\n",
        "class_names = ['Normal', 'Cataract', 'Glaucoma', 'Myopia']                      # For this notebook\n",
        "classes = ['Normal', 'Diabetes', 'Glaucoma', 'Cataract', 'Age related Macular Degeneration', 'Hypertension', 'Pathological Myopia', 'Other abnormalities']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j5W53X6RkEz3"
      },
      "source": [
        "## Exploratory Data Analysis"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i26iOBs7p0aC"
      },
      "source": [
        "data = pd.read_csv('/content/full_df.csv')          # Path to full_df.csv\n",
        "print(data.head(3))                                 # Displaying the top three rows and all the columns of the dataset"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yTRkNwX9_WKF"
      },
      "source": [
        "print(data.columns)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "57acduNSDQ2C"
      },
      "source": [
        "print(data.info())\n",
        "\n",
        "# It can be seen that there no null values in the dataset"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a1zj9HyabC18"
      },
      "source": [
        "DATA_PATH = '/content/ODIR-5K/ODIR-5K/data.xlsx'            # Path to data.xlsx\n",
        "main_df = pd.read_excel(DATA_PATH)\n",
        "sample_in_classes = main_df.iloc[:, -8:]\n",
        "all_classes = sample_in_classes.sum() \n",
        "print(all_classes)\n",
        "\n",
        "# The data set is highly imbalanced. \n",
        "# Only 103 images are there for Hypertension (H) while 1000+ images are there Diabetic(D) or Normal(N) class"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "In7DvzKRMBp2"
      },
      "source": [
        "# Displaying some random images from the dataset\n",
        "# Initially the images are in the BGR color format and hence they appear blue.\n",
        "show_images = ['/content/ODIR-5K/ODIR-5K/Training Images/1006_left.jpg', '/content/ODIR-5K/ODIR-5K/Training Images/0_right.jpg', '/content/ODIR-5K/ODIR-5K/Training Images/1537_left.jpg', '/content/ODIR-5K/ODIR-5K/Training Images/1613_right.jpg']\n",
        "count = 0\n",
        "\n",
        "plt.figure(figsize = (10, 5))\n",
        "for iter in show_images:\n",
        "    im = cv2.imread(iter)\n",
        "    im = cv2.cvtColor(im, cv2.COLOR_BGR2RGB)\n",
        "    plt.subplot(2, 2, 1 + count)\n",
        "    plt.imshow(im)\n",
        "    plt.axis('off')\n",
        "    count += 1\n",
        "plt.tight_layout()\n",
        "\n",
        "# It can be observed that the images are of different sizes."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8BC9VYuhdjBq"
      },
      "source": [
        "# Visualizing the dataset\n",
        "plt.figure(figsize = (10, 6))\n",
        "plt.pie(all_classes, labels = classes, startangle = 90, autopct='%1.1f%%', shadow = True)\n",
        "plt.axis('equal')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2UnVfGuKSpJe"
      },
      "source": [
        "# Function to plot the countplot\n",
        "\n",
        "def cplot(variable, data_f, hue = None):\n",
        "    sns.countplot(x = data_f[variable], hue = hue, palette = 'Paired')\n",
        "    plt.title(\"{} distribution\".format(variable))\n",
        "    plt.tight_layout()\n",
        "    plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_d5Zm1wBDnNq"
      },
      "source": [
        "plt.figure(figsize = (20, 4))\n",
        "cplot('Patient Age', data)                                          # It can be observed that most of the patients are fairly old"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Iu-w3pBrENci"
      },
      "source": [
        "cplot('Patient Sex', data)                                          # More number of male patients than female patients"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X4DA1hq77r5t"
      },
      "source": [
        "abbrevated = ['N', 'D', 'G', 'C', 'A', 'H', 'M', 'O']\n",
        "for plot in range(8):\n",
        "    plt.figure(figsize = (8, 4))\n",
        "    cplot('Patient Sex', data, hue = data[abbrevated[plot]])\n",
        "    plt.show() \n",
        "\n",
        "# 0- Disease not present, 1- Disease present"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "98ouGQuvc2Yr"
      },
      "source": [
        "# Plotting the correlation matrix to check if a patient is suffering from two or more than two diseases at a time.\n",
        "# For example: if we consider first row, it can be observed that 15% patient suffering from glaucoma are also Diabetic.\n",
        "\n",
        "columns = main_df.iloc[:,-8:]\n",
        "normalize = columns.sum()                                           # Finding the sum of each column\n",
        "correlation = columns.T.dot(columns)                                # Correlation = C.(C)^T; (C)^T is the transpose is the transpose of C, and '.' indicates a dot product\n",
        "correlation_2 = correlation / normalize                             # Normalizing the values to be in between 0 and 1; 0- No correlation, 1- 100% correlated\n",
        "correlation_2.style.background_gradient().set_precision(3)          # Rounding up to 3 decimal points."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OVhlaYVODRf7"
      },
      "source": [
        "## Dataset Creation\n",
        "### Making a separate dataset having images from cataract and normal class."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "j1FYScHyrTcW"
      },
      "source": [
        "# Defining a function to find a keyword in the text\n",
        "\n",
        "def if_keyword(text, keyword):\n",
        "    if keyword in text:\n",
        "        return 1\n",
        "    else:\n",
        "        return 0"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wUuD2_Dpsgul"
      },
      "source": [
        "# Finding the keyword in left and right diagnostic keywords.\n",
        "\n",
        "data[\"left_cataract\"] = data[\"Left-Diagnostic Keywords\"].apply(lambda x: if_keyword(x, 'cataract'))\n",
        "data[\"right_cataract\"] = data[\"Right-Diagnostic Keywords\"].apply(lambda x: if_keyword(x, 'cataract'))\n",
        "\n",
        "data['left_normal'] = data['Left-Diagnostic Keywords'].apply(lambda x: if_keyword(x, 'normal fundus'))\n",
        "data['right_normal'] = data['Right-Diagnostic Keywords'].apply(lambda x: if_keyword(x, 'normal fundus'))\n",
        "\n",
        "data['left_glaucoma'] = data['Left-Diagnostic Keywords'].apply(lambda x: if_keyword(x, 'glaucoma'))\n",
        "data['right_glaucoma'] = data['Right-Diagnostic Keywords'].apply(lambda x: if_keyword(x, 'glaucoma'))\n",
        "\n",
        "data['left_myopia'] = data['Left-Diagnostic Keywords'].apply(lambda x: if_keyword(x, 'pathological myopia'))\n",
        "data['right_myopia'] = data['Right-Diagnostic Keywords'].apply(lambda x: if_keyword(x, 'pathological myopia'))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UrV15KL-uBtx"
      },
      "source": [
        "# Images having 'cataract' associated with their Diagnostic Keywords\n",
        "\n",
        "left_cataract = data.loc[(data.C == 1) & (data.left_cataract == 1)][\"Left-Fundus\"].values\n",
        "right_cataract = data.loc[(data.C == 1) & (data.right_cataract == 1)][\"Right-Fundus\"].values\n",
        "\n",
        "# 300-Images having 'normal' associated with their Diagnostic Keywords\n",
        "# All or more images can be taken into consideration by removing or changing the sample function \n",
        "\n",
        "left_normal = data.loc[(data.N == 1) & (data.left_normal == 1)][\"Left-Fundus\"].sample(300, random_state = 11).values\n",
        "right_normal = data.loc[(data.N == 1) & (data.right_normal == 1)][\"Right-Fundus\"].sample(300, random_state = 11).values\n",
        "\n",
        "left_glaucoma = data.loc[(data.G == 1) & (data.left_glaucoma == 1)][\"Left-Fundus\"].values\n",
        "right_glaucoma = data.loc[(data.G == 1) & (data.right_glaucoma == 1)][\"Right-Fundus\"].values\n",
        "\n",
        "left_myopia = data.loc[(data.M == 1) & (data.left_myopia == 1)][\"Left-Fundus\"].values\n",
        "right_myopia = data.loc[(data.M == 1) & (data.right_myopia == 1)][\"Right-Fundus\"].values"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d68W7MZnx8E-"
      },
      "source": [
        "# Joining both the arrays (left eye's and right eye's diagnosys) into one single arrays for each of the classes.\n",
        "\n",
        "if join:\n",
        "    cataract = np.concatenate((left_cataract, right_cataract), axis = 0)\n",
        "    normal = np.concatenate((left_normal, right_normal), axis = 0)\n",
        "    glaucoma = np.concatenate((left_glaucoma, right_glaucoma), axis = 0)\n",
        "    myopia = np.concatenate((left_myopia, right_myopia), axis = 0)\n",
        "    join = False\n",
        "\n",
        "print(f'Cataract: {len(cataract)} \\t Normal: {len(normal)} \\t Glaucoma: {len(glaucoma)} \\t Myopia: {len(myopia)}')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ntpq3qLBAJxw"
      },
      "source": [
        "# Visualizing the dataset\n",
        "our_classes = [600, 594, 616, 457]\n",
        "plt.figure(figsize = (10, 6))\n",
        "plt.pie(our_classes, labels = class_names, startangle = 90, autopct='%1.1f%%', shadow = True)\n",
        "plt.axis('equal')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XbCw0k4_dTeW"
      },
      "source": [
        "def CLAHE(img, clipLimit, tileGridSize):\n",
        "  clahe = cv2.createCLAHE(clipLimit = clipLimit, tileGridSize = tileGridSize)\n",
        "  lab = cv2.cvtColor(img, cv2.COLOR_BGR2LAB)                    # convert from BGR to LAB color space\n",
        "  l, a, b = cv2.split(lab)                                      # split on 3 different channels\n",
        "  l2 = clahe.apply(l)                                           # apply CLAHE to the L-channel\n",
        "  lab = cv2.merge((l2,a,b))                                     # merge channels\n",
        "  img = cv2.cvtColor(lab, cv2.COLOR_LAB2BGR)                    # convert from LAB to BGR\n",
        "  img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)\n",
        "  return img"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mDIc6fLozqqa"
      },
      "source": [
        "# Creating one single dataset by combing the images from normal, cataract, glaucoma and myopia diagnosys\n",
        "\n",
        "dataset_dir = '/content/preprocessed_images'                    # Pre-processed by the author (cropping, centralization and color space- BGR)\n",
        "dataset = []\n",
        "def create_dataset(image_category, label):\n",
        "    for img in tqdm(image_category):                            # Showing the progress bar\n",
        "        image_path = os.path.join(dataset_dir, img)\n",
        "        try:\n",
        "            image = cv2.imread(image_path, cv2.IMREAD_COLOR)    # cv2.IMREAD_COLOR loads a color image.\n",
        "            image = cv2.resize(image, (image_size, image_size)) # Resizing the image to (224, 224) - same as default input to the VGG19 model\n",
        "        except:\n",
        "            continue\n",
        "        image = CLAHE(image, 20, (10, 10))                      # Applying CLAHE processing\n",
        "        dataset.append([np.array(image), np.array(label)])\n",
        "    random.shuffle(dataset)\n",
        "    return dataset"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vYS2_7Pw03C3"
      },
      "source": [
        "data_set1 = create_dataset(normal, 0)\n",
        "data_set1 = create_dataset(cataract, 1)\n",
        "data_set1 = create_dataset(glaucoma, 2)\n",
        "data_set1 = create_dataset(myopia, 3)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3gVjQG3hyeDz"
      },
      "source": [
        "# For Cataract v/s Normal v/s Glaucoma v/s Myopia prediction we have thus created a mini-dataset from the original dataset\n",
        "print(len(data_set1))        "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Kf-sJKTF1Sxl"
      },
      "source": [
        "# Displaying randomly selected eight images from the dataset\n",
        "\n",
        "plt.figure(figsize = (10, 5))\n",
        "for iter in range(8):\n",
        "    im = random.choice(range(len(dataset)))\n",
        "    image = dataset[im][0]\n",
        "    category = dataset[im][1]\n",
        "    plt.subplot(2, 4, 1 + iter)\n",
        "    plt.imshow(image)\n",
        "    plt.axis('off')\n",
        "    plt.title(\"Label: %s\" %class_names[category])\n",
        "plt.tight_layout()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4gKf8o-c11lo"
      },
      "source": [
        "# Splitting the dataset into data and associated labels\n",
        "\n",
        "x_dev = np.array([i[0] for i in dataset]).reshape(-1, image_size, image_size, 3)\n",
        "y_dev = np.array([i[1] for i in dataset])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ISIA-cHCnIWR"
      },
      "source": [
        "## Train, Validation, Test - Split"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1Nxq4v4f4jMu"
      },
      "source": [
        "# Shuffling the samples\n",
        "\n",
        "indexes = np.arange(x_dev.shape[0])\n",
        "np.random.shuffle(indexes)\n",
        "X_dev = x_dev[indexes,:]\n",
        "Y_dev = y_dev[indexes]\n",
        "\n",
        "# Then, we split our data into train/val/test sets\n",
        "train_split = np.int(0.6 * Y_dev.size)          \n",
        "val_split = np.int(0.9 * Y_dev.size)          \n",
        "\n",
        "X_train = X_dev[: train_split, : ]              # 60% of the data for training\n",
        "Y_train = Y_dev[: train_split]\n",
        "\n",
        "X_val = X_dev[train_split : val_split, : ]      # 30% for validation\n",
        "Y_val = Y_dev[train_split : val_split]\n",
        "\n",
        "X_test = X_dev[val_split: , :]                  # 10% for testing\n",
        "Y_test = Y_dev[val_split : ]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-4s30Eraf2MQ"
      },
      "source": [
        "print(f'X_train: {X_train.shape} \\tX_val: {X_val.shape} \\tX_test: {X_test.shape}')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2bFDd11GncM9"
      },
      "source": [
        "## Data Scaling"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sr4-QlSHIVdc"
      },
      "source": [
        "# Min-Max Normalization\n",
        "Train_min = X_train.min()\n",
        "Train_max = X_train.max()\n",
        "\n",
        "X_train = (X_train - Train_min)/(Train_max - Train_min)\n",
        "X_val = (X_val - Train_min)/(Train_max - Train_min)\n",
        "X_test = (X_test - Train_min)/(Train_max - Train_min)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TqhogzimnoWe"
      },
      "source": [
        "## One Hot Encoding"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ht3CxnhBNytd"
      },
      "source": [
        "# Representing the labels as one-hot encoded\n",
        "\n",
        "Y_train_oh = to_categorical(Y_train)\n",
        "Y_val_oh = to_categorical(Y_val)\n",
        "Y_test_oh = to_categorical(Y_test)\n",
        "\n",
        "print(f'Labels: {Y_train[:2]}')\n",
        "print(f'One hot encoded labels: \\n{Y_train_oh[:2]}' )"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sKaNXTF3oQbt"
      },
      "source": [
        "## Data Augmentation"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RaDa5IwQKFwm"
      },
      "source": [
        "# Data Augmentation is used to feed the model with minor changes. This technique is very useful when\n",
        "# we have a dataset which is very imbalannced and/or small in size.\n",
        "# Here, we are using techniques such as rotating the images by 15 degrees in a random fashion, flipping the images horizontally. \n",
        "# At a time, a batch of 32 images are fed. \n",
        "\n",
        "batch_size = 32\n",
        "gen_params = {\"rotation_range\":15, \"horizontal_flip\":True, \"fill_mode\":'constant', 'cval':0 }\n",
        "train_gen = ImageDataGenerator(**gen_params)\n",
        "val_gen = ImageDataGenerator(**gen_params)\n",
        "\n",
        "train_gen.fit(X_train, seed = 1)\n",
        "val_gen.fit(X_val, seed = 1)\n",
        "\n",
        "train_flow = train_gen.flow(X_train, Y_train_oh, batch_size = batch_size)\n",
        "val_flow = val_gen.flow(X_val, Y_val_oh, batch_size = batch_size) "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aFzmCzvyK5ii"
      },
      "source": [
        "# The keras ImageDataGenerator returns a generator and thus we have to use getitem().\n",
        "\n",
        "plt.figure(figsize = (14, 6))\n",
        "X_batch, Y_batch = train_flow.__getitem__(0)\n",
        "print(f'Minimum pixel value: {X_batch.min()} \\t Maximum pixel value: {X_batch.max()}')\n",
        "for i in range(8):\n",
        "    plt.subplot(2, 4, i+1)\n",
        "    plt.imshow(X_batch[i])\n",
        "    plt.title(\"Label: %s\" %class_names[int(Y_batch[i].argmax())])\n",
        "    plt.axis('off')\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CvIEBAMyoaJr"
      },
      "source": [
        "## Defining the models"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "goIe5dGQi2dj"
      },
      "source": [
        "# Here we define our experimental model. This model was finalized after doing a lot of manipulations.\n",
        "# Starting with the learning rate to the number of neurons, kernel size, number of kernels everything was changed.\n",
        "# We used Average Pooling, Max Pooling, batch normalization- no batch normalization, with and without dropouts\n",
        "# and find out this model was performing better if not the best.\n",
        "\n",
        "def our_cnn(lr = lr_cnn):\n",
        "  print(f'\\nYou chose our cnn model. \\n')\n",
        "\n",
        "  inputs = Input(shape = i_shape)\n",
        "  c1 = Conv2D(256, (3, 3), activation = 'relu', padding = 'same')(inputs)       # Padding is used to keep the dimensions same before and after convolution\n",
        "  c2 = Conv2D(256, (3, 3), activation = 'relu', padding = 'same')(c1)\n",
        "  mp1 = MaxPool2D(2, 2)(c2)\n",
        "  bn1 = BatchNormalization()(mp1)\n",
        "  dp1 = Dropout(0.3)(bn1)\n",
        "\n",
        "  c3 = Conv2D(512, (3, 3), activation = 'relu', padding = 'same')(dp1)\n",
        "  c4 = Conv2D(512, (3, 3), activation = 'relu', padding = 'same')(c3)\n",
        "  mp2 = MaxPool2D(2, 2)(c4)\n",
        "  bn2 = BatchNormalization()(mp2)\n",
        "  dp2 = Dropout(0.25)(bn2)\n",
        "\n",
        "  c5 = Conv2D(1024, (3, 3), activation = 'relu', padding = 'same')(dp2)\n",
        "  c6 = Conv2D(1024, (3, 3), activation = 'relu', padding = 'same')(c5)\n",
        "  mp3 = MaxPool2D(2, 2)(c6)\n",
        "  bn3 = BatchNormalization()(mp3)\n",
        "  dp3 = Dropout(0.25)(bn3)\n",
        "\n",
        "  c7 = Conv2D(1024, (3, 3), activation = 'relu', padding = 'same')(dp3)\n",
        "  c8 = Conv2D(1024, (3, 3), activation = 'relu', padding = 'same')(c7)\n",
        "  mp4 = MaxPool2D(2, 2)(c8)\n",
        "  bn4 = BatchNormalization()(mp4)\n",
        "  dp4 = Dropout(0.25)(bn4)\n",
        "\n",
        "  c9 = Conv2D(2048, (3, 3), activation = 'relu', padding = 'same')(dp4)\n",
        "  c10 = Conv2D(2048, (3, 3), activation = 'relu', padding = 'same')(c9)\n",
        "  mp5 = MaxPool2D(2, 2)(c10)\n",
        "  bn5 = BatchNormalization()(mp5)\n",
        "  dp5 = Dropout(0.3)(bn5)\n",
        "\n",
        "  flat = Flatten()(dp5)                                                         # Since Dense layers or fully connected layers require input in a single dimension, flatten was used.\n",
        "  d1 = Dense(1024, activation = 'relu')(flat)\n",
        "  d2 = Dense(512, activation = 'relu')(d1)\n",
        "  d3 = Dense(128, activation = 'relu')(d2)\n",
        "  \n",
        "  out = Dense(4, activation = 'softmax')(d3)                                    # Softmax outputs each class as an equivalent of probability. 4 classes each corresponding to a single disease\n",
        "\n",
        "  our_model = Model(inputs = inputs, outputs = out)                             # Finalizing the model\n",
        "  our_model.compile(optimizer = SGD(learning_rate = lr), loss = 'categorical_crossentropy', metrics = ['accuracy', Precision(), Recall(), AUC_value])\n",
        "  # Model is compiled with loss as categorical crossentropy, optimizer being stochastic gradient descent with initial learing rate = 1e-5, and four metrics to monitor the training.\n",
        "    \n",
        "  return our_model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CNKoLZK4aTlt"
      },
      "source": [
        "def vgg_model(train_l = False, lr = lr_cnn):\n",
        "    print(f'\\nYou chose VGG19. \\n')\n",
        "        \n",
        "    vgg = VGG19(weights = \"imagenet\", include_top = False, input_shape = (image_size, image_size, 3))     # Importing the model with the weights achieved on Imagenet dataset\n",
        "    # Classification network/fully connected layers are not imported. \n",
        "    vgg.trainable = train_l                                                     # Freezing the trainable paramters\n",
        "    input_image = Input(shape = i_shape)\n",
        "    x1 = vgg(input_image, training = False)\n",
        "    x2 = Flatten()(x1)\n",
        "    x3 = Dense(256, activation = 'relu')\n",
        "    out = Dense(4, activation = 'softmax')(x2)\n",
        "        \n",
        "    vgg_model = Model(inputs = input_image, outputs = out)\n",
        "    vgg_model.compile(optimizer = Adam(learning_rate = lr), loss = 'categorical_crossentropy', metrics = ['accuracy', Precision(), Recall(), AUC_value])   \n",
        "    return vgg_model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-qL69Z7ejZdK"
      },
      "source": [
        "def inceptionV3(train_l = False, lr = lr_cnn):\n",
        "  print(f'\\nYou chose InceptionV3. \\n')\n",
        "  # Importing/Calling inception from keras applications\n",
        "\n",
        "  inception = InceptionV3(weights = 'imagenet', include_top = False, input_shape = i_shape)\n",
        "  inception.trainable = train_l\n",
        "\n",
        "  input_image = Input(shape = i_shape)\n",
        "  x1 = inception(input_image, training = False)\n",
        "  x2 = Flatten()(x1)\n",
        "  x3 = Dense(256, activation = 'relu')(x2)\n",
        "  out = Dense(4, activation = 'softmax')(x3)\n",
        "\n",
        "  inception_model = Model(inputs = input_image, outputs = out)\n",
        "  inception_model.compile(optimizer = Adam(learning_rate = lr), loss = 'categorical_crossentropy', metrics = ['accuracy', Precision(), Recall(), AUC_value])\n",
        "  \n",
        "  return inception_model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "twJMPfywjhG4"
      },
      "source": [
        "def xception(train_l = False, lr = lr_cnn):\n",
        "  print(f'\\nYou chose Xception. \\n')\n",
        "  # Here, we use the improved version of Inception model on our dataset.\n",
        "  \n",
        "  xception = Xception(weights = 'imagenet', include_top = False, input_shape = i_shape)\n",
        "  xception.trainable = train_l\n",
        "\n",
        "  input_image = Input(shape = i_shape)\n",
        "  x1 = xception(input_image, training = False)\n",
        "  x2 = Flatten()(x1)\n",
        "  x3 = Dense(256, activation = 'relu')(x2)\n",
        "  out = Dense(4, activation = 'softmax')(x3)\n",
        "\n",
        "  xception_model = Model(inputs = input_image, outputs = out)\n",
        "  xception_model.compile(optimizer = Adam(learning_rate = lr), loss = 'categorical_crossentropy', metrics = ['accuracy', Precision(), Recall(), AUC_value])\n",
        "  \n",
        "  return xception_model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_9f9klmtoiLT"
      },
      "source": [
        "## Calling the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hjy5pEuY_u33"
      },
      "source": [
        "# This function lets you select one model out of all the models.\n",
        "# An input will be asked from the user.\n",
        "\n",
        "def model_s():\n",
        "  model_selection = int(input(f'Choose a model please- \\n1: Our CNN model \\n2: VGG19 model \\n3: InceptionV3 model \\n4: Xception Model \\n'))\n",
        "\n",
        "  if model_selection == 1:\n",
        "    model_name = 'our_cnn.h5'                     # Path to save and load the model at.\n",
        "    model = our_cnn()\n",
        "    \n",
        "    model.summary()\n",
        "    return model, model_name, model_selection\n",
        "\n",
        "  elif model_selection == 2:\n",
        "    model_name = 'vgg19.h5'                       # Path to save and load the vgg19 model at.\n",
        "    model = vgg_model()\n",
        "    \n",
        "    model.summary()\n",
        "    return model, model_name, model_selection\n",
        "\n",
        "  elif model_selection == 3:\n",
        "    model_name = 'inceptionV3.h5'                 # Path to save and load the inceptionV3 model at.\n",
        "    model = inceptionV3()\n",
        "    \n",
        "    model.summary()\n",
        "    return model, model_name, model_selection\n",
        "\n",
        "  elif model_selection == 4:\n",
        "    model_name = 'xception.h5'                    # Path to save and load the xception model at.\n",
        "    model = xception()\n",
        "    \n",
        "    model.summary()                               # This function also prints the summary of the model, displaying the structure, paramteres.\n",
        "    return model, model_name, model_selection\n",
        "\n",
        "  else:\n",
        "    print('Wrong Choice, please choose again')    # In case a wrong input is given, you are again asked to select a proper input.\n",
        "    model_s()\n",
        "\n",
        "model, model_name, model_selection = model_s()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OljYAmjEorly"
      },
      "source": [
        "## Defining the callbacks"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-7RPNQE77Fj5"
      },
      "source": [
        "# These callbacks monitor the training. From changing the learing rate throughout the training process to stopping\n",
        "# the training if the validation loss starts increasing, everything is being controlled by these callbacks.\n",
        "# Here, we save the best model at the path specified by the variable 'model_name'. This saved model can be later\n",
        "# directly used without training for the classifying 4 classes.\n",
        "\n",
        "model_checkpoint = ModelCheckpoint(model_name, monitor = 'val_loss', save_best_only = True, save_weights_only = False, mode = 'min')\n",
        "\n",
        "early_stop = EarlyStopping(monitor = 'val_loss', mode = 'min', restore_best_weights = True, patience = 5, min_delta = 0.0001)\n",
        "\n",
        "def scheduler(epoch, lr):\n",
        "    if (epoch + 1) % 5 == 0 and epoch < 20:\n",
        "        lr /= 2\n",
        "    elif (epoch + 1) % 15 == 0:\n",
        "        lr /= 2\n",
        "    return lr\n",
        "\n",
        "lr_schedule = LearningRateScheduler(scheduler, verbose = 1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ulAICrH-oynV"
      },
      "source": [
        "## Training the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f42yJ9Jf7Vk3"
      },
      "source": [
        "# This cell starts the training of the model and saves the statistics during the training to the variable 'history'.\n",
        "\n",
        "history = model.fit(train_flow, batch_size = 32, epochs = 50, validation_data = (val_flow), verbose = 1, callbacks = [lr_schedule, model_checkpoint, early_stop])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pZ3jNkG0ZdWu"
      },
      "source": [
        "## Fine Tuning the Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lBlQVwQHebL3"
      },
      "source": [
        "# This block of code is used to fine tune the model, i.e. we even update the weights of kernels of convolutional\n",
        "# layers to let them adapt better to our classifier.\n",
        "\n",
        "if model_selection == 1:\n",
        "    model = load_model('our_cnn.h5')\n",
        "elif model_selection == 2:\n",
        "    model = load_model('vgg19.h5')\n",
        "elif model_selection == 3:\n",
        "    model = load_model('inceptionV3.h5')\n",
        "else:\n",
        "    model = load_model('xception.h5')\n",
        "\n",
        "model.trainable = True\n",
        "model.compile(optimizer = Adam(learning_rate = lr_ft), loss = 'categorical_crossentropy', metrics = ['accuracy', Precision(), Recall(), AUC_value])\n",
        "\n",
        "model.summary()\n",
        "\n",
        "# Same callbacks are used for fine-tuning as well, so as to stop the model from over-fitting.\n",
        "tune_history = model.fit(train_flow, batch_size = 32, epochs = 20, validation_data = (val_flow), verbose = 1, callbacks = [lr_schedule, model_checkpoint, early_stop])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eVT0JGnOo8pk"
      },
      "source": [
        "## Testing the best saved model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T4Nnr6wO7t2G"
      },
      "source": [
        "# Evaluating the model\n",
        "      \n",
        "# We use the weights from the best saved model.\n",
        "model.load_weights(model_name)\n",
        "Y_pred = model.predict(X_test)\n",
        "Y_pred = Y_pred.argmax(axis = 1)                       # The maximum value of all the values is chosen for each prediction.\n",
        "\n",
        "loss, accuracy, auc, precision, recall = model.evaluate(X_test, Y_test_oh, verbose = 0)\n",
        "print(f'loss: {loss} \\tAccuracy: {accuracy} \\tAUC_value: {auc} \\tPrecision: {precision} \\tRecall: {recall}')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6fvAn28F_AsA"
      },
      "source": [
        "# Here, F1 score is calculated on the predicted values.\n",
        "\n",
        "F1_score = f1_score(y_true = Y_test, y_pred = Y_pred, average = 'weighted')\n",
        "\n",
        "print(f'F1 score: {F1_score}')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4Un9gTI42P-1"
      },
      "source": [
        "# This function to calculate confusion matrix was taken from StackOverFlow\n",
        "\n",
        "classes=['N', 'C', 'G', 'M']                           # Labels for the confusion matrix\n",
        "true_classes = Y_test\n",
        "print('Confusion Matrix')\n",
        "cm = confusion_matrix(true_classes, Y_pred)            # From sklearn metrics\n",
        "\n",
        "def plot_confusion_matrix(cm, classes, title = 'Confusion matrix', cmap = plt.cm.Oranges):\n",
        "    plt.figure(figsize=(6, 6))\n",
        "    plt.imshow(cm, interpolation = 'nearest', cmap = cmap)\n",
        "    plt.title(title)\n",
        "    tick_marks = np.arange(len(classes))\n",
        "    plt.xticks(tick_marks, classes, rotation = 30)\n",
        "    plt.yticks(tick_marks, classes)\n",
        "\n",
        "    thresh = cm.max() / 2.\n",
        "    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):\n",
        "        plt.text(j, i, cm[i, j], horizontalalignment=\"center\",  color=\"white\" if cm[i, j] > thresh else \"black\")\n",
        "    plt.tight_layout()\n",
        "    plt.ylabel('True label')\n",
        "    plt.xlabel('Predicted label')\n",
        "\n",
        "plot_confusion_matrix(cm, classes)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v-LRcUj13ZZy"
      },
      "source": [
        "# True positive (TP): correct positive prediction\n",
        "# False positive (FP): incorrect positive prediction\n",
        "# True negative (TN): correct negative prediction\n",
        "# False negative (FN): incorrect negative prediction\n",
        "\n",
        "# Calculating the error rates: TPR, TNR, FPR\n",
        "\n",
        "def calculate_tpr_tnr_fpr(y_test, y_pred_test):\n",
        "    actual_pos = y_test == 1\n",
        "    actual_neg = y_test == 0\n",
        "    \n",
        "    true_pos = (y_pred_test == 1) & (actual_pos)\n",
        "    false_pos = (y_pred_test == 1) & (actual_neg)\n",
        "    true_neg = (y_pred_test == 0) & (actual_neg)\n",
        "    false_neg = (y_pred_test == 0) & (actual_pos)\n",
        "    \n",
        "    tpr = np.sum(true_pos) / np.sum(actual_pos)\n",
        "    tnr = np.sum(true_neg) / np.sum(actual_neg)\n",
        "    fpr = np.sum(false_pos) / np.sum(actual_neg)\n",
        "    \n",
        "    return tpr, tnr, fpr\n",
        "\n",
        "tpr, tnr, fpr = calculate_tpr_tnr_fpr(Y_test, Y_pred)\n",
        "print(f'True postive rate, Sensitivity: {tpr}')\n",
        "print(f'True negative rate, Specitivity: {tnr}')\n",
        "print(f'False positive rate: {fpr}')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "okA2_mGb-H_c"
      },
      "source": [
        "# Plotting the graphs for accuracy, loss, auc_value, precision, recall v/s epochs\n",
        "\n",
        "plt.figure(figsize = (20, 15))\n",
        "epochs = np.arange(len(history.history[\"accuracy\"]))\n",
        "\n",
        "# accuracy v/s epochs\n",
        "plt.subplot(3, 2, 1)\n",
        "plt.plot(epochs, history.history[\"accuracy\"])\n",
        "plt.plot(epochs, history.history[\"val_accuracy\"])\n",
        "plt.title(\"Accuracy\")\n",
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Accuracy\")\n",
        "plt.legend([\"Train\", \"Val\"])\n",
        "plt.ylim(0, 1)\n",
        "\n",
        "# loss v/s epochs\n",
        "plt.subplot(3, 2, 2)\n",
        "plt.plot(epochs, history.history[\"loss\"])\n",
        "plt.plot(epochs, history.history[\"val_loss\"])\n",
        "plt.title(\"Loss\")\n",
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.legend([\"Train\", \"Val\"])\n",
        "\n",
        "# auc_value v/s epochs\n",
        "plt.subplot(3, 2, 3)\n",
        "plt.plot(epochs, history.history[\"auc_value\"])\n",
        "plt.plot(epochs, history.history[\"val_auc_value\"])\n",
        "plt.title(\"ROC_AUC_Value\")\n",
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"AUC_Value\")\n",
        "plt.legend([\"Train\", \"Val\"])\n",
        "\n",
        "# precision v/s epochs\n",
        "plt.subplot(3, 2, 4)\n",
        "plt.plot(epochs, history.history[\"precision\"])\n",
        "plt.plot(epochs, history.history[\"val_precision\"])   \n",
        "plt.title(\"Precision\")\n",
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Precision\")\n",
        "plt.legend([\"Train\", \"Val\"])\n",
        "\n",
        "# recall v/s epochs\n",
        "plt.subplot(3, 2, 5)\n",
        "plt.plot(epochs, history.history[\"recall\"])\n",
        "plt.plot(epochs, history.history[\"val_recall\"])\n",
        "plt.title(\"Recall\")\n",
        "plt.xlabel(\"Epochs\")\n",
        "plt.ylabel(\"Recall\")\n",
        "plt.legend([\"Train\", \"Val\"])\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TjJrp6g8-8-p"
      },
      "source": [
        "# Plotting 8 random images which were classified wrongly\n",
        "\n",
        "wrong_indexes = np.where(Y_pred != Y_test)[0]\n",
        "print(f'Number of test images: {Y_test.size}')\n",
        "print(f'Number of wrong predictions: {wrong_indexes.size}')\n",
        "\n",
        "sample_indexes = np.random.choice(np.arange(wrong_indexes.shape[0], dtype = int), size = 8, replace = False)\n",
        "plt.figure(figsize = (10, 5))\n",
        "for (ii,jj) in enumerate(sample_indexes):\n",
        "    plt.subplot(2, 4 , ii+1)\n",
        "    plt.imshow(X_test[wrong_indexes[jj]], cmap = \"gray\")\n",
        "    plt.title(f\"Label: {class_names[Y_test[wrong_indexes[jj]]]} \\n Predicted: {class_names[Y_pred[wrong_indexes[jj]]]}\")\n",
        "    plt.axis('off')\n",
        "plt.tight_layout()\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "G8_d0g9X1SxV"
      },
      "source": [
        "# Plotting 8 random images which were classified rightly\n",
        "\n",
        "right_indexes = np.where(Y_pred == Y_test)[0]\n",
        "print(f'Number of test images: {Y_test.size}')\n",
        "print(f'Number of right predictions: {right_indexes.size}')\n",
        "\n",
        "sample_indexes = np.random.choice(np.arange(right_indexes.shape[0], dtype = int), size = 8, replace = False)\n",
        "plt.figure(figsize = (10, 5))\n",
        "for (ii,jj) in enumerate(sample_indexes):\n",
        "    plt.subplot(2, 4 , ii+1)\n",
        "    plt.imshow(X_test[right_indexes[jj]], cmap = \"gray\")\n",
        "    plt.title(f\"Label: {class_names[Y_test[right_indexes[jj]]]} \\n Predicted: {class_names[Y_pred[right_indexes[jj]]]}\")\n",
        "    plt.axis('off')\n",
        "plt.tight_layout()\n",
        "plt.show()"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}