Elsevier-chapter-code.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#imports\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.colors import ListedColormap\n",
    "import seaborn as sns\n",
    "\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.discriminant_analysis import LinearDiscriminantAnalysis\n",
    "from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis as QDA\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import (confusion_matrix,accuracy_score)\n",
    "from sklearn.preprocessing  import LabelEncoder, OneHotEncoder\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import sklearn.metrics as skm\n",
    "import sklearn.model_selection as skms\n",
    "import sklearn.preprocessing as skp\n",
    "\n",
    "import math\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.67\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/srv/conda/envs/notebook/lib/python3.6/site-packages/sklearn/utils/validation.py:63: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  return f(*args, **kwargs)\n"
     ]
    }
   ],
   "source": [
    "#LinearDiscriminantAnalysis (Modelling)\n",
    "\n",
    "df3 = pd.read_csv(\"features_3_sec.csv\")\n",
    "X = df3.drop(columns = ['filename','length', 'label'] )\n",
    "Y = pd.DataFrame(data = df3.label)\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state = 0)\n",
    "\n",
    "sc = StandardScaler()\n",
    "X_train = sc.fit_transform(X_train)\n",
    "X_test = sc.transform(X_test)\n",
    "\n",
    "lda = LinearDiscriminantAnalysis(n_components = 3,solver = 'eigen')\n",
    "\n",
    "X_lda = lda.fit_transform(X_train, Y_train)\n",
    "Y_pred = lda.predict(X_test)\n",
    "\n",
    "print(round(accuracy_score(Y_test, Y_pred), 2))\n",
    "print(lda.explained_variance_ratio_)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LinearDiscriminantAnalysis (Evaluation)\n",
    "\n",
    "cm = confusion_matrix(Y_test, Y_pred)\n",
    "\n",
    "from matplotlib.colors import ListedColormap\n",
    "ax= plt.subplots(figsize=(15,15))\n",
    "sns.set(font_scale=3.4)\n",
    "with sns.axes_style('white'):\n",
    "    sns.heatmap(cm, cbar=False, square=True,annot=True,fmt='g',\n",
    "                cmap=ListedColormap(['gray']),linewidths=2.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "X_lda2 = lda.fit_transform(X,Y)      #Creating a common dataframe of resolved components and labels\n",
    "Y1 = pd.DataFrame(data = df3.label)\n",
    "LDA_df = np.hstack((X_lda2,Y1))\n",
    "\n",
    "ld1 = np.array(LDA_df[:,0])\n",
    "ld2 = np.array(LDA_df[:,1])\n",
    "label = np.array(LDA_df[:,2])\n",
    "\n",
    "ax= plt.subplots(figsize=(10,10))\n",
    "sns.set(font_scale=1.5)\n",
    "sns.scatterplot(ld1,ld2,hue = label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/srv/conda/envs/notebook/lib/python3.6/site-packages/sklearn/utils/validation.py:63: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().\n",
      "  return f(*args, **kwargs)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.74\n"
     ]
    }
   ],
   "source": [
    "#Quadratic Discriminant Analysis (Modelling)\n",
    "\n",
    "df3 = pd.read_csv(\"features_3_sec.csv\")\n",
    "X = df3.drop(columns = ['filename','length', 'label'] )\n",
    "Y = pd.DataFrame(data = df3.label)\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state = 0)\n",
    "\n",
    "sc = StandardScaler()\n",
    "X_train = sc.fit_transform(X_train)\n",
    "X_test = sc.transform(X_test)\n",
    "\n",
    "qda_model=QDA()\n",
    "qda_model.fit(X_train,Y_train)\n",
    "\n",
    "y_pred= qda_model.predict(X_test)\n",
    "\n",
    "print(round(accuracy_score(Y_test, y_pred), 2))\n",
    "\n",
    " \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Quadratic Discriminant Analysis (Analysis)\n",
    "cm = confusion_matrix(Y_test, y_pred)\n",
    "\n",
    "ax= plt.subplots(figsize=(15,15))\n",
    "sns.set(font_scale=3.4)\n",
    "with sns.axes_style('white'):\n",
    "    sns.heatmap(cm, cbar=False, square=True,annot=True,fmt='g',\n",
    "                cmap=ListedColormap(['gray']),linewidths=2.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/srv/conda/envs/notebook/lib/python3.6/site-packages/sklearn/neighbors/_classification.py:179: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples,), for example using ravel().\n",
      "  return self._fit(X, y)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.875\n"
     ]
    }
   ],
   "source": [
    "#K Nearest Neighbors (Modelling)\n",
    "df3 = pd.read_csv(\"features_3_sec.csv\")\n",
    "X = df3.drop(columns = ['filename','length', 'label'] )\n",
    "Y = pd.DataFrame(data = df3.label)\n",
    "X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state = 0)\n",
    "\n",
    "sc = StandardScaler()\n",
    "X_train = sc.fit_transform(X_train)\n",
    "X_test = sc.transform(X_test)\n",
    "\n",
    "KNN = KNeighborsClassifier(n_neighbors = 3)\n",
    "KNN.fit(X_train, Y_train)\n",
    "Y_pred = KNN.predict(X_test)\n",
    "\n",
    "print(round(accuracy_score(Y_test, Y_pred), 3))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#K Nearest Neighbors (Analysis)\n",
    "\n",
    "cm = confusion_matrix(Y_test, Y_pred)\n",
    "from matplotlib.colors import ListedColormap\n",
    "ax= plt.subplots(figsize=(15,15))\n",
    "sns.set(font_scale=3.4)\n",
    "with sns.axes_style('white'):\n",
    "    sns.heatmap(cm, cbar=False, square=True,annot=True,fmt='g',\n",
    "                cmap=ListedColormap(['gray']),linewidths=2.5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Neural Network\n",
    "# shuffle samples\n",
    "df = pd.read_csv(\"features_3_sec.csv\")\n",
    "\n",
    "# map labels to index\n",
    "label_index = dict()\n",
    "index_label = dict()\n",
    "for i, x in enumerate(df.label.unique()):\n",
    "    label_index[x] = i\n",
    "    index_label[i] = x\n",
    "print(label_index)\n",
    "print(index_label)\n",
    "\n",
    "# update labels in df to index\n",
    "df.label = [label_index[l] for l in df.label]\n",
    "\n",
    "# shuffle samples\n",
    "df_shuffle = df\n",
    "\n",
    "# remove irrelevant columns\n",
    "df_shuffle.drop(['filename', 'length'], axis=1, inplace=True)\n",
    "df_y = df_shuffle.pop('label')\n",
    "df_X = df_shuffle\n",
    "\n",
    "# split into train dev and test\n",
    "X_train, df_test_valid_X, y_train, df_test_valid_y = skms.train_test_split(df_X, df_y, train_size=0.7,  stratify=df_y)\n",
    "X_dev, X_test, y_dev, y_test = skms.train_test_split(df_test_valid_X, df_test_valid_y, train_size=0.66,  stratify=df_test_valid_y)\n",
    "\n",
    "\n",
    "\n",
    "scaler = skp.StandardScaler()\n",
    "X_train = pd.DataFrame(scaler.fit_transform(X_train), columns=X_train.columns)\n",
    "X_dev = pd.DataFrame(scaler.transform(X_dev), columns=X_train.columns)\n",
    "X_test = pd.DataFrame(scaler.transform(X_test), columns=X_train.columns)\n",
    "\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras as k\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras.models import Sequential, load_model\n",
    "\n",
    "from tensorflow.keras.models import Sequential\n",
    "\n",
    "ACCURACY_THRESHOLD = 0.94\n",
    "\n",
    "class myCallback(k.callbacks.Callback):\n",
    "    def on_epoch_end(self, epoch, logs={}):\n",
    "        if(logs.get('val_accuracy') is not None and logs.get('val_accuracy') >= ACCURACY_THRESHOLD):\n",
    "            print(\"\\n\\nStopping training as we have reached %2.2f%% accuracy!\" %(ACCURACY_THRESHOLD*100))   \n",
    "            self.model.stop_training = True\n",
    "\n",
    "def trainModel(model, epochs, optimizer):\n",
    "    batch_size = 128\n",
    "    callback = myCallback()\n",
    "    model.compile(optimizer=optimizer,\n",
    "                  loss='sparse_categorical_crossentropy',\n",
    "                  metrics=['accuracy']\n",
    "    )\n",
    "    return model.fit(X_train, y_train, validation_data=(X_dev, y_dev), epochs=epochs, \n",
    "                     batch_size=batch_size, callbacks=[callback])\n",
    "\n",
    "def plotHistory(history):\n",
    "    print(\"Max. Validation Accuracy\",max(history.history))\n",
    "    pd.DataFrame(history.history).plot(figsize=(12,6))\n",
    "    plt.show()\n",
    "\n",
    "#Model 1\n",
    "model_1 = k.models.Sequential([\n",
    "    k.layers.Dense(256, activation='relu', input_shape=(X_train.shape[1],)),\n",
    "    k.layers.Dense(128, activation='relu'),\n",
    "    k.layers.Dense(64, activation='relu'),\n",
    "    k.layers.Dense(10, activation='softmax'),\n",
    "])\n",
    "print(model_1.summary())\n",
    "model_1_history = trainModel(model=model_1, epochs=70, optimizer='adam')\n",
    "\n",
    "plotHistory(model_1_history)\n",
    "\n",
    "#Model 2\n",
    "model_2 = k.models.Sequential([\n",
    "    k.layers.Dense(512, activation='relu', input_shape=(X_train.shape[1],)),\n",
    "    k.layers.Dropout(0.2),\n",
    "    \n",
    "    k.layers.Dense(256, activation='relu'),\n",
    "    k.layers.Dropout(0.2),\n",
    "\n",
    "    k.layers.Dense(128, activation='relu'),\n",
    "    k.layers.Dropout(0.2),\n",
    "\n",
    "    k.layers.Dense(64, activation='relu'),\n",
    "    k.layers.Dropout(0.2),\n",
    "\n",
    "    k.layers.Dense(10, activation='softmax'),\n",
    "])\n",
    "print(model_2.summary())\n",
    "model_2_history = trainModel(model=model_2, epochs=100, optimizer='adam')\n",
    "\n",
    "plotHistory(model_2_history)\n",
    "\n",
    "#Model 3\n",
    "model_3 = k.models.Sequential([\n",
    "    k.layers.Dense(512, activation='relu', input_shape=(X_train.shape[1],)),\n",
    "    k.layers.Dropout(0.2),\n",
    "    \n",
    "    k.layers.Dense(256, activation='relu'),\n",
    "    k.layers.Dropout(0.2),\n",
    "\n",
    "    k.layers.Dense(128, activation='relu'),\n",
    "    k.layers.Dropout(0.2),\n",
    "\n",
    "    k.layers.Dense(64, activation='relu'),\n",
    "    k.layers.Dropout(0.2),\n",
    "\n",
    "    k.layers.Dense(10, activation='softmax'),\n",
    "])\n",
    "print(model_3.summary())\n",
    "model_3_history = trainModel(model=model_3, epochs=700, optimizer='sgd')\n",
    "\n",
    "plotHistory(model_3_history)\n",
    "\n",
    "#Model 4\n",
    "model_4 = k.models.Sequential([\n",
    "    k.layers.Dense(1024, activation='relu', input_shape=(X_train.shape[1],)),\n",
    "    k.layers.Dropout(0.3),\n",
    "    \n",
    "    k.layers.Dense(512, activation='relu'),\n",
    "    k.layers.Dropout(0.3),\n",
    "\n",
    "    k.layers.Dense(256, activation='relu'),\n",
    "    k.layers.Dropout(0.3),\n",
    "\n",
    "    k.layers.Dense(128, activation='relu'),\n",
    "    k.layers.Dropout(0.3),\n",
    "\n",
    "    k.layers.Dense(64, activation='relu'),\n",
    "    k.layers.Dropout(0.3),\n",
    "\n",
    "    k.layers.Dense(10, activation='softmax'),\n",
    "])\n",
    "print(model_4.summary())\n",
    "model_4_history = trainModel(model=model_4, epochs=500, optimizer='rmsprop')\n",
    "\n",
    "\n",
    "plotHistory(model_4_history)\n",
    "\n",
    "test_loss, test_acc  = model_1.evaluate(X_test, y_test, batch_size=128)\n",
    "print(\"The test Loss is :\",test_loss)\n",
    "print(\"\\nThe Best test Accuracy is :\",test_acc*100)\n",
    "\n",
    "test_loss, test_acc  = model_2.evaluate(X_test, y_test, batch_size=128)\n",
    "print(\"The test Loss is :\",test_loss)\n",
    "print(\"\\nThe Best test Accuracy is :\",test_acc*100)\n",
    "\n",
    "test_loss, test_acc  = model_3.evaluate(X_test, y_test, batch_size=128)\n",
    "print(\"The test Loss is :\",test_loss)\n",
    "print(\"\\nThe Best test Accuracy is :\",test_acc*100)\n",
    "\n",
    "test_loss, test_acc  = model_4.evaluate(X_test, y_test, batch_size=128)\n",
    "print(\"The test Loss is :\",test_loss)\n",
    "print(\"\\nThe Best test Accuracy is :\",test_acc*100)\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "3f98a48246f38fbcc250405817991eb47b98a99ad9d7e1da91ba80a9502c031b"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}