Ensemble Models Diabetes Dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ensemble Methods - Diabetes Dataset\n",
    "\n",
    "### Models Created \n",
    "\n",
    "1. Decision Tree\n",
    "\n",
    "2. Random Forest\n",
    "\n",
    "3. Support Vector Machines\n",
    "\n",
    "4. Logistics Regression\n",
    "\n",
    "5. Bagging Model\n",
    "\n",
    "6. Adaboost Model\n",
    "\n",
    "7. Gradient Boosting Method\n",
    "\n",
    "8. Applied the Voting Classifier Model\n",
    "\n",
    "Accuracy 80% and Kappa - 0.53"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing the Dataset\n",
    "diabetes = pd.read_csv(\"~/Downloads/diabetes.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>148</td>\n",
       "      <td>72</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>33.6</td>\n",
       "      <td>0.627</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>85</td>\n",
       "      <td>66</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>26.6</td>\n",
       "      <td>0.351</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>183</td>\n",
       "      <td>64</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>23.3</td>\n",
       "      <td>0.672</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>89</td>\n",
       "      <td>66</td>\n",
       "      <td>23</td>\n",
       "      <td>94</td>\n",
       "      <td>28.1</td>\n",
       "      <td>0.167</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>137</td>\n",
       "      <td>40</td>\n",
       "      <td>35</td>\n",
       "      <td>168</td>\n",
       "      <td>43.1</td>\n",
       "      <td>2.288</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  \\\n",
       "0            6      148             72             35        0  33.6   \n",
       "1            1       85             66             29        0  26.6   \n",
       "2            8      183             64              0        0  23.3   \n",
       "3            1       89             66             23       94  28.1   \n",
       "4            0      137             40             35      168  43.1   \n",
       "\n",
       "   DiabetesPedigreeFunction  Age  Outcome  \n",
       "0                     0.627   50        1  \n",
       "1                     0.351   31        0  \n",
       "2                     0.672   32        1  \n",
       "3                     0.167   21        0  \n",
       "4                     2.288   33        1  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "diabetes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Scaling is required because the data is in different scales of measurement\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "sc = StandardScaler()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Defined the X and Ys\n",
    "x = diabetes.drop(\"Outcome\", axis = 1)\n",
    "y = diabetes.Outcome"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/preprocessing/data.py:645: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.partial_fit(X, y)\n",
      "/anaconda3/lib/python3.6/site-packages/sklearn/base.py:464: DataConversionWarning: Data with input dtype int64, float64 were all converted to float64 by StandardScaler.\n",
      "  return self.fit(X, **fit_params).transform(X)\n"
     ]
    }
   ],
   "source": [
    "# Apply Standard Scaling on X\n",
    "scaled_x = pd.DataFrame(sc.fit_transform(x), columns=diabetes.columns[:8])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.639947</td>\n",
       "      <td>0.848324</td>\n",
       "      <td>0.149641</td>\n",
       "      <td>0.907270</td>\n",
       "      <td>-0.692891</td>\n",
       "      <td>0.204013</td>\n",
       "      <td>0.468492</td>\n",
       "      <td>1.425995</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-1.123396</td>\n",
       "      <td>-0.160546</td>\n",
       "      <td>0.530902</td>\n",
       "      <td>-0.692891</td>\n",
       "      <td>-0.684422</td>\n",
       "      <td>-0.365061</td>\n",
       "      <td>-0.190672</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.233880</td>\n",
       "      <td>1.943724</td>\n",
       "      <td>-0.263941</td>\n",
       "      <td>-1.288212</td>\n",
       "      <td>-0.692891</td>\n",
       "      <td>-1.103255</td>\n",
       "      <td>0.604397</td>\n",
       "      <td>-0.105584</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-0.998208</td>\n",
       "      <td>-0.160546</td>\n",
       "      <td>0.154533</td>\n",
       "      <td>0.123302</td>\n",
       "      <td>-0.494043</td>\n",
       "      <td>-0.920763</td>\n",
       "      <td>-1.041549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-1.141852</td>\n",
       "      <td>0.504055</td>\n",
       "      <td>-1.504687</td>\n",
       "      <td>0.907270</td>\n",
       "      <td>0.765836</td>\n",
       "      <td>1.409746</td>\n",
       "      <td>5.484909</td>\n",
       "      <td>-0.020496</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pregnancies   Glucose  BloodPressure  SkinThickness   Insulin       BMI  \\\n",
       "0     0.639947  0.848324       0.149641       0.907270 -0.692891  0.204013   \n",
       "1    -0.844885 -1.123396      -0.160546       0.530902 -0.692891 -0.684422   \n",
       "2     1.233880  1.943724      -0.263941      -1.288212 -0.692891 -1.103255   \n",
       "3    -0.844885 -0.998208      -0.160546       0.154533  0.123302 -0.494043   \n",
       "4    -1.141852  0.504055      -1.504687       0.907270  0.765836  1.409746   \n",
       "\n",
       "   DiabetesPedigreeFunction       Age  \n",
       "0                  0.468492  1.425995  \n",
       "1                 -0.365061 -0.190672  \n",
       "2                  0.604397 -0.105584  \n",
       "3                 -0.920763 -1.041549  \n",
       "4                  5.484909 -0.020496  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scaled_x.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The purpose of scaling is to make sure that the different scales of measurement are done away with and we have a single scale of measurement. \n",
    "\n",
    "Further, the Standard Scaling is based on Z Score Transformation (Continuous Prob Distt). \n",
    "\n",
    "It means that the transformed dataset will have the mean == 0 and SD == 1.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>min</th>\n",
       "      <th>25%</th>\n",
       "      <th>50%</th>\n",
       "      <th>75%</th>\n",
       "      <th>max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Pregnancies</th>\n",
       "      <td>768.0</td>\n",
       "      <td>2.544261e-17</td>\n",
       "      <td>1.000652</td>\n",
       "      <td>-1.141852</td>\n",
       "      <td>-0.844885</td>\n",
       "      <td>-0.250952</td>\n",
       "      <td>0.639947</td>\n",
       "      <td>3.906578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Glucose</th>\n",
       "      <td>768.0</td>\n",
       "      <td>3.614007e-18</td>\n",
       "      <td>1.000652</td>\n",
       "      <td>-3.783654</td>\n",
       "      <td>-0.685236</td>\n",
       "      <td>-0.121888</td>\n",
       "      <td>0.605771</td>\n",
       "      <td>2.444478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BloodPressure</th>\n",
       "      <td>768.0</td>\n",
       "      <td>-1.327244e-17</td>\n",
       "      <td>1.000652</td>\n",
       "      <td>-3.572597</td>\n",
       "      <td>-0.367337</td>\n",
       "      <td>0.149641</td>\n",
       "      <td>0.563223</td>\n",
       "      <td>2.734528</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SkinThickness</th>\n",
       "      <td>768.0</td>\n",
       "      <td>7.994184e-17</td>\n",
       "      <td>1.000652</td>\n",
       "      <td>-1.288212</td>\n",
       "      <td>-1.288212</td>\n",
       "      <td>0.154533</td>\n",
       "      <td>0.719086</td>\n",
       "      <td>4.921866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Insulin</th>\n",
       "      <td>768.0</td>\n",
       "      <td>-3.556183e-17</td>\n",
       "      <td>1.000652</td>\n",
       "      <td>-0.692891</td>\n",
       "      <td>-0.692891</td>\n",
       "      <td>-0.428062</td>\n",
       "      <td>0.412008</td>\n",
       "      <td>6.652839</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>BMI</th>\n",
       "      <td>768.0</td>\n",
       "      <td>2.295979e-16</td>\n",
       "      <td>1.000652</td>\n",
       "      <td>-4.060474</td>\n",
       "      <td>-0.595578</td>\n",
       "      <td>0.000942</td>\n",
       "      <td>0.584771</td>\n",
       "      <td>4.455807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <td>768.0</td>\n",
       "      <td>2.398978e-16</td>\n",
       "      <td>1.000652</td>\n",
       "      <td>-1.189553</td>\n",
       "      <td>-0.688969</td>\n",
       "      <td>-0.300128</td>\n",
       "      <td>0.466227</td>\n",
       "      <td>5.883565</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Age</th>\n",
       "      <td>768.0</td>\n",
       "      <td>1.857600e-16</td>\n",
       "      <td>1.000652</td>\n",
       "      <td>-1.041549</td>\n",
       "      <td>-0.786286</td>\n",
       "      <td>-0.360847</td>\n",
       "      <td>0.660206</td>\n",
       "      <td>4.063716</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                          count          mean       std       min       25%  \\\n",
       "Pregnancies               768.0  2.544261e-17  1.000652 -1.141852 -0.844885   \n",
       "Glucose                   768.0  3.614007e-18  1.000652 -3.783654 -0.685236   \n",
       "BloodPressure             768.0 -1.327244e-17  1.000652 -3.572597 -0.367337   \n",
       "SkinThickness             768.0  7.994184e-17  1.000652 -1.288212 -1.288212   \n",
       "Insulin                   768.0 -3.556183e-17  1.000652 -0.692891 -0.692891   \n",
       "BMI                       768.0  2.295979e-16  1.000652 -4.060474 -0.595578   \n",
       "DiabetesPedigreeFunction  768.0  2.398978e-16  1.000652 -1.189553 -0.688969   \n",
       "Age                       768.0  1.857600e-16  1.000652 -1.041549 -0.786286   \n",
       "\n",
       "                               50%       75%       max  \n",
       "Pregnancies              -0.250952  0.639947  3.906578  \n",
       "Glucose                  -0.121888  0.605771  2.444478  \n",
       "BloodPressure             0.149641  0.563223  2.734528  \n",
       "SkinThickness             0.154533  0.719086  4.921866  \n",
       "Insulin                  -0.428062  0.412008  6.652839  \n",
       "BMI                       0.000942  0.584771  4.455807  \n",
       "DiabetesPedigreeFunction -0.300128  0.466227  5.883565  \n",
       "Age                      -0.360847  0.660206  4.063716  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scaled_x.describe().T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Decision Tree Model\n",
    "\n",
    "1. It is based on Entropy or Gini\n",
    "\n",
    "2. Decision Tree Models are easier to understand\n",
    "\n",
    "3. Disadv of Decision Tree Model is that it tends to Overfit the Training Data\n",
    "\n",
    "Overfitting means that the machine is able to learn very good patterns from TRAIN data \n",
    "but when it comes to applying all the learning on the TEST Data, it does not perform well.\n",
    "\n",
    "This means that the Variance Error of the Model is very High.\n",
    "\n",
    "Our objective then becomes to reduce the error of variance. If this error is reduced, the model will become more stable.\n",
    "\n",
    "How will I identify whether the problem is due to data or my model?\n",
    "\n",
    "Specifically, in classification problems, we need to see one thing is the count of Target Variable.\n",
    "\n",
    "The target variable 0 and 1 values should be evenly distributed otherwise the data is said be to suffering from Imbalance. Such kinda of datasets are called Imbalanced Datasets.\n",
    "\n",
    "1. You need to see if the data has different scales of measurements.\n",
    "\n",
    "2. Also check if the data has lots of categories and hence Dummy Variables (0 and 1) due to Label Encoding\n",
    "\n",
    "3. In such cases, its recommended to scale the Dataset.\n",
    "\n",
    "4. Target Variable space should be evenly distributed and there should not be any Imbalance in terms of classes.\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Importing the required library\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "dtree = DecisionTreeClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Splitting the Data in Train and Test\n",
    "from sklearn.model_selection import train_test_split\n",
    "xtrain,xtest,ytrain,ytest = train_test_split(scaled_x, y, test_size = 0.30, random_state = 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, presort=False, random_state=None,\n",
       "            splitter='best')"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit the Model on Training Dataset\n",
    "dtree.fit(xtrain,ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make Predictions\n",
    "model_tree = dtree.predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Check the Accuracy of the Model\n",
    "from sklearn.metrics import accuracy_score, classification_report, cohen_kappa_score, confusion_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7229437229437229"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(ytest, model_tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.81      0.78      0.79       157\n",
      "           1       0.56      0.61      0.58        74\n",
      "\n",
      "   micro avg       0.72      0.72      0.72       231\n",
      "   macro avg       0.69      0.69      0.69       231\n",
      "weighted avg       0.73      0.72      0.73       231\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(ytest, model_tree))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[122,  35],\n",
       "       [ 29,  45]])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(ytest, model_tree)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3770961489845791"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cohen_kappa_score(ytest, model_tree)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Why Kappa is a very good indicator of classification models\n",
    "\n",
    "1. Kappa actually tells you the model's effectiveness in terms of implementing in real life scenarios.\n",
    "\n",
    "2. If the data is suffering from Outliers and Imbalanced Class, Kappa penalizes such kind of things and hence Kappa becomes a neutral metric on which you can rely.\n",
    "\n",
    "3. Kappa Score as it appears has the values in the range of 0 and 1. If the values are closer to 1, better the model and if the Kappa score has values closer to 0, worse the model it is.\n",
    "\n",
    "\n",
    "Taking about Decision Trees\n",
    "\n",
    "if the decision tree is suffering from Overfitting, the best way to reduce the error of variance is to create a Random Forest Model.\n",
    "\n",
    "A RF Model is a collection of Decision Trees. It builts several alike Decision Trees and averages the prediction from the trees. This way it reduces the error of variance. \n",
    "\n",
    "So, the RF would be a better model which reduces the error of variance and hence the overall model's performance gets improved. \n",
    "\n",
    "In a nutshell, the RF Model is a type of Ensemble which takes multiple decision trees into consideration and outputs the labels basis the average of all the Trees behind.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier\n",
    "rf = RandomForestClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(bootstrap=True, class_weight=None, criterion='gini',\n",
       "            max_depth=None, max_features='auto', max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_leaf=0.0, n_estimators=10, n_jobs=None,\n",
       "            oob_score=False, random_state=None, verbose=0,\n",
       "            warm_start=False)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit the Random Forest Model\n",
    "rf.fit(xtrain,ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fit the Random Forest Model\n",
    "rf_model = rf.predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.7835497835497836"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(ytest, rf_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.4687701223438506"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cohen_kappa_score(ytest, rf_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Logistics Regression\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "lg = LogisticRegression()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='warn',\n",
       "          n_jobs=None, penalty='l2', random_state=None, solver='warn',\n",
       "          tol=0.0001, verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lg.fit(xtrain,ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "lg_model = lg.predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7792207792207793\n",
      "0.4560690705942102\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(ytest,lg_model))\n",
    "print(cohen_kappa_score(ytest,lg_model))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Ensembling the Random Forest and Decision Tree Classifier Models\n",
    "\n",
    "It means I would combine both the models and will fit it on my training data and will try to make the predictions\n",
    "\n",
    "Voting Classifier Algo in Sklearn."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import VotingClassifier\n",
    "v = VotingClassifier(estimators=[(\"Tree\", dtree), (\"LG\", lg), \n",
    "                                 (\"RF\", rf), (\"GBM\", gb),(\"Bag\", bagg)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/linear_model/logistic.py:433: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "VotingClassifier(estimators=[('Tree', DecisionTreeClassifier(class_weight=None, criterion='gini', max_depth=None,\n",
       "            max_features=None, max_leaf_nodes=None,\n",
       "            min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "            min_samples_leaf=1, min_samples_split=2,\n",
       "            min_weight_fraction_le...imators=10, n_jobs=None, oob_score=False, random_state=None,\n",
       "         verbose=0, warm_start=False))],\n",
       "         flatten_transform=None, n_jobs=None, voting='hard', weights=None)"
      ]
     },
     "execution_count": 151,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "v.fit(xtrain, ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "vott_model = v.predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8051948051948052"
      ]
     },
     "execution_count": 153,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(ytest, vott_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5343367826904986"
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cohen_kappa_score(ytest, vott_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply SVM\n",
    "from sklearn.svm import SVC\n",
    "svm = SVC()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/anaconda3/lib/python3.6/site-packages/sklearn/svm/base.py:196: FutureWarning: The default value of gamma will change from 'auto' to 'scale' in version 0.22 to account better for unscaled features. Set gamma explicitly to 'auto' or 'scale' to avoid this warning.\n",
      "  \"avoid this warning.\", FutureWarning)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "SVC(C=1.0, cache_size=200, class_weight=None, coef0=0.0,\n",
       "  decision_function_shape='ovr', degree=3, gamma='auto_deprecated',\n",
       "  kernel='rbf', max_iter=-1, probability=False, random_state=None,\n",
       "  shrinking=True, tol=0.001, verbose=False)"
      ]
     },
     "execution_count": 65,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svm.fit(xtrain,ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "svm_model = svm.predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7532467532467533\n",
      "0.3920771965464702\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(ytest,svm_model))\n",
    "print(cohen_kappa_score(ytest,svm_model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Apply Gradient Boosting\n",
    "from sklearn.ensemble import AdaBoostClassifier, GradientBoostingClassifier, BaggingClassifier\n",
    "ada = AdaBoostClassifier()\n",
    "gb = GradientBoostingClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "AdaBoostClassifier(algorithm='SAMME.R', base_estimator=None,\n",
       "          learning_rate=1.0, n_estimators=50, random_state=None)"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ada.fit(xtrain,ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "ada_model = ada.predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7489177489177489\n",
      "0.3930415873878771\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(ytest,ada_model))\n",
    "print(cohen_kappa_score(ytest,ada_model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
       "              learning_rate=0.1, loss='deviance', max_depth=3,\n",
       "              max_features=None, max_leaf_nodes=None,\n",
       "              min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "              min_samples_leaf=1, min_samples_split=2,\n",
       "              min_weight_fraction_leaf=0.0, n_estimators=100,\n",
       "              n_iter_no_change=None, presort='auto', random_state=None,\n",
       "              subsample=1.0, tol=0.0001, validation_fraction=0.1,\n",
       "              verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gb.fit(xtrain, ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "gb_model = gb.predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7878787878787878\n",
      "0.5003751931141028\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(ytest,gb_model))\n",
    "print(cohen_kappa_score(ytest,gb_model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "bagg = BaggingClassifier()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "BaggingClassifier(base_estimator=None, bootstrap=True,\n",
       "         bootstrap_features=False, max_features=1.0, max_samples=1.0,\n",
       "         n_estimators=10, n_jobs=None, oob_score=False, random_state=None,\n",
       "         verbose=0, warm_start=False)"
      ]
     },
     "execution_count": 142,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bagg.fit(xtrain,ytrain)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "bagg_model = bagg.predict(xtest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7705627705627706\n",
      "0.451552210724365\n"
     ]
    }
   ],
   "source": [
    "print(accuracy_score(ytest,bagg_model))\n",
    "print(cohen_kappa_score(ytest,bagg_model))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}