Chapter_8/Lab-8.3.4_Boosting.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 8.3.4 Boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.datasets import load_boston\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.model_selection import train_test_split,GridSearchCV\n",
    "from sklearn.ensemble import AdaBoostRegressor\n",
    "from xgboost import XGBRegressor\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "      <th>MEDV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "      <td>21.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "      <td>34.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "      <td>33.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "      <td>36.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CRIM    ZN  INDUS  CHAS    NOX     RM   AGE     DIS  RAD    TAX  \\\n",
       "0  0.00632  18.0   2.31   0.0  0.538  6.575  65.2  4.0900  1.0  296.0   \n",
       "1  0.02731   0.0   7.07   0.0  0.469  6.421  78.9  4.9671  2.0  242.0   \n",
       "2  0.02729   0.0   7.07   0.0  0.469  7.185  61.1  4.9671  2.0  242.0   \n",
       "3  0.03237   0.0   2.18   0.0  0.458  6.998  45.8  6.0622  3.0  222.0   \n",
       "4  0.06905   0.0   2.18   0.0  0.458  7.147  54.2  6.0622  3.0  222.0   \n",
       "\n",
       "   PTRATIO       B  LSTAT  MEDV  \n",
       "0     15.3  396.90   4.98  24.0  \n",
       "1     17.8  396.90   9.14  21.6  \n",
       "2     17.8  392.83   4.03  34.7  \n",
       "3     18.7  394.63   2.94  33.4  \n",
       "4     18.7  396.90   5.33  36.2  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# loading the dataset\n",
    "boston_data = load_boston()\n",
    "boston = pd.DataFrame(boston_data.data,columns = boston_data['feature_names'])\n",
    "boston['MEDV'] = boston_data['target']\n",
    "boston.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## In this notebook i am also going to compare the two boosting techniques - Adaboosting vs XGB Boosting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "to know more about the difference between the two - https://datascience.stackexchange.com/questions/39193/adaboost-vs-gradient-boosting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(253, 13) (253, 13)\n"
     ]
    }
   ],
   "source": [
    "# splitting the data into train and test\n",
    "X = boston.drop('MEDV',axis = 1)\n",
    "y = boston['MEDV']\n",
    "X_train,X_test,y_train,y_test = train_test_split(X,y,test_size = 0.5,random_state = 1)\n",
    "print(X_train.shape,X_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Adaboost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training score  0.9602475897306324\n",
      "Test score  0.8585549637358928\n"
     ]
    }
   ],
   "source": [
    "base_reg = DecisionTreeRegressor(max_depth = 4)\n",
    "reg = AdaBoostRegressor(base_reg,n_estimators=5000)\n",
    "reg.fit(X_train,y_train)\n",
    "print('Training score ',reg.score(X_train,y_train))\n",
    "print('Test score ',reg.score(X_test,y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x2c8c3a914a8>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlMAAAGFCAYAAADZznZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAddklEQVR4nO3de7StdVkv8O8jSJm3Y7FPFhdBpTqklopaaRdTC0IFLynkKXFYaIlpasnpQiPKMk3teDkplVmOFLVSUUkztU5WKtu7qBQRKno8Yml1slL0OX+8c8disfbea/Nba813bj6fMRi8N+Z6WJc5v+/v/V2quwMAwHVzg2UXAACwyoQpAIABwhQAwABhCgBggDAFADDg0GV94cMPP7yPOeaYZX15AIBNe+c73/np7t610bmlhaljjjkmu3fvXtaXBwDYtKr6yN7OecwHADBAmAIAGCBMAQAMEKYAAAYIUwAAA4QpAIABwhQAwABhCgBggDAFADBAmAIAGCBMAQAMEKYAAAYIUwAAA4QpAIABhy67gM045uzXbflrXv7Uk7f8NQGA6x8tUwAAA4QpAIABwhQAwABhCgBggDAFADBAmAIAGCBMAQAMEKYAAAYIUwAAA4QpAIABwhQAwABhCgBggDAFADBAmAIAGCBMAQAMEKYAAAYIUwAAA4QpAIABwhQAwABhCgBggDAFADBAmAIAGCBMAQAMEKYAAAYIUwAAA4QpAIABwhQAwABhCgBggDAFADBAmAIAGCBMAQAMEKYAAAYIUwAAA4QpAIABwhQAwABhCgBggDAFADBAmAIAGCBMAQAMEKYAAAYIUwAAA4QpAIABwhQAwABhCgBggDAFADBgU2Gqqk6sqkuq6tKqOnsf1z24qrqqTti6EgEA5mu/YaqqDknyvCQnJTk+yelVdfwG1900yY8neftWFwkAMFebaZm6a5JLu/uy7v58kvOTnLLBdb+Y5GlJ/n0L6wMAmLXNhKkjknxszf4Vi2P/qarumOSo7n7tvl6oqs6sqt1VtfvKK6884GIBAOZmM2GqNjjW/3my6gZJnpXkift7oe4+r7tP6O4Tdu3atfkqAQBmajNh6ookR63ZPzLJJ9bs3zTJ7ZL8WVVdnuRbklygEzoAcH2wmTB1UZLjqurYqjosyWlJLthzsrv/qbsP7+5juvuYJG9Lcv/u3r0tFQMAzMh+w1R3X5XkrCRvSPKhJC/v7our6tyquv92FwgAMGeHbuai7r4wyYXrjp2zl2u/a7wsAIDVYAZ0AIABwhQAwABhCgBggDAFADBAmAIAGCBMAQAMEKYAAAYIUwAAA4QpAIABwhQAwABhCgBggDAFADBAmAIAGCBMAQAMEKYAAAYIUwAAA4QpAIABwhQAwABhCgBggDAFADBAmAIAGCBMAQAMEKYAAAYIUwAAA4QpAIABwhQAwABhCgBggDAFADBAmAIAGCBMAQAMEKYAAAYIUwAAA4QpAIABwhQAwABhCgBggDAFADBAmAIAGCBMAQAMEKYAAAYIUwAAA4QpAIABwhQAwABhCgBggDAFADBAmAIAGCBMAQAMEKYAAAYIUwAAA4QpAIABwhQAwABhCgBggDAFADBAmAIAGCBMAQAMEKYAAAYIUwAAAzYVpqrqxKq6pKouraqzNzj/6Kp6f1W9p6reWlXHb32pAADzs98wVVWHJHlekpOSHJ/k9A3C0ku6+/bd/c1JnpbkmVteKQDADG2mZequSS7t7su6+/NJzk9yytoLuvuf1+zeOElvXYkAAPN16CauOSLJx9bsX5HkbusvqqrHJHlCksOSfPdGL1RVZyY5M0mOPvroA60VAGB2NtMyVRscu1bLU3c/r7tvk+TJSX52oxfq7vO6+4TuPmHXrl0HVikAwAxtJkxdkeSoNftHJvnEPq4/P8mpI0UBAKyKzYSpi5IcV1XHVtVhSU5LcsHaC6rquDW7Jyf5260rEQBgvvbbZ6q7r6qqs5K8IckhSV7Y3RdX1blJdnf3BUnOqqp7J/lCks8kefh2Fg0AMBeb6YCe7r4wyYXrjp2zZvtxW1wXAMBKMAM6AMAAYQoAYIAwBQAwQJgCABggTAEADBCmAAAGCFMAAAOEKQCAAcIUAMAAYQoAYIAwBQAwQJgCABggTAEADBCmAAAGCFMAAAOEKQCAAcIUAMAAYQoAYIAwBQAwQJgCABggTAEADBCmAAAGCFMAAAOEKQCAAcIUAMAAYQoAYIAwBQAwQJgCABggTAEADBCmAAAGCFMAAAOEKQCAAcIUAMAAYQoAYIAwBQAwQJgCABggTAEADBCmAAAGCFMAAAOEKQCAAcIUAMAAYQoAYMChyy7gYHPM2a/b8te8/Kknb/lrAgBbQ8sUAMAAYQoAYIAwBQAwQJgCABggTAEADBCmAAAGCFMAAAOEKQCAAcIUAMAAYQoAYIAwBQAwQJgCABggTAEADBCmAAAGbCpMVdWJVXVJVV1aVWdvcP4JVfXBqnpfVb2pqm619aUCAMzPfsNUVR2S5HlJTkpyfJLTq+r4dZe9O8kJ3X2HJH+Q5GlbXSgAwBxtpmXqrkku7e7LuvvzSc5PcsraC7r7Ld39ucXu25IcubVlAgDM02bC1BFJPrZm/4rFsb15ZJI/3uhEVZ1ZVburaveVV165+SoBAGZqM2GqNjjWG15Y9d+TnJDk6Rud7+7zuvuE7j5h165dm68SAGCmDt3ENVckOWrN/pFJPrH+oqq6d5KfSfKd3f0fW1MeAMC8baZl6qIkx1XVsVV1WJLTklyw9oKqumOSFyS5f3d/auvLBACYp/2Gqe6+KslZSd6Q5ENJXt7dF1fVuVV1/8VlT09ykySvqKr3VNUFe3k5AICDymYe86W7L0xy4bpj56zZvvcW1wUAsBLMgA4AMECYAgAYIEwBAAwQpgAABghTAAADhCkAgAHCFADAAGEKAGCAMAUAMECYAgAYIEwBAAwQpgAABghTAAADhCkAgAHCFADAAGEKAGCAMAUAMECYAgAYIEwBAAwQpgAABghTAAADhCkAgAHCFADAAGEKAGCAMAUAMECYAgAYIEwBAAwQpgAABghTAAADhCkAgAHCFADAAGEKAGCAMAUAMECYAgAYIEwBAAwQpgAABghTAAADhCkAgAHCFADAAGEKAGCAMAUAMECYAgAYIEwBAAwQpgAABghTAAADhCkAgAHCFADAAGEKAGCAMAUAMECYAgAYIEwBAAwQpgAABghTAAADhCkAgAHCFADAAGEKAGDApsJUVZ1YVZdU1aVVdfYG57+jqt5VVVdV1YO3vkwAgHnab5iqqkOSPC/JSUmOT3J6VR2/7rKPJjkjyUu2ukAAgDk7dBPX3DXJpd19WZJU1flJTknywT0XdPfli3Nf2oYaAQBmazNh6ogkH1uzf0WSu12XL1ZVZyY5M0mOPvro6/ISbKFjzn7dlr/m5U89ectfEwDmbDN9pmqDY31dvlh3n9fdJ3T3Cbt27bouLwEAMCubCVNXJDlqzf6RST6xPeUAAKyWzYSpi5IcV1XHVtVhSU5LcsH2lgUAsBr2G6a6+6okZyV5Q5IPJXl5d19cVedW1f2TpKruUlVXJPn+JC+oqou3s2gAgLnYTAf0dPeFSS5cd+ycNdsXZXr8BwBwvWIGdACAAcIUAMAAYQoAYIAwBQAwQJgCABggTAEADBCmAAAGCFMAAAOEKQCAAcIUAMAAYQoAYIAwBQAwQJgCABggTAEADBCmAAAGCFMAAAOEKQCAAcIUAMAAYQoAYIAwBQAwQJgCABggTAEADBCmAAAGCFMAAAOEKQCAAcIUAMAAYQoAYIAwBQAwQJgCABggTAEADBCmAAAGCFMAAAOEKQCAAcIUAMAAYQoAYIAwBQAwQJgCABhw6LILgM045uzXbflrXv7Uk7f8NQG4/tEyBQAwQMsUbKHtaEFLtqcVbZVqBZgzYQqYPcEPmDNhCmALCX5w/aPPFADAAC1TANdTWtFga2iZAgAYIEwBAAwQpgAABghTAAADhCkAgAHCFADAAGEKAGCAeaYAmD1zYjFnWqYAAAYIUwAAA4QpAIABwhQAwIBNhamqOrGqLqmqS6vq7A3Of1lVvWxx/u1VdcxWFwoAMEf7Hc1XVYckeV6S+yS5IslFVXVBd39wzWWPTPKZ7r5tVZ2W5FeTPHQ7CgaAOduOkYdGHc7bZlqm7prk0u6+rLs/n+T8JKesu+aUJL+72P6DJPeqqtq6MgEA5qm6e98XVD04yYnd/cOL/R9McrfuPmvNNR9YXHPFYv/vFtd8et1rnZnkzMXu1ye5ZKv+R9Y4PMmn93vV8q1KnYlat4tat4dat4dat4dat8d21Hqr7t610YnNTNq5UQvT+gS2mWvS3eclOW8TX/M6q6rd3X3Cdn6NrbAqdSZq3S5q3R5q3R5q3R5q3R47XetmHvNdkeSoNftHJvnE3q6pqkOT3DzJP25FgQAAc7aZMHVRkuOq6tiqOizJaUkuWHfNBUkevth+cJI39/6eHwIAHAT2+5ivu6+qqrOSvCHJIUle2N0XV9W5SXZ39wVJfjvJi6vq0kwtUqdtZ9H7sa2PEbfQqtSZqHW7qHV7qHV7qHV7qHV77Git++2ADgDA3pkBHQBggDAFADBAmAIAGCBMsZKq6ib7OHebnawFgOu3lQ5TVfXLy67hYFdVX1VVD6iqOy+7lnXeW1UPWXugqr68qn4pyeuXVNNBoaoetJfjh1XVz+10PQeiqm5YVXesqv+67FpWVVV9w5rtL1t37lt2vqIDV1WHW9KMnbTSo/mq6l3dfadl17E/VfXsfZ3v7h/fqVr2p6pem+Ts7v5AVX1Nkncl2Z3kNknO6+5fX2qBC4vWp+dmmt7jR5N8Y5JfS/KqJL/Q3f9vieVtqKpul+SnkhyfaYWADyZ5Rne/b6mFrVNVb0jypSQ/1t1/vzh2UpJnJXl9dz9+mfWtVVXPT/KcxXQtN0/y10m+mOQrkzypu1+61AIXquqB+zrf3X+0U7Xsz9r31fXvsXN8z10EvKdmmpbnF5O8ONNSIjdI8kPdPaubq6q6Z5LHZlpSLUk+lOS53f1nSytqnar6oX2d7+7f26la9qeqfiTJn3X33y4C9AuTPCjJ5UnO6O537UQdm1lOZs4OqapbZOPlbNLdc5mF/dFJPpDk5Zlmj5/zHdOx3f2BxfYjkryxu3+oqm6a5C+TzCJMdfffJTmpqn4yyYeTfDLJ93b3xcutbGNVdUqmsPcri39Xkjsn+cOqelJ3v3qZ9a3V3d9bVacn+dOqekmS2yXZleSh3f3e5VZ3Ld/e3Y9ebD8iyd9096lVdcskf5xkFmEq0wLw71n8k1zzPaCTzCZM5Zq1rX+vmuN713OT/HSmlTfenOSk7n7booXtpZlRS3VVnZyp3nOT/EKm7+edkrywqs7q7guXWd8ad9ngWCW5X5IjkswmTCV5XJIXLbZPT3KHJMcmuWOS/5nk23eiiFUPU9+Q5J3Z+9qAt97Zcvbqa5J8f5KHJrkqycuS/GF3f2apVW3sC2u275XkN5Oku/+lqr60nJKubbFs0U8meWSSH0vyfUmeXVU/1t3bsYD2qHOT3Ke7L19z7L1V9eYkr178Mycvz9Ta9xNJPpvku7v7b5Zb0oY+v2b7PklekSTd/cmZPeV5UKa//ztk+lm/tLsvXW5Je9V72d5ofw4O7e4/SZKqOre735Yk3f3hmf0OJNN71qnrbkreU1W7kzwnySzCVHc/ds/2orXnYUmenORtSZ6yrLr24qru3vO5dd8kv9fd/5DpZvBpO1XEqoepD3b3HZddxP4sfrDPT/L8qjoiU3q+uKqe3N0vXm511/KxqnpspvUW75TFXV1V3SjJDZdZ2DrvTvLnSe7c3f+U5Lyqum+SV1fVK7v7fyy3vGu54boglSTp7surak7f11TVPZL8r0wtkUcl+c4kr6mqlyV5Snf/xzLrW+ezi5/7x5PcPVO43hO2b7TMwtbq7lcmeWVV3TjJKUmeUVVfleRnuvvPl1vdtRy56JpQa7az2D9ieWXt1dqbvH9bd25u4e+WG7Xudvf7quqrl1HQ3iz+hs5I8sQkb0/y4JneqH5p0SXlM5kaANaGvR17D1j1MLVSqupOmYLUfTI9gnjnciva0CMztaLcO9Njnc8ujn9Lkt9ZWlXXdkZ3X+P7192vrao/TfKzS6ppX75QVUd390fXHqyqW2VqrZyTX0/yw939jsX+q6rqjUnOSfLeTC3Cc/GoJM9Ocsskj+/uTy6O3yvJ65ZW1d79e5J/SvLPSY5O8uXLLWdDP7lme/e6c+v35+CbquqfM4W9Gy22s9if2/f3X6/juR1VVY/J9PjsTUlO7O6PLLmkfTkn0+/lIUku2NPVo6q+M8llO1XEqndAP6O7X7TB8S9Pcr/ufsXOV3VtVfULmZofP5Tk/EydeOf2AXpQqKq7J/mB7n7MsmtZq6pOTfK0JL+cKUR3pn4JZyd5cne/aonlXUNV3aC7N3ykW1X/rbs/tNM1rbpFp+PTk9w1yZ8mOb+75xhM2EZV9dkk/3ujU0nu0d232OGSNrTo0vGpJFfmmq17laS7+w5LKWwvFq1oN13bdaaqviLJId39LztSwyqHqbWq6pAk35PpDet7k/xFdz94uVVNFr+Yl+XqJug93/TZ/WJW1Wuyj6bx7r7/DpazKVX1zUl+IMlDkvx9pv5oz11uVddWVd+Uqcn8GzP97C9O8msz7NSdxdQCj8lU656Rh8/r7k8ttbB1quo5uXYfn08neUt3v3U5VV3b4j3gfUnemqnGa/yNzWxE7z2S3HrPiK2q+oNMoyOT5Je6+81LK27FLVpL9mouj3yr6tGZunhs9Fnw0O7esb5IB2rRx+uemT4T7tfdO/L4dOXDVFV9R6Zv2slJ3pGp38Stu/tzSy1sjcWjnL2aUxPqCv2xf12S0zKF53/I1Kn/Sd29z+81+7do3XtJphEyewZ43CnJw5M8rLv/cnnVXVNVPXyDw1+ZKVi/bEZTeZyRfd+k/O7OVbNvVfWmJI/t7g8u9t+fqe/MjZP8dHefuMTyDkpVdVSS07r76cuuJUmq6ouZ+qT+YHd/fN252U2PkSRVdbdMWeABmd4DHpPpsd+ODPRa6TBVVVck+WiS30jyqsWIs7/v7mOXXNqmLFrTTuvu3192LRupql1J0t1XLruW9RZ3+n+R5JF7RkVV1WXdPZcRnNewSi1+VfW2JD/a3e9ed/ybk7ygu++2nMo2bzFg4q9WYYDK3FTVRd19lzX7f9TdD1xs/2V333151R08qurwTKO8T8/Usf+V3f2k5VY1qap3ZxqEck6SJ6ztMlNV757T31VVPSXTzdNHM02F8coku3c6B6x6B/Q/THJqpiHHX6yqV2d+ozdSVTfLlJKPSHJBkjcmOSvJkzLNOzOrMFVVP59pUrlKcoOquirTxIjnLreya3hQppapt1TV6zP1RZvdOOg1fm3ZBRyAm60PUknS3e9ZzDc2e939b3MaFr9KYTrJf1m7sydILcxqxNmqWfz9PCBTC8rXZfrgv3V3H7nUwq6tu/s3q+rPk/x+VX1fkscsnvjM7TP2zCSXZGpUeW13/3tV7XiNKx2muvtxVfX4TM9HT0/y9CQ3q2mZkQt7PrNgvzjTsM2/TvLDmUbLHJbklO5+z77+w51WVT+R5B5J7tJXz3596yS/UVU/0d3PWmqBV3tNd+8Zan5qpvmQvrqqfiPTHd6fLLe8a3lEd5+x7CI2qarqFuubx6vqK7MCS1AtOqP+YKbpPeZilcL0h6vq5O6+xmjIxRQUcxwav0o+lak7ys8meWt3d1U9YMk17VV3/01VfWuSX0ry7trPzOhLcstc3V/616vqLZlGdR66kwO9Vvox33qL+XpOzPRN/Z7uPnzJJSWZ+hx09+0X24dk6iB79E6NMjgQi+bd+3T3p9cd35XkT+bSvLvRc/vFh/33Z+og+d3LqWxjc+1nsJGqOjPJj2RqOd2zFMOdk/xqkhd29wuWVdt6VfUvme6U1zZDfS5Tf4/Hd/cnllLYAaiqu8+sH9ptM00r8Ve55s//25Lct+c5eetKWNysnpap/9lLMvX1fOPcuids9Civqr4r01Itu7p7li3Ui5H8982UAe6R5E3d/QM78rVXOUxV1Yv2drdfVTfq7vUTuC3F+g/SOX+wVtUHuvt2B3pup83tuf3+VNWHM/2B723pox1ZP2qzFq0QP5VpNF8yjTx8ene/ZnlVra7FTdRDMj3qf31Pa1/eN9MyKDea2+9yTQscPyxXj+a8OMnfJjl9btOOrKJFa//pmYLVcUl+PlOL+iyCalWdutF0LTUt3/ao7n7qEso6IItHqg/cqcEdqx6mZhtK1lqMjNgzIVtlmpX1c7l6aoSbLau29fb1PZ3T93sx+OCZezvf3Xs9twyLFpSLsnGY6rm1pK2SxWO9k3L1ZKIfTPKGOc3lVlUvyjSb/DuS3C3JR5J8a6ZFxWczx9h6VXXHTB/6s552ZJVV1e2zmNqlu2+z7HpWTVU9YV/nd+qzYKX7TCX5isUf+6zv9rv7kGXXcAD2zCa83txmEz4kyU0y707na126KoGpqs7Zx+nu7l/csWL2o6q+NslbkvyfTEsMVaZm/mdW1T1n9JjvhCR36O4vLR5FfDrJbfvqGdtnYy/TjlR333OphR2kuvv9VfVzmVr/OHBrHzk+Ksnabgg71lq06i1T7vavp+bUSrYZq/RYsqqeuMHhG2daauiruvsmO1zSXi1afN6zfj6pqvrxTOs2bjQP1Y5bsUf9KzXtyCrZx8juJyZ5b3efssTyVt4y32dXPUytzAcUW2vVfvZV9T0bjTCc22R96y36HTwuU5B6eZJn9IxmQa+qD3f3hmsFVtUl3f31O13TRqrqc0ku3bOb5DZr9tPzWgXhAZlapr4t0yzY5yf5rVWZv2/OFtP37BnZfa8kt8g0svtxcxvZvYqWeZOy6o/5uP6617ILOBBrg9RGk/Utq669WYyMfEKmTsi/m+RO66dKmIl9DTKZzSoISb4p0xxNH1t3/FZJ5vIoMknS3a9MskrTjqySW68Z2f1bmfHIbg7MqoepJ6/dWUyNcLskH5/T3TNbr7v/cdk1HIgVmqwvVfX0JA9Mcl6S289ovraN3LyqHrjB8Uoym4EdSZ6VaSmWaywdtZhy5FlJ7reUqvahu/8104TCv79m2pGzkwhT190X9mx09xdrWrFDkBpQ03JHex6x3baq3rf2/E61+q76Y77nZ5qZ++KqunmmptMvZlqX50nd/dKlFggLVfVvufZkfbPsh7LoM/MfSa7KxivGzyakVNXv7Ot8dz9ip2rZl/1MOfKf89BxcFulkd2roqqOyz5afff0+9v2OlY8TF3c3d+42H58ku/q7lOr6pZJ/niV+tRwcFuVyfrYHlV1aXff9kDPAftWVa/N1Or7vnXHT0jy8929I62+q/6Y7/Nrtu+T5BVJ0t2frBmtywU9LcPzrDWT9b0qyddW1ZMzo8n6Vs1+lrfo7n7xjhWzbxdV1Y9092+uPVhVj0zyziXVBAeDY9YHqSTp7t1VdcxOFbHqLVNvSfKMJB/PNNfMNyyC1KFJPrC3UT4wBybrG1dVz9nocKY+SEd09yxuGKvqqzP1k/t8rg5PJ2QayfWAOc43BatgLq2+s3ijGfCoJM/OtNDh49e8Id0r09pSMFsm6xvX3Y/ds11Tc/TDMg1MeVuSpyyrrvW6+/8m+baqumemQTJJ8rrufvMSy4KDwSxafVe6ZWpfqurx6yfyg2UxWd/2WbREn5Hpe/n2JL/S3ZcstShgR8yl1fdgDlMf7e6jl10HJCbr2y5V9ZhMk4q+KclT1089AFw/rGv1vXinW30P5jD1se4+atl1QHLN4e9VdUhM1rclFtM4fCrJldl4GofZzCwOHLxWvc/UvhycKZFVZbK+7WGJE2DpVrplarHQ8Ub/A5XkRnMZyQMm6wM4eK10mIJVUVU37O4v7P9KDsR+bqiEVGBHCFOwA5a5mjkA2+sGyy4AridMyQ9wkNKnCHbGrqp6wt5Odvczd7IYALaOMAU745AkN4kWKoCDjj5TsAP0mQI4eOkzBTtDixTAQUrLFOyAqvraJA9Jctsk70/y29191XKrAmArCFOwA6rqZZlmQf+LJCcl+Uh3P265VQGwFYQp2AHr1uY7NMk79KECODjoMwU7Y+3afB7vARxEtEzBDrA2H8DBS5gCABjgMR8AwABhCgBggDAFADBAmAIAGPD/AckRu75eFq2wAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# feature importance\n",
    "pd.Series(reg.feature_importances_,index = X_train.columns).sort_values(ascending=False).plot.bar(figsize = (10,6))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# XGB Classifier \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Lenovo\\Anaconda3\\lib\\site-packages\\xgboost\\core.py:587: FutureWarning: Series.base is deprecated and will be removed in a future version\n",
      "  if getattr(data, 'base', None) is not None and \\\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[09:40:42] WARNING: src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
      "Training score is  0.9826216624003152\n",
      "Test score is  0.8880422216668562\n"
     ]
    }
   ],
   "source": [
    "xgb = XGBRegressor()\n",
    "xgb.fit(X_train,y_train)\n",
    "print('Training score is ',xgb.score(X_train,y_train))\n",
    "print('Test score is ',xgb.score(X_test,y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x2c8c54f4d68>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlMAAAGFCAYAAADZznZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAfiklEQVR4nO3dfbildV3v8ffHGSnzKZUpiwcHFeuQWuqIlZaaUhAK+JAxeUq8VLRAxacce6ATZZmmdlROimaaVwpoqaNSlEqdtNDZKCIDUhOijByPo2l1ssTR7/njXlvWLNbeew2/vfd9r5n367q4WPcDa3/Ze+29Put3/37fO1WFJEmSbplb9V2AJEnSPDNMSZIkNTBMSZIkNTBMSZIkNTBMSZIkNdjY1xc+9NBDa/PmzX19eUmSpJlddtllX6yqTdOO9RamNm/ezMLCQl9fXpIkaWZJPrPUMS/zSZIkNTBMSZIkNTBMSZIkNTBMSZIkNTBMSZIkNTBMSZIkNZgpTCU5Psk1SXYl2bbEOU9IclWSnUneurplSpIkDdOKfaaSbADOBY4DdgM7kmyvqqvGzjkaeBHw4Kr6cpLvWquCJUmShmSWkaljgV1VdW1V3QicD5w8cc7TgHOr6ssAVfWF1S1TkiRpmGYJU4cB149t7x7tG3cv4F5JPpzk0iTHT3uiJKcnWUiysGfPnltWsSRJ0oDMEqYyZV9NbG8EjgYeBmwF3pDkO2/2H1WdV1VbqmrLpk1Tb28jSZI0V2YJU7uBI8a2DwdumHLOu6vq61X1aeAaunAlSZJ0QJslTO0Ajk5yVJJDgFOB7RPnvAt4OECSQ+ku+127moVKkiQN0Yphqqr2AmcCFwNXAxdW1c4k5yQ5aXTaxcCXklwFXAK8oKq+tFZFS5IkDUWqJqc/rY8tW7bUwsLCTOdu3va+Vf/6173kxFV/TkmSdGBKcllVbZl2zA7okiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDQxTkiRJDWYKU0mOT3JNkl1Jtk05flqSPUkuH/3z1NUvVZIkaXg2rnRCkg3AucBxwG5gR5LtVXXVxKkXVNWZa1CjJEnSYM0yMnUssKuqrq2qG4HzgZPXtixJkqT5MEuYOgy4fmx792jfpMcluSLJO5IcMe2JkpyeZCHJwp49e25BuZIkScMyS5jKlH01sf0eYHNV3Rd4P/DmaU9UVedV1Zaq2rJp06b9q1SSJGmAZglTu4HxkabDgRvGT6iqL1XV10abrwcesDrlSZIkDdssYWoHcHSSo5IcApwKbB8/Icn3jG2eBFy9eiVKkiQN14qr+apqb5IzgYuBDcAbq2pnknOAharaDjwryUnAXuBfgNPWsGZJkqTBWDFMAVTVRcBFE/vOHnv8IuBFq1uaJEnS8NkBXZIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqcFMYSrJ8UmuSbIrybZlznt8kkqyZfVKlCRJGq4Vw1SSDcC5wAnAMcDWJMdMOe/2wLOAj6x2kZIkSUM1y8jUscCuqrq2qm4EzgdOnnLebwEvBf5rFeuTJEkatFnC1GHA9WPbu0f7viXJ/YAjquq9yz1RktOTLCRZ2LNnz34XK0mSNDSzhKlM2VffOpjcCngl8LyVnqiqzquqLVW1ZdOmTbNXKUmSNFCzhKndwBFj24cDN4xt3x64N/A3Sa4DfhjY7iR0SZJ0MJglTO0Ajk5yVJJDgFOB7YsHq+pfq+rQqtpcVZuBS4GTqmphTSqWJEkakBXDVFXtBc4ELgauBi6sqp1Jzkly0loXKEmSNGQbZzmpqi4CLprYd/YS5z6svSxJkqT5YAd0SZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBoYpSZKkBjOFqSTHJ7kmya4k26Ycf0aSTya5PMmHkhyz+qVKkiQNz4phKskG4FzgBOAYYOuUsPTWqrpPVf0Q8FLgFateqSRJ0gDNMjJ1LLCrqq6tqhuB84GTx0+oqn8b27wtUKtXoiRJ0nBtnOGcw4Drx7Z3Aw+aPCnJGcBzgUOAn5j2RElOB04HOPLII/e3VkmSpMGZZWQqU/bdbOSpqs6tqnsALwR+bdoTVdV5VbWlqrZs2rRp/yqVJEkaoFnC1G7giLHtw4Ebljn/fOCUlqIkSZLmxSxhagdwdJKjkhwCnApsHz8hydFjmycC/7R6JUqSJA3XinOmqmpvkjOBi4ENwBurameSc4CFqtoOnJnkkcDXgS8DT1rLoiVJkoZilgnoVNVFwEUT+84ee/zsVa5LkiRpLtgBXZIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqcFMYSrJ8UmuSbIrybYpx5+b5KokVyT5QJK7rX6pkiRJw7NimEqyATgXOAE4Btia5JiJ0z4ObKmq+wLvAF662oVKkiQN0SwjU8cCu6rq2qq6ETgfOHn8hKq6pKq+Otq8FDh8dcuUJEkaplnC1GHA9WPbu0f7lvIU4C+mHUhyepKFJAt79uyZvUpJkqSBmiVMZcq+mnpi8t+BLcDLph2vqvOqaktVbdm0adPsVUqSJA3UxhnO2Q0cMbZ9OHDD5ElJHgn8KvDQqvra6pQnSZI0bLOMTO0Ajk5yVJJDgFOB7eMnJLkf8DrgpKr6wuqXKUmSNEwrhqmq2gucCVwMXA1cWFU7k5yT5KTRaS8Dbge8PcnlSbYv8XSSJEkHlFku81FVFwEXTew7e+zxI1e5LkmSpLlgB3RJkqQGhilJkqQGhilJkqQGhilJkqQGhilJkqQGhilJkqQGhilJkqQGhilJkqQGhilJkqQGhilJkqQGhilJkqQGhilJkqQGhilJkqQGhilJkqQGG/su4ECzedv7Vv05r3vJiav+nJIkaXU4MiVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktTAMCVJktRgpjCV5Pgk1yTZlWTblOM/nuRjSfYmefzqlylJkjRMK4apJBuAc4ETgGOArUmOmTjts8BpwFtXu0BJkqQh2zjDOccCu6rqWoAk5wMnA1ctnlBV142OfXMNapQkSRqsWS7zHQZcP7a9e7RvvyU5PclCkoU9e/bckqeQJEkalFnCVKbsq1vyxarqvKraUlVbNm3adEueQpIkaVBmCVO7gSPGtg8HblibciRJkubLLGFqB3B0kqOSHAKcCmxf27IkSZLmw4phqqr2AmcCFwNXAxdW1c4k5yQ5CSDJA5PsBn4GeF2SnWtZtCRJ0lDMspqPqroIuGhi39ljj3fQXf6TJEk6qNgBXZIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqYFhSpIkqcHGvgtQfzZve9+qP+d1Lzlx1Z9TkqQhc2RKkiSpgSNTmguOokmShsqRKUmSpAaGKUmSpAaGKUmSpAaGKUmSpAaGKUmSpAaGKUmSpAaGKUmSpAb2mZJW0Vr0w4K16Yk1T7VK0pAZpiQNnsFP0pB5mU+SJKmBYUqSJKmBYUqSJKmBc6YkaRXN0/yueapVGjJHpiRJkho4MiVJGjxH0TRkhilJklbRWgQ/Q9+weZlPkiSpgSNTkiQdpBxFWx0zjUwlOT7JNUl2Jdk25fi3JblgdPwjSTavdqGSJElDtGKYSrIBOBc4ATgG2JrkmInTngJ8uaruCbwS+L3VLlSSJGmIZrnMdyywq6quBUhyPnAycNXYOScD/2P0+B3Aa5KkqmoVa5UkSQepIV+SzEp5J8njgeOr6qmj7Z8HHlRVZ46dc+XonN2j7X8enfPFiec6HTh9tPl9wDWr8n+xr0OBL654Vv/mpU6w1rVirWvDWteGta4Na10ba1Hr3apq07QDs4xMZcq+yQQ2yzlU1XnAeTN8zVssyUJVbVnLr7Ea5qVOsNa1Yq1rw1rXhrWuDWtdG+td6ywT0HcDR4xtHw7csNQ5STYCdwT+ZTUKlCRJGrJZwtQO4OgkRyU5BDgV2D5xznbgSaPHjwc+6HwpSZJ0MFjxMl9V7U1yJnAxsAF4Y1XtTHIOsFBV24E/At6SZBfdiNSpa1n0Ctb0MuIqmpc6wVrXirWuDWtdG9a6Nqx1baxrrStOQJckSdLSvJ2MJElSA8OUJElSA8OUJElSA8OUpLmU5HbLHLvHetYi6eA212Eqye/0XcOBLsldkjwmyQP6rkXrI8njlth/SJJfX+96lvGJJE8Y35Hk25P8NvCXPdU0syS3TnK/JN/Vdy3jknz/2ONvmzj2w+tf0YEtyaFJpjW+1hyZ69V8ST5WVffvu46VJHnVcser6lnrVctKkrwX2FZVVyb5HuBjwAJwD+C8qvqDXguckOTewC/T3YS76O4Z+fKquqLXwiYkeexyx6vqz9erlpUkuRj4JvBLVfXp0b4T6G5i/pdVdVaf9S0ajT69hq7Fyy8CPwD8PvAu4Der6v/1WN7NJHkt8OpRa5k7Av8AfAO4M/D8qnpbrwWOjP9dnfwbO9S/uUkeDjyT7jZlAFcDr6mqv+mtqClGYfQldC2Efgt4C91tT24F/EJVDeJDQJKnAX9TVf80CnpvBB4HXAecVlUf67O+cUl+YbnjVfUn61HHLLeTGbINSe7E9NvZUFVD6cL+DOBK4EK67vFD/hRyVFVdOXr8ZOCvq+oXktwe+DAwmDCV5GS6N8/fHf07wAOAP0vy/Kp6d5/1TXgHcPnoH9j3NVDAYMJUVf1Ukq3A+5O8Fbg3sAn42ar6RL/V3aSq/hk4IckLgE8Bnwd+qqp29lvZkn6sqp4xevxk4B+r6pQkdwX+AhhEmGLf1+bk36rB/e1KciJdqD4H+E26Gu8PvDHJmVV1UZ/1TXgN8Ct0dwn5IHBCVV06Gg18G8MZUX028KbR463AfYGjgPsB/xP4sX7KmuqBU/YFeDRwGGCYmsH3A5ex9L0B776+5Szpe4CfAX4W2AtcAPxZVX2516qm+/rY40cArweoqn9P8s1+SlrSOcBxVXXd2L5PJPkg8O7RP0PxOLqf/33p6npbVe3qt6RlXUg30vMc4CvAT1TVP/Zb0r5Gt656AfAU4JeAnwZeleSXqmotbqLe6saxx8cBbweoqs8P7CpPLfF42vYQvAA4ZSLoX55kAXg1MKQwtbGq/gogyTlVdSlAVX1qYK+BvVW1+F7wKOBPqupLdB+wXtpjXTdTVc9cfDwaRXsi8ELgUuDF61XHvIepq6rqfn0XsZLRi/C1wGuTHEaX9HcmeWFVvaXf6m7m+iTPpLvf4v0ZfVJKchvg1n0WNsWtJ4IUAFV1XZJB1VpV7wTemeS2wMnAy5PcBfjVqvrbfqvbV5KHAP+LbiTyCOChwHuSXAC8uKq+1md9Yz4O/C3wgKr6V+C8JI8C3p3knVX1on7Lu5mvjOr7HPBguhC4GApv02dhEw4fTU3I2GNG24f1V9aS7jptxLSqrkjy3X0UtIzxD6T/OXFsSEH1m6NpHl+m+1A9HkqG9FoFvvU7dBrwPOAjwOPX+wPVvIepuZLk/nRB6ji6Yf3L+q1oqqfQjfg8ku6yzldG+38Y+OPeqpru60mOrKrPju9Mcje6EcAh+i/gX4F/A44Evr3fcqb6A+CpVfXR0fa7kvw1cDbwCboR4SE4rar2+R2qqvcmeT/waz3VtJynA68C7gqcVVWfH+1/BPC+3qq6uReMPV6YODa5PQT/cQuP9eEHk/wbXTC9zegxo+0h/S04m+5nvQHYvnjpPMlDgWv7LGxSkjPoLkt+ADi+qj7TSx1zPgH9tKp605T93w48uqrevv5V3VyS36QbKr0aOJ9uEu9Q3+znRpJTgJcCv0MXTIvu+vk24IVV9a4ey9vHaILsVuBY4P3A+VU1xDcmktyqqqZe0k3y36rq6vWuaX8keTDwc1V1Rt+1aO0l+Qrwv6cdAh5SVXda55IOCKPRntuPT0dJ8h3Ahqr69/4q29do+skXgD3sO7oXoKrqvutSxzyHqXFJNgA/SfeG9VPA31XV4/utqjP6YV/LTcO6i9/0df1hzyLJe1hmuLmqTlrHclaU5AfphnZ/gO77uRP4/SFNlIZvvQauAD5E9/3d53s8pBWdAKPl+mfQfV8XV0meW1Vf6LWwJST5IeDngCcAn6abk/iafqvaV5JXc/P5SF8ELqmqD/VT1c2NLvPefXEVVJJ30K04BPjtqvpgb8VNMRotWdLQLqPPo9FcpIfT/Y49uqoGc/k0yTPopqNMe9/62apalzlecx+mkvw43Q/4ROCjdHMR7l5VX+21sDGjy05L6mtYchr/MK2NJKexfEh98/pVs7zRyM5b6VbzLC7wuD/wJOCJVfXh/qq7SZJ7AafSfYD6Et3CjudX1bK/b31J8qQpu+9MFwAvGErbkSQfAJ5ZVVeNtj9JNx/ltsCvVNXxPZY3syRHAKdW1cv6rmVeJXkQ3fvrY+heq2fQXfYbzOKpJN+gmzv581X1uYlj69bKY67DVJLdwGeBPwTeNVpx9umqOqrn0mYyGk07tar+tO9apkmyCaCq9vRdyzTzNoo2L5JcCvxiVX18Yv8PAa+rqgf1U9m+RqN9fwc8ZXFlZJJrq2ooq3hnMlrc8fdDWUyTZEdVPXBs+8+r6rGjxx+uqgf3V93ykhxKt3J6K91k+XdW1fP7rWr+JHkxXcj/LF3LhncCC0N8b03ycboFM2cDzx2f3pPk4+v1ezXvE9D/DDiFbsn5N5K8m2GtiAAgyR3oEv1hwHbgr4EzgefT9R0aVJhK8ht0DfAC3CrJXrpmg+f0W9nN/H7fBcxqzoLfHSaDFEBVXT7qNzYUj6MbmbokyV/SzUcc1PryWVTVfw5sWfx3jm8sBqmRwVzeWTR6TT6GbgTlXnRv/HevqsN7LWy+nQ5cQzdQ8d6q+q8kg3tvHamqen2SvwX+NMlPA2eMrk6tW81zHaaq6tlJzqK7lrsVeBlwh3S3mLiohtMB+S10S0z/AXgq3WqZQ4CTq+ry5f7D9ZbkOcBDgAfWTd2v7w78YZLnVNUrey1wX0+uqtP6LmJGcxP86KZI3GlyKD/JnRnWLajeU1WL7SZOoeuJ9d1J/pBuROKv+i1vZaNJvj9P14pkKD6V5MSq2meF4aitwxD7d32BborHrwEfqqpK8piea5p3d+WmOch/kOQSutWHG4e6eKqq/jHJjwC/DXw8K3RGX21zfZlv0qi30PF0L4CfrKpDey4J6OYcVNV9Ro830E06PXJIKyIWjYZMj6uqL07s3wT81VAuRcBwb22xv5I8eCjzkACSnA48jW7kdPG2EQ8Afg94Y1W9rq/axk37+Y8C38/QTTz9iX4qmy7Jv9N9Uh4fhvoq3XyPs6rqhl4Km5DknnStGv6efX/+Pwo8qobXvPU5dCOUt6Wb63cB3Z0b5upy71CNVsc/iu599SHAB6rq5/qt6ibTLuUleRjdLXA2VdW6jKbPdZhK8qalRiaS3KaqJpui9WLyj/6QQ0CSK6vq3vt7rA9JPkX3C77U7YSGdP+oDXRzEA6ja41x5eiT/q8AtxlSSIVvjUL8Mt1qPuhWSb6sqt7TX1X7Ws/5EAebdDc4fiI3rebcCfwTsHWoLSdGI+hb6YLV0cBv0I1QDir8zbPRJdXHDmzBzCnT2uCku9Xc06vqJetSx5yHqcGGknGj1QaLzeNC10H2q9zUGuEOfdU2abnv6dC+36NP+juYHqZqSCMTSd5E1038o8CDgM8AP0J3U+nB9MOaJ6MFKK9Y6nhVLXmsL6PLeidwU+PTq4CLh3rpJMn96ALKYFtOTJPkPoxaZVTVPfquZ94kee5yx4f4u9W3uZ4zBXzH6Jd90CMTVbWh7xr2w2KH3klD69ALsGtIgWkFW4D7VtU3R8PmXwTuWTd1wR6MJGcvc7iq6rfWrZjlbQBux5xMOk/yvcAlwP+huxVO6C6fvCLJwwd0mW9ay4lU1cN7LWw/VNUnk/w63Yia9t/4pbGnA+OX9ud3BGYNzfvI1NyMTGj1zdNlnjm71Pu8KbtvS3erobtU1e3WuaSphvw9nGY0Onn5ZD+pJM+iu7/gtD5U627eWk4ss1r6ecAnqurkHsube/P0d7ZP8x6m/CEfxJL85LQVW0Ns1pfkq8CuxU3gHmPb1IC64I8bzZF4Nl2QuhB4eQ2kC/q8/f4n+VRVTb2vYZJrqur71rumaUYr4U6lm3C+2HLiDUPsMQQwaomzuFr6EcCd6FZLP3toq6Xn0bx9aOnLvF/m00FsPEhNa9bXV11L+EG6Hj3XT+y/GzCIyzvjRqvinks3CfnNwP0nWyUMwCP6LmA/LbcgZjB3bKiqdwLz1HLi7mOrpd/AgFdL68A172HqheMbo9YI9wY+N5RPz1o7c9as75V0t+LY59ZBo5YTrwQe3UtVUyR5GfBY4DzgPgPq17aPqvqXvmvYT3dM8tgp+wMMZhHKoqr6D7qGwn861nJiGzC0MPX1xQdV9Y10d8EwSDVIdwuhxctW90xyxfjxoY6k92neL/O9lq4z984kd6Qb5v0G3T2Enl9Vb+u1QK2pJP/JzZv1DXJuxwotJ77Vh2wIRnNmvgbsZfpd2Af3xj8Pkvzxcser6snrVcuBZJ5WS8+LJEezzEj64lw63WTew9TOqvqB0eOzgIdV1SlJ7gr8xTzNp9D+m6dmfUl2VdU99/eYJK23JO+lG0m/YmL/FuA3qmowI+lDMe+X+W4ce3wc8HaAqvp8hnWvK62B6m5t88qxZn3vAr43yQsZXrO+HUmeVlWvH9+Z5CnAZT3VpHW0wu0tqqresm7FSMvbPBmkAKpqIcnm9S9n+OZ9ZOoS4OXA5+j6t3z/KEhtBK5cauWMDlxDbdaX5Lvp5nTdyE3haQvdqqPHDLHflFZXkldP2003X+6wqpr3D7c6QDiSvv/m/Zf36cCr6G7KeNbYG9Ij6O4tpYPMUJv1VdX/BX40ycPpFkkAvK+qPthjWVpHVfXMxcfphs6fSLeI5lLgxX3VJU3hSPp+muuRqeUkOWuyOZ4OLDbr07wZjZqfRvca/Qjwu1V1Ta9FSRMcSd9/B3KY+mxVHdl3HVo7NuvTPElyBl0D1A8AL5lskyENzcRI+k5H0pd2IIep66vqiL7r0NoZbymQZAM269OAjVpOfAHYw/SWE/bukebUvM+ZWs6BmRI1zmZ9mieDvB2LpHZzPTI1utHxtP+BALdxdcyBzWZ9kqQhmOswpYNbkltX1ddXPlPq3wof/gz/0hwzTGlueTdzSdIQ3KrvAqQGtrmXJPXOOUWaZ5uSPHepg1X1ivUsRpJ0cDJMaZ5tAG6HI1SSpB45Z0pzyzlTkqQhcM6U5pkjUpKk3jkypbmV5HuBJwD3BD4J/FFV7e23KknSwcYwpbmV5AK6Luh/B5wAfKaqnt1vVZKkg41hSnNr4t58G4GPOodKkrTenDOleTZ+bz4v70mSeuHIlOaW9+aTJA2BYUqSJKmBl/kkSZIaGKYkSZIaGKYkSZIaGKYkSZIa/H/dcohkG4dSFQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# feature importance\n",
    "pd.Series(xgb.feature_importances_,index = X_train.columns).sort_values(ascending=False).plot.bar(figsize = (10,6))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "XGB performs the best,"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}