Chapter_8/Lab_8.3.3_Bagging_and_Random_Forests.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 8.3.3 Bagging and Random Forests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Lenovo\\Anaconda3\\lib\\importlib\\_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n",
      "C:\\Users\\Lenovo\\Anaconda3\\lib\\importlib\\_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n",
      "C:\\Users\\Lenovo\\Anaconda3\\lib\\importlib\\_bootstrap.py:219: RuntimeWarning: numpy.ufunc size changed, may indicate binary incompatibility. Expected 192 from C header, got 216 from PyObject\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.datasets import load_boston\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import train_test_split,cross_val_score\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CRIM</th>\n",
       "      <th>ZN</th>\n",
       "      <th>INDUS</th>\n",
       "      <th>CHAS</th>\n",
       "      <th>NOX</th>\n",
       "      <th>RM</th>\n",
       "      <th>AGE</th>\n",
       "      <th>DIS</th>\n",
       "      <th>RAD</th>\n",
       "      <th>TAX</th>\n",
       "      <th>PTRATIO</th>\n",
       "      <th>B</th>\n",
       "      <th>LSTAT</th>\n",
       "      <th>MEDV</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.00632</td>\n",
       "      <td>18.0</td>\n",
       "      <td>2.31</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.538</td>\n",
       "      <td>6.575</td>\n",
       "      <td>65.2</td>\n",
       "      <td>4.0900</td>\n",
       "      <td>1.0</td>\n",
       "      <td>296.0</td>\n",
       "      <td>15.3</td>\n",
       "      <td>396.90</td>\n",
       "      <td>4.98</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.02731</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>6.421</td>\n",
       "      <td>78.9</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>396.90</td>\n",
       "      <td>9.14</td>\n",
       "      <td>21.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.02729</td>\n",
       "      <td>0.0</td>\n",
       "      <td>7.07</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.469</td>\n",
       "      <td>7.185</td>\n",
       "      <td>61.1</td>\n",
       "      <td>4.9671</td>\n",
       "      <td>2.0</td>\n",
       "      <td>242.0</td>\n",
       "      <td>17.8</td>\n",
       "      <td>392.83</td>\n",
       "      <td>4.03</td>\n",
       "      <td>34.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.03237</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>6.998</td>\n",
       "      <td>45.8</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>394.63</td>\n",
       "      <td>2.94</td>\n",
       "      <td>33.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.06905</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.18</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.458</td>\n",
       "      <td>7.147</td>\n",
       "      <td>54.2</td>\n",
       "      <td>6.0622</td>\n",
       "      <td>3.0</td>\n",
       "      <td>222.0</td>\n",
       "      <td>18.7</td>\n",
       "      <td>396.90</td>\n",
       "      <td>5.33</td>\n",
       "      <td>36.2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      CRIM    ZN  INDUS  CHAS    NOX     RM   AGE     DIS  RAD    TAX  \\\n",
       "0  0.00632  18.0   2.31   0.0  0.538  6.575  65.2  4.0900  1.0  296.0   \n",
       "1  0.02731   0.0   7.07   0.0  0.469  6.421  78.9  4.9671  2.0  242.0   \n",
       "2  0.02729   0.0   7.07   0.0  0.469  7.185  61.1  4.9671  2.0  242.0   \n",
       "3  0.03237   0.0   2.18   0.0  0.458  6.998  45.8  6.0622  3.0  222.0   \n",
       "4  0.06905   0.0   2.18   0.0  0.458  7.147  54.2  6.0622  3.0  222.0   \n",
       "\n",
       "   PTRATIO       B  LSTAT  MEDV  \n",
       "0     15.3  396.90   4.98  24.0  \n",
       "1     17.8  396.90   9.14  21.6  \n",
       "2     17.8  392.83   4.03  34.7  \n",
       "3     18.7  394.63   2.94  33.4  \n",
       "4     18.7  396.90   5.33  36.2  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# loading the dataset\n",
    "boston_data = load_boston()\n",
    "boston = pd.DataFrame(boston_data.data,columns = boston_data['feature_names'])\n",
    "boston['MEDV'] = boston_data['target']\n",
    "boston.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Again, i would advice you to get familier with the data, since its an inbuilt dataset, we can use boston_data.DESCR to get mode information"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- CRIM - per capita crime rate by town\n",
    "- ZN - proportion of residential land zoned for lots over 25,000 sq.ft.\n",
    "- INDUS - proportion of non-retail business acres per town.\n",
    "- CHAS - Charles River dummy variable (1 if tract bounds river; 0 otherwise)\n",
    "- NOX - nitric oxides concentration (parts per 10 million)\n",
    "- RM - average number of rooms per dwelling\n",
    "- AGE - proportion of owner-occupied units built prior to 1940\n",
    "- DIS - weighted distances to five Boston employment centres\n",
    "- RAD - index of accessibility to radial highways\n",
    "- TAX - full-value property-tax rate per $10,000\n",
    "\n",
    "- PTRATIO - pupil-teacher ratio by town\n",
    "- B - 1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town\n",
    "- LSTAT - % lower status of the population\n",
    "- MEDV - Median value of owner-occupied homes in $1000's"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "soucrs - [https://www.cs.toronto.edu/~delve/data/boston/bostonDetail.html]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Its better that we go through the above list once, so that we know what we are dealing with here."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(253, 13) (253, 13)\n"
     ]
    }
   ],
   "source": [
    "#splitting the data into train and test \n",
    "X_train,X_test,y_train,y_test = train_test_split(boston.drop('MEDV',axis = 1),boston['MEDV'],test_size = 0.5,random_state = 0)\n",
    "print(X_train.shape,X_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Bagging\n",
    "- We have learnt it earlier that random forest is a special case of bagging algorithms, and that random forest uses a subspace of featurs that are considered for each split. Generally random forest considers m features at each split, out of total feature space of p features. \n",
    "- Usually the value of m is sqrt(p), but if m = p, it becomes same as baaging.\n",
    "- Although there is a separate library for bagging in sklearn, we are going to follow the same approach in the book, and use random forests with m = p."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In random forests, we use max_features paramter to control the number of features considered for each split, so, for m = p, we just pass max_features as None, or the size of predictor space - \n",
    "for more on this check out the max_features paramtere from here - https://scikit-learn.org/stable/modules/generated/sklearn.ensemble.RandomForestClassifier.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Lenovo\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:245: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training score  0.9767990266869814\n",
      "Test Error  0.7609357558107139\n"
     ]
    }
   ],
   "source": [
    "bagging = RandomForestRegressor(max_features=None)# or we can pass max_features = len(X_train.columns) = 13\n",
    "bagging.fit(X_train,y_train)\n",
    "print('Training score ',bagging.score(X_train,y_train))\n",
    "print('Test Error ',bagging.score(X_test,y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here the score means the r2 score\n",
    "So, we can see there is a vast imporvement from the score we were getting the last time when we used decision tree regressor"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NOTE: In book, the errors are mentioned in RSS, which can be calculated by sklearn.metrics.mean_squared_error than multiplying it by number of observations, i have used R2 score here. (Doesn;t have any effect in comparison though)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Random Forests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training score  0.9766441092887623\n",
      "Test Error  0.7704984744736889\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Lenovo\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:245: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    }
   ],
   "source": [
    "rf = RandomForestRegressor(max_features = 6) # same as book\n",
    "rf.fit(X_train,y_train)\n",
    "print('Training score ',rf.score(X_train,y_train))\n",
    "print('Test Error ',rf.score(X_test,y_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we are getting more improved results than bagging. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature importance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x1f74e4f5780>"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAlkAAAGFCAYAAADO7OaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAgAElEQVR4nO3de7hdV13u8e9LSgG5nUJzRNuGpBDUcpFiaFUQ5FJIBZtyT721PGhFWwQLSFEsxyCKoOBRq1C1ohyh3AQCBCpyO6CnkEDLJYVIKEhD5VBsAY9AS8rv/DFn6MrO2nuvNHvsNdfu9/M8eTLnmHOu/PbOvrxrzDHGTFUhSZKkpXWLaRcgSZK0EhmyJEmSGjBkSZIkNWDIkiRJasCQJUmS1MBh0y5griOPPLLWrl077TIkSZIW9ZGPfOQrVbV63LHBhay1a9eyY8eOaZchSZK0qCT/Nt8xbxdKkiQ1YMiSJElqwJAlSZLUgCFLkiSpAUOWJElSA4YsSZKkBgxZkiRJDRiyJEmSGjBkSZIkNWDIkiRJasCQJUmS1IAhS5IkqQFDliRJUgOGLEmSpAYOm3YBh2LtuW9v8rqff9GjmryuJEm6+bAnS5IkqQFDliRJUgOGLEmSpAYmCllJNibZlWR3knMXOO/xSSrJhpG25/bX7UryyKUoWpIkaegWHfieZBVwPnASsAfYnmRrVV0+57zbA78GfGik7ThgM3BP4PuBf0pyj6q6Yek+BEmSpOGZpCfrBGB3VV1RVdcDFwGbxpz3AuDFwLdG2jYBF1XVdVX1OWB3/3qSJEkr2iQh6yjgypH9PX3bdyU5Hjimqt52sNf215+ZZEeSHVdfffVEhUuSJA3ZJCErY9rquweTWwAvA555sNd+t6HqgqraUFUbVq9ePUFJkiRJwzbJYqR7gGNG9o8GrhrZvz1wL+B9SQDuAmxNcsoE10qSJK1Ik/RkbQfWJ1mX5HC6gexb9x2sqq9V1ZFVtbaq1gKXAKdU1Y7+vM1JbpVkHbAe+PCSfxSSJEkDs2hPVlXtTXI2cDGwCriwqnYm2QLsqKqtC1y7M8nrgMuBvcBZziyUJEk3BxM9u7CqtgHb5rSdN8+5Pzln/4XAC29ifZIkSTPJFd8lSZIaMGRJkiQ1YMiSJElqwJAlSZLUgCFLkiSpAUOWJElSA4YsSZKkBgxZkiRJDRiyJEmSGjBkSZIkNWDIkiRJasCQJUmS1IAhS5IkqQFDliRJUgOGLEmSpAYMWZIkSQ0YsiRJkhowZEmSJDVgyJIkSWrAkCVJktSAIUuSJKkBQ5YkSVIDhixJkqQGDFmSJEkNGLIkSZIaMGRJkiQ1YMiSJElqwJAlSZLUgCFLkiSpgYlCVpKNSXYl2Z3k3DHHn5rkE0kuS/LBJMf17WuTfLNvvyzJy5f6A5AkSRqiwxY7Ickq4HzgJGAPsD3J1qq6fOS0V1fVy/vzTwFeCmzsj322qu67tGVLkiQN2yQ9WScAu6vqiqq6HrgI2DR6QlV9fWT3tkAtXYmSJEmzZ9GeLOAo4MqR/T3AiXNPSnIWcA5wOPDQkUPrklwKfB14XlV9YMy1ZwJnAqxZs2bi4mfJ2nPf3uR1P/+iRzV5XUmSdGgm6cnKmLYDeqqq6vyquhvwHOB5ffO/A2uq6ni6APbqJHcYc+0FVbWhqjasXr168uolSZIGapKQtQc4ZmT/aOCqBc6/CDgVoKquq6r/6Lc/AnwWuMdNK1WSJGl2TBKytgPrk6xLcjiwGdg6ekKS9SO7jwI+07ev7gfOk+RYYD1wxVIULkmSNGSLjsmqqr1JzgYuBlYBF1bVziRbgB1VtRU4O8nDgW8D1wKn95c/CNiSZC9wA/DUqrqmxQciSZI0JJMMfKeqtgHb5rSdN7L99HmueyPwxkMpUJIkaRa54rskSVIDhixJkqQGDFmSJEkNGLIkSZIaMGRJkiQ1YMiSJElqwJAlSZLUgCFLkiSpAUOWJElSA4YsSZKkBgxZkiRJDRiyJEmSGjBkSZIkNWDIkiRJasCQJUmS1IAhS5IkqQFDliRJUgOGLEmSpAYMWZIkSQ0YsiRJkhowZEmSJDVgyJIkSWrAkCVJktSAIUuSJKkBQ5YkSVIDhixJkqQGDFmSJEkNGLIkSZIaMGRJkiQ1MFHISrIxya4ku5OcO+b4U5N8IsllST6Y5LiRY8/tr9uV5JFLWbwkSdJQLRqykqwCzgdOBo4DThsNUb1XV9W9q+q+wIuBl/bXHgdsBu4JbAT+vH89SZKkFW2SnqwTgN1VdUVVXQ9cBGwaPaGqvj6ye1ug+u1NwEVVdV1VfQ7Y3b+eJEnSinbYBOccBVw5sr8HOHHuSUnOAs4BDgceOnLtJXOuPWrMtWcCZwKsWbNmkrolSZIGbZKerIxpqwMaqs6vqrsBzwGed5DXXlBVG6pqw+rVqycoSZIkadgmCVl7gGNG9o8Grlrg/IuAU2/itZIkSSvCJCFrO7A+ybokh9MNZN86ekKS9SO7jwI+029vBTYnuVWSdcB64MOHXrYkSdKwLTomq6r2JjkbuBhYBVxYVTuTbAF2VNVW4OwkDwe+DVwLnN5fuzPJ64DLgb3AWVV1Q6OPRZIkaTAmGfhOVW0Dts1pO29k++kLXPtC4IU3tUBJkqRZ5IrvkiRJDRiyJEmSGjBkSZIkNWDIkiRJasCQJUmS1IAhS5IkqQFDliRJUgOGLEmSpAYMWZIkSQ0YsiRJkhowZEmSJDVgyJIkSWrAkCVJktSAIUuSJKkBQ5YkSVIDhixJkqQGDFmSJEkNGLIkSZIaMGRJkiQ1YMiSJElqwJAlSZLUgCFLkiSpAUOWJElSA4YsSZKkBgxZkiRJDRiyJEmSGjBkSZIkNWDIkiRJasCQJUmS1MBEISvJxiS7kuxOcu6Y4+ckuTzJx5O8O8ldR47dkOSy/s/WpSxekiRpqA5b7IQkq4DzgZOAPcD2JFur6vKR0y4FNlTVN5L8CvBi4En9sW9W1X2XuG5JkqRBm6Qn6wRgd1VdUVXXAxcBm0ZPqKr3VtU3+t1LgKOXtkxJkqTZMknIOgq4cmR/T982n6cA7xjZv3WSHUkuSXLquAuSnNmfs+Pqq6+eoCRJkqRhW/R2IZAxbTX2xOTngA3Ag0ea11TVVUmOBd6T5BNV9dn9XqzqAuACgA0bNox9bUmSpFkySU/WHuCYkf2jgavmnpTk4cBvAadU1XX72qvqqv7vK4D3AccfQr2SJEkzYZKQtR1Yn2RdksOBzcB+swSTHA+8gi5gfXmk/Ygkt+q3jwQeAIwOmJckSVqRFr1dWFV7k5wNXAysAi6sqp1JtgA7qmor8BLgdsDrkwB8oapOAX4IeEWS79AFuhfNmZUoSZK0Ik0yJouq2gZsm9N23sj2w+e57l+Aex9KgZIkSbPIFd8lSZIaMGRJkiQ1YMiSJElqwJAlSZLUgCFLkiSpAUOWJElSA4YsSZKkBgxZkiRJDRiyJEmSGjBkSZIkNWDIkiRJasCQJUmS1IAhS5IkqQFDliRJUgOGLEmSpAYMWZIkSQ0YsiRJkhowZEmSJDVgyJIkSWrAkCVJktSAIUuSJKkBQ5YkSVIDhixJkqQGDFmSJEkNGLIkSZIaMGRJkiQ1YMiSJElqwJAlSZLUwEQhK8nGJLuS7E5y7pjj5yS5PMnHk7w7yV1Hjp2e5DP9n9OXsnhJkqShWjRkJVkFnA+cDBwHnJbkuDmnXQpsqKr7AG8AXtxfeyfg+cCJwAnA85McsXTlS5IkDdMkPVknALur6oqquh64CNg0ekJVvbeqvtHvXgIc3W8/EnhXVV1TVdcC7wI2Lk3pkiRJwzVJyDoKuHJkf0/fNp+nAO+4iddKkiStCIdNcE7GtNXYE5OfAzYADz6Ya5OcCZwJsGbNmglKkiRJGrZJerL2AMeM7B8NXDX3pCQPB34LOKWqrjuYa6vqgqraUFUbVq9ePWntkiRJgzVJyNoOrE+yLsnhwGZg6+gJSY4HXkEXsL48cuhi4BFJjugHvD+ib5MkSVrRFr1dWFV7k5xNF45WARdW1c4kW4AdVbUVeAlwO+D1SQC+UFWnVNU1SV5AF9QAtlTVNU0+EkmSpAGZZEwWVbUN2Dan7byR7YcvcO2FwIU3tUBJkqRZ5IrvkiRJDRiyJEmSGjBkSZIkNWDIkiRJasCQJUmS1IAhS5IkqQFDliRJUgOGLEmSpAYMWZIkSQ0YsiRJkhowZEmSJDVgyJIkSWrAkCVJktSAIUuSJKkBQ5YkSVIDhixJkqQGDFmSJEkNGLIkSZIaMGRJkiQ1YMiSJElqwJAlSZLUgCFLkiSpAUOWJElSA4YsSZKkBgxZkiRJDRiyJEmSGjBkSZIkNWDIkiRJasCQJUmS1MBEISvJxiS7kuxOcu6Y4w9K8tEke5M8fs6xG5Jc1v/ZulSFS5IkDdlhi52QZBVwPnASsAfYnmRrVV0+ctoXgDOAZ415iW9W1X2XoFZJkqSZsWjIAk4AdlfVFQBJLgI2Ad8NWVX1+f7YdxrUKEmSNHMmuV14FHDlyP6evm1St06yI8klSU49qOokSZJm1CQ9WRnTVgfxb6ypqquSHAu8J8knquqz+/0DyZnAmQBr1qw5iJeWJEkapkl6svYAx4zsHw1cNek/UFVX9X9fAbwPOH7MORdU1Yaq2rB69epJX1qSJGmwJglZ24H1SdYlORzYDEw0SzDJEUlu1W8fCTyAkbFckiRJK9WiIauq9gJnAxcDnwJeV1U7k2xJcgpAkvsn2QM8AXhFkp395T8E7EjyMeC9wIvmzEqUJElakSYZk0VVbQO2zWk7b2R7O91txLnX/Qtw70OsUZIkaea44rskSVIDhixJkqQGDFmSJEkNGLIkSZIaMGRJkiQ1YMiSJElqwJAlSZLUgCFLkiSpAUOWJElSA4YsSZKkBgxZkiRJDRiyJEmSGjBkSZIkNWDIkiRJasCQJUmS1IAhS5IkqQFDliRJUgOGLEmSpAYMWZIkSQ0YsiRJkhowZEmSJDVgyJIkSWrAkCVJktSAIUuSJKkBQ5YkSVIDhixJkqQGDFmSJEkNGLIkSZIaMGRJkiQ1MFHISrIxya4ku5OcO+b4g5J8NMneJI+fc+z0JJ/p/5y+VIVLkiQN2aIhK8kq4HzgZOA44LQkx8057QvAGcCr51x7J+D5wInACcDzkxxx6GVLkiQN2yQ9WScAu6vqiqq6HrgI2DR6QlV9vqo+DnxnzrWPBN5VVddU1bXAu4CNS1C3JEnSoE0Sso4CrhzZ39O3TWKia5OcmWRHkh1XX331hC8tSZI0XJOErIxpqwlff6Jrq+qCqtpQVRtWr1494UtLkiQN1yQhaw9wzMj+0cBVE77+oVwrSZI0syYJWduB9UnWJTkc2AxsnfD1LwYekeSIfsD7I/o2SZKkFW3RkFVVe4Gz6cLRp4DXVdXOJFuSnAKQ5P5J9gBPAF6RZGd/7TXAC+iC2nZgS98mSZK0oh02yUlVtQ3YNqftvJHt7XS3AsddeyFw4SHUKEmSNHNc8V2SJKkBQ5YkSVIDhixJkqQGDFmSJEkNGLIkSZIaMGRJkiQ1YMiSJElqwJAlSZLUgCFLkiSpAUOWJElSA4YsSZKkBgxZkiRJDRiyJEmSGjBkSZIkNWDIkiRJasCQJUmS1IAhS5IkqQFDliRJUgOGLEmSpAYMWZIkSQ0YsiRJkhowZEmSJDVgyJIkSWrAkCVJktSAIUuSJKkBQ5YkSVIDhixJkqQGDFmSJEkNGLIkSZIamChkJdmYZFeS3UnOHXP8Vkle2x//UJK1ffvaJN9Mcln/5+VLW74kSdIwHbbYCUlWAecDJwF7gO1JtlbV5SOnPQW4tqrunmQz8AfAk/pjn62q+y5x3ZIkSYM2SU/WCcDuqrqiqq4HLgI2zTlnE/C3/fYbgIclydKVKUmSNFsmCVlHAVeO7O/p28aeU1V7ga8Bd+6PrUtyaZL3J/mJcf9AkjOT7Eiy4+qrrz6oD0CSJGmIJglZ43qkasJz/h1YU1XHA+cAr05yhwNOrLqgqjZU1YbVq1dPUJIkSdKwLTomi67n6piR/aOBq+Y5Z0+Sw4A7AtdUVQHXAVTVR5J8FrgHsONQC1c7a899+5K/5udf9Kglf01JkoZskp6s7cD6JOuSHA5sBrbOOWcrcHq//XjgPVVVSVb3A+dJciywHrhiaUqXJEkarkV7sqpqb5KzgYuBVcCFVbUzyRZgR1VtBf4aeFWS3cA1dEEM4EHAliR7gRuAp1bVNS0+EEmSpCGZ5HYhVbUN2Dan7byR7W8BTxhz3RuBNx5ijZIkSTNnopAlDZXjxyRJQ+VjdSRJkhowZEmSJDVgyJIkSWrAkCVJktSAIUuSJKkBQ5YkSVIDLuEgLROXm5Ckmxd7siRJkhowZEmSJDVgyJIkSWrAkCVJktSAA98lHWCWBulbq5MfpKGyJ0uSJKkBQ5YkSVIDhixJkqQGDFmSJEkNGLIkSZIaMGRJkiQ1YMiSJElqwJAlSZLUgCFLkiSpAUOWJElSAz5WR5J0AB8BJB06e7IkSZIasCdLkjSzWvS4gb1uWhqGLEmSloGB8ObH24WSJEkN2JMlSZL2Y6/b0pioJyvJxiS7kuxOcu6Y47dK8tr++IeSrB059ty+fVeSRy5d6ZIkScO1aMhKsgo4HzgZOA44Lclxc057CnBtVd0deBnwB/21xwGbgXsCG4E/719PkiRpRZvkduEJwO6qugIgyUXAJuDykXM2Af+j334D8GdJ0rdfVFXXAZ9Lsrt/vf+zNOVLkqSbsyHf2kxVLXxC8nhgY1X9Yr//88CJVXX2yDmf7M/Z0+9/FjiRLnhdUlX/q2//a+AdVfWGOf/GmcCZ/e4PALsO+SM70JHAVxq8bgvW2oa1tjErtc5KnWCtrVhrGzf3Wu9aVavHHZikJytj2uYms/nOmeRaquoC4IIJarnJkuyoqg0t/42lYq1tWGsbs1LrrNQJ1tqKtbZhrfObZOD7HuCYkf2jgavmOyfJYcAdgWsmvFaSJGnFmSRkbQfWJ1mX5HC6gexb55yzFTi933488J7q7kNuBTb3sw/XAeuBDy9N6ZIkScO16O3Cqtqb5GzgYmAVcGFV7UyyBdhRVVuBvwZe1Q9sv4YuiNGf9zq6QfJ7gbOq6oZGH8timt6OXGLW2oa1tjErtc5KnWCtrVhrG9Y6j0UHvkuSJOng+VgdSZKkBgxZkiRJDRiyJEmSGjBkSZJWlCS3W+DY3ZazFt28GbKmLMnvTbuGm4Mkd07ymCQ/Mu1aRiX5wZHtW8059qPLX9HCkjxunvbDk/z2ctezEiW5ZZLjk/z3adcyiSRH9o9RG5KPJXniaEOSWyf5XeCdU6pJN0MrcnZhkj9Z6HhV/dpy1bKYJB+tqvtNu47FJHnsQser6h+Wq5ZJJHkbcG5VfTLJ9wEfBXYAdwMuqKo/nmqBvdH//7lfC0P82khyMfAd4Fer6nN928l0D4Z/Z1U9Y5r1jUpyL+A36B5sX3RLyfxRVX18qoXNkeTlwJ/2S97cke7ZrjcAdwKeVVWvmWqBI/rg/yK6pXpeALyK7jEltwB+oaoGEWD63qo/o1um6FeAewJ/CLwZ+J2q+n9TLG+sJA8Bnkb3aDmATwF/VlXvm1pRYyT5JeB9VfWZPlxfCDwO+DxwRlV9dJr17ZPkFxY6XlV/txx1TPJYnVn0VOCTwOvoVpgf2rusUauSHME8NVbVNctcz3zeAFzW/4H96y1gUCELWFdVn+y3nwy8q6p+IcntgX8GBhGy2P/zOPdrYHBft1X1yCSnAf+U5NXAvYDVwJOq6mPTre5GSTbR/VL9/f7vAD8CvDHJs6rqLdOsb46fqKqn9ttPBv61qk5NchfgHcBgQhZdcPlNuqd6vAc4uaou6XtkX8NAeomq6rPAyUmeDXwa+BLwyKraOd3KxkvyKLrP7Rbgd+i+Xu8HXJjk7KraNs365ng68Mp++zTgPsA64HjgfwI/MZ2yDnD/MW0Bfho4CjBkHYLvA54APIluEdTXAm+sqmunWtV4Pwh8hPmf83js8pYzr8fRfT7vA7wFeE1V7Z5uSQv69sj2w4C/BKiq/0zynemUNFbNsz1ufyheR9cz8OvAV4GHVtW/TrekA2wBTqqqz4+0fSzJe+i+focUsq4f2T4JeD1AVX1peHfhOKyq/hEgyZaqugSgqj49pFr7x7s9G3gK8KvATwF/kuRXq2rXVIsb79nAqXPeqFyWZAfwp8CQQtbeqtr38/XRwN9V1X/QvfF68RTr2k9VPW3fdt/j9rPAc4BLgBcuVx0rMmT1/+EvB16e5Ci6tL0zyXOq6lXTre4Al1fV8dMuYjFV9SbgTUluC2wC/ijJnYHfqqr3T7e6sa5M8jS652fej/4ddpLbALecZmFzHN3f3s7INv3+UdMra7wkDwT+nK438BjgwcBbk7wWeGFVXTfN+kbcck7AAqCqPp9kSP//AF9N8mjgi8AD6ILBvqBwm2kWNsboG5Rvzjk2pDcFlwLvB36kqr4GXNB/jt+S5E1V9dzplneAu4zrCa6qjyf53mkUtIDv9EMwrqV7AzsaWAb19dp/D50BPBP4EPD45Q7ZKzJk7ZPkfnQB6yS6bvePTLeiFeFbwNeArwNrgFtPt5x5PYWuN+PhdLeyvtq3/yjwN1Or6kDPHtneMefY3P0h+GPgF6tq3zNI35zkXcB5wMfoemaH4NtJ1lTVF0Ybk9yVrnd7SH4Z+BPgLsAzqupLffvDgLdPrarxfjjJ1+neBNym36bfH9LPgjOqar+f91X1tiT/BDxvSjUt5L9u4rFpOI/uZ9MqYOu+W7BJHgxcMc3CRiU5i+7W5ruBjVX1b1OpY4UOfP8dum7MTwEX0Q3IHdoPVgCSnFFVrxzTfmvgp6vq9ctf1YH6QZmnAScA/wRcVFVDDAFqKMktqmrs7dYkP1RVn1rumsZJcirwYuD36N5cFd0YjXOB51TVm6dYnqYkyQOAn6mqs6Zdy6gkXwX+97hDwAOr6ohlLmlBfQ/R7UeH4CT5HmBVVf3n9Cq7UT8s5MvA1ezfyxqgquo+y1LHCg1Z36FL1Pu6s/d9kMv6yT1YSVYBj6ALM48EPlBVj59uVZ3+c/px4IN0n8/9vnCGNGMTIMlbWeD2RVWdsozlzKu//XbsvpkuSd5AN6sM4Her6j1TK24e/dICZ9GNy9o3a+/8qvryVAubI8kP090muCfd9/5O4A+HNEAfIMmfcuDYvK8A762qD06nqpUjyX2BnwGeCHyObnzun023qv31vUDzGuiQDOC7450eQvc5/umqGsTtzSRPpRsmMu73wJOqalnGj63UkHXXhY5Pq9twPkkeRPcF+ijgw3TjMo6tqm9MtbARSc5g4dDyt8tXzeJm5YdWkncDT6uqy/v9T9CNIbgt8JtVtXGK5R2g7wl4Nd3son0TNu4HnA78bFX98/Sqm01JTh/TfCe6UPDaoSw3MkuS3APYTPeG9T/oJj89q6oW/N0wNEmOATZX1UumXctcSU6k+731GLqv17Pobh8OYoJZkhvoxuX9fFV9cc6xZVseZ0WGrPn0PUWbq+rvp13LPkn2AF8A/gJ4cz/77XNVtW7Kpa0YSVYDVNXV065lriTbq+r+I/v/UFWP7bf/uaoeML3qDpTkEuBXqurSOe33BV5RVSdOp7L9zUpP5kL6SRr/MgsTY4am73n/APCUfbOgk1xRVUOZrT2vJEfSzY4/jW7yy5uq6lnTrepGSV5I9wbgC3TLdrwJ2DG031lJLqWbpHMecM7o0Jskly7X99WKHPie5A50qfooYCvwLuBs4Fl06zwNJmQBbwROpVse4YYkb2FYs3SA2fylleT5dIv7BbhFkr10iz5umW5l+/lvozv7AlZvEN3uc9xhbsACqKrL+jXIhuIPp13Aoaqqbw5pWYQZ8zi6nqz3Jnkn3djcwX4y+++dx9D1DN2DLrgcW1VHT7Ww8c4EdtF1DLytqr6VZHC/s+iGBv1lkvcDf5/kp4Cz+jtEy1bvigxZdKsQX0u3cvIv0s3gOhzYVFWXLXThcquqpyd5Bt097dOAlwB3SPdIiG0DWpl4pn5pJfl14IHA/evGlcmPBf4iya9X1cumWuCNPp3kUVW13yyyfrr5ENfzSZIj5t4SSHInhvWYridX1RnTLuKm6gcW/zzdEiQ6eG+tqn1LzpxKt6bb9yb5C7qeoX+cbnkH+DLdUJHnAR+sqkrymCnXNJ+7cOPY4T9O8l66maaHDXGCWVX9a5IfA34XuDSLrAS/1Fbk7cIkn6iqe/fbq+gGka4ZyqyHhfRr+Gyk+wJ+RFUdOeWSFpXkAUMbi9N3FZ9UVV+Z074a+Meh3IJJcne6afr/QvfoH+hWJv9x4NFDW+QzyZnAL9H1Co/W+wfAhVX1imnVNmo5x1wcqiT/SffOerSn5Rt040meUVVXTaWwGTbu/79/I/AEukHPD51OZeP1bwo3043FfDXdGLJ3Df32Zj8L/tF0v68eCLy7qn5mulV1xt0STPKTdI8BWl1Vy9LzvlJD1uCfAbdPklfO9447yW2qau6Cf1PRh9Un0t2CfWd1zwR8NN0jNm4zlNCyT5JPVtW9DvbYNKR7MPTPcuNsvZ3AZ4DThjbVHL7by/YbdPVCV+9Lquqt06tqf0k+TfeDf77HVQ3i+WpqYznH3Cylvrf9NLrAtR54Pl3P26DebI3T3/J87FAmQSU5ddxSLekeY/fLVfWiZaljhYasG7hxAbfQrUL7DW5cwuEO06ptriEHwFFJXkm3wveHgROBfwN+jO4hzINbc2ihz+tQP+dJjqf7ATvYqeazou8d2s74kFUD7Mk4DDiZGxdzvRy4eIi3X2ZBP6HopfMdr6p5jw1FknvTLz1RVXebdj37JDlnoeOz8LldTityTFZVrZp2DQfhe/pfrkN/x70BuE9VfafvIv4KcPe6cXXqodm3MvVcg1qZep6p5qmqh0y1sHkkOW+Bw1VVL1i2Yha2e2hBaj5Jvh94L/DvdI+DCd0tmJcmeYi3C2+SVcDtGPBg98VU1SeS/DZdT/GQjN5m+2VgdIjAyuu1OUQrsidrlszKO+5ZugU7S2ZtqnmSZ45pvi3dY4zuXFW3W+aSxpql20V9L/Flc9fDSmFDcJIAAARGSURBVPJrdM/eG7eOlhYwaz+fFpgR/0zgY1W1aYrlzWuWvs+mxZA1ZbPyRZrkG8DufbvA3Ub2qYGuoj90/QyizXQD3fdNNf+roa05M04/BuPpdAHrdcAf1UBWfU/yiHEzyIa4uGOST1fV2Gc+JtlVVT+w3DXNuln5ubpPv3TPvhnxDwOOoJsR//ShzYgfNWthdhpW5O1CNfHDdOs2XTmn/a6AtzNuoqp6EzBLU833zdI6h26w/t8C95u7pMO0jX7exi3uOK265rHQ5JbBPPVhxjxs2gUcpGNHZsT/FTM0I14LM2RN33NGd/olHO4FfHEovQK9l9E95mW/RxL1SyK8DPjpqVS1QlTVf9Etkvv3I1PNzwUGFbKSvAR4LHABcO8BreO2nxlb3PGOSR47pj3AYCbpzJKqumbaNRykb+/bqKob0j31Y5ABK92jv/bdArt7ko+PHveuxv68XThlSV5Otwr5ziR3pOsuvoHuWVDPqqrXTLXA3iJLInx3XTKtbP0YsuuAvYx/sv0gQkGSb3Lg4o6DHOuW5G8WOl5VT16uWjQdMzYjfj0L3NXYN7ZUHUPWlCXZWVX37LefAfxkVZ2a5C7AO4YyriDJ7qq6+8Eek6ZhVhd3lIYuydvo7mp8fE77BuD5VeVdjRHeLpy+60e2TwJeD1BVX8qwnlu2PckvVdVfjjYmeQrwkSnVJI1V3WOTXjayuOObge9P8hwGtrjjIo/5qKp61bIVIy1u7dyABVBVO5KsXf5yhs2erCnrn/v0R8AX6dbK+cE+YB0GfHK+WUfLLcn30o1ruZ4bQ9UGuhkwjxnwelkSMOjFHf90XDPdOMejqso3wxoM72ocHL95p++XgT+he+jmM0bCysPonmk3CFX1f4EfT/IQuoH5AG+vqvdMsSxpYkNd3LGqnrZvO1339c/STYi5BHjhtOqS5uFdjYNgT9aAJXnG3AUKJS1u1hZ37Huuz6Cr70PA71fVrqkWJY3hXY2DY8gasCRfqKo1065DmjWztLhjkrPoFnV9N/CiucukSEM0567GTu9qjGfIGrAkV1bVMdOuQ5o1o8uKJFnFgBd37JfF+DJwNeOXxXDdIWlGOSZr2EzA0k0zM4s7AoN/hJKkm8aerCnrHxA97j8hwG2cWSQdvFla3FHSymXIkrTiJLllVX178TOnb5E3WgZCaYYZsiStOEk+WlX3m3Ydkm7ebjHtAiSpgUE9LkHSzZPjfSStRKuTnDPfwap66XIWI+nmyZAlaSVaBdwOe7QkTZFjsiStOI7JkjQEjsmStBLZgyVp6uzJkrTiJPl+4InA3YFPAH9dVXunW5WkmxtDlqQVJ8lr6VZ9/wBwMvBvVfX06VYl6ebGkCVpxZnz7MLDgA87RkvScnNMlqSVaPTZhd4mlDQV9mRJWnF8dqGkITBkSZIkNeDtQkmSpAYMWZIkSQ0YsiRJkhowZEmSJDXw/wECa4gwypRgvQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 720x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "pd.Series(rf.feature_importances_,index = X_train.columns).sort_values(ascending=False).plot.bar(figsize = (10,6))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "From the above graph we can see that RM and LSTAT are by far the most important features of all."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Happy Learning :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}