Stacking on Boston_Dataset.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Stacking - Boston Dataset.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPpP6cPb0Y6LQPwmPil03bY",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/mukul-mschauhan/Data-Science-Projects/blob/master/Stacking%20on%20Boston_Dataset.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-37FNDAvDNNX",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        },
        "outputId": "ab670272-52e3-47da-ff6e-07bfda96009b"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "from sklearn.datasets import load_boston\n",
        "import warnings"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/statsmodels/tools/_testing.py:19: FutureWarning: pandas.util.testing is deprecated. Use the functions in the public API at pandas.testing instead.\n",
            "  import pandas.util.testing as tm\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NLr6edhGDmls",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Function Approach to do the same\n",
        "def _df(load):\n",
        "  # Creating Dataframe\n",
        "  df = pd.DataFrame(data = load.data, columns = load.feature_names)\n",
        "  # Adding Output to the DF\n",
        "  df[\"label\"] = load.target\n",
        "  # Return df\n",
        "  return(df)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BTC6ZvUkDwTc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df = _df(load_boston()) # Running Function Created Above"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ii3DphiPFRfk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "9f08479d-0231-474d-cdea-4079804115ea"
      },
      "source": [
        "df.columns"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Index(['CRIM', 'ZN', 'INDUS', 'CHAS', 'NOX', 'RM', 'AGE', 'DIS', 'RAD', 'TAX',\n",
              "       'PTRATIO', 'B', 'LSTAT', 'label'],\n",
              "      dtype='object')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KR8RB6W3Efi0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 904
        },
        "outputId": "d52efd79-8e3f-4d1d-939f-c5aef0337dc6"
      },
      "source": [
        "# Data Dictionary\n",
        "print(load_boston().DESCR)"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            ".. _boston_dataset:\n",
            "\n",
            "Boston house prices dataset\n",
            "---------------------------\n",
            "\n",
            "**Data Set Characteristics:**  \n",
            "\n",
            "    :Number of Instances: 506 \n",
            "\n",
            "    :Number of Attributes: 13 numeric/categorical predictive. Median Value (attribute 14) is usually the target.\n",
            "\n",
            "    :Attribute Information (in order):\n",
            "        - CRIM     per capita crime rate by town\n",
            "        - ZN       proportion of residential land zoned for lots over 25,000 sq.ft.\n",
            "        - INDUS    proportion of non-retail business acres per town\n",
            "        - CHAS     Charles River dummy variable (= 1 if tract bounds river; 0 otherwise)\n",
            "        - NOX      nitric oxides concentration (parts per 10 million)\n",
            "        - RM       average number of rooms per dwelling\n",
            "        - AGE      proportion of owner-occupied units built prior to 1940\n",
            "        - DIS      weighted distances to five Boston employment centres\n",
            "        - RAD      index of accessibility to radial highways\n",
            "        - TAX      full-value property-tax rate per $10,000\n",
            "        - PTRATIO  pupil-teacher ratio by town\n",
            "        - B        1000(Bk - 0.63)^2 where Bk is the proportion of blacks by town\n",
            "        - LSTAT    % lower status of the population\n",
            "        - MEDV     Median value of owner-occupied homes in $1000's\n",
            "\n",
            "    :Missing Attribute Values: None\n",
            "\n",
            "    :Creator: Harrison, D. and Rubinfeld, D.L.\n",
            "\n",
            "This is a copy of UCI ML housing dataset.\n",
            "https://archive.ics.uci.edu/ml/machine-learning-databases/housing/\n",
            "\n",
            "\n",
            "This dataset was taken from the StatLib library which is maintained at Carnegie Mellon University.\n",
            "\n",
            "The Boston house-price data of Harrison, D. and Rubinfeld, D.L. 'Hedonic\n",
            "prices and the demand for clean air', J. Environ. Economics & Management,\n",
            "vol.5, 81-102, 1978.   Used in Belsley, Kuh & Welsch, 'Regression diagnostics\n",
            "...', Wiley, 1980.   N.B. Various transformations are used in the table on\n",
            "pages 244-261 of the latter.\n",
            "\n",
            "The Boston house-price data has been used in many machine learning papers that address regression\n",
            "problems.   \n",
            "     \n",
            ".. topic:: References\n",
            "\n",
            "   - Belsley, Kuh & Welsch, 'Regression diagnostics: Identifying Influential Data and Sources of Collinearity', Wiley, 1980. 244-261.\n",
            "   - Quinlan,R. (1993). Combining Instance-Based and Model-Based Learning. In Proceedings on the Tenth International Conference of Machine Learning, 236-243, University of Massachusetts, Amherst. Morgan Kaufmann.\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vXhQnqB7Elh_",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Splitting the Data in Train and Test\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "# Getting X and Ys\n",
        "X = df.drop(\"label\", axis = 1)\n",
        "y = df.label\n",
        "\n",
        "# Creating Train and Test\n",
        "xtrain,xtest,ytrain,ytest = train_test_split(X, y, test_size = 0.33, random_state = None)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CQZEEpryVAqo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "ba869696-93d8-4eaf-9fdc-f09829a2624b"
      },
      "source": [
        "# Apply Linear Regression Model as Base Model\n",
        "from sklearn.linear_model import LinearRegression\n",
        "lr = LinearRegression()\n",
        "pred_lr = lr.fit(xtrain, ytrain).predict(xtest)\n",
        "\n",
        "# Checking Model Metrics\n",
        "from sklearn.metrics import r2_score, mean_squared_error\n",
        "print(\"R2 Score: \", r2_score(ytest, pred_lr))\n",
        "print(\"RMSE: \", np.sqrt(mean_squared_error(ytest, pred_lr)))"
      ],
      "execution_count": 125,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "R2 Score:  0.6754610877705005\n",
            "RMSE:  5.635157057595285\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rm8HWea2VnW4",
        "colab_type": "text"
      },
      "source": [
        "# Base Model: Linear Regression\n",
        "\n",
        "The Base Model reports R Sqaured Value: 0.67 and RMSE: 5.6.\n",
        "Now, we will use Stacking to Improve the Performance of the Model(s).\n",
        "\n",
        "Stacking generally uses lots of models and finds the best predicted values from each model and creates a meta learner to learn from the training set + predicted values to generate a Super Model that improves the overall performance."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "17zOqr_bVi3U",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "ac9ba86b-3934-44e9-b046-5577b8949359"
      },
      "source": [
        "# Checking Sklearn Version\n",
        "import sklearn\n",
        "sklearn.__version__"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'0.22.2.post1'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E2Xk07iwD79O",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Get a List of Models as Base Models\n",
        "from sklearn.ensemble import StackingRegressor, RandomForestRegressor, BaggingRegressor, GradientBoostingRegressor\n",
        "\n",
        "from sklearn.linear_model import Ridge, Lasso\n",
        "def base_models():\n",
        "  models = dict()\n",
        "  models['lr'] = LinearRegression()\n",
        "  models[\"Ridge\"] = Ridge()\n",
        "  models[\"Lasso\"] = Lasso()\n",
        "  models[\"Tree\"] = DecisionTreeRegressor()\n",
        "  models[\"Random Forest\"] = RandomForestRegressor()\n",
        "  models[\"Bagging\"] = BaggingRegressor()\n",
        "  models[\"GBM\"] = GradientBoostingRegressor()\n",
        "  return models\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bgZKS02WEuHc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Now we will apply K Fold Cross Validation. We will now create a evaluate function with Repeated Stratified K Fold\n",
        "# And Capture the Cross Val Score\n",
        "from sklearn.model_selection import cross_val_score, RepeatedStratifiedKFold, StratifiedKFold, KFold, RepeatedKFold\n",
        "from matplotlib import pyplot\n",
        "\n",
        "# Function to evaluate the list of models\n",
        "def eval_models(model):\n",
        "  cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
        "  scores = -cross_val_score(model, X, y, scoring='neg_mean_absolute_error', cv=cv, n_jobs=-1, error_score='raise')\n",
        "  return scores"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B7VQfsDxFIeL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        },
        "outputId": "c2d704a9-b364-46a3-bb8a-3c73fea028a0"
      },
      "source": [
        "# Lets Apply and Calculate the Scores\n",
        "# Getting X and Ys\n",
        "X = df.drop(\"label\", axis = 1)\n",
        "y = df.label\n",
        "\n",
        "# get the models to evaluate\n",
        "models = base_models()\n",
        "# evaluate the models and store results\n",
        "results, names = list(), list() \n",
        "\n",
        "for name, model in models.items():\n",
        "  scores = eval_models(model)\n",
        "  results.append(scores)\n",
        "  names.append(name)\n",
        "  print('>%s %.3f (%.3f)' % (name, scores.mean(), scores.std()))\n"
      ],
      "execution_count": 64,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            ">lr 3.391 (0.530)\n",
            ">Ridge 3.382 (0.519)\n",
            ">Lasso 3.711 (0.549)\n",
            ">Tree 2.875 (0.391)\n",
            ">Random Forest 2.197 (0.353)\n",
            ">Bagging 2.251 (0.343)\n",
            ">GBM 2.088 (0.278)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uQSNPAXKFsw6",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 346
        },
        "outputId": "6e4bfb41-fb3b-4365-e868-4e98f1a70ef0"
      },
      "source": [
        "# plot model performance for comparison\n",
        "pyplot.boxplot(results, labels=names, showmeans=True)\n",
        "pyplot.xticks(rotation = 90)\n",
        "pyplot.ylabel(\"MSE\")\n",
        "pyplot.title(\"Model Performance\")\n",
        "pyplot.show()"
      ],
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAFJCAYAAACW1Sr+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de5xddXnv8c83QyRoDMyQqMgl8dKjIykXnSpIKgatRxSDR+mRVKrYsZRWAhxbPdXhCGhTa49aaWidUsYLguMl5RJUqHgcwFFBJyGBQEpNVYyIMklGLobgkHnOH2vtsDPs2bP3zOy99pr1fb9e68W6z7OHzHr277J+P0UEZmZWXHOyDsDMzLLlRGBmVnBOBGZmBedEYGZWcE4EZmYF50RgZlZwTgSWa5KWSApJ+9Vw7pmSBpsU1wmSfiTpUUlvbsbPNJsqJwJrGkk/lfRbSQvH7b8jfZgvySayfRLKo+nyU0l/PY1bfhi4NCLmR8S1MxWnWSM4EViz/QRYWdqQ9LvA07ML5ykOioj5JDF+SNLr67m4rGSyGLh7KgHUUroxm0lOBNZsXwDeUbb9TuCK8hMkHSjpCknDku6TdIGkOemxNkkfl7Rd0o+BN1a4tk/SA5Lul/Q3ktrqDTIivk/yIF+a3vdPJG2RNCLp3yUtLvuZIek9kn4E/EjSfwHPB65PSxf7S3qupHWSdkraKulPy66/SNJaSVdKehg4U9LNaezfS+9xvaSDJV0l6WFJPywvQUm6RNK29Nh6Sb8/7v5fSX+nj0i6W1JX2fHDJV2d/r53SLq07NiEn9tmDycCa7bbgAWSOtMH9OnAlePOWQMcSPIwPZEkcbwrPfanwCnAsUAXcNq4az8HPAG8MD3ndcC76wlQiROAI4E7JJ0KfBB4C7AI+A7QP+6yNwOvAF4SES8Afga8Ka0aehz4EvBz4LlpzH8r6aSy608F1gIHAVel+04H/hg4FHgB8H3gs0AHsAW4sOz6HwLHpMe+CHxV0ryy4yvSGA4C1gGXpp+1DfgacB+wJP1ZX0qP1fK5bTaICC9emrIAPwVeC1wAfBR4PXATsB8QJA+iNuC3JA/U0nV/Btycrn8bOLvs2OvSa/cDng08DhxQdnwlMJCunwkMThDbkvQ+vwZGSB6056bHbgC6y86dA+wCFqfbAZxU6bOm64cDe4Bnlh3/KPC5dP0i4NZx198M9JRtfwK4oWz7TcDGKr/rEeDosvt/q+zYS4DH0vXjgWFgvwr3qPq5vcyexXWRloUvALcCz2NctRCwEJhL8g215D6Sb6qQfKPeNu5YyeL02gcklfbNGXf+ZBZGxBPj9i0GLpH0ibJ9SmMq/fxqP+O5wM6IeGRc3F1l25Wu/1XZ+mMVtufvDUb6K6A7/VkBLCD5XZb8smx9FzAvbYs4HLivwmeG2j63zQJOBNZ0EXGfpJ8AbyB5eJXbDoySPITuSfcdAdyfrj9A8vCi7FjJNpISQaWH+XRsA1ZHxFVVzqk2jO8vgA5JzyxLBuWfabLrq0rbA94PvAa4OyLGJI2QPLQnsw04QtJ+FX5ntXxumwXcRmBZ6SapTvlN+c6I2AN8BVgt6Zlp4+R7ebId4SvAuZIOk9QO/HXZtQ8A3wQ+IWmBpDmSXiDpxGnG2gt8QNKRsLdB+g9rvTgitgHfAz4qaZ6ko0g+//i2kal6Jkm7yDCwn6QPkZQIavEDkuT6d5KekcZ3QnpsWp/b8sOJwDIREf8VEUMTHF4F/Ab4MTBI0vj5mfTYvwL/DmwCNgBXj7v2HcDTSEoTIyQNsIdMM9ZrgI8BX0p79WwGTq7zNitJ2iF+AVwDXBgR35pOXGX+HbgR+E+SKpvd1FgdlibeN5E0rv+MpEH7bemxmfjclgOK8MQ0ZmZF5hKBmVnBORGYmRWcE4GZWcE5EZiZFZwTgZlZweXuhbKFCxfGkiVLsg7DzCxX1q9fvz0iFlU6lrtEsGTJEoaGJup+bmZmlUiacFgQVw2ZmRWcE4GZWcE5EZiZFZwTgZlZwTkRmJkVnBOBmVnBORGYmRVc7t4jsNmhbCrJmnnIdLPGcCKwTEz0UJfkB75Zk7lqyMys4JwIzMwKzonAzKzgGp4IJLVJukPS1yocO1PSsKSN6fLuRsdjZmb7akZj8XnAFmDBBMe/HBHnNCEOMzOroKElAkmHAW8ELm/kzzEzs6lrdNXQp4D3A2NVznmrpDslrZV0eKUTJJ0laUjS0PDwcEMCNTMrqoYlAkmnAA9GxPoqp10PLImIo4CbgM9XOikiLouIrojoWrSo4gQ7ZmY2RY0sEZwArJD0U+BLwEmSriw/ISJ2RMTj6eblwMsaGM+sIqnuxcyskoYlgoj4QEQcFhFLgNOBb0fEGeXnSDqkbHMFSaOy1SAiKi6THTMzG6/pQ0xI+jAwFBHrgHMlrQCeAHYCZzY7HjOzolPevil2dXWFJ6+fWN7H6sl7/GatStL6iOiqdKywg8559Eszs0RhE4FHvzQzS3isITOzgnMiMDMrOCcCM7OCcyIwMys4JwIzs4JzIjAzKzgnAjOzgnMiMDMrOCcCsxnQ39/P0qVLaWtrY+nSpfT392cdklnNCvtmsdlM6e/vp6enh76+PpYtW8bg4CDd3d0ArFy5MuPozCbnEoHZNK1evZq+vj6WL1/O3LlzWb58OX19faxevTrr0Mxq4tFHx8n7WEOOv/na2trYvXs3c+fO3btvdHSUefPmsWfPngwjM3tStdFHXSIwm6bOzk4GBwf32Tc4OEhnZ2dGEZnVx4nAbJp6enro7u5mYGCA0dFRBgYG6O7upqenJ+vQzGrixmKzaSo1CK9atYotW7bQ2dnJ6tWr3VBsueE2gnHyWEddrtXi7+joYGRkpGH3b29vZ+fOnQ27v9ls4RnKLDMjIyMNTUxTmWnOzPblNgIzs4JzIjAzKzgnAjOzgnMiMDMruIYnAkltku6Q9LUKx/aX9GVJWyXdLmlJo+MxM7N9NaNEcB6wZYJj3cBIRLwQ+AfgY02Ix8zMyjQ0EUg6DHgjcPkEp5wKfD5dXwu8Ru4PaGbWVI0uEXwKeD8wNsHxQ4FtABHxBPAQcHCDYzIzszINSwSSTgEejIj1M3CvsyQNSRoaHh6egejMzKykkSWCE4AVkn4KfAk4SdKV4865HzgcQNJ+wIHAjvE3iojLIqIrIroWLVrUwJDNzIqnYYkgIj4QEYdFxBLgdODbEXHGuNPWAe9M109Lz2mdgXLMzAqg6WMNSfowMBQR64A+4AuStgI7SRKGmZk1UVMSQUTcDNycrn+obP9u4A+bEYOZmVXmN4vNzArOicDMrOCcCMzMCs6JwMys4JwIzMwKzonAzKzgnAjMzArOicDMrOCcCMzMCs6JwMys4JwIzMwKrumDzlmxxIUL4KIDazp3uG0O71u0kI8Pb2fhnonmMqpwfzObllmfCDo6OhgZGanrmnpmy2xvb2fnzp31hlUYuvhhah1ZvPe2j7Dh3q/S+wd/yQXHXVDb/SXiomkEaGazv2poZGSEiGjYUm+SscqGdw1z3dbrCIJrt17L9se2Zx2SWWHM+kRg+dB7Zy9jkVQHjcUYvZt6M47IrDicCFpcR0cHkmpegLrO7+joyPgTPlkaGB0bBWB0bNSlArMmciJocUWo2iovDZS4VGDWPE4ElrlND27aWxooGR0bZeODGzOKyKxYZn2vIWt9a1eszToEs0JzicDMrOCcCMzMCs6JwMys4JwIzMwKzonAzKzgGpYIJM2T9ANJmyTdLeniCuecKWlY0sZ0eXej4jEzs8oa2X30ceCkiHhU0lxgUNINEXHbuPO+HBHnNDAOMzOromGJIJIhJx9NN+emS23DUJqZWdM0tI1AUpukjcCDwE0RcXuF094q6U5JayUdPsF9zpI0JGloeHi4kSGbmRVOQxNBROyJiGOAw4CXS1o67pTrgSURcRRwE/D5Ce5zWUR0RUTXokWLGhmymVnhNKXXUET8GhgAXj9u/46IeDzdvBx4WTPiMTOzJzWy19AiSQel6wcAfwD8x7hzDinbXAFsaVQ8ZmZWWSNLBIcAA5LuBH5I0kbwNUkflrQiPefctGvpJuBc4MwGxjOp4V3DnHnjmR4H38wKpWGJICLujIhjI+KoiFgaER9O938oItal6x+IiCMj4uiIWB4R/1H9ro3Ve2cvG361Ibfj4DuRmdlU+M3i1GyYMzfviczMsjHr5yOICxfARQdOel7vwe2MzZ8Pc8TY6G56L+/igh2Tz94VFy6YiTCnbXwiO/vos1l4wMKswzKzHFDy3ld+dHV1xdDQUM3nS2Kyzzi8a5iTrz6Zx/c8vnff/m37c+Nbb5z0YVrL/aelhiQG8JGD27lm/nxG54i5Y8FbHn20pkSW/IyHphFgdY3+/TT89282S0haHxFdlY7N+hJBLarNmXvBcRdkFFVCFz9cUyK77uqTGU0T2egccW37Qs5+91BtieyimYrW8qi/v5/Vq1ezZcsWOjs76enpYeXKlVmHZU3kNgLyP2euJ3+3qerv76enp4c1a9awe/du1qxZQ09PD/39/VmHZk3kqqFpaoX7n7buNO4dufcp+1/U/qJJ5wNuhfhb+f6z3dKlS1mzZg3Lly/fu29gYIBVq1axefPmDCOrnUs0tXHV0Cznyd+bT1Ld17RiwtqyZQvLli3bZ9+yZcvYsiUf73aWSjR9fX0sW7aMwcFBuru7AZwM6uCqIbMpiIiKy2THWk1nZyeDg4P77BscHKSzszOjiOqzevVq+vr6WL58OXPnzmX58uX09fWxevXqrEPLFScCswLr6emhu7ubgYEBRkdHGRgYoLu7m56enqxDq0neSzStwlVDZgVWqj5ZtWrV3jr21atX56ZapVSiKW/jyFOJplU4EZgV3MqVK3Pz4B+vVKIZ30bgqqH6OBFYw02lYbVW7e3tDbu3tb68l2haRdXuo5LOiIgr0/UTIuK7ZcfOiYhLmxDjPtx9NF/3r1erxVOvvMdvs1e17qOTNRa/t2x9zbhjfzKtqMzMrCVMlgg0wXqlbTMzy6HJEkFMsF5p28zMcmiyxuIXpzOMCXhBuk66/fyGRmZmZk0xWSJwZ1wzs1muaiKIiPvKtyUdDLwK+FlErG9kYGZm1hxV2wgkfU3S0nT9EGAzSW+hL0g6vwnxmZlZg03WWPy8iCiNRfsu4KaIeBPwCtx91MxsVpgsEZTP1vIa4BsAEfEIMFbxCjMzy5XJGou3SVoF/Bx4KXAjgKQDgLkNjs0scx0dHYyM1Dj3c6qeITXa29vZuXNnvWGZzajJEkE38GHgtcDbIuLX6f7jgM82MjCzVjAyMtLwIT7MsjZZr6EHgbMr7B8ABqpdK2kecCuwf/pz1kbEhePO2R+4AngZsIMk2fy0jvjNzGyaqiYCSeuqHY+IFVUOPw6cFBGPSpoLDEq6ISJuKzunGxiJiBdKOh34GPC2GmM3M7MZMFnV0PHANqAfuJ06xheKpDz9aLo5N13Gl7FPBS5K19cCl0pSePhGM7OmmazX0HOADwJLgUuAPwC2R8QtEXHLZDeX1CZpI/AgSdfT28edcihJoiEingAeAg6ucJ+zJA1JGhoeHp7sx5qZWR2qJoKI2BMRN0bEO0kaiLcCN0s6p5abp9cfAxwGvLz0clq9IuKyiOiKiK5FixZN5RZmZjaBSWcoSxt03wisBJYA/whcU88PiYhfSxoAXk/ydnLJ/cDhwM8l7QccSNJoPKM8Q5aZ2cQmayy+gqRa6BvAxWVvGU9K0iJgNE0CB5BUK31s3GnrgHcC3wdOA7490+0D9d7OM0yZWdFMViI4A/gNcB5wbtk3a5G0By+ocu0hwOcltZFUQX0lIr4m6cPAUESsA/pIxi3aCuwETp/6R5m9XKIxs0aa7D2CyRqTq117J3Bshf0fKlvfDfzhVH9GEbhEkx/Du4Z5363v4+MnfpyFByzMOpxZaypfjPw3Ud2UH/Rmtq/eO3vZ8KsN9G7qzTqUWS0iKi6THbOJORGYzYDhXcNct/U6guDardey/bHtWYdkVjMnArMZ0HtnL2ORDMg7FmMuFViuOBGYTVOpNDA6lozaPjo26lKB5cqk7xGYFVlcuAAuOrDqOb0HtzM2fz7MebIRc2x0N72Xd3HBjupDWMeF1TremTWHE4FZFbr44UkbGzetO43RkXv32Tc6R2xc3AWr1la/v0RcNN0ozabHicBsmtauqP6wN2t1biMwMys4JwIzs4JzIjAzKzgnAjOzgnMiMDMrOPcaMisYD9pm4zkRmBXMRA91j1zbHK2YiJ0IzMyaqBUTsROB2SQ8MZDNdk4ElolqD9eJjmXxbckTA1kROBFYJvywNGsd7j5qZlZwTgRmZgXnRGBmVnBOBGZmBedEYGZWcA1LBJIOlzQg6R5Jd0s6r8I5r5b0kKSN6fKhRsVjZmaVNbL76BPAX0bEBknPBNZLuiki7hl33nci4pQGxmFmZlU0rEQQEQ9ExIZ0/RFgC3Boo36emZlNTVPaCCQtAY4Fbq9w+HhJmyTdIOnICa4/S9KQpKHh4eEGRmpmVjwNTwSS5gP/BpwfEQ+PO7wBWBwRRwNrgGsr3SMiLouIrojoWrRoUWMDNjMrmIYmAklzSZLAVRFx9fjjEfFwRDyarn8DmCtpYSNjMjOzfTWy15CAPmBLRHxygnOek56HpJen8exoVExmZvZUjew1dALwx8Bdkjam+z4IHAEQEb3AacCfS3oCeAw4PTwamZlZUzUsEUTEIFB1IPeIuBS4tFExmBVZR0cHIyMjdV1Tz9wL7e3t7Ny5s96wapb3+PPEw1CbzVIjIyMNHe67kRP2QP7jzxMPMWFmVnBOBGZmBedEYGZWcE4EZmYF50RgZlZw7jVkNgXVepxMdMyvyFirKmwi8B+yTYf/LdhsUtiqoYioezGz1jS8a5gzbzyT7Y9tzzqUvTo6OpBU8wLUdX5HR8eMxVrYRJB3U/nHZDZb9d7Zy4ZfbaB3U2/WoexVeiGuUUu9b11X40SQUy7RmCWGdw1z3dbrCIJrt17bUqWCvHAiMLOWrFqpVe+dvYzFGABjMdZSpYK8cCIws5asWqlFqTQwOjYKwOjYqEsFU+BEYFZwea5aKS8NlLhUUD8nArOCy3PVyqYHN+0tDZSMjo2y8cGNE1xhlShvjYhdXV0xNDSUdRhmLU/SpJ0EhncNc/LVJ/P4nsf37tu/bX9ufOuNLDyg+qyxtdx/Wi46sHH33vszHmrYrRv9+6n3/pLWR0RXpWOFfaHMzKpXrVxw3AUZRZXQxQ83/kF6UcNunyuuGjIrMFetGLhEYFZoa1eszToEI6mie9+t7+PjJ3580iq5RnAiMJul4sIFDa1njwsXNOzeRVPefTeLKjknArNZynXs+TC+++7ZR5/d9FKB2wjMzDLUCt13nQjMZkB/fz9Lly6lra2NpUuX0t/fn3VIlgOt8ma0q4bMpqm/v5+enh76+vpYtmwZg4ODdHd3A7By5cqMo7Os1NJG03twO2Pz58OcJ0cHHhvdTe/lXVywo/roojPZRtOwF8okHQ5cATwbCOCyiLhk3DkCLgHeAOwCzoyIDdXu6xfKrNUsXbqUNWvWsHz58r37BgYGWLVqFZs3b84srlZ7oWkq92+k9vZ2du7c2bD71/L7OW3dadw7cu9T9r+o/UWT9uiayRfKGpkIDgEOiYgNkp4JrAfeHBH3lJ3zBmAVSSJ4BXBJRLyi2n2dCKzVtLW1sXv3bubOnbt33+joKPPmzWPPnj2ZxZX3RFCvosUzk4mgYW0EEfFA6dt9RDwCbAEOHXfaqcAVkbgNOChNIGa50dnZyeDg4D77BgcH6ezszCiiJ9Uz41W9S3t7e9Yfz2ZIUxqLJS0BjgVuH3foUGBb2fbPeWqyQNJZkoYkDQ0PDzcqTLMp6enpobu7m4GBAUZHRxkYGKC7u5uenp5M45rKxEX1nN/IahVrroY3FkuaD/wbcH5EPDyVe0TEZcBlkFQNzWB4ZtNWahBetWoVW7ZsobOzk9WrV7uh2HKjoYlA0lySJHBVRFxd4ZT7gcPLtg9L95nlysqVK/3gt9xqWNVQ2iOoD9gSEZ+c4LR1wDuUOA54KCIeaFRMZmb2VI0sEZwA/DFwl6TSUIYfBI4AiIhe4BskPYa2knQffVcD4zEzswoalggiYhCo2hE4khaq9zQqBjMzm5yHmDAzKzgnAjOzgvNYQ2ZmDdLIYTJm8oU+JwIzswaod3iJLIfIcNWQmVnBORGYmRWcE4GZWcE5EZiZFZwTgZlZwTkRmJkVnBOBmVnBORGYmRWcE4GZWcE5EZiZFZwTgZlZwTkRmJkVnBOBmVnBORGYmRWch6E2M2uianMUTHSs0cNTOxGYWa604oO0Hq0US4kTgZnlSis+SPPOicCsYPL+jdpmnhOBWcH4oW7jNazXkKTPSHpQ0uYJjr9a0kOSNqbLhxoVi5mZTayRJYLPAZcCV1Q55zsRcUoDYzAzs0k0rEQQEbcCOxt1fzMzmxlZv1B2vKRNkm6QdGTGsZiZFVKWjcUbgMUR8aikNwDXAr9T6URJZwFnARxxxBHNi9DMrAAyKxFExMMR8Wi6/g1grqSFE5x7WUR0RUTXokWLmhqnmdlsl1kikPQcpZ2WJb08jWVHVvGYmRVVw6qGJPUDrwYWSvo5cCEwFyAieoHTgD+X9ATwGHB6uIOzmVnTKW/PXknDwH0N/BELge0NvH+jOf5s5Tn+PMcOjn8yiyOiYt167hJBo0kaioiurOOYKsefrTzHn+fYwfFPR9bdR83MLGNOBGZmBedE8FSXZR3ANDn+bOU5/jzHDo5/ytxGYGZWcC4RmJkVnBOBmVnBORGYzQBJT886hnpJOq+WfTb7ORHMInl8GOWdpFdKugf4j3T7aEn/nHFYtXpnhX1nNjsIy17hp6qU1AbcHREvzjqWqZL0SuByYD5whKSjgT+LiL/INrLaSVoBvCrdvCUirs8ynjr8A/DfgXUAEbFJ0quqX5ItSSuBPwKeJ2ld2aEF5GwOEUnvrbD7IWB9RGxsdjy1kvSWascj4upmxQJOBETEHkn3SjoiIn6WdTxTlLuHUTlJHwVeDlyV7jpX0vER8cEMw6pZRGwbN+n7nqxiqdH3gAdIhjT4RNn+R4A7M4lo6rrSpfTF4RSSz3C2pK9GxN9nFll1a4GN6QJQ/g8oACeCDLQDd0v6AfCb0s6IWJFdSPXJ4cOo3BuBYyJiDEDS54E7gDwkgm1piSwkzQXOA7ZkHFNVEXEfcJ+k1wKPRcSYpP8GvBi4K9vo6nYY8NLSkPaSLgS+TlK6XA+0aiJ4C3A6cBRwHdAfEVuzCsaJIPF/sg5gmnL3MKrgIJ6sljgwy0DqdDZwCXAocD/wTeA9mUZUu1uB35fUThL3D4G3AW/PNKr6PAt4vGx7FHh2RDwm6fEJrslcRFwLXCvpGcCpwCckHQz0RMQtzY7HiQDI4hc/w/L8MAL4KHCHpAGSIvKrgL/ONqTaRMR28vXgLKeI2CWpG/jniPh7SS1brz6Bq4DbJV2Xbr8J+GL6gL0nu7BqtpukTeNhYDEwL4sgCv1msaRHSOrjnnIIiIhY0OSQCkvSIcDvpZs/iIhfZhlPrdIqlU+TfAtdKukoYEVE/E3GoU1K0h3AX5C0MXVHxN2S7oqI3804tLpI+j3glenmdyNiKMt4aiHpJJKqoZcD3wK+lGXchU4Es4Wkf6yw+yFgKCKuq3CspUg6AdgYEb+RdAbwUuCStC67pUm6BXgf8C8RcWy6b3NELM02sslJOhH4S5KH58ckPR84PyLOzTi0uqQ9/55NWQ1Hq3f8kDRG0qg9SPJldJ8HcbP/H7hqaHaYR9LQ99V0+63AT4CjJS2PiPMzi6w2nyaJ9WjgvUAfcAVwYqZR1ebpEfGDcQ31T2QVTD3SKtFbSu+fRMSPgbwlgVUksx/+iqSDhEgeqkdlGVcN3pV1AOWcCGaHo4ATImIPgKRPA98BlpGPXiBPRERIOhX4p4joS+ut82C7pBeQfqOTdBpJ18yWJ+l4kqSb2/dPSDpGvCgicjXfeUR8PusYyvnN4tmhneSPueQZQEeaGFq250SZRyR9ADgD+LqkOaTzW+fAe4B/AV4s6X7gfJLG+zz4FMn7Jzsgef+EJ1/qy4ttJNWguSJpoaQLJZ0rab6kT0vaLOk6SS9sdjwuEcwOfw9slHQzT/a6+du058S3sgysRm8jedO1OyJ+KekI4P9mHNOk0rrpv4iI16a/6zkR8UjWcdUj5++fAPwYuFnS1yn70hMRn8wupJp8ERgCfgf4AfBZkp5/v08ySsCrmxmMG4tnibTXzcvTzR9GxC+yjKce6UN0d/qWd+nFphsiYjTj0CYkab+IeELSbRFxXNbxTIWktcAngUuBV5BUs3RFxOmZBlaH9AWyp4iIi5sdSz0kbYqIo5Vk4fsi4oiyYxsj4pimxuNEkF+SXlrteERsaFYs0yFpPck3oXbguyQvNv02Ilq2f76kDRHx0rQ95lCShvryt9KbOkTAVEhaSPIt9LUkJclvAuflrb49j0r/fsavV9puBlcN5VtpnJh5JOOtbCL5gz6KpNh5fEZx1avSi02bsg6qRvNI6thPImkwLvVaaelEkFZrXdLKybYaSZ+KiPMlXU+Fd4FyMDzM89MB/1S2Trr9vGYH40SQYxGxHEDS1STjrdyVbi8FLsowtHop7cHydqDUW6jVOzI8Kx35cjNPJoCSli9mp9VwiyU9LSJ+m3U8U/CF9L8fzzSKqTu1bL30GWLcdtM4EcwOLyolAYCI2CypM8uA6nQe8AHgmvTt1ucDAxnHNJk2kp5aqnCs5RNB6sfAd9Nvo+XVWq3e0EpErE//m9fhYQ4CDouIfwJIB7xcRPJv5383Oxi3EcwCkvpJ/pCvTHe9HZgfESuzi2p2y6Ied6bltaG1nKS7eGrifYikavRvWrW9Q9J3gdMjYlu6vRF4DUnX789GxGuaGY9LBLPDu4A/J/lmDcmokp/OLpz6SFoEvB84krJBtyLipMyCmlylkkCulB74kuan249mG9GU3EDS5fWL6fbpwNOBXwKfIxmErhU9rZQEUoNp0tqR9qJrKpcILHOSvgl8Gfgrkpex3gkMR0TTi8i1ksjMuKwAAAZCSURBVNQREbmazWu8tC3pC0BHums78I6IuDu7qOpTqWRW1qOrZQfQk7Q1Iiq+OCbpvyLiBc2Mp9Ub5KwKSV9J/3uXpDvHL1nHV4eDI6IPGI2IWyLiT0h64bSsvCeB1GXAeyNicUQsJhmA7l8zjqlebZJK78+URiJtSzdbecyn2yX96fidkv6M5AWzpnLVUL6VqoJOyTSK6Su9OPaApDcCv+DJb6nWOM+IiL2N8hFxcxbVEtP0buAzafWWSMb1f3f6OT6aaWTV/S+SiWn+CCi97/MyYH/gzc0OxlVDs1A6Vs/KiLhq0pNbgKRTSAbJOxxYQzKJ+kWRnwnsc0nSNSQPoVJXzDOAl0XE/8guqqmRdCBARORq3KF0XoIj0827I+LbmcThRJBfkhaQDHp2KMnE9TcB55AU8TdFxKlVLm9pks6PiE9lHcdspmSKyotJRqkNkmR8cUSMZBpYHdJ3OcZ7CFgfEXmbbS0zTgQ5pmR6vhHg+yRdz55FUjw+L+9/BJJ+Vj7+is0cSW8pDYEhqT1PD/7xJH2R5K36UunxFJIJX5YAX42IVp28vqU4EeRYea+IdMiAB4AjImJ3tpFNn6RtEXF41nHMRtXGuckbSbcCbyh1fU3bCr4OvJ6kVPCSLOPLC/cayre9o3Omcw/8fDYkgZS/oTSOJljPo2ex75wboyTzRz9GPubiaAnuNZRvR0t6OF0XcEC6LSAiYkF2oU1O0iNUfuALOKDJ4RTJAZKOJfkiOC9d35sQ8jJqbeoqkq6Ypbm53wR8Me01dE92YeWLq4bMCkZStXGcosXf6H6K9N2BV6ab342IoSzjySMnAjPLPUnPYt/hSX6WYTi54zYCM8stSSsk/Qj4CXBL+t8bso0qf5wIzCzPPgIcB/xnRDyPZLa127INKX+cCMwsz0bTUTvnSJqTDpnRlXVQeeNeQ2YFJukokpev9j4L8jDfcplfp+8O3ApcJelByibZsdq4sdisoCR9hmR+67uBsXR3pKO/5kLaTfQxktqNtwMHAle16oQ0rcqJwKygJN0zm968lbQQ2BF+qNXNbQRmxfV9SblMBJKOk3SzpKslHStpM7AZ+JWk12cdX964RGBWUJJOJBm19pckwzGU3kg/KtPAaiBpCPggSVXQZcDJEXGbpBcD/RFxbKYB5owTgVlBSdoKvBe4iyfbCIiI+zILqkaSNkbEMen6lojoLDt2hxNBfdxryKy4hiNiXdZBTNFY2fpj4475222dXCIwKyhJ/wwcRDKW/96ROvPQfVTSHpJuoqUBCneVDgHzImJuVrHlkUsEZsV1AEkCeF3ZvgBaPhFERNvkZ1mtXCIwMys4dx81KyhJh0m6RtKD6fJvkg7LOi5rPicCs+L6LEn30eemy/XpPisYVw2ZFVR5F8xq+2z2c4nArLh2SDpDUlu6nAF4jJ4CconArKAkLQbWAMeT9Bb6HnCuZ/cqHicCM7OC83sEZgUjaQ1V3r6NiHObGI61ALcRmBXPELCeZLL3lwI/SpdjgKdlGJdlxFVDZgUl6TZgWUQ8kW7PBb4TEcdlG5k1m0sEZsXVDiwo256f7rOCcRuBWXH9HXCHpAGSwdpeBVyUaUSWCVcNmRWYpOcAr0g3b4+IX2YZj2XDicCswCQdCiymrHYgIm7NLiLLgquGzApK0seAtwF38+RELwE4ERSMSwRmBSXpXuCoiHh80pNtVnOvIbPi+jHgmbzMVUNmBbYL2Cjp/7HvVJV+s7hgnAjMimtduljBuY3AzKzgXCIwKyhJvwN8FHgJybhDAETE8zMLyjLhxmKz4vos8GngCWA5cAVwZaYRWSZcNWRWUJLWR8TLJN0VEb9bvi/r2Ky5XDVkVlyPS5oD/EjSOcD9JAPPWcG4RGBWUJJ+D9gCHAR8BDgQ+FhE3J5pYNZ0TgRmBoCkNuD0iLgq61isudxYbFYwkhZI+oCkSyW9TolzgK3A/8w6Pms+lwjMCkbSdcAI8H3gNcCzSOYjOC8iNmYZm2XDicCsYMb1EmoDHgCOiIjd2UZmWXHVkFnxjJZWImIP8HMngWJzicCsYCTtAX5T2gQOIBmATkBExIKJrrXZyYnAzKzgXDVkZlZwTgRmZgXnRGBmVnBOBGZmBedEYGZWcE4EZmYF9/8BU0dvwBWf+HIAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XdRUydIIH4If",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# get a stacking ensemble of models\n",
        "from xgboost import XGBRegressor\n",
        "from lightgbm import LGBMRegressor\n",
        "def get_stacking():\n",
        "\t# define the base models\n",
        "  level0 = list()\n",
        "  level0.append(('Tree', DecisionTreeRegressor()))\n",
        "  level0.append(('RF', RandomForestRegressor()))\n",
        "  level0.append(('XGB', XGBRegressor()))\n",
        "  level0.append(('Bagging', BaggingRegressor()))\n",
        "\t# define meta learner model\n",
        "  level1 = LGBMRegressor()\n",
        "\t# define the stacking ensemble\n",
        "  model = StackingRegressor(estimators=level0, final_estimator=level1, cv=5)\n",
        "  return model"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_0cYWFnnPUQu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def base_models():\n",
        "  models = dict()\n",
        "  models[\"Tree\"] = DecisionTreeRegressor()\n",
        "  models[\"Random Forest\"] = RandomForestRegressor()\n",
        "  models[\"Bagging\"] = BaggingRegressor()\n",
        "  models[\"XGB\"] = XGBRegressor()\n",
        "  models[\"Stacked Model\"] = get_stacking()\n",
        "  return models"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UB_qe9-xPgzr",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 431
        },
        "outputId": "592d9955-405f-4726-da98-93ec06d31ee6"
      },
      "source": [
        "# Function to evaluate the list of models\n",
        "def eval_models(model):\n",
        "  cv = RepeatedKFold(n_splits=10, n_repeats=3, random_state=1)\n",
        "  scores = -cross_val_score(model, X, y, scoring='neg_mean_absolute_error', cv=cv, n_jobs=-1, error_score='raise')\n",
        "  return scores\n",
        "\n",
        "\n",
        "# get the models to evaluate\n",
        "models = base_models()\n",
        "# evaluate the models and store results\n",
        "results, names = list(), list() \n",
        "\n",
        "for name, model in models.items():\n",
        "  scores = eval_models(model)\n",
        "  results.append(scores)\n",
        "  names.append(name)\n",
        "  print('>%s %.3f (%.3f)' % (name, scores.mean(), scores.std()))\n",
        "\n",
        "\n",
        "# plot model performance for comparison\n",
        "pyplot.boxplot(results, labels=names, showmeans=True)\n",
        "pyplot.xticks(rotation = 90)\n",
        "pyplot.ylabel(\"MSE\")\n",
        "pyplot.title(\"Model Performance\")\n",
        "pyplot.show()"
      ],
      "execution_count": 117,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            ">Tree 2.924 (0.514)\n",
            ">Random Forest 2.190 (0.370)\n",
            ">Bagging 2.296 (0.358)\n",
            ">XGB 2.146 (0.328)\n",
            ">Stacked Model 2.708 (0.358)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAFJCAYAAACW1Sr+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3de5icdX338fcnm5igECAmCgJJEH10ZeUgq6KkakB9pGJslRbyFDEapbYasNpaZXkE1K2lj1ZpuGweJCogrAcEDFSitAngqlA3IeG0KhHBgCgLRE4hYZP99o/7nuxkmD3vzG9278/ruubiPs3Md4bN/Z3fWRGBmZkV15TUAZiZWVpOBGZmBedEYGZWcE4EZmYF50RgZlZwTgRmZgXnRGATmqT5kkLS1GFcu0RSZ53iOkbS3ZKelPRn9XhPs9FyIrC6kXSvpGckza44fmt+M5+fJrLdEsqT+eNeSZ8cw0t+BrggIvaMiKvHK06zWnAisHr7DbC4tCPplcBz04XzLPtExJ5kMX5a0ttG8uSyksk84M7RBDCc0o3ZeHIisHq7FDi1bP+9wCXlF0jaW9Ilknok3SfpLElT8nNNkr4g6WFJ9wBvr/LclZIelPSApM9JahppkBHxM7IbeUv+uu+X1C1pi6QfSppX9p4h6cOS7gbulvRr4MXANXnpYrqkF0laJelRSZskfbDs+edIukLSNyU9DiyRdEMe+0/z17hG0vMlXSbpcUk/Ly9BSTpf0ub83DpJf1Lx+t/Jv9MnJN0pqbXs/EGSrsy/70ckXVB2bsDPbZOHE4HV283ATEnN+Q36ZOCbFdcsB/Ymu5m+kSxxvC8/90HgBOBIoBU4seK53wB2AC/Jr3kr8IGRBKjMMcChwK2S3gmcCbwLmAP8GOioeNqfAa8FXhERhwC/Bd6RVw1tB74F3A+8KI/5nyQdW/b8dwJXAPsAl+XHTgbeAxwAHAL8DPg6MAvoBs4ue/7PgSPyc5cD35U0o+z8ojyGfYBVwAX5Z20CrgXuA+bn7/Wt/NxwPrdNBhHhhx91eQD3Am8GzgI+D7wNuB6YCgTZjagJeIbshlp63l8DN+Tba4APlZ17a/7cqcALge3AHmXnFwNr8+0lQOcAsc3PX+ePwBayG+3p+bnrgKVl104BtgLz8v0Ajq32WfPtg4CdwF5l5z8PfCPfPge4qeL5NwBtZftfBK4r238HsGGQ73oLcHjZ6/9n2blXAE/n268DeoCpVV5j0M/tx+R5uC7SUrgUuAk4mIpqIWA2MI3sF2rJfWS/VCH7Rb254lzJvPy5D0oqHZtScf1QZkfEjopj84DzJX2x7JjymErvP9h7vAh4NCKeqIi7tWy/2vP/ULb9dJX9PXcFI/09sDR/rwBmkn2XJb8v294KzMjbIg4C7qvymWF4n9smAScCq7uIuE/Sb4A/Jbt5lXsY6CW7Cd2VH5sLPJBvP0h286LsXMlmshJBtZv5WGwG2iPiskGuGWwa398BsyTtVZYMyj/TUM8fVN4e8AngOODOiOiTtIXspj2UzcBcSVOrfGfD+dw2CbiNwFJZSlad8lT5wYjYCXwHaJe0V944+TH62xG+A5wu6UBJ+wKfLHvug8CPgC9KmilpiqRDJL1xjLGuAD4l6VDY1SD9F8N9ckRsBn4KfF7SDEmHkX3+yraR0dqLrF2kB5gq6dNkJYLh+G+y5PrPkp6Xx3dMfm5Mn9smDicCSyIifh0RXQOcXgY8BdwDdJI1fn4tP/dV4IfARmA9cGXFc08FnkNWmthC1gC7/xhjvQo4D/hW3qvnDuD4Eb7MYrJ2iN8BVwFnR8R/jiWuMj8EVgO/Iquy2cYwq8PyxPsOssb135I1aJ+UnxuPz20TgCK8MI2ZWZG5RGBmVnBOBGZmBedEYGZWcE4EZmYF50RgZlZwE25A2ezZs2P+/PmpwzAzm1DWrVv3cETMqXZuwiWC+fPn09U1UPdzMzOrRtKA04K4asjMrOCcCMzMCq7miSBfSORWSddWObckXwxjQ/4Y0bzxZmY2dvVoIziDbG73gSbB+nZEfKQOcZiZWRU1LRFIOpBsKcGLavk+9dTR0UFLSwtNTU20tLTQ0eEFm8xsYqt11dCXyeZJ7xvkmndLui1fs/WgahdIOk1Sl6Sunp6emgQ6HB0dHbS1tbF8+XK2bdvG8uXLaWtrczIwswmtZolA0gnAQxGxbpDLrgHmR8RhZEsWXlztooi4MCJaI6J1zpyq3WDror29nZUrV7Jw4UKmTZvGwoULWblyJe3t7cliMjMbq5pNQy3p82QLb+8AZpC1EVwZEacMcH0T2XJ+ew/2uq2trZFqHEFTUxPbtm1j2rRpu4719vYyY8YMdu7cmSQmM7PhkLQuIlqrnatZiSAiPhURB0bEfOBkYE1lEpBUvmDIIrJG5YbV3NxMZ2fnbsc6Oztpbm5OFJGZ2djVfRyBpM9IWpTvni7pTkkbgdOBJfWOZyTa2tpYunQpa9eupbe3l7Vr17J06VLa2tpSh2ZmNmoTboWylFVDkDUYt7e3093dTXNzM21tbSxevDhZPGZmwzFY1ZATgZlZASRpIzAzs4nBicDMrOCcCMzMCs6JwMys4JwIzMwKzonAzKzgnAjMzArOicDMrOCcCMzMCs6JwMys4JwIzMwKzonAzKzgnAjMzArOicDMrOCcCMzMCs6JwMys4JwIzMwKzonAzKzgnAjMzArOicDMrOCcCMzMCs6JwMys4JwIzMwKruaJQFKTpFslXVvl3HRJ35a0SdItkubXOh4zM9tdPUoEZwDdA5xbCmyJiJcAXwLOq0M8ZmZWpqaJQNKBwNuBiwa45J3Axfn2FcBxklTLmMzMbHe1LhF8GfgE0DfA+QOAzQARsQN4DHh+jWMyM7MyNUsEkk4AHoqIdePwWqdJ6pLU1dPTMw7RmZlZSS1LBMcAiyTdC3wLOFbSNyuueQA4CEDSVGBv4JHKF4qICyOiNSJa58yZU8OQzcyKp2aJICI+FREHRsR84GRgTUScUnHZKuC9+faJ+TVRq5jMzOzZptb7DSV9BuiKiFXASuBSSZuAR8kShpmZ1VFdEkFE3ADckG9/uuz4NuAv6hGDmZlV55HFZmYFV/eqoYlgvIYyuLnDzCYCJ4IqhnMDl+QbvZlNCq4aMjMrOCcCM7OCcyIwMys4JwIzs4JzIjAzKzgnAjOzgnMiMDMrOCcCM7OCcyIwMys4JwIzs4JzIjAzKzgnAjOzgnMiMDMrOCcCM7OCcyIwMys4JwIzs4JzIjAzKzgnAjOzgnMiMDMrOCcCM7OCcyIwMys4JwIzs4KrWSKQNEPSf0vaKOlOSedWuWaJpB5JG/LHB2oVj5mZVTe1hq+9HTg2Ip6UNA3olHRdRNxccd23I+IjNYzDzMwGUbNEEBEBPJnvTssfUav3MzOz0alpG4GkJkkbgIeA6yPiliqXvVvSbZKukHTQAK9zmqQuSV09PT21DNnMrHBqmggiYmdEHAEcCLxGUkvFJdcA8yPiMOB64OIBXufCiGiNiNY5c+bUMmQzs8KpS6+hiPgjsBZ4W8XxRyJie757EXBUPeIxM7N+tew1NEfSPvn2HsBbgF9UXLN/2e4ioLtW8ZiZWXW17DW0P3CxpCayhPOdiLhW0meArohYBZwuaRGwA3gUWFLDeMzMrAplnXsmjtbW1ujq6kodBpKYaN+dmRWXpHUR0VrtnEcWm42Djo4OWlpaaGpqoqWlhY6OjtQhmQ1bLauGzAqho6ODtrY2Vq5cyYIFC+js7GTp0qUALF68OHF0VmuSxuV1UtYwuGpolFw1ZCUtLS0sX76chQsX7jq2du1ali1bxh133JEwMmsUjXC/cNWQ1YSrQzLd3d0sWLBgt2MLFiygu9ud4GxicCKwUSlVhyxfvpxt27axfPly2traCpkMmpub6ezs3O1YZ2cnzc3NiSIyG6GImFCPo446KhpB9tUV16GHHhpr1qzZ7diaNWvi0EMPTRRROpdffnkcfPDBsWbNmnjmmWdizZo1cfDBB8fll1+eOjRrEI1wvyDrtl/1vuo2glFqhDq/lJqamti2bRvTpk3bday3t5cZM2awc+fOhJGl0dHRQXt7O93d3TQ3N9PW1uaGYtulEe4Xg7URuNeQjUqpOqS8gbTI1SGLFy/2jd8mLLcR2Ki0tbWxdOlS1q5dS29vL2vXrmXp0qW0tbWlDs3MRsglAhuV0q/fZcuW7aoOaW9v969iswnIbQSj1Ah1fmY2MTTC/cLjCMzMbEBOBGZmBedEYGZWcE4EZmYF50RgZlZwTgRmZgVXyEQwa9YsJI3pAYzp+bNmzUr8LZiZZQo5oGzLli2N0Kc36fubmZUUskRgZmb9Bk0Ekk4p2z6m4txHahWUmZnVz1Algo+VbS+vOPf+cY7FzMwSGCoRaIDtavtmZjYBDZUIYoDtavtmZjYBDdVr6OWSbiP79X9Ivk2+/+LBnihpBnATMD1/nysi4uyKa6YDlwBHAY8AJ0XEvSP9EGZmNnpDJYKxLDe1HTg2Ip6UNA3olHRdRNxcds1SYEtEvETSycB5wEljeE8zMxuhQauGIuK+8gfwJPAqYHa+P9hzIyKezHen5Y/K6qR3Ahfn21cAx8kd7M2sgRRhAOpQ3UevldSSb+8P3EHWW+hSSR8d6sUlNUnaADwEXB8Rt1RccgCwGSAidgCPAc8f8acws4bR0dFBS0sLTU1NtLS00NHRkTqkMSkNQE352LJlS00/41BVQwdHxB359vvIbuanStoL+Anw5cGeHBE7gSMk7QNcJaml7PWGTdJpwGkAc+fOHenTzcbFeBVWU49qr6WOjg7a2tpYuXIlCxYsoLOzk6VLlwJ4GdMGNlSvod6y7eOAHwBExBNA33DfJCL+CKwF3lZx6gHgIABJU4G9yRqNK59/YUS0RkTrnDlzhvu2ZuNqOL/chnPdZNbe3s7KlStZuHAh06ZNY+HChaxcuZL29vbUodkghkoEmyUtk/TnZG0DqwEk7UFW5z8gSXPykkDp+rcAv6i4bBXw3nz7RGBNTIB/KT1be1iyegkPP/1w6lDMGkp3dzcLFizY7diCBQvo7u5OFJENx1CJYClwKLCErGvnH/PjRwNfH+K5+wNr8y6nPyerVrpW0mckLcqvWQk8X9ImslHMnxzFZ6i7FbetYP0f1rNi44rUoZg1lObmZjo7O3c71tnZSXPzWDogWq1pAvwA301ra2t0dXWN6TUkjbqI3rO1h+OvPJ7tO7czvWk6q9+9mtl7zK5rDNa4iv7/daA2gvb29gnbRtAI/0/HIwZJ6yKitdq5QRuLJa0a7HxELBrs/GS04rYV9EXWPNIXfazYuIKzjj4rcVRmjaF0s1+2bBnd3d00NzdP6CRQFIOWCCT1kHXv7ABuoWJ+oYi4sabRVZGyRFBeGigZbamgEX5l2Pjz/9fJpxH+n9a6RDBUG8F+wJlAC3A+WYPvwxFxY4okkFp5aaCkVCowM5uohhpZvDMiVkfEe8kaiDcBNxR1LYKND22kt693t2O9fb1seGhDoojMzMZuyKUq84nh3g4sBuYD/wZcVduwGtMVi65IHYKZ2bgbqrH4ErJqoR8A545mVHAjirNnwjl7p49hAvBoWrPJb6jG4j7gqXy3/EKRzStX97tZ6u6j46URYhgvk+mzjJW/i8mnEf6fJu0+GhFe3N7MnsUlxcllyDYCM7NKw7mBN8IvaRse/+I3Mys4JwIzs4JzIjAzq6GJMFuxE4GZWQ1NhNmKnQjMzGqkZ2sP39/0fYLg6k1XN2ypwInAzKxGqs1W3Ii8HkEijRDDeJlMn2Ws/F30mzTfxShnIehpmsLxB76I7VP6f29P7+tj9f2/Y/bOYa/0WxbHY6OKo2TUA8rMzIpO5z4+qoS24ubP0nf3VVA2UWXf1OmseMvHR7yGiSTinBGHMGyuGjIzq4GJNFuxSwRmZjUwkWYrdonAzKzgClsiGK9Js0Zr3333Tfr+ZmYlhUwE49GTYdL0iDCzwnPVkJlZwTkRmJkVnBOBmVnB1SwRSDpI0lpJd0m6U9IZVa55k6THJG3IH5+uVTxmZlZdLRuLdwAfj4j1kvYC1km6PiLuqrjuxxFxQg3jMDOzQdSsRBARD0bE+nz7CaAbOKBW72dmZqNTlzYCSfOBI4Fbqpx+naSNkq6TdOgAzz9NUpekrp6enhpGamZWPDVPBJL2BL4HfDQiHq84vR6YFxGHA8uBq6u9RkRcGBGtEdE6Z86c2gZshTVr1iwkjekBjOn5s2bNSvwtWBHVdECZpGlkSeCyiLiy8nx5YoiIH0j6iqTZEdGYqzfYpLZly5bkgwRTj3i3YqplryEBK4HuiPjXAa7ZL78OSa/J43mkVjGZmdmz1bJEcAzwHuB2SaV5V88E5gJExArgROBvJO0AngZOjtQ/yczMCqZmiSAiOoFBy7kRcQFwQa1iMDMbD6mr7Go9SWUhJ50zMxuuIkxS6SkmCs49ZczMJYKCc08ZM3OJwMys4JwIzMwKzonAbJz0bO1hyeolPPy0x0PaxOJEYDZOVty2gvV/WM+KjStSh2I2Ik4EZuOgZ2sP39/0fYLg6k1XT/hSgXuTFYsTgdk4WHHbCvqiD4C+6JvwpYJSb7KUjy1btqT+GgrDicBsjEqlgd6+XgB6+3onRanAisPjCMxycfZMOGfvET9vxfP3pW/PPWFK/3iIvt5trLiolbMeGdmv2jh75ojf32ysnAjMcjr38VENrtu46kR6t/xyt2O9U8SGea2w7IqRxSAR54w4BLMxcSKwMenZ2sM/3PQPfOGNX2D2HrNTh5PEFYtGdrM3azRuI7AxcZdJs4nPicBGbbJ1mTQrKicCG7XJ1mXSxo9HWU8sTgQ2Ku4yaYNxleHE4kRgo1JeGihxqcDAVYYTkXsNFdxo+85vfNF+9E5/zm7Hevt62XDbpbD6/408Bps0qlUZnnX0WYmjssEo9aIkI9Xa2hpdXV2pw2j4peeGqxE+RyPE0ChxNEIMY4mjZ2sPx195PNt3bt91bHrTdFa/e/WIuxc3yncxHhrhs0haFxGt1c65RGBmz+JR1sXiRGBmz+JR1sXiRGBm48ajrCcm9xoyMyu4miUCSQdJWivpLkl3SjqjyjWS9G+SNkm6TdKrahWPmZlVV8uqoR3AxyNivaS9gHWSro+Iu8quOR54af54LfDv+X/NzKxOalYiiIgHI2J9vv0E0A0cUHHZO4FLInMzsI+k/WsVk5mZPVtdGoslzQeOBG6pOHUAsLls//782IP1iMusUmmt3VT23XffpO9vxVTzRCBpT+B7wEcj4vFRvsZpwGkAc+fOHcfoDHzzKxmPAT+NMHDIbKRqmggkTSNLApdFxJVVLnkAOKhs/8D82G4i4kLgQshGFtcg1MLyzc/MatlrSMBKoDsi/nWAy1YBp+a9h44GHosIVwuZmdVRLUsExwDvAW6XtCE/diYwFyAiVgA/AP4U2ARsBd5Xw3jMzMbdcKtWh7ouZam6ZokgIjqBQT95ZJ/8w7WKwcys1iZDtahHFpuZFZznGjKzqtybrDicCMzsWdybrFhcNWRmVnBOBGZmBeeqoSomQ3cwM7PhciKowjdwMysSVw2ZmRWcE4GZWcE5EZiZFZwTgZlZwTkRmJkVnBOBmVnBORGYmRWcE4GZWcE5EZiZFZwTgZlZwXmKCbNh8hxUNlk5EZgNk2/gNlm5asjMrOCcCMzMCs6JwMys4JwIzMwKzonAzKzgnAjMzAquZolA0tckPSTpjgHOv0nSY5I25I9P1yoWMzMbWC3HEXwDuAC4ZJBrfhwRJ9QwBjMzG0LNEkFE3CRpfq1e3+rDo2mtGv9dTC6pRxa/TtJG4HfA30fEndUuknQacBrA3Llz6xie+R+qVeO/i8klZWPxemBeRBwOLAeuHujCiLgwIlojonXOnDl1C9DMrAiSJYKIeDwinsy3fwBMkzQ7VTxmZkWVLBFI2k95BaKk1+SxPJIqHjOzoqpZG4GkDuBNwGxJ9wNnA9MAImIFcCLwN5J2AE8DJ4crHs3M6q6WvYYWD3H+ArLupWZmlpBHFpuZFZwTgZlZwTkRmJkVnCZa+6ykHuC+1HEAs4GHUwfRIPxd9PN30c/fRb9G+C7mRUTVgVgTLhE0CkldEdGaOo5G4O+in7+Lfv4u+jX6d+GqITOzgnMiMDMrOCeC0bswdQANxN9FP38X/fxd9Gvo78JtBGZmBecSgZlZwTkRmJkVnBPBCEl6buoYUpN0xnCOmdnE4DaCYZL0euAiYM+ImCvpcOCvI+JvE4dWd5LWR8SrKo7dGhFHporJrBFIuh2odlMVEBFxWJ1DGpbUS1VOJF8C/jewCiAiNkp6Q9qQ6kvSYuD/AAdLWlV2aibwaJqo0pP0sSqHHwPWRcSGesdTb5JmACcBW4BrgE8AfwL8GvhsRKQeUVtPJ6QOYDScCEYgIjZXLMa9M1UsifwUeJBsuPwXy44/AdyWJKLG0Jo/rsn3TyD7Pj4k6bsR8S/JIquPS4Be4HnAx4E7yKaYXwB8gwl6cxyNiNg1/Y2kecBLI+I/Je1BA99vGzawBrQ5rx4KSdOAM4DuxDHVVf5Hfp+kNwNPR0SfpP8FvBy4PW10SR0IvKq09Kqks4H/AN4ArAMmeyJ4RUS0SJoK3B8Rb8yPr5a0MWVgqUj6IHAaMAs4hOxvZAVwXMq4BuLG4uH7EPBh4ADgAeCIfL+IbgJmSDoA+BHwHrJffkX1AmB72X4v8MKIeLri+GT1DEBE7AB+V3GuaKXmkg8DxwCPA0TE3WR/Jw3JJYJhyus5/yp1HA1CEbFV0lLgKxHxL5ImfV34IC4DbpH0/Xz/HcDlkp4H3JUurLo5UNK/kTWIlrbJ9w9IF1ZS2yPimVJVcl5aatieOU4Ew5RXgfw72S+9FkmHAYsi4nOJQ0tBkl5HlhiX5seaEsaTVER8VtJq4PX5oQ9FRFe+XYQfD/9Qtt1Vca5yvyhulHQmsIektwB/S38bUsNx99FhknQj2R/8/y91k5R0R0S0pI2s/iS9kaxR8CcRcZ6kFwMfjYjTE4eWjKQm4IWU/biKiN+mi8hSkjSF7EfSW8lKRj8ELooGveE6EQyTpJ9HxKvL+8tL2hARR6SOLRVJz42IranjSE3SMuBs4A9kdeIN3Wd8vElaALw4Ii7J968gayQF+FxErEkWnA2Lq4aG72FJh5DX80k6kawrZeHk1UIrgT2BQg+uy50BvCwiHkkdSCLnAsvK9l8GLCHrTnomUJhEMMiAMgAa9ceBE8HwfZhsKtmXS3oA+A3FqP+t5ssUfHBdhc1kA8iKamZElDeK3x0R6wAkfT5RTKmUxkyUehRemv/3FNxYPLHl9b9/GxFvznuCTImIJ1LHlZIH1+3mHuAGSf9BWXfRiPjXdCHV1T7lOxHxrrLdF9Y5lqRKA8okvaViypV/lLQe+GSayAbncQRDkDQ1InaSjZIkIp4qehKgYnCdpL+nYIPrKvwWuB54DrBX2aMofiHp7ZUHJZ0A/DJBPI1Ako4p23k9DXy/dWPxEEoTrEn6d7I+0d8Fniqdj4grkwWXiKTZwPnAm8kaRn8EnFHgOvJCk/RS4FqyKUjW54ePIutOe0JE/CpVbKlIOgr4GrA32b+RLcD7I2L9oE9MxFVDwzcDeAQ4lqyuT/l/C5UI8mqy8yOiqO0ju0j6ckR8VNI1VKn/jYhFCcJKYTtwGFmb2aH5sZvIRuO/GihcIsjbSA6XtHe+39BtSE4EQ3tBPrvkHfQngJLCFaciYqekeZKeExHPpI4nsVJD4BeSRpHeDWTz6Hwxr0ZF0gvJpm1/OdmEfIWSJ4CzyeabKo1D+kyjJgQngqE1kXWTVJVzhUsEuXuAn+RTUZdXkxWlcRTY9auPiLgxdSyJHQX8M7AhX6DolcDHyCbbOzVlYAl9jezH41/m++8Bvg68a8BnJOQ2giFUW4Sl6PLZNZ8lIs6tdyyNYIC+44+RTa/wuaK0neRJ4EtkE88dHRH3Jw4pmWqDTRt5AKpLBEOrVhIotNINX9Ke+f6TaSNK7jqy7rOX5/snA88Ffk82K+s70oRVH5L2Ac4DXgu8DfhT4DpJZxR4VPHTkhZERCdA3oPo6cQxDcglgiFImhURhV19qxpJLWT146VpBB4GTo2IO9NFlc4AS3eWepvdHhGvTBVbPUi6B/gK8OV8KmokHZEfuy8iFqeML4X8819Mf6+hR4ElEdGQ6zO4RDAEJ4GqLgQ+FhFrASS9Cfgq/bNvFk2TpNdExH8DSHo1/bOx7kgXVt28obIaKF+i8/X5Ai2Fk3/+wyXNzPcfTxzSoFwisBGTtDEiDh/qWFHkN/6v0d+p4HHgA8CdwNsj4jsJw7M6GmD96l0atUOFE4GNmKSryAYOlc+jclRE/Hm6qNKbKH3GrXYk9QEbyNqNtlPRxtioHSqcCGzEJO1LNuPkArLeMj8Gzo2ILUkDS2SAX4GPAevyKgIriHwm3sVkjebrgA7gvxp1HYISJwIbNknvKk2pIWnfot74K0m6nGzQVGkFqhOA24D5wHcjYrIvXm9V5PMLLSabiuUfI2JV4pAG1LCTIFlDOqts+7+SRdF4DgReFREfj4iPkw2wegHZqNIlKQOzNCTNAY4kG1x3P/BQ2ogG515DNhIaYLvoXkDZ9NNAL9na1k9L2j7Ac2wSkvR+stHEM4ArgL+MiIZOAuBEYCOzh6QjyUqSM/LtXQmhUWdWrIPLgFskfT/ffwdweb52xV0DP80moYvIppa4j2zxpreWr9vRqBMRuo3Ahk3S2kFOR0QcW7dgGkzehbQ0juInEdGVMh5LQ9IbBzvfqPNSORGYjRNJLyCrEgAgIn6bMByzYXNjsdkYSVok6W6ydaxvzP97XdqozIbPicBs7D4LHA38KiIOJusueHPakMyGz4nAbOx686mmp0iaks/BVLjFWGzicq8hGxVJh5ENmNr1N1TE9Ztzf8yn5L4JuEzSQ5Qt2GPFMdCypSXuNWSThqSvka1ReyfQlx+OiHh/uqjSybuJPk1Wwv4rsqmHLyvKgjTWr6zX0LuA/YBv5vuLgT9ExN8lCWwITgQ2YpLuiohXpI6jEUmaDTzS6HPLWG1J6oqI1qGONQq3Edho/EEJx24AAAV5SURBVExS4ROBpKMl3SDpSklHSrqDbDDRHyS9LXV8ltTzJL24tCPpYOB5CeMZlNsIbDQuIUsGv6d/qt2IiMPShlV3FwBnklUFrQGOj4ibJb2cbNbJ1SmDs6T+DrghX71NwDzgr9OGNDBXDdmISdoEfAy4nf42AiLivmRBJVC+GLmk7ohoLjt3a0QcmS46S03SdODl+e4vIqJh551yicBGo6eRp9Sto76y7cqFyf0Lq8AkPZfsx9K8iPigpJdKellEXJs6tmpcIrARk/QVYB+y+fd3/copWvdRSTvJuokK2APYWjoFzIiIaalis7QkfZtsYZpTI6IlTww/LZUgG41LBDYae5AlgLeWHQugUIkgIpqGvsoK6pCIOEnSYoCI2KryaUgbjBOBjVhEvC91DGYN7hlJe5BXEUo6hN3XrGgo7j5qIybpQElXSXoof3xP0oGp4zJrIGeT9Ro7SNJlZCv6fSJtSANzG4GNmKTrgcuBS/NDpwB/FRFvSReVWeOQNIusrejo/L83A3tFxG+SBjYAJwIbsfJuk4MdMysqST8hG1fyeL7fDHw3IlrSRladq4ZsNB6RdIqkpvxxCuB5dcz6/RNwjaTnSTqKbP3iUxLHNCCXCGzEJM0DlgOvI2sM+ylwulfkMusn6c/I2gX2At4dEb9KHNKAnAjMzMaJpOXsPpjwOODXwL0AEXF6grCG5O6jNmxV/sh306h/5GZ11FWxvy5JFCPkRGAjUfojPwZ4BfDtfP8vgLuSRGTWQCLiYti1RsW2iNiZ7zcB01PGNhhXDdmISboZWBARO/L9acCPI+LotJGZNYb838ibI+LJfH9P4EcR8fq0kVXnXkM2GvsCM8v298yPmVlmRikJAOTbz00Yz6BcNWSj8c/ArZLWkg2WeQNwTtKIzBrLU5JeFRHrAfIupJUz1DYMVw3ZqEjaD3htvntLRPw+ZTxmjUTSq4FvAb8j+7G0H3BSRDRk47ETgY2KpAPIVl3aVaqMiJvSRWTWWPK2s5flu7+MiN6U8QzGicBGTNJ5wEnAnfQvzhIRsShdVGaNRVILWe+6GaVjEXFJuogG5kRgIybpl8Bhjbz0nllKks4G3kSWCH4AHA90RsSJKeMaiHsN2WjcA3j1LbOBnUg2qvj3+fodhwN7pw1pYO41ZKOxFdgg6b/YfalKjyw2yzwdEX2SdkiaCTwEHJQ6qIE4EdhorMofZlZdl6R9gK+STTPxJPCztCENzG0EZmY1JGk+MDMibkscyoDcRmAjJumlkq6QdJeke0qP1HGZNYq82hSAiLg3Im4rP9ZoXDVko/F1sjVZvwQsBN6Hf1SYIWkG2VQSsyXtSzaYDLIpWQ5IFtgQXDVkIyZpXUQcJen2iHhl+bHUsZmlJOkM4KPAi4AH6E8EjwNfjYgLUsU2GJcIbDS2S5oC3C3pI2R/8HsmjsksuYg4Hzhf0rKIWJ46nuFyicBGLJ9HpRvYB/gsWf/o8yLilqSBmSWW/9vYXJp7S9KpwLuB+4BzIuLRlPENxInAxixfdOPkiLgsdSxmKUlaT7YOwaOS3kA28dwy4Aig2SOLbcKTNFPSpyRdIOmtynwE2AT8Zer4zBpAU9mv/pOACyPiexHxf4GXJIxrUG4jsJG4FNhCNjDmA8CZZI1hfx4RG1IGZtYgmiRNzVfvOw44rexcw95vGzYwa0gvLusldBHwIDA3IralDcusYXQAN0p6mGwhmh8DSHoJ8FjKwAbjRGAjsWs+9YjYKel+JwGzfhHRng8c259sjeJSI+wUsraChuTGYhs2STuBp0q7wB5kE9CJbD2CmQM918walxOBmVnBudeQmVnBORGYmRWcE4GZWcE5EZiZFZwTgZlZwTkRmJkV3P8A/dl03RNlW/YAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3KX0eS9fJh3l",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Lets apply on Train and Test Split\n",
        "level0 = list()\n",
        "level0.append(('Tree', DecisionTreeRegressor()))\n",
        "level0.append(('RF', RandomForestRegressor()))\n",
        "level0.append(('GBM', GradientBoostingRegressor()))\n",
        "level0.append(('Bagging', BaggingRegressor()))\n",
        "level0.append((\"XGB\", XGBRegressor()))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dn12mTjtI0xg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "level1 = LGBMRegressor()\n",
        "model = StackingRegressor(estimators=level0, final_estimator=level1, cv=10)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VBzRgKnjJP94",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "outputId": "d61e0275-6e16-4109-cba3-0b636eb381ad"
      },
      "source": [
        "# Making Predictions on Train & Test\n",
        "stacked_pred = model.fit(xtrain, ytrain).predict(xtest)"
      ],
      "execution_count": 122,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[21:31:26] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[21:31:30] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[21:31:30] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[21:31:30] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[21:31:30] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[21:31:30] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[21:31:30] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[21:31:30] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[21:31:30] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[21:31:30] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n",
            "[21:31:30] WARNING: /workspace/src/objective/regression_obj.cu:152: reg:linear is now deprecated in favor of reg:squarederror.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sTJ7D-sPOzNX",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import r2_score, mean_squared_error"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gbbK3iqDPAJf",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "b20cb924-0cbf-4654-a82e-3b6cb4f7a610"
      },
      "source": [
        "# Checking Model Metrics\n",
        "print(r2_score(ytest, stacked_pred))\n",
        "print(np.sqrt(mean_squared_error(ytest, stacked_pred)))"
      ],
      "execution_count": 123,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0.8202170270477344\n",
            "4.194178557256572\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YkbmmQ-ITWCW",
        "colab_type": "text"
      },
      "source": [
        "## Conclusion\n",
        "\n",
        "Hence, it is clear that LightGBM is a better stacked model as it has increased the R2 Score to 0.82"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nSVZ6lVeTfQu",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}