3 polynomial_regression_05.2021.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Polynomial Regression\n",
    "Outline:\n",
    "1. Data Import\n",
    "2. Modeling\n",
    "3. RS, TPOT\n",
    "4. BO, GS\n",
    "5. Postmodeling & others"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data & Library Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import libraries\n",
    "import pickle\n",
    "import pandas as pd\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.multioutput import MultiOutputRegressor\n",
    "from sklearn.model_selection import RandomizedSearchCV, GridSearchCV\n",
    "# Modeling related libraries\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "from sklearn.metrics import r2_score, mean_squared_error, mean_absolute_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import Training dataset  \n",
    "X_train_dev = pd.read_csv('Dataset for metamodeling/X_train_dev.csv', header=0).copy()\n",
    "X_val = pd.read_csv('Dataset for metamodeling/X_val.csv', header=0).copy()\n",
    "y_train_dev = pd.read_csv('Dataset for metamodeling/y_train_dev.csv', header=0).copy()\n",
    "y_val = pd.read_csv('Dataset for metamodeling/y_val.csv', header=0).copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 8320 entries, 0 to 8319\n",
      "Data columns (total 7 columns):\n",
      " #   Column                Non-Null Count  Dtype  \n",
      "---  ------                --------------  -----  \n",
      " 0   $volume_{oo.15min}$   8320 non-null   float64\n",
      " 1   $volume_{oo.60min}$   8320 non-null   float64\n",
      " 2   $volume_{oo.total}$   8320 non-null   float64\n",
      " 3   $height_{damage.ih}$  8320 non-null   float64\n",
      " 4   $length_{damage.ih}$  8320 non-null   float64\n",
      " 5   $height_{damage.oh}$  8320 non-null   float64\n",
      " 6   $length_{damage.oh}$  8320 non-null   float64\n",
      "dtypes: float64(7)\n",
      "memory usage: 455.1 KB\n"
     ]
    }
   ],
   "source": [
    "y_val.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "r2_pr, rmse_pr 0.35661629962902125 0.030805240515907637\n"
     ]
    }
   ],
   "source": [
    "# Initial modeling\n",
    "pr_1 = make_pipeline(PolynomialFeatures(degree=1),\n",
    "                      LinearRegression())\n",
    "pr_1.fit(X_train_dev,y_train_dev)\n",
    "y_pred = pr_1.predict(X_val)\n",
    "\n",
    "# Metrics\n",
    "from sklearn.metrics import r2_score, mean_squared_error\n",
    "\n",
    "r2_pr = r2_score(y_val, y_pred)\n",
    "rmse_pr = mean_squared_error(y_val, y_pred, squared=True)\n",
    "\n",
    "print('r2_pr, rmse_pr', r2_pr, rmse_pr)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Random Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'degree': 2, 'include_bias': True, 'interaction_only': False, 'order': 'C'}\n",
      "{'copy_X': True, 'fit_intercept': True, 'n_jobs': None, 'normalize': False}\n"
     ]
    }
   ],
   "source": [
    "print(PolynomialFeatures().get_params())\n",
    "print(LinearRegression().get_params())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Develop RandomSearchCV for the DNN\n",
    "# Set up the sample space for Random Search\n",
    "parameter_grid = { 'degree' : list(range(1, 10, 1))}#,\n",
    "#'hidden_layer_sizes' : list(range(1, 60, 5)) }      # range(start, stop, step) #(start=1,stop=1000, num=100))\n",
    "# Define how many samples\n",
    "number_models = 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ------------------\n",
    "# Random try is enough (since search space is small) \n",
    "# we dont need Random search from sklearn!\n",
    "degree=2\n",
    "polyreg = make_pipeline(PolynomialFeatures(degree, order='F'),\n",
    "                      LinearRegression(fit_intercept=True))\n",
    "polyreg.fit(X_train_dev,y_train_dev)\n",
    "y_pred = polyreg.predict(X_val)\n",
    "r2_pr = r2_score(y_val, y_pred)\n",
    "print('r2_pr', r2_pr)\n",
    "rmse_pr = mean_squared_error(y_val, y_pred, squared=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Grid Search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 2 folds for each of 32 candidates, totalling 64 fits\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.021, test=-0.022), neg_median_absolute_error=(train=-0.073, test=-0.074), r2=(train=0.547, test=0.533), total=   2.3s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:    2.7s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.021, test=-0.021), neg_median_absolute_error=(train=-0.075, test=-0.075), r2=(train=0.539, test=0.541), total=   2.4s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed:    5.5s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.021, test=-0.022), neg_median_absolute_error=(train=-0.073, test=-0.074), r2=(train=0.547, test=0.533), total=   3.5s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed:    9.9s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.021, test=-0.021), neg_median_absolute_error=(train=-0.075, test=-0.075), r2=(train=0.539, test=0.541), total=   3.4s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed:   14.1s remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.010, test=-0.010), neg_median_absolute_error=(train=-0.050, test=-0.051), r2=(train=0.788, test=0.781), total=   4.0s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.010, test=-0.010), neg_median_absolute_error=(train=-0.051, test=-0.051), r2=(train=0.785, test=0.784), total=   4.0s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.010, test=-0.010), neg_median_absolute_error=(train=-0.050, test=-0.051), r2=(train=0.788, test=0.781), total=   5.9s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.010, test=-0.010), neg_median_absolute_error=(train=-0.051, test=-0.051), r2=(train=0.785, test=0.784), total=   5.9s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.021, test=-0.022), neg_median_absolute_error=(train=-0.073, test=-0.074), r2=(train=0.547, test=0.533), total=   2.4s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.021, test=-0.021), neg_median_absolute_error=(train=-0.075, test=-0.075), r2=(train=0.539, test=0.541), total=   2.4s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.021, test=-0.022), neg_median_absolute_error=(train=-0.073, test=-0.074), r2=(train=0.547, test=0.533), total=   3.4s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.021, test=-0.021), neg_median_absolute_error=(train=-0.075, test=-0.075), r2=(train=0.539, test=0.541), total=   3.5s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.010, test=-0.010), neg_median_absolute_error=(train=-0.050, test=-0.051), r2=(train=0.788, test=0.781), total=   4.0s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.010, test=-0.010), neg_median_absolute_error=(train=-0.051, test=-0.051), r2=(train=0.785, test=0.784), total=   4.1s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.010, test=-0.010), neg_median_absolute_error=(train=-0.050, test=-0.051), r2=(train=0.788, test=0.781), total=   6.0s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.010, test=-0.010), neg_median_absolute_error=(train=-0.051, test=-0.051), r2=(train=0.785, test=0.784), total=   5.9s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.019, test=-0.020), neg_median_absolute_error=(train=-0.064, test=-0.066), r2=(train=0.581, test=0.559), total=   6.5s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.020, test=-0.020), neg_median_absolute_error=(train=-0.066, test=-0.066), r2=(train=0.571, test=0.569), total=   6.5s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.019, test=-0.020), neg_median_absolute_error=(train=-0.064, test=-0.066), r2=(train=0.581, test=0.559), total=   9.5s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.020, test=-0.020), neg_median_absolute_error=(train=-0.066, test=-0.066), r2=(train=0.571, test=0.569), total=   9.7s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.005, test=-0.005), neg_median_absolute_error=(train=-0.032, test=-0.034), r2=(train=0.889, test=0.880), total=  17.6s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.005, test=-0.005), neg_median_absolute_error=(train=-0.032, test=-0.033), r2=(train=0.887, test=0.882), total=  17.9s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.005, test=-0.005), neg_median_absolute_error=(train=-0.032, test=-0.034), r2=(train=0.889, test=0.880), total=  24.6s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.005, test=-0.005), neg_median_absolute_error=(train=-0.032, test=-0.033), r2=(train=0.887, test=0.882), total=  24.3s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.019, test=-0.020), neg_median_absolute_error=(train=-0.064, test=-0.066), r2=(train=0.581, test=0.559), total=   6.6s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.020, test=-0.020), neg_median_absolute_error=(train=-0.066, test=-0.066), r2=(train=0.571, test=0.569), total=   6.6s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.019, test=-0.020), neg_median_absolute_error=(train=-0.064, test=-0.066), r2=(train=0.581, test=0.559), total=   9.7s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.020, test=-0.020), neg_median_absolute_error=(train=-0.066, test=-0.066), r2=(train=0.571, test=0.569), total=   9.7s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.005, test=-0.005), neg_median_absolute_error=(train=-0.032, test=-0.034), r2=(train=0.889, test=0.880), total=  17.8s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.005, test=-0.005), neg_median_absolute_error=(train=-0.032, test=-0.033), r2=(train=0.887, test=0.882), total=  17.3s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.005, test=-0.005), neg_median_absolute_error=(train=-0.032, test=-0.034), r2=(train=0.889, test=0.880), total=  24.4s\n",
      "[CV] estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=True, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.005, test=-0.005), neg_median_absolute_error=(train=-0.032, test=-0.033), r2=(train=0.887, test=0.882), total=  24.4s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.021, test=-0.022), neg_median_absolute_error=(train=-0.073, test=-0.074), r2=(train=0.547, test=0.533), total=   2.4s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.021, test=-0.021), neg_median_absolute_error=(train=-0.075, test=-0.075), r2=(train=0.539, test=0.541), total=   2.3s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.021, test=-0.022), neg_median_absolute_error=(train=-0.073, test=-0.074), r2=(train=0.547, test=0.533), total=   3.8s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.021, test=-0.021), neg_median_absolute_error=(train=-0.075, test=-0.075), r2=(train=0.539, test=0.541), total=   3.6s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.010, test=-0.010), neg_median_absolute_error=(train=-0.050, test=-0.051), r2=(train=0.788, test=0.781), total=   4.0s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.010, test=-0.010), neg_median_absolute_error=(train=-0.051, test=-0.051), r2=(train=0.785, test=0.784), total=   4.0s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.010, test=-0.010), neg_median_absolute_error=(train=-0.050, test=-0.051), r2=(train=0.788, test=0.781), total=   5.8s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.010, test=-0.010), neg_median_absolute_error=(train=-0.051, test=-0.051), r2=(train=0.785, test=0.784), total=   5.7s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.021, test=-0.022), neg_median_absolute_error=(train=-0.073, test=-0.074), r2=(train=0.547, test=0.533), total=   2.4s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.021, test=-0.021), neg_median_absolute_error=(train=-0.075, test=-0.075), r2=(train=0.539, test=0.541), total=   2.4s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.021, test=-0.022), neg_median_absolute_error=(train=-0.073, test=-0.074), r2=(train=0.547, test=0.533), total=   3.4s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.021, test=-0.021), neg_median_absolute_error=(train=-0.075, test=-0.075), r2=(train=0.539, test=0.541), total=   3.4s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.010, test=-0.010), neg_median_absolute_error=(train=-0.050, test=-0.051), r2=(train=0.788, test=0.781), total=   3.9s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.010, test=-0.010), neg_median_absolute_error=(train=-0.051, test=-0.051), r2=(train=0.785, test=0.784), total=   4.0s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.010, test=-0.010), neg_median_absolute_error=(train=-0.050, test=-0.051), r2=(train=0.788, test=0.781), total=   5.7s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=3, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.010, test=-0.010), neg_median_absolute_error=(train=-0.051, test=-0.051), r2=(train=0.785, test=0.784), total=   5.7s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.019, test=-0.020), neg_median_absolute_error=(train=-0.064, test=-0.066), r2=(train=0.581, test=0.559), total=   6.3s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.020, test=-0.020), neg_median_absolute_error=(train=-0.066, test=-0.066), r2=(train=0.571, test=0.569), total=   6.7s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.019, test=-0.020), neg_median_absolute_error=(train=-0.064, test=-0.066), r2=(train=0.581, test=0.559), total=   9.3s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.020, test=-0.020), neg_median_absolute_error=(train=-0.066, test=-0.066), r2=(train=0.571, test=0.569), total=   9.3s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.005, test=-0.005), neg_median_absolute_error=(train=-0.032, test=-0.034), r2=(train=0.889, test=0.880), total=  17.2s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.005, test=-0.005), neg_median_absolute_error=(train=-0.032, test=-0.033), r2=(train=0.887, test=0.882), total=  17.1s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.005, test=-0.005), neg_median_absolute_error=(train=-0.032, test=-0.034), r2=(train=0.889, test=0.880), total=  23.6s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=True, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.005, test=-0.005), neg_median_absolute_error=(train=-0.032, test=-0.033), r2=(train=0.887, test=0.882), total=  26.6s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.019, test=-0.020), neg_median_absolute_error=(train=-0.064, test=-0.066), r2=(train=0.581, test=0.559), total=   6.6s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.020, test=-0.020), neg_median_absolute_error=(train=-0.066, test=-0.066), r2=(train=0.571, test=0.569), total=   6.5s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.019, test=-0.020), neg_median_absolute_error=(train=-0.064, test=-0.066), r2=(train=0.581, test=0.559), total=   9.4s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=True, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.020, test=-0.020), neg_median_absolute_error=(train=-0.066, test=-0.066), r2=(train=0.571, test=0.569), total=   9.6s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.005, test=-0.005), neg_median_absolute_error=(train=-0.032, test=-0.034), r2=(train=0.889, test=0.880), total=  17.3s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=F, neg_mean_squared_error=(train=-0.005, test=-0.005), neg_median_absolute_error=(train=-0.032, test=-0.033), r2=(train=0.887, test=0.882), total=  17.6s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.005, test=-0.005), neg_median_absolute_error=(train=-0.032, test=-0.034), r2=(train=0.889, test=0.880), total=  23.9s\n",
      "[CV] estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C \n",
      "[CV]  estimator__linearregression__fit_intercept=False, estimator__polynomialfeatures__degree=4, estimator__polynomialfeatures__include_bias=False, estimator__polynomialfeatures__interaction_only=False, estimator__polynomialfeatures__order=C, neg_mean_squared_error=(train=-0.005, test=-0.005), neg_median_absolute_error=(train=-0.032, test=-0.033), r2=(train=0.887, test=0.882), total=  25.5s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  64 out of  64 | elapsed: 11.7min finished\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Length mismatch: Expected axis has 8 elements, new values have 7 elements",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-11-afc3c12ed4dc>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     25\u001b[0m df_grids_pr_perfm = pd.DataFrame([degree, order, interaction_only, include_bias, intercept,\n\u001b[0;32m     26\u001b[0m                                     r2_pr, neg_mean_squared_error_pr, neg_median_absolute_error_pr]).T\n\u001b[1;32m---> 27\u001b[1;33m df_grids_pr_perfm.columns = ['degree','order', 'interaction_only', 'include_bias','intercept'\n\u001b[0m\u001b[0;32m     28\u001b[0m                                'r2_pr', 'neg_mean_squared_error_pr','neg_median_absolute_error_pr']\n\u001b[0;32m     29\u001b[0m \u001b[1;31m# df_rands_dnn_perfm.groupby(['Learning Rate']).mean()\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36m__setattr__\u001b[1;34m(self, name, value)\u001b[0m\n\u001b[0;32m   5150\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   5151\u001b[0m             \u001b[0mobject\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__getattribute__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 5152\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mobject\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__setattr__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mname\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   5153\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   5154\u001b[0m             \u001b[1;32mpass\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mpandas\\_libs\\properties.pyx\u001b[0m in \u001b[0;36mpandas._libs.properties.AxisProperty.__set__\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\generic.py\u001b[0m in \u001b[0;36m_set_axis\u001b[1;34m(self, axis, labels)\u001b[0m\n\u001b[0;32m    562\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_set_axis\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0maxis\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mint\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mIndex\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m->\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    563\u001b[0m         \u001b[0mlabels\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mensure_index\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mlabels\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 564\u001b[1;33m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_mgr\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mset_axis\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0maxis\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlabels\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    565\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_clear_item_cache\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    566\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mC:\\ProgramData\\Anaconda3\\lib\\site-packages\\pandas\\core\\internals\\managers.py\u001b[0m in \u001b[0;36mset_axis\u001b[1;34m(self, axis, new_labels)\u001b[0m\n\u001b[0;32m    224\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    225\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mnew_len\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[0mold_len\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 226\u001b[1;33m             raise ValueError(\n\u001b[0m\u001b[0;32m    227\u001b[0m                 \u001b[1;34mf\"Length mismatch: Expected axis has {old_len} elements, new \"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    228\u001b[0m                 \u001b[1;34mf\"values have {new_len} elements\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mValueError\u001b[0m: Length mismatch: Expected axis has 8 elements, new values have 7 elements"
     ]
    }
   ],
   "source": [
    "# 04.05.2021 Tanmoy at 1pm\n",
    "model = MultiOutputRegressor(make_pipeline(PolynomialFeatures(),\n",
    "                      LinearRegression()))\n",
    "hyperparameters = dict(estimator__polynomialfeatures__degree=[3,4], # 5 will take enormous amount of time\n",
    "                    estimator__polynomialfeatures__order =['F', 'C'],\n",
    "                       estimator__polynomialfeatures__interaction_only = [True, False],\n",
    "                       estimator__polynomialfeatures__include_bias = [True, False],\n",
    "                      estimator__linearregression__fit_intercept = [True, False])\n",
    "#number_models = 10\n",
    "grid_search_model_pr = GridSearchCV(model, hyperparameters, #n_iter=number_models,  #random_state=0, \n",
    "                                                  scoring=['r2', 'neg_mean_squared_error', 'neg_median_absolute_error'],\n",
    "                                                   n_jobs=1, refit=False, cv=2, verbose=5, #n_jobs=1 to get Verbose output (runtime of each model) for each iteration\n",
    "                                                   pre_dispatch='2*n_jobs', error_score='raise', return_train_score=True)\n",
    "grid_search_model_pr = grid_search_model_pr.fit(X_train_dev, y_train_dev)\n",
    "# Calculate scores on validation training set +++ \n",
    "degree = [item['estimator__polynomialfeatures__degree'] for item in grid_search_model_pr.cv_results_['params']]\n",
    "order = [item['estimator__polynomialfeatures__order'] for item in grid_search_model_pr.cv_results_['params']]\n",
    "interaction_only = [item['estimator__polynomialfeatures__interaction_only'] for item in grid_search_model_pr.cv_results_['params']]\n",
    "include_bias = [item['estimator__polynomialfeatures__include_bias'] for item in grid_search_model_pr.cv_results_['params']]\n",
    "intercept = [item['estimator__linearregression__fit_intercept'] for item in grid_search_model_pr.cv_results_['params']]\n",
    "\n",
    "r2_pr = list(grid_search_model_pr.cv_results_['mean_test_r2'])\n",
    "neg_mean_squared_error_pr = list(grid_search_model_pr.cv_results_['mean_test_neg_mean_squared_error'])\n",
    "neg_median_absolute_error_pr = list(grid_search_model_pr.cv_results_['mean_test_neg_median_absolute_error'])\n",
    "df_grids_pr_perfm = pd.DataFrame([degree, order, interaction_only, include_bias, intercept,\n",
    "                                    r2_pr, neg_mean_squared_error_pr, neg_median_absolute_error_pr]).T\n",
    "df_grids_pr_perfm.columns = ['degree','order', 'interaction_only', 'include_bias','intercept'\n",
    "                               'r2_pr', 'neg_mean_squared_error_pr','neg_median_absolute_error_pr']\n",
    "# df_rands_dnn_perfm.groupby(['Learning Rate']).mean()\n",
    "display(df_grids_pr_perfm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_rands_pr_perfm.to_csv('df_rands_pr_perfm.csv')\n",
    "#model.get_params().keys()\n",
    "#PolynomialFeatures().get_params"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}