main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "id": "initial_id",
   "metadata": {
    "collapsed": true,
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.445517Z",
     "start_time": "2024-06-01T15:03:55.907567Z"
    }
   },
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.impute import SimpleImputer\n",
    "%matplotlib inline"
   ],
   "outputs": [],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.486669Z",
     "start_time": "2024-06-01T15:03:57.446527Z"
    }
   },
   "cell_type": "code",
   "source": [
    "test_df = pd.read_csv(\"test.csv\")\n",
    "train_df = pd.read_csv(\"train.csv\")"
   ],
   "id": "c905a1590dc85c79",
   "outputs": [],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.504123Z",
     "start_time": "2024-06-01T15:03:57.488679Z"
    }
   },
   "cell_type": "code",
   "source": "train_df.head() ",
   "id": "988c944cee2254c8",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "   Id  MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \\\n",
       "0   1          60       RL         65.0     8450   Pave   NaN      Reg   \n",
       "1   2          20       RL         80.0     9600   Pave   NaN      Reg   \n",
       "2   3          60       RL         68.0    11250   Pave   NaN      IR1   \n",
       "3   4          70       RL         60.0     9550   Pave   NaN      IR1   \n",
       "4   5          60       RL         84.0    14260   Pave   NaN      IR1   \n",
       "\n",
       "  LandContour Utilities  ... PoolArea PoolQC Fence MiscFeature MiscVal MoSold  \\\n",
       "0         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      2   \n",
       "1         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      5   \n",
       "2         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      9   \n",
       "3         Lvl    AllPub  ...        0    NaN   NaN         NaN       0      2   \n",
       "4         Lvl    AllPub  ...        0    NaN   NaN         NaN       0     12   \n",
       "\n",
       "  YrSold  SaleType  SaleCondition  SalePrice  \n",
       "0   2008        WD         Normal     208500  \n",
       "1   2007        WD         Normal     181500  \n",
       "2   2008        WD         Normal     223500  \n",
       "3   2006        WD        Abnorml     140000  \n",
       "4   2008        WD         Normal     250000  \n",
       "\n",
       "[5 rows x 81 columns]"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>MSZoning</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>Street</th>\n",
       "      <th>Alley</th>\n",
       "      <th>LotShape</th>\n",
       "      <th>LandContour</th>\n",
       "      <th>Utilities</th>\n",
       "      <th>...</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>PoolQC</th>\n",
       "      <th>Fence</th>\n",
       "      <th>MiscFeature</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SaleType</th>\n",
       "      <th>SaleCondition</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8450</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>208500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>RL</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9600</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Reg</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>2007</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>181500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>68.0</td>\n",
       "      <td>11250</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>9</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>223500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>70</td>\n",
       "      <td>RL</td>\n",
       "      <td>60.0</td>\n",
       "      <td>9550</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2006</td>\n",
       "      <td>WD</td>\n",
       "      <td>Abnorml</td>\n",
       "      <td>140000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>84.0</td>\n",
       "      <td>14260</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>12</td>\n",
       "      <td>2008</td>\n",
       "      <td>WD</td>\n",
       "      <td>Normal</td>\n",
       "      <td>250000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 81 columns</p>\n",
       "</div>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 3
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "<h2>Dealing with outliers</h2>",
   "id": "43c558d91f1510c1"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.509626Z",
     "start_time": "2024-06-01T15:03:57.506137Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# outlier notes:\n",
    "# TotalBsmtSF > 3000 price < 200000 BsmtQual = Ex\n",
    "# GrLivArea > 4000 price < 200000\n",
    "# take a look at PoolArea > 550 price > 700000"
   ],
   "id": "6247cb2bcb7e4e88",
   "outputs": [],
   "execution_count": 4
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.515360Z",
     "start_time": "2024-06-01T15:03:57.510637Z"
    }
   },
   "cell_type": "code",
   "source": "outlier_indexes = train_df[(train_df[\"SalePrice\"]<200000) & (train_df[\"TotalBsmtSF\"]>3000)].index",
   "id": "a128504363146009",
   "outputs": [],
   "execution_count": 5
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.521504Z",
     "start_time": "2024-06-01T15:03:57.516370Z"
    }
   },
   "cell_type": "code",
   "source": "outlier_indexes",
   "id": "c8c8138daca0e9d6",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index([523, 1298], dtype='int64')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 6
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.527266Z",
     "start_time": "2024-06-01T15:03:57.522514Z"
    }
   },
   "cell_type": "code",
   "source": "train_df.drop(outlier_indexes,inplace=True)",
   "id": "91cb759a14e13845",
   "outputs": [],
   "execution_count": 7
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.538388Z",
     "start_time": "2024-06-01T15:03:57.528274Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# everything about this house is perfect it`s a novelty here\n",
    "train_df[(train_df[\"PoolArea\"]>550) & (train_df[\"SalePrice\"]>700000)]"
   ],
   "id": "cddaba324c89031b",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "        Id  MSSubClass MSZoning  LotFrontage  LotArea Street Alley LotShape  \\\n",
       "1182  1183          60       RL        160.0    15623   Pave   NaN      IR1   \n",
       "\n",
       "     LandContour Utilities  ... PoolArea PoolQC  Fence MiscFeature MiscVal  \\\n",
       "1182         Lvl    AllPub  ...      555     Ex  MnPrv         NaN       0   \n",
       "\n",
       "     MoSold YrSold  SaleType  SaleCondition  SalePrice  \n",
       "1182      7   2007        WD        Abnorml     745000  \n",
       "\n",
       "[1 rows x 81 columns]"
      ],
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>MSZoning</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>Street</th>\n",
       "      <th>Alley</th>\n",
       "      <th>LotShape</th>\n",
       "      <th>LandContour</th>\n",
       "      <th>Utilities</th>\n",
       "      <th>...</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>PoolQC</th>\n",
       "      <th>Fence</th>\n",
       "      <th>MiscFeature</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SaleType</th>\n",
       "      <th>SaleCondition</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1182</th>\n",
       "      <td>1183</td>\n",
       "      <td>60</td>\n",
       "      <td>RL</td>\n",
       "      <td>160.0</td>\n",
       "      <td>15623</td>\n",
       "      <td>Pave</td>\n",
       "      <td>NaN</td>\n",
       "      <td>IR1</td>\n",
       "      <td>Lvl</td>\n",
       "      <td>AllPub</td>\n",
       "      <td>...</td>\n",
       "      <td>555</td>\n",
       "      <td>Ex</td>\n",
       "      <td>MnPrv</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>2007</td>\n",
       "      <td>WD</td>\n",
       "      <td>Abnorml</td>\n",
       "      <td>745000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 81 columns</p>\n",
       "</div>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 8
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": [
    "<h2>Handling missing values\n",
    "</h2>"
   ],
   "id": "aee4f8bf83706765"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.543382Z",
     "start_time": "2024-06-01T15:03:57.540478Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def percent_missing(df):\n",
    "    percent_nan = df.isna().sum() / len(df)\n",
    "    percent_nan = percent_nan[percent_nan > 0].sort_values()\n",
    "    return percent_nan"
   ],
   "id": "8d04d34fb4920a75",
   "outputs": [],
   "execution_count": 9
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.553084Z",
     "start_time": "2024-06-01T15:03:57.544394Z"
    }
   },
   "cell_type": "code",
   "source": "percent_missing(train_df)",
   "id": "d0b31705b8e31621",
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Electrical      0.000686\n",
       "MasVnrArea      0.005487\n",
       "BsmtQual        0.025377\n",
       "BsmtCond        0.025377\n",
       "BsmtFinType1    0.025377\n",
       "BsmtExposure    0.026063\n",
       "BsmtFinType2    0.026063\n",
       "GarageCond      0.055556\n",
       "GarageQual      0.055556\n",
       "GarageFinish    0.055556\n",
       "GarageYrBlt     0.055556\n",
       "GarageType      0.055556\n",
       "LotFrontage     0.177641\n",
       "FireplaceQu     0.473251\n",
       "MasVnrType      0.598080\n",
       "Fence           0.807270\n",
       "Alley           0.937586\n",
       "MiscFeature     0.962963\n",
       "PoolQC          0.995885\n",
       "dtype: float64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 10
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.558228Z",
     "start_time": "2024-06-01T15:03:57.553084Z"
    }
   },
   "cell_type": "code",
   "source": "train_df.drop(\"Utilities\", axis=1, inplace=True)",
   "id": "f4ccfe5fb84c767b",
   "outputs": [],
   "execution_count": 11
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.574473Z",
     "start_time": "2024-06-01T15:03:57.559241Z"
    }
   },
   "cell_type": "code",
   "source": [
    "train_df[\"LotFrontage\"] = train_df.groupby([\"Neighborhood\",\"MSZoning\"])[\"LotFrontage\"].transform(lambda x:x.fillna(x.mean()))\n",
    "train_df[\"LotFrontage\"] = train_df.groupby(\"Neighborhood\")[\"LotFrontage\"].transform(lambda x:x.fillna(x.mean()))\n",
    "train_df.dropna(subset=\"Electrical\",axis=0,inplace=True)"
   ],
   "id": "22ed837b69555d3a",
   "outputs": [],
   "execution_count": 12
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "<h2>preprocessing data</h2>",
   "id": "469146929a1ccb85"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.578544Z",
     "start_time": "2024-06-01T15:03:57.575481Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler, FunctionTransformer, OneHotEncoder\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.model_selection import train_test_split"
   ],
   "id": "1499752073a5247f",
   "outputs": [],
   "execution_count": 13
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.587763Z",
     "start_time": "2024-06-01T15:03:57.580554Z"
    }
   },
   "cell_type": "code",
   "source": [
    "X = train_df.drop(\"SalePrice\",axis=1)\n",
    "y = np.log(train_df[\"SalePrice\"])\n",
    "# y = train_df[\"SalePrice\"]\n",
    "X_train,X_val,y_train,y_val = train_test_split(X,y,test_size=0.2,random_state=42)"
   ],
   "id": "c1509ca8c5f40228",
   "outputs": [],
   "execution_count": 14
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.593918Z",
     "start_time": "2024-06-01T15:03:57.588774Z"
    }
   },
   "cell_type": "code",
   "source": [
    "categorical_columns = X_train.select_dtypes(include=[\"object\",\"category\"]).columns\n",
    "numerical_columns = X_train.select_dtypes(exclude=[\"object\",\"category\"]).columns"
   ],
   "id": "c730ea012c9a737c",
   "outputs": [],
   "execution_count": 15
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.598093Z",
     "start_time": "2024-06-01T15:03:57.594930Z"
    }
   },
   "cell_type": "code",
   "source": [
    "numerical_transformer = Pipeline([\n",
    "    (\"imputer\", SimpleImputer(strategy=\"constant\",fill_value=0)),\n",
    "    (\"scaler\", StandardScaler()),\n",
    "])\n",
    "categorical_transformer = Pipeline([\n",
    "    (\"imputer\",SimpleImputer(strategy=\"constant\",fill_value=\"None\")),\n",
    "    (\"onehot\", OneHotEncoder(handle_unknown=\"ignore\",sparse_output =False)),\n",
    "])"
   ],
   "id": "4832f71ab7187f4a",
   "outputs": [],
   "execution_count": 16
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.603447Z",
     "start_time": "2024-06-01T15:03:57.599103Z"
    }
   },
   "cell_type": "code",
   "source": [
    "preprocessor = ColumnTransformer([\n",
    "    (\"num\",numerical_transformer,numerical_columns),\n",
    "    (\"cat\",categorical_transformer,categorical_columns)\n",
    "],remainder=\"passthrough\")"
   ],
   "id": "ab809d7885cc1021",
   "outputs": [],
   "execution_count": 17
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.607874Z",
     "start_time": "2024-06-01T15:03:57.604456Z"
    }
   },
   "cell_type": "code",
   "source": [
    "pipeline = Pipeline([\n",
    "    (\"preprocessor\",preprocessor)\n",
    "])"
   ],
   "id": "cd8193d64cda8e7d",
   "outputs": [],
   "execution_count": 18
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:57.639411Z",
     "start_time": "2024-06-01T15:03:57.608883Z"
    }
   },
   "cell_type": "code",
   "source": [
    "X_train_preprocessed = pipeline.fit_transform(X_train)\n",
    "X_val_preprocessed = pipeline.transform(X_val)"
   ],
   "id": "80950bf4b99a3297",
   "outputs": [],
   "execution_count": 19
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "<h3>Base features pred</h3>",
   "id": "43e833953c319207"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:58.064998Z",
     "start_time": "2024-06-01T15:03:57.639411Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from sklearn.svm import LinearSVR,SVR\n",
    "from sklearn.ensemble import RandomForestRegressor,GradientBoostingRegressor,StackingRegressor,AdaBoostRegressor\n",
    "from sklearn.linear_model import ElasticNet\n",
    "from sklearn.metrics import mean_absolute_error,root_mean_squared_error,accuracy_score\n",
    "from sklearn.model_selection import GridSearchCV,KFold\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.decomposition import PCA\n",
    "from xgboost import XGBRegressor"
   ],
   "id": "c3f76092d3b34fea",
   "outputs": [],
   "execution_count": 20
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:03:59.178382Z",
     "start_time": "2024-06-01T15:03:59.171651Z"
    }
   },
   "cell_type": "code",
   "source": [
    "models = {\n",
    "    \"ElasticNet\": ElasticNet(random_state=42),\n",
    "    \"RandomForest\": RandomForestRegressor(random_state=42,n_jobs=-2),\n",
    "    \"GradientBoosting\":GradientBoostingRegressor(random_state=42),\n",
    "    \"SVR\":SVR(),\n",
    "    \"Ada\":AdaBoostRegressor(random_state=42),\n",
    "    \"XGBoost\":XGBRegressor(random_state=42,n_jobs=-2),\n",
    "}\n",
    "param_grids = {\n",
    "    \"ElasticNet\":{\n",
    "        \"alpha\": np.logspace(0,100,10),\n",
    "        \"l1_ratio\":np.linspace(.5,1,10)\n",
    "    },\n",
    "    \"RandomForest\":{\n",
    "        \"n_estimators\":[200,300,350,400],\n",
    "        # \"min_samples_split\":[2,3,4],\n",
    "        \"min_samples_split\":[2],\n",
    "        \"min_samples_leaf\":[1,2],\n",
    "        \"max_depth\":[None,10],\n",
    "    },\n",
    "    \"GradientBoosting\":{\n",
    "        # \"learning_rate\":[0.01,0.08,0.09,0.1,0.2],\n",
    "        \"learning_rate\":[0.08,0.09],\n",
    "        \"n_estimators\":[300,350,400],\n",
    "        # \"min_samples_split\":[2,10,15],\n",
    "        \"min_samples_split\":[2,5],\n",
    "        \"max_depth\":[3,4],\n",
    "    },\n",
    "    \"SVR\":{\n",
    "        # \"kernel\":[\"linear\", \"poly\", \"rbf\",],\n",
    "        \"kernel\":[\"rbf\"],\n",
    "        # \"gamma\":[\"scale\",\"auto\"],\n",
    "        \"gamma\":[\"auto\"],\n",
    "        # \"C\":[.1,1.5,10,80,100],\n",
    "        \"C\":[0.1,0.7,1,1.5],\n",
    "        \"epsilon\":[0.07,0.08,0.1],\n",
    "    },\n",
    "    \"Ada\":{\n",
    "        \"loss\" : ['linear','exponential'],\n",
    "        \"n_estimators\":[500,600],\n",
    "        \"learning_rate\":[1.0,4,5,8],\n",
    "    },\n",
    "    \"XGBoost\":{\n",
    "        \"n_estimators\": [100,200,400,600,700,800,900,1000,1100,1200],\n",
    "        \"objective\": [\"reg:squarederror\"],\n",
    "        \"max_depth\": [2,3,4,5,6,8,10],\n",
    "        \"learning_rate\": [0.01,0.03,0.04,0.05,0.6,0.1],\n",
    "        \"subsample\":[0.4,0.6,0.8,1],\n",
    "    },\n",
    "}"
   ],
   "id": "dfbaabcbe0c53aac",
   "outputs": [],
   "execution_count": 21
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:04:00.727283Z",
     "start_time": "2024-06-01T15:04:00.718595Z"
    }
   },
   "cell_type": "code",
   "source": "cv = KFold(n_splits=5,shuffle=True,random_state=42)",
   "id": "625673727871990e",
   "outputs": [],
   "execution_count": 22
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T10:01:31.129169Z",
     "start_time": "2024-06-01T09:21:00.177378Z"
    }
   },
   "cell_type": "code",
   "source": [
    "grids = {}\n",
    "for model_name, model in models.items():\n",
    "    grids[model_name] = GridSearchCV(model,param_grids[model_name],cv=cv,scoring='neg_mean_squared_error',n_jobs=-2)\n",
    "    grids[model_name].fit(X_train_preprocessed,y_train)\n",
    "    best_params = grids[model_name].best_params_\n",
    "    best_score = np.sqrt(-1 * grids[model_name].best_score_)\n",
    "    print(f\"Best parameters for {model_name}: {best_params}\")\n",
    "    print(f\"Best RMSE for {model_name}: {best_score}\\n\")"
   ],
   "id": "54d2292ca7653ba9",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters for ElasticNet: {'alpha': 1.0, 'l1_ratio': 0.5}\n",
      "Best RMSE for ElasticNet: 0.3956761919827401\n",
      "\n",
      "Best parameters for RandomForest: {'max_depth': None, 'min_samples_leaf': 2, 'min_samples_split': 2, 'n_estimators': 400}\n",
      "Best RMSE for RandomForest: 0.13974646761602755\n",
      "\n",
      "Best parameters for GradientBoosting: {'learning_rate': 0.08, 'max_depth': 3, 'min_samples_split': 2, 'n_estimators': 400}\n",
      "Best RMSE for GradientBoosting: 0.12106129775384859\n",
      "\n",
      "Best parameters for SVR: {'C': 1, 'epsilon': 0.07, 'gamma': 'auto', 'kernel': 'rbf'}\n",
      "Best RMSE for SVR: 0.11752158667808911\n",
      "\n",
      "Best parameters for Ada: {'learning_rate': 4, 'loss': 'exponential', 'n_estimators': 600}\n",
      "Best RMSE for Ada: 0.15748570296100853\n",
      "\n",
      "Best parameters for XGBoost: {'learning_rate': 0.03, 'max_depth': 3, 'n_estimators': 1200, 'objective': 'reg:squarederror', 'subsample': 0.4}\n",
      "Best RMSE for XGBoost: 0.1160657492238416\n",
      "\n"
     ]
    }
   ],
   "execution_count": 23
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "<h2>Creating featured columns</h2>",
   "id": "9d43e5ef8eaaa798"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:04:31.413374Z",
     "start_time": "2024-06-01T15:04:31.391969Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def get_target_mean_columns(X1,X2):\n",
    "    \n",
    "    df_train = pd.concat([X1,y_train],axis=1)\n",
    "    df_valid = X2.copy()\n",
    "    \n",
    "    df_train[\"YrMoSoldMeanEncoded\"] = df_train[\"YrSold\"].apply(str) + \"-\" + df_train[\"MoSold\"].apply(str)\n",
    "    df_valid[\"YrMoSoldMeanEncoded\"] = df_valid[\"YrSold\"].apply(str) + \"-\" + df_valid[\"MoSold\"].apply(str)\n",
    "    dt = df_train.groupby([\"YrMoSoldMeanEncoded\"])[\"SalePrice\"].agg([\"mean\"]).reset_index(drop=False)\n",
    "    df_train[\"YrMoSoldMeanEncoded\"] = df_train.merge(dt, on=\"YrMoSoldMeanEncoded\",how=\"left\")[\"mean\"].values\n",
    "    df_valid[\"YrMoSoldMeanEncoded\"] = df_valid.merge(dt, on=\"YrMoSoldMeanEncoded\",how=\"left\")[\"mean\"].values\n",
    "    \n",
    "    df_train[\"ContourSlopeMeanEncoded\"] = df_train[\"LandContour\"].apply(str) + \"-\" + df_train[\"LandSlope\"].apply(str)\n",
    "    df_valid[\"ContourSlopeMeanEncoded\"] = df_valid[\"LandContour\"].apply(str) + \"-\" + df_valid[\"LandSlope\"].apply(str)\n",
    "    dt = df_train.groupby([\"ContourSlopeMeanEncoded\"])[\"SalePrice\"].agg([\"mean\"])\n",
    "    df_train[\"ContourSlopeMeanEncoded\"] = df_train.merge(dt,on=[\"ContourSlopeMeanEncoded\"],how=\"left\")[\"mean\"].values\n",
    "    df_valid[\"ContourSlopeMeanEncoded\"] = df_valid.merge(dt,on=[\"ContourSlopeMeanEncoded\"],how=\"left\")[\"mean\"].values\n",
    "    \n",
    "    df_train[\"ExterQualCondMeanEncoded\"] = df_train[\"ExterQual\"].apply(str) + \"-\" + df_train[\"ExterCond\"].apply(str)\n",
    "    df_valid[\"ExterQualCondMeanEncoded\"] = df_valid[\"ExterQual\"].apply(str) + \"-\" + df_valid[\"ExterCond\"].apply(str)\n",
    "    dt = df_train.groupby([\"ExterQualCondMeanEncoded\"])[\"SalePrice\"].agg([\"mean\"])\n",
    "    df_train[\"ExterQualCondMeanEncoded\"] = df_train.merge(dt,on=[\"ExterQualCondMeanEncoded\"],how=\"left\")[\"mean\"].values\n",
    "    df_valid[\"ExterQualCondMeanEncoded\"] = df_valid.merge(dt,on=[\"ExterQualCondMeanEncoded\"],how=\"left\")[\"mean\"].values\n",
    "    \n",
    "    df_train[\"HeatingHeatQCMeanEncoded\"] = df_train[\"Heating\"].apply(str) + \"-\" + df_train[\"HeatingQC\"].apply(str)\n",
    "    df_valid[\"HeatingHeatQCMeanEncoded\"] = df_valid[\"Heating\"].apply(str) + \"-\" + df_valid[\"HeatingQC\"].apply(str)\n",
    "    dt = df_train.groupby([\"HeatingHeatQCMeanEncoded\"])[\"SalePrice\"].agg([\"mean\"])\n",
    "    df_train[\"HeatingHeatQCMeanEncoded\"] = df_train.merge(dt,on=[\"HeatingHeatQCMeanEncoded\"],how=\"left\")[\"mean\"].values\n",
    "    df_valid[\"HeatingHeatQCMeanEncoded\"] = df_valid.merge(dt,on=[\"HeatingHeatQCMeanEncoded\"],how=\"left\")[\"mean\"].values\n",
    "    \n",
    "    df_train[\"FirePlacesFQuMeanEncoded\"] = df_train[\"Fireplaces\"].apply(str) + \"-\" + df_train[\"FireplaceQu\"].apply(str)\n",
    "    df_valid[\"FirePlacesFQuMeanEncoded\"] = df_valid[\"Fireplaces\"].apply(str) + \"-\" + df_valid[\"FireplaceQu\"].apply(str)\n",
    "    dt = df_train.groupby([\"FirePlacesFQuMeanEncoded\"])[\"SalePrice\"].agg([\"mean\"])\n",
    "    df_train[\"FirePlacesFQuMeanEncoded\"] = df_train.merge(dt,on=[\"FirePlacesFQuMeanEncoded\"],how=\"left\")[\"mean\"].values\n",
    "    df_valid[\"FirePlacesFQuMeanEncoded\"] = df_valid.merge(dt,on=[\"FirePlacesFQuMeanEncoded\"],how=\"left\")[\"mean\"].values\n",
    "    \n",
    "    dt = df_train.groupby(\"Neighborhood\")[\"SalePrice\"].agg([\"mean\"])\n",
    "    df_train[\"NeighborhoodMeanEncoded\"] = df_train.merge(dt,on=[\"Neighborhood\"],how=\"left\")[\"mean\"].values\n",
    "    df_valid[\"NeighborhoodMeanEncoded\"] = df_valid.merge(dt,on=[\"Neighborhood\"],how=\"left\")[\"mean\"].values\n",
    "    X_train_mean_encoded = df_train.drop(\"SalePrice\",axis=1)\n",
    "    X_valid_mean_encoded = df_valid\n",
    "    return X_train_mean_encoded, X_valid_mean_encoded"
   ],
   "id": "45e0e7fa6810b580",
   "outputs": [],
   "execution_count": 23
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:04:32.421842Z",
     "start_time": "2024-06-01T15:04:32.381046Z"
    }
   },
   "cell_type": "code",
   "source": "X_train_mean_encoded,X_val_mean_encoded = get_target_mean_columns(X_train,X_val)",
   "id": "c339970e84e51d44",
   "outputs": [],
   "execution_count": 24
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:04:32.760637Z",
     "start_time": "2024-06-01T15:04:32.747397Z"
    }
   },
   "cell_type": "code",
   "source": [
    "def custom_features(df):\n",
    "    df_out = df.copy()\n",
    "    remod_cat = []\n",
    "    shed_val = []\n",
    "    for i,row in df_out.iterrows():\n",
    "        if row[\"YearRemodAdd\"] - row[\"YearBuilt\"] == 0:\n",
    "            remod_cat.append(0)\n",
    "        elif row[\"YearRemodAdd\"] - row[\"YearBuilt\"] <= 5:\n",
    "            remod_cat.append(1)\n",
    "        elif row[\"YearRemodAdd\"] - row[\"YearBuilt\"] <= 15:\n",
    "            remod_cat.append(2)\n",
    "        elif row[\"YearRemodAdd\"] - row[\"YearBuilt\"] <= 30:\n",
    "            remod_cat.append(3)\n",
    "        else:\n",
    "            remod_cat.append(4)\n",
    "        if row[\"3SsnPorch\"] > 0:\n",
    "            row[\"ScreenPorch\"] = row[\"3SsnPorch\"]\n",
    "            \n",
    "        if row[\"MiscFeature\"] == \"Shed\":\n",
    "            shed_val.append(row[\"MiscVal\"])\n",
    "        else:\n",
    "            shed_val.append(0)\n",
    "        \n",
    "    df_out[\"Age\"] = df_out[\"YrSold\"] - df_out[\"YearBuilt\"]\n",
    "    df_out[\"HasBsmtBath\"] = ((df_out[\"BsmtFullBath\"] > 0) | (df_out[\"BsmtHalfBath\"] > 0)).astype(object)\n",
    "    df_out[\"NonBedroomRoomsAbvGrd\"] = df_out[\"TotRmsAbvGrd\"] - df_out[\"BedroomAbvGr\"]\n",
    "    df_out[\"HasGarage\"] = (df_out[\"GarageArea\"] > 0).astype(object)\n",
    "    df_out[\"HasBsmt\"] = (df_out[\"TotalBsmtSF\"] > 0).astype(object)\n",
    "    df_out[\"Has2ndFloor\"] = (df_out[\"2ndFlrSF\"] > 0).astype(object)\n",
    "    df_out[\"YrSold_cat\"] = df_out[\"YrSold\"].astype(object)\n",
    "    df_out[\"MoSold_cat\"] = df_out[\"MoSold\"].astype(object)\n",
    "    df_out[\"YearBuilt_cat\"] = df_out[\"YearBuilt\"].astype(object)\n",
    "    df_out[\"MSSubClass_cat\"] = df_out[\"MSSubClass\"].astype(object)\n",
    "    df_out[\"RemodCat\"] = remod_cat\n",
    "    df_out[\"RemodCat\"] = df_out[\"RemodCat\"].astype(object)\n",
    "    df_out[\"ShedVal\"] = shed_val\n",
    "    df_out.drop(\"3SsnPorch\",axis=1,inplace=True)\n",
    "    df_out.drop(\"MiscVal\",axis=1,inplace=True)\n",
    "    return df_out\n",
    "feature_engineering_transformer = FunctionTransformer(custom_features)"
   ],
   "id": "63d0d95c8c4cca1d",
   "outputs": [],
   "execution_count": 25
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:04:33.709254Z",
     "start_time": "2024-06-01T15:04:33.701694Z"
    }
   },
   "cell_type": "code",
   "source": [
    "new_cols_categorical = pd.Index([\"HasBsmtBath\",\"YrSold_cat\",\"MoSold_cat\",\"YearBuilt_cat\",\"MSSubClass_cat\",\"RemodCat\",\"Has2ndFloor\",\"HasBsmt\",\"HasGarage\",])\n",
    "new_cols_numerical = pd.Index([\"Age\",\"ShedVal\",\"NonBedroomRoomsAbvGrd\"])\n",
    "categorical_columns = X_train_mean_encoded.drop([\"MiscVal\",\"3SsnPorch\"],axis=1).select_dtypes(include=[\"object\",\"category\"]).columns.append(new_cols_categorical)\n",
    "numerical_columns = X_train_mean_encoded.drop([\"3SsnPorch\",\"MiscVal\"],axis=1).select_dtypes(exclude=[\"object\",\"category\"]).columns.append(new_cols_numerical)"
   ],
   "id": "615ac06bcd44d1b6",
   "outputs": [],
   "execution_count": 26
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:04:34.072495Z",
     "start_time": "2024-06-01T15:04:34.063195Z"
    }
   },
   "cell_type": "code",
   "source": [
    "preprocessor = ColumnTransformer([\n",
    "    (\"num\",numerical_transformer,numerical_columns),\n",
    "    (\"cat\",categorical_transformer,categorical_columns)\n",
    "],remainder=\"passthrough\")"
   ],
   "id": "13176f5840b2b323",
   "outputs": [],
   "execution_count": 27
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:04:34.404595Z",
     "start_time": "2024-06-01T15:04:34.392635Z"
    }
   },
   "cell_type": "code",
   "source": [
    "pipeline_fe = Pipeline([\n",
    "    (\"fe\",feature_engineering_transformer),\n",
    "    (\"preprocessor\",preprocessor),\n",
    "])"
   ],
   "id": "c399a88a390120d",
   "outputs": [],
   "execution_count": 28
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:04:35.722699Z",
     "start_time": "2024-06-01T15:04:35.645700Z"
    }
   },
   "cell_type": "code",
   "source": [
    "X_train_fe_preprocessed = pipeline_fe.fit_transform(X_train_mean_encoded)\n",
    "X_val_fe_preprocessed = pipeline_fe.transform(X_val_mean_encoded)"
   ],
   "id": "9bee72c8f19e4679",
   "outputs": [],
   "execution_count": 29
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "<h3>finding hyperparameters</h3>",
   "id": "a2a7e9d84683d78"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T10:50:13.180181Z",
     "start_time": "2024-06-01T10:01:31.363913Z"
    }
   },
   "cell_type": "code",
   "source": [
    "grids = {}\n",
    "for model_name, model in models.items():\n",
    "    grids[model_name] = GridSearchCV(model,param_grids[model_name],cv=cv,scoring='neg_mean_squared_error',n_jobs=-2)\n",
    "    grids[model_name].fit(X_train_fe_preprocessed,y_train)\n",
    "    best_params = grids[model_name].best_params_\n",
    "    best_score = np.sqrt(-1 * grids[model_name].best_score_)\n",
    "    print(f\"Best parameters for {model_name}: {best_params}\")\n",
    "    print(f\"Best RMSE for {model_name}: {best_score}\\n\")"
   ],
   "id": "92ed36352a50e096",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters for ElasticNet: {'alpha': 1.0, 'l1_ratio': 0.5}\n",
      "Best RMSE for ElasticNet: 0.3956761919827401\n",
      "\n",
      "Best parameters for RandomForest: {'max_depth': None, 'min_samples_leaf': 2, 'min_samples_split': 2, 'n_estimators': 400}\n",
      "Best RMSE for RandomForest: 0.13730683746497369\n",
      "\n",
      "Best parameters for GradientBoosting: {'learning_rate': 0.09, 'max_depth': 3, 'min_samples_split': 2, 'n_estimators': 350}\n",
      "Best RMSE for GradientBoosting: 0.12151664619314677\n",
      "\n",
      "Best parameters for SVR: {'C': 1.5, 'epsilon': 0.07, 'gamma': 'auto', 'kernel': 'rbf'}\n",
      "Best RMSE for SVR: 0.11410775154143743\n",
      "\n",
      "Best parameters for Ada: {'learning_rate': 5, 'loss': 'exponential', 'n_estimators': 600}\n",
      "Best RMSE for Ada: 0.15143689522894868\n",
      "\n",
      "Best parameters for XGBoost: {'learning_rate': 0.01, 'max_depth': 4, 'n_estimators': 1200, 'objective': 'reg:squarederror', 'subsample': 0.4}\n",
      "Best RMSE for XGBoost: 0.11657105557808814\n",
      "\n"
     ]
    }
   ],
   "execution_count": 31
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "<h3>testing with pca</h3>",
   "id": "af5e031947973bad"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T10:50:13.192774Z",
     "start_time": "2024-06-01T10:50:13.183248Z"
    }
   },
   "cell_type": "code",
   "source": [
    "pipeline_fe_pca = Pipeline([\n",
    "    (\"fe\",feature_engineering_transformer),\n",
    "    (\"preprocessor\",preprocessor),\n",
    "    (\"pca\",PCA(n_components=4,random_state=42))\n",
    "])"
   ],
   "id": "a056303fc4c22578",
   "outputs": [],
   "execution_count": 32
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T10:50:13.335874Z",
     "start_time": "2024-06-01T10:50:13.193300Z"
    }
   },
   "cell_type": "code",
   "source": [
    "X_train_fe_pca_preprocessed = pipeline_fe_pca.fit_transform(X_train_mean_encoded)\n",
    "X_val_fe_pca_preprocessed = pipeline_fe_pca.transform(X_val_mean_encoded)"
   ],
   "id": "488fb9e3a3b4ddfd",
   "outputs": [],
   "execution_count": 33
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T11:00:45.525744Z",
     "start_time": "2024-06-01T10:50:13.335874Z"
    }
   },
   "cell_type": "code",
   "source": [
    "grids = {}\n",
    "for model_name, model in models.items():\n",
    "    grids[model_name] = GridSearchCV(model,param_grids[model_name],cv=cv,scoring='neg_mean_squared_error',n_jobs=-2)\n",
    "    grids[model_name].fit(X_train_fe_pca_preprocessed,y_train)\n",
    "    best_params = grids[model_name].best_params_\n",
    "    best_score = np.sqrt(-1 * grids[model_name].best_score_)\n",
    "    print(f\"Best parameters for {model_name}: {best_params}\")\n",
    "    print(f\"Best RMSE for {model_name}: {best_score}\\n\")"
   ],
   "id": "fcb2f54d568cb327",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best parameters for ElasticNet: {'alpha': 1.0, 'l1_ratio': 0.5}\n",
      "Best RMSE for ElasticNet: 0.2233227494258543\n",
      "\n",
      "Best parameters for RandomForest: {'max_depth': 10, 'min_samples_leaf': 2, 'min_samples_split': 2, 'n_estimators': 350}\n",
      "Best RMSE for RandomForest: 0.14611938064591923\n",
      "\n",
      "Best parameters for GradientBoosting: {'learning_rate': 0.08, 'max_depth': 3, 'min_samples_split': 5, 'n_estimators': 300}\n",
      "Best RMSE for GradientBoosting: 0.14842168729080868\n",
      "\n",
      "Best parameters for SVR: {'C': 1, 'epsilon': 0.07, 'gamma': 'auto', 'kernel': 'rbf'}\n",
      "Best RMSE for SVR: 0.17277036636337503\n",
      "\n",
      "Best parameters for Ada: {'learning_rate': 5, 'loss': 'exponential', 'n_estimators': 500}\n",
      "Best RMSE for Ada: 0.1565437869349402\n",
      "\n",
      "Best parameters for XGBoost: {'learning_rate': 0.03, 'max_depth': 3, 'n_estimators': 200, 'objective': 'reg:squarederror', 'subsample': 0.6}\n",
      "Best RMSE for XGBoost: 0.14092388068456058\n",
      "\n"
     ]
    }
   ],
   "execution_count": 34
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "<h3>validating models</h3>",
   "id": "71a5c2a5a8d77e9f"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:09:18.811579Z",
     "start_time": "2024-06-01T15:09:18.804656Z"
    }
   },
   "cell_type": "code",
   "source": [
    "tuned_models = {\n",
    "    \"RandomForest\":RandomForestRegressor(min_samples_leaf = 2,n_estimators=400,random_state=42,n_jobs=-2),\n",
    "    \"GradientBoosting\": GradientBoostingRegressor(n_estimators=350,learning_rate=.09,max_depth=3,min_samples_split=2,random_state=42),\n",
    "    \"SVR\":SVR(C=1.5,epsilon=0.07,gamma=\"auto\"),\n",
    "    \"XGBoost\":XGBRegressor(learning_rate=0.01,n_estimators=1200,subsample=0.4,max_depth=3,random_state=42,n_jobs=-2,objective='reg:squarederror'),\n",
    "    # \"KNN\":KNeighborsRegressor(n_neighbors=11),\n",
    "}"
   ],
   "id": "f68e7863309477d1",
   "outputs": [],
   "execution_count": 36
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:09:25.148935Z",
     "start_time": "2024-06-01T15:09:19.408208Z"
    }
   },
   "cell_type": "code",
   "source": [
    "grids = {}\n",
    "for model_name,model in tuned_models.items():\n",
    "    grids[model_name] = model\n",
    "    grids[model_name].fit(X_train_fe_preprocessed,y_train)\n",
    "    pred = np.exp(grids[model_name].predict(X_val_fe_preprocessed))\n",
    "    print(f\"RMSE for {model_name} is {root_mean_squared_error(np.exp(y_val),pred)}\")"
   ],
   "id": "b8ad557581880853",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE for RandomForest is 24733.76261688844\n",
      "RMSE for GradientBoosting is 21009.198249106594\n",
      "RMSE for SVR is 22697.09249409938\n",
      "RMSE for XGBoost is 20812.369113423763\n"
     ]
    }
   ],
   "execution_count": 37
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "<h3>creating a stacked regressor</h3>",
   "id": "dbecd2758abc7082"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:24:00.039641Z",
     "start_time": "2024-06-01T15:23:41.036400Z"
    }
   },
   "cell_type": "code",
   "source": [
    "stacked_model = StackingRegressor([(model_name,model) for model_name,model in tuned_models.items()],n_jobs=-2)\n",
    "stacked_model.fit(X_train_fe_preprocessed,y_train)\n",
    "pred = np.exp(stacked_model.predict(X_val_fe_preprocessed))\n",
    "print(f\"RMSE: {root_mean_squared_error(np.exp(y_val),pred)}\")"
   ],
   "id": "2687c9fcef873f2f",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RMSE: 20206.32141045509\n"
     ]
    }
   ],
   "execution_count": 46
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "<h3>predicting final test set</h3>",
   "id": "6174ee1282868134"
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:24:00.417460Z",
     "start_time": "2024-06-01T15:24:00.039641Z"
    }
   },
   "cell_type": "code",
   "source": [
    "df_test = pd.read_csv(\"test.csv\")\n",
    "X_full_mean_encoded, df_test_mean_encoded = get_target_mean_columns(X,df_test)\n",
    "X_full_fe_preprocessed = pipeline_fe.fit_transform(X_full_mean_encoded)\n",
    "df_test_fe_preprocessed = pipeline_fe.transform(df_test_mean_encoded)"
   ],
   "id": "de1f8cb8f0d50c3a",
   "outputs": [],
   "execution_count": 47
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:24:31.828605Z",
     "start_time": "2024-06-01T15:24:00.418469Z"
    }
   },
   "cell_type": "code",
   "source": [
    "stacked_model.fit(X_full_fe_preprocessed,y)\n",
    "final_pred = np.exp(stacked_model.predict(df_test_fe_preprocessed))"
   ],
   "id": "9fbb6b8af0caf225",
   "outputs": [],
   "execution_count": 48
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:24:31.833764Z",
     "start_time": "2024-06-01T15:24:31.830629Z"
    }
   },
   "cell_type": "code",
   "source": [
    "# grids[\"XGBoost\"].fit(X_full_fe_preprocessed,y)\n",
    "# final_pred = np.exp(grids[\"XGBoost\"].predict(df_test_fe_preprocessed))"
   ],
   "id": "88749a773d2b1ded",
   "outputs": [],
   "execution_count": 49
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:24:31.844057Z",
     "start_time": "2024-06-01T15:24:31.835013Z"
    }
   },
   "cell_type": "code",
   "source": [
    "df_xgboost_out = df_test[[\"Id\"]].copy()\n",
    "df_xgboost_out[\"SalePrice\"] = final_pred"
   ],
   "id": "47752a933aeba62e",
   "outputs": [],
   "execution_count": 50
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:24:31.858916Z",
     "start_time": "2024-06-01T15:24:31.845093Z"
    }
   },
   "cell_type": "code",
   "source": "df_xgboost_out.to_csv(\"submission_stacked_with_XGB_normalized_full.csv\",index=False)",
   "id": "870587a10d0f9c1e",
   "outputs": [],
   "execution_count": 51
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-06-01T15:16:10.429564Z",
     "start_time": "2024-06-01T15:16:10.418009Z"
    }
   },
   "cell_type": "code",
   "source": "# df_xgboost_out.to_csv(\"XGBRegressor_2_full_no_leakage.csv\",index=False)",
   "id": "e97b08090bec6855",
   "outputs": [],
   "execution_count": 45
  },
  {
   "metadata": {},
   "cell_type": "markdown",
   "source": "<h3>final score on kaggle: 0.12870</h3>",
   "id": "9be16b9cdc1dbb76"
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}