Feature Engineering.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "16227b9a-53e0-4e59-ad4d-2f1d740d64df",
   "metadata": {},
   "outputs": [],
   "source": [
    "## feature engineering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b26e7d8a-cb6f-4139-b4ab-39103dd77312",
   "metadata": {},
   "outputs": [],
   "source": [
    "## import libs\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.impute import SimpleImputer ## to fill numerical nan values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4396bf6c-b98a-40ff-a800-8481f6fdc836",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset=pd.read_csv('train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2a9e3bf2-92d7-4902-89a5-9e59ab2ed0f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "## handle numerical NANs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "2445bdfa-5829-4dac-8e1b-96c5c1e4c918",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['MSSubClass',\n",
       " 'LotFrontage',\n",
       " 'LotArea',\n",
       " 'OverallQual',\n",
       " 'OverallCond',\n",
       " 'YearBuilt',\n",
       " 'YearRemodAdd',\n",
       " 'MasVnrArea',\n",
       " 'BsmtFinSF1',\n",
       " 'BsmtFinSF2',\n",
       " 'BsmtUnfSF',\n",
       " 'TotalBsmtSF',\n",
       " '1stFlrSF',\n",
       " '2ndFlrSF',\n",
       " 'LowQualFinSF',\n",
       " 'GrLivArea',\n",
       " 'BsmtFullBath',\n",
       " 'BsmtHalfBath',\n",
       " 'FullBath',\n",
       " 'HalfBath',\n",
       " 'BedroomAbvGr',\n",
       " 'KitchenAbvGr',\n",
       " 'TotRmsAbvGrd',\n",
       " 'Fireplaces',\n",
       " 'GarageYrBlt',\n",
       " 'GarageCars',\n",
       " 'GarageArea',\n",
       " 'WoodDeckSF',\n",
       " 'OpenPorchSF',\n",
       " 'EnclosedPorch',\n",
       " '3SsnPorch',\n",
       " 'ScreenPorch',\n",
       " 'PoolArea',\n",
       " 'MiscVal',\n",
       " 'MoSold',\n",
       " 'YrSold',\n",
       " 'SalePrice']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_features=[feature for feature in dataset.columns if dataset[feature].dtype!='O' and feature!='Id']\n",
    "num_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "69650ade-95b8-4043-8abe-0f059e669fb7",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_features_nan=[feature for feature in num_features if dataset[feature].isnull().sum()>0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7e30074f-9be1-4bc7-9e7f-76de8715a3e1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['LotFrontage', 'MasVnrArea', 'GarageYrBlt']"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_features_nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3468f41b-cc92-4ad4-a881-215e537b0d10",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "5b09866f-3f88-4da9-8e90-95fbb80607d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "for feature in num_features_nan:\n",
    "    dataset[feature+\"_nan\"]=np.where(dataset[feature].isnull(),1,0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5481fe0d-561c-46ae-b698-62df67848c4c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b10fad74-3802-4dc9-96df-0a469b08e50e",
   "metadata": {},
   "outputs": [],
   "source": [
    "## use SimpleImputer to replace NAN with median\n",
    "imputer=SimpleImputer(strategy='median')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "9aa620e4-cb35-4b3c-ada0-4cb139c3c805",
   "metadata": {},
   "outputs": [],
   "source": [
    "# here instead of utilising whole dataset we take numerical section of it\n",
    "num_imputed_df=pd.DataFrame(data=imputer.fit_transform(dataset[num_features]),columns=num_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "17e8caba-0c11-4b70-b750-b93b8e8aa0e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>OverallQual</th>\n",
       "      <th>OverallCond</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>YearRemodAdd</th>\n",
       "      <th>MasVnrArea</th>\n",
       "      <th>BsmtFinSF1</th>\n",
       "      <th>BsmtFinSF2</th>\n",
       "      <th>...</th>\n",
       "      <th>WoodDeckSF</th>\n",
       "      <th>OpenPorchSF</th>\n",
       "      <th>EnclosedPorch</th>\n",
       "      <th>3SsnPorch</th>\n",
       "      <th>ScreenPorch</th>\n",
       "      <th>PoolArea</th>\n",
       "      <th>MiscVal</th>\n",
       "      <th>MoSold</th>\n",
       "      <th>YrSold</th>\n",
       "      <th>SalePrice</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>60.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8450.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2003.0</td>\n",
       "      <td>2003.0</td>\n",
       "      <td>196.0</td>\n",
       "      <td>706.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>61.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>208500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9600.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1976.0</td>\n",
       "      <td>1976.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>978.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>298.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2007.0</td>\n",
       "      <td>181500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>60.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>11250.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2001.0</td>\n",
       "      <td>2002.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>486.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>42.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>223500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>70.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>9550.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1915.0</td>\n",
       "      <td>1970.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>35.0</td>\n",
       "      <td>272.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2006.0</td>\n",
       "      <td>140000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>60.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>14260.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>350.0</td>\n",
       "      <td>655.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>192.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>250000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1455</th>\n",
       "      <td>60.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>7917.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1999.0</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>40.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>2007.0</td>\n",
       "      <td>175000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1456</th>\n",
       "      <td>20.0</td>\n",
       "      <td>85.0</td>\n",
       "      <td>13175.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1978.0</td>\n",
       "      <td>1988.0</td>\n",
       "      <td>119.0</td>\n",
       "      <td>790.0</td>\n",
       "      <td>163.0</td>\n",
       "      <td>...</td>\n",
       "      <td>349.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>2010.0</td>\n",
       "      <td>210000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1457</th>\n",
       "      <td>70.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>9042.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1941.0</td>\n",
       "      <td>2006.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>275.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2500.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2010.0</td>\n",
       "      <td>266500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1458</th>\n",
       "      <td>20.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>9717.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1950.0</td>\n",
       "      <td>1996.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>1029.0</td>\n",
       "      <td>...</td>\n",
       "      <td>366.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>112.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2010.0</td>\n",
       "      <td>142125.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1459</th>\n",
       "      <td>20.0</td>\n",
       "      <td>75.0</td>\n",
       "      <td>9937.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1965.0</td>\n",
       "      <td>1965.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>830.0</td>\n",
       "      <td>290.0</td>\n",
       "      <td>...</td>\n",
       "      <td>736.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>2008.0</td>\n",
       "      <td>147500.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1460 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      MSSubClass  LotFrontage  LotArea  OverallQual  OverallCond  YearBuilt  \\\n",
       "0           60.0         65.0   8450.0          7.0          5.0     2003.0   \n",
       "1           20.0         80.0   9600.0          6.0          8.0     1976.0   \n",
       "2           60.0         68.0  11250.0          7.0          5.0     2001.0   \n",
       "3           70.0         60.0   9550.0          7.0          5.0     1915.0   \n",
       "4           60.0         84.0  14260.0          8.0          5.0     2000.0   \n",
       "...          ...          ...      ...          ...          ...        ...   \n",
       "1455        60.0         62.0   7917.0          6.0          5.0     1999.0   \n",
       "1456        20.0         85.0  13175.0          6.0          6.0     1978.0   \n",
       "1457        70.0         66.0   9042.0          7.0          9.0     1941.0   \n",
       "1458        20.0         68.0   9717.0          5.0          6.0     1950.0   \n",
       "1459        20.0         75.0   9937.0          5.0          6.0     1965.0   \n",
       "\n",
       "      YearRemodAdd  MasVnrArea  BsmtFinSF1  BsmtFinSF2  ...  WoodDeckSF  \\\n",
       "0           2003.0       196.0       706.0         0.0  ...         0.0   \n",
       "1           1976.0         0.0       978.0         0.0  ...       298.0   \n",
       "2           2002.0       162.0       486.0         0.0  ...         0.0   \n",
       "3           1970.0         0.0       216.0         0.0  ...         0.0   \n",
       "4           2000.0       350.0       655.0         0.0  ...       192.0   \n",
       "...            ...         ...         ...         ...  ...         ...   \n",
       "1455        2000.0         0.0         0.0         0.0  ...         0.0   \n",
       "1456        1988.0       119.0       790.0       163.0  ...       349.0   \n",
       "1457        2006.0         0.0       275.0         0.0  ...         0.0   \n",
       "1458        1996.0         0.0        49.0      1029.0  ...       366.0   \n",
       "1459        1965.0         0.0       830.0       290.0  ...       736.0   \n",
       "\n",
       "      OpenPorchSF  EnclosedPorch  3SsnPorch  ScreenPorch  PoolArea  MiscVal  \\\n",
       "0            61.0            0.0        0.0          0.0       0.0      0.0   \n",
       "1             0.0            0.0        0.0          0.0       0.0      0.0   \n",
       "2            42.0            0.0        0.0          0.0       0.0      0.0   \n",
       "3            35.0          272.0        0.0          0.0       0.0      0.0   \n",
       "4            84.0            0.0        0.0          0.0       0.0      0.0   \n",
       "...           ...            ...        ...          ...       ...      ...   \n",
       "1455         40.0            0.0        0.0          0.0       0.0      0.0   \n",
       "1456          0.0            0.0        0.0          0.0       0.0      0.0   \n",
       "1457         60.0            0.0        0.0          0.0       0.0   2500.0   \n",
       "1458          0.0          112.0        0.0          0.0       0.0      0.0   \n",
       "1459         68.0            0.0        0.0          0.0       0.0      0.0   \n",
       "\n",
       "      MoSold  YrSold  SalePrice  \n",
       "0        2.0  2008.0   208500.0  \n",
       "1        5.0  2007.0   181500.0  \n",
       "2        9.0  2008.0   223500.0  \n",
       "3        2.0  2006.0   140000.0  \n",
       "4       12.0  2008.0   250000.0  \n",
       "...      ...     ...        ...  \n",
       "1455     8.0  2007.0   175000.0  \n",
       "1456     2.0  2010.0   210000.0  \n",
       "1457     5.0  2010.0   266500.0  \n",
       "1458     4.0  2010.0   142125.0  \n",
       "1459     6.0  2008.0   147500.0  \n",
       "\n",
       "[1460 rows x 37 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "num_imputed_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f15efc7d-5a32-4497-b3b1-2aaabeeb5a27",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset[num_features]=num_imputed_df[num_features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "97eaef99-e875-4f90-9c6e-566e948cdb12",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n"
     ]
    }
   ],
   "source": [
    "## cross verify numerical null values\n",
    "print([feature for feature in num_features if dataset[feature].isnull().sum()>0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b9b8fca2-fca6-459f-88b1-f49d2c2aecb3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1460 entries, 0 to 1459\n",
      "Data columns (total 84 columns):\n",
      " #   Column           Non-Null Count  Dtype  \n",
      "---  ------           --------------  -----  \n",
      " 0   Id               1460 non-null   int64  \n",
      " 1   MSSubClass       1460 non-null   float64\n",
      " 2   MSZoning         1460 non-null   object \n",
      " 3   LotFrontage      1460 non-null   float64\n",
      " 4   LotArea          1460 non-null   float64\n",
      " 5   Street           1460 non-null   object \n",
      " 6   Alley            91 non-null     object \n",
      " 7   LotShape         1460 non-null   object \n",
      " 8   LandContour      1460 non-null   object \n",
      " 9   Utilities        1460 non-null   object \n",
      " 10  LotConfig        1460 non-null   object \n",
      " 11  LandSlope        1460 non-null   object \n",
      " 12  Neighborhood     1460 non-null   object \n",
      " 13  Condition1       1460 non-null   object \n",
      " 14  Condition2       1460 non-null   object \n",
      " 15  BldgType         1460 non-null   object \n",
      " 16  HouseStyle       1460 non-null   object \n",
      " 17  OverallQual      1460 non-null   float64\n",
      " 18  OverallCond      1460 non-null   float64\n",
      " 19  YearBuilt        1460 non-null   float64\n",
      " 20  YearRemodAdd     1460 non-null   float64\n",
      " 21  RoofStyle        1460 non-null   object \n",
      " 22  RoofMatl         1460 non-null   object \n",
      " 23  Exterior1st      1460 non-null   object \n",
      " 24  Exterior2nd      1460 non-null   object \n",
      " 25  MasVnrType       1452 non-null   object \n",
      " 26  MasVnrArea       1460 non-null   float64\n",
      " 27  ExterQual        1460 non-null   object \n",
      " 28  ExterCond        1460 non-null   object \n",
      " 29  Foundation       1460 non-null   object \n",
      " 30  BsmtQual         1423 non-null   object \n",
      " 31  BsmtCond         1423 non-null   object \n",
      " 32  BsmtExposure     1422 non-null   object \n",
      " 33  BsmtFinType1     1423 non-null   object \n",
      " 34  BsmtFinSF1       1460 non-null   float64\n",
      " 35  BsmtFinType2     1422 non-null   object \n",
      " 36  BsmtFinSF2       1460 non-null   float64\n",
      " 37  BsmtUnfSF        1460 non-null   float64\n",
      " 38  TotalBsmtSF      1460 non-null   float64\n",
      " 39  Heating          1460 non-null   object \n",
      " 40  HeatingQC        1460 non-null   object \n",
      " 41  CentralAir       1460 non-null   object \n",
      " 42  Electrical       1459 non-null   object \n",
      " 43  1stFlrSF         1460 non-null   float64\n",
      " 44  2ndFlrSF         1460 non-null   float64\n",
      " 45  LowQualFinSF     1460 non-null   float64\n",
      " 46  GrLivArea        1460 non-null   float64\n",
      " 47  BsmtFullBath     1460 non-null   float64\n",
      " 48  BsmtHalfBath     1460 non-null   float64\n",
      " 49  FullBath         1460 non-null   float64\n",
      " 50  HalfBath         1460 non-null   float64\n",
      " 51  BedroomAbvGr     1460 non-null   float64\n",
      " 52  KitchenAbvGr     1460 non-null   float64\n",
      " 53  KitchenQual      1460 non-null   object \n",
      " 54  TotRmsAbvGrd     1460 non-null   float64\n",
      " 55  Functional       1460 non-null   object \n",
      " 56  Fireplaces       1460 non-null   float64\n",
      " 57  FireplaceQu      770 non-null    object \n",
      " 58  GarageType       1379 non-null   object \n",
      " 59  GarageYrBlt      1460 non-null   float64\n",
      " 60  GarageFinish     1379 non-null   object \n",
      " 61  GarageCars       1460 non-null   float64\n",
      " 62  GarageArea       1460 non-null   float64\n",
      " 63  GarageQual       1379 non-null   object \n",
      " 64  GarageCond       1379 non-null   object \n",
      " 65  PavedDrive       1460 non-null   object \n",
      " 66  WoodDeckSF       1460 non-null   float64\n",
      " 67  OpenPorchSF      1460 non-null   float64\n",
      " 68  EnclosedPorch    1460 non-null   float64\n",
      " 69  3SsnPorch        1460 non-null   float64\n",
      " 70  ScreenPorch      1460 non-null   float64\n",
      " 71  PoolArea         1460 non-null   float64\n",
      " 72  PoolQC           7 non-null      object \n",
      " 73  Fence            281 non-null    object \n",
      " 74  MiscFeature      54 non-null     object \n",
      " 75  MiscVal          1460 non-null   float64\n",
      " 76  MoSold           1460 non-null   float64\n",
      " 77  YrSold           1460 non-null   float64\n",
      " 78  SaleType         1460 non-null   object \n",
      " 79  SaleCondition    1460 non-null   object \n",
      " 80  SalePrice        1460 non-null   float64\n",
      " 81  LotFrontage_nan  1460 non-null   int64  \n",
      " 82  MasVnrArea_nan   1460 non-null   int64  \n",
      " 83  GarageYrBlt_nan  1460 non-null   int64  \n",
      "dtypes: float64(37), int64(4), object(43)\n",
      "memory usage: 958.2+ KB\n"
     ]
    }
   ],
   "source": [
    "dataset.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "23acffce-6d56-423d-a889-ae31400d69b1",
   "metadata": {},
   "outputs": [],
   "source": [
    "## fill Categorical Null Values with 'Missing' keyword"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8cb4018f-ecf9-45d4-8826-9b9ccd01362e",
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_features=[feature for feature in dataset.columns if feature not in num_features and feature!='Id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "dc818f9c-4e65-4447-ae0c-b139bc288fe4",
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_features_nan=[feature for feature in cat_features if dataset[feature].isnull().sum()>0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "ced645b2-7a26-40c6-b751-3864ebb849ba",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Alley',\n",
       " 'MasVnrType',\n",
       " 'BsmtQual',\n",
       " 'BsmtCond',\n",
       " 'BsmtExposure',\n",
       " 'BsmtFinType1',\n",
       " 'BsmtFinType2',\n",
       " 'Electrical',\n",
       " 'FireplaceQu',\n",
       " 'GarageType',\n",
       " 'GarageFinish',\n",
       " 'GarageQual',\n",
       " 'GarageCond',\n",
       " 'PoolQC',\n",
       " 'Fence',\n",
       " 'MiscFeature']"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cat_features_nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0d268442-a8f1-4c62-b8a4-94f207ac19a2",
   "metadata": {},
   "outputs": [],
   "source": [
    "for feature in cat_features_nan:\n",
    "    dataset[feature+'_nan']=np.where(dataset[feature].isnull(),1,0)\n",
    "    dataset[feature].fillna('Missing',inplace=True) \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f0d7a60d-83c0-43f5-8056-e0795c33fa2b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1460 entries, 0 to 1459\n",
      "Data columns (total 100 columns):\n",
      " #   Column            Non-Null Count  Dtype  \n",
      "---  ------            --------------  -----  \n",
      " 0   Id                1460 non-null   int64  \n",
      " 1   MSSubClass        1460 non-null   float64\n",
      " 2   MSZoning          1460 non-null   object \n",
      " 3   LotFrontage       1460 non-null   float64\n",
      " 4   LotArea           1460 non-null   float64\n",
      " 5   Street            1460 non-null   object \n",
      " 6   Alley             1460 non-null   object \n",
      " 7   LotShape          1460 non-null   object \n",
      " 8   LandContour       1460 non-null   object \n",
      " 9   Utilities         1460 non-null   object \n",
      " 10  LotConfig         1460 non-null   object \n",
      " 11  LandSlope         1460 non-null   object \n",
      " 12  Neighborhood      1460 non-null   object \n",
      " 13  Condition1        1460 non-null   object \n",
      " 14  Condition2        1460 non-null   object \n",
      " 15  BldgType          1460 non-null   object \n",
      " 16  HouseStyle        1460 non-null   object \n",
      " 17  OverallQual       1460 non-null   float64\n",
      " 18  OverallCond       1460 non-null   float64\n",
      " 19  YearBuilt         1460 non-null   float64\n",
      " 20  YearRemodAdd      1460 non-null   float64\n",
      " 21  RoofStyle         1460 non-null   object \n",
      " 22  RoofMatl          1460 non-null   object \n",
      " 23  Exterior1st       1460 non-null   object \n",
      " 24  Exterior2nd       1460 non-null   object \n",
      " 25  MasVnrType        1460 non-null   object \n",
      " 26  MasVnrArea        1460 non-null   float64\n",
      " 27  ExterQual         1460 non-null   object \n",
      " 28  ExterCond         1460 non-null   object \n",
      " 29  Foundation        1460 non-null   object \n",
      " 30  BsmtQual          1460 non-null   object \n",
      " 31  BsmtCond          1460 non-null   object \n",
      " 32  BsmtExposure      1460 non-null   object \n",
      " 33  BsmtFinType1      1460 non-null   object \n",
      " 34  BsmtFinSF1        1460 non-null   float64\n",
      " 35  BsmtFinType2      1460 non-null   object \n",
      " 36  BsmtFinSF2        1460 non-null   float64\n",
      " 37  BsmtUnfSF         1460 non-null   float64\n",
      " 38  TotalBsmtSF       1460 non-null   float64\n",
      " 39  Heating           1460 non-null   object \n",
      " 40  HeatingQC         1460 non-null   object \n",
      " 41  CentralAir        1460 non-null   object \n",
      " 42  Electrical        1460 non-null   object \n",
      " 43  1stFlrSF          1460 non-null   float64\n",
      " 44  2ndFlrSF          1460 non-null   float64\n",
      " 45  LowQualFinSF      1460 non-null   float64\n",
      " 46  GrLivArea         1460 non-null   float64\n",
      " 47  BsmtFullBath      1460 non-null   float64\n",
      " 48  BsmtHalfBath      1460 non-null   float64\n",
      " 49  FullBath          1460 non-null   float64\n",
      " 50  HalfBath          1460 non-null   float64\n",
      " 51  BedroomAbvGr      1460 non-null   float64\n",
      " 52  KitchenAbvGr      1460 non-null   float64\n",
      " 53  KitchenQual       1460 non-null   object \n",
      " 54  TotRmsAbvGrd      1460 non-null   float64\n",
      " 55  Functional        1460 non-null   object \n",
      " 56  Fireplaces        1460 non-null   float64\n",
      " 57  FireplaceQu       1460 non-null   object \n",
      " 58  GarageType        1460 non-null   object \n",
      " 59  GarageYrBlt       1460 non-null   float64\n",
      " 60  GarageFinish      1460 non-null   object \n",
      " 61  GarageCars        1460 non-null   float64\n",
      " 62  GarageArea        1460 non-null   float64\n",
      " 63  GarageQual        1460 non-null   object \n",
      " 64  GarageCond        1460 non-null   object \n",
      " 65  PavedDrive        1460 non-null   object \n",
      " 66  WoodDeckSF        1460 non-null   float64\n",
      " 67  OpenPorchSF       1460 non-null   float64\n",
      " 68  EnclosedPorch     1460 non-null   float64\n",
      " 69  3SsnPorch         1460 non-null   float64\n",
      " 70  ScreenPorch       1460 non-null   float64\n",
      " 71  PoolArea          1460 non-null   float64\n",
      " 72  PoolQC            1460 non-null   object \n",
      " 73  Fence             1460 non-null   object \n",
      " 74  MiscFeature       1460 non-null   object \n",
      " 75  MiscVal           1460 non-null   float64\n",
      " 76  MoSold            1460 non-null   float64\n",
      " 77  YrSold            1460 non-null   float64\n",
      " 78  SaleType          1460 non-null   object \n",
      " 79  SaleCondition     1460 non-null   object \n",
      " 80  SalePrice         1460 non-null   float64\n",
      " 81  LotFrontage_nan   1460 non-null   int64  \n",
      " 82  MasVnrArea_nan    1460 non-null   int64  \n",
      " 83  GarageYrBlt_nan   1460 non-null   int64  \n",
      " 84  Alley_nan         1460 non-null   int64  \n",
      " 85  MasVnrType_nan    1460 non-null   int64  \n",
      " 86  BsmtQual_nan      1460 non-null   int64  \n",
      " 87  BsmtCond_nan      1460 non-null   int64  \n",
      " 88  BsmtExposure_nan  1460 non-null   int64  \n",
      " 89  BsmtFinType1_nan  1460 non-null   int64  \n",
      " 90  BsmtFinType2_nan  1460 non-null   int64  \n",
      " 91  Electrical_nan    1460 non-null   int64  \n",
      " 92  FireplaceQu_nan   1460 non-null   int64  \n",
      " 93  GarageType_nan    1460 non-null   int64  \n",
      " 94  GarageFinish_nan  1460 non-null   int64  \n",
      " 95  GarageQual_nan    1460 non-null   int64  \n",
      " 96  GarageCond_nan    1460 non-null   int64  \n",
      " 97  PoolQC_nan        1460 non-null   int64  \n",
      " 98  Fence_nan         1460 non-null   int64  \n",
      " 99  MiscFeature_nan   1460 non-null   int64  \n",
      "dtypes: float64(37), int64(20), object(43)\n",
      "memory usage: 1.1+ MB\n"
     ]
    }
   ],
   "source": [
    "dataset.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "74591e25-08d2-4d3f-8571-5004adf682c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "year_features=[feature for feature in num_features if 'Yr' in feature or 'Year' in feature]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "a3aecd30-cc64-4d49-a5c0-e7ac3816906b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['YearBuilt', 'YearRemodAdd', 'GarageYrBlt', 'YrSold']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "year_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "7beb64e1-f29d-4e88-9d3e-d359fb98af6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "for feature in year_features:\n",
    "    if feature!='YrSold':\n",
    "        dataset[feature+'Diff']=dataset['YrSold']-dataset[feature]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "2381ba60-2a41-4dc0-aa3d-afe0a4618184",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0        5.0\n",
       "1       31.0\n",
       "2        7.0\n",
       "3       91.0\n",
       "4        8.0\n",
       "        ... \n",
       "1455     8.0\n",
       "1456    32.0\n",
       "1457    69.0\n",
       "1458    60.0\n",
       "1459    43.0\n",
       "Name: YearBuiltDiff, Length: 1460, dtype: float64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset['YearBuiltDiff']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35142aec-2417-4e4e-bfed-cc3fb86a0be0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "3547c4da-752a-4e6b-b049-3cf83731fe17",
   "metadata": {},
   "outputs": [],
   "source": [
    "## now we have all numerical and non numerical nan values filled with either median or 'Missing' keyword"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "00c1247c-ce8e-4fc6-aa58-594106ae3ab4",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataframe_final=pd.get_dummies(dataset,prefix=cat_features,columns=cat_features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "41b74b2b-17d1-4ebd-ad14-06dc16c71de5",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataframe_final.drop('SalePrice',axis=1,inplace=True) ## we have to remove the label from our training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "5446b6c5-dfe1-4db7-afc1-9fef05d57dfa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Id</th>\n",
       "      <th>MSSubClass</th>\n",
       "      <th>LotFrontage</th>\n",
       "      <th>LotArea</th>\n",
       "      <th>OverallQual</th>\n",
       "      <th>OverallCond</th>\n",
       "      <th>YearBuilt</th>\n",
       "      <th>YearRemodAdd</th>\n",
       "      <th>MasVnrArea</th>\n",
       "      <th>BsmtFinSF1</th>\n",
       "      <th>...</th>\n",
       "      <th>SaleCondition_Alloca</th>\n",
       "      <th>SaleCondition_Family</th>\n",
       "      <th>SaleCondition_Normal</th>\n",
       "      <th>SaleCondition_Partial</th>\n",
       "      <th>LotFrontage_nan_0</th>\n",
       "      <th>LotFrontage_nan_1</th>\n",
       "      <th>MasVnrArea_nan_0</th>\n",
       "      <th>MasVnrArea_nan_1</th>\n",
       "      <th>GarageYrBlt_nan_0</th>\n",
       "      <th>GarageYrBlt_nan_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>60.0</td>\n",
       "      <td>65.0</td>\n",
       "      <td>8450.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2003.0</td>\n",
       "      <td>2003.0</td>\n",
       "      <td>196.0</td>\n",
       "      <td>706.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>20.0</td>\n",
       "      <td>80.0</td>\n",
       "      <td>9600.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>1976.0</td>\n",
       "      <td>1976.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>978.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>60.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>11250.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2001.0</td>\n",
       "      <td>2002.0</td>\n",
       "      <td>162.0</td>\n",
       "      <td>486.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>70.0</td>\n",
       "      <td>60.0</td>\n",
       "      <td>9550.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1915.0</td>\n",
       "      <td>1970.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>216.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>60.0</td>\n",
       "      <td>84.0</td>\n",
       "      <td>14260.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>350.0</td>\n",
       "      <td>655.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1455</th>\n",
       "      <td>1456</td>\n",
       "      <td>60.0</td>\n",
       "      <td>62.0</td>\n",
       "      <td>7917.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>1999.0</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1456</th>\n",
       "      <td>1457</td>\n",
       "      <td>20.0</td>\n",
       "      <td>85.0</td>\n",
       "      <td>13175.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1978.0</td>\n",
       "      <td>1988.0</td>\n",
       "      <td>119.0</td>\n",
       "      <td>790.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1457</th>\n",
       "      <td>1458</td>\n",
       "      <td>70.0</td>\n",
       "      <td>66.0</td>\n",
       "      <td>9042.0</td>\n",
       "      <td>7.0</td>\n",
       "      <td>9.0</td>\n",
       "      <td>1941.0</td>\n",
       "      <td>2006.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>275.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1458</th>\n",
       "      <td>1459</td>\n",
       "      <td>20.0</td>\n",
       "      <td>68.0</td>\n",
       "      <td>9717.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1950.0</td>\n",
       "      <td>1996.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>49.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1459</th>\n",
       "      <td>1460</td>\n",
       "      <td>20.0</td>\n",
       "      <td>75.0</td>\n",
       "      <td>9937.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>1965.0</td>\n",
       "      <td>1965.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>830.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1460 rows × 330 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        Id  MSSubClass  LotFrontage  LotArea  OverallQual  OverallCond  \\\n",
       "0        1        60.0         65.0   8450.0          7.0          5.0   \n",
       "1        2        20.0         80.0   9600.0          6.0          8.0   \n",
       "2        3        60.0         68.0  11250.0          7.0          5.0   \n",
       "3        4        70.0         60.0   9550.0          7.0          5.0   \n",
       "4        5        60.0         84.0  14260.0          8.0          5.0   \n",
       "...    ...         ...          ...      ...          ...          ...   \n",
       "1455  1456        60.0         62.0   7917.0          6.0          5.0   \n",
       "1456  1457        20.0         85.0  13175.0          6.0          6.0   \n",
       "1457  1458        70.0         66.0   9042.0          7.0          9.0   \n",
       "1458  1459        20.0         68.0   9717.0          5.0          6.0   \n",
       "1459  1460        20.0         75.0   9937.0          5.0          6.0   \n",
       "\n",
       "      YearBuilt  YearRemodAdd  MasVnrArea  BsmtFinSF1  ...  \\\n",
       "0        2003.0        2003.0       196.0       706.0  ...   \n",
       "1        1976.0        1976.0         0.0       978.0  ...   \n",
       "2        2001.0        2002.0       162.0       486.0  ...   \n",
       "3        1915.0        1970.0         0.0       216.0  ...   \n",
       "4        2000.0        2000.0       350.0       655.0  ...   \n",
       "...         ...           ...         ...         ...  ...   \n",
       "1455     1999.0        2000.0         0.0         0.0  ...   \n",
       "1456     1978.0        1988.0       119.0       790.0  ...   \n",
       "1457     1941.0        2006.0         0.0       275.0  ...   \n",
       "1458     1950.0        1996.0         0.0        49.0  ...   \n",
       "1459     1965.0        1965.0         0.0       830.0  ...   \n",
       "\n",
       "      SaleCondition_Alloca  SaleCondition_Family  SaleCondition_Normal  \\\n",
       "0                        0                     0                     1   \n",
       "1                        0                     0                     1   \n",
       "2                        0                     0                     1   \n",
       "3                        0                     0                     0   \n",
       "4                        0                     0                     1   \n",
       "...                    ...                   ...                   ...   \n",
       "1455                     0                     0                     1   \n",
       "1456                     0                     0                     1   \n",
       "1457                     0                     0                     1   \n",
       "1458                     0                     0                     1   \n",
       "1459                     0                     0                     1   \n",
       "\n",
       "      SaleCondition_Partial  LotFrontage_nan_0  LotFrontage_nan_1  \\\n",
       "0                         0                  1                  0   \n",
       "1                         0                  1                  0   \n",
       "2                         0                  1                  0   \n",
       "3                         0                  1                  0   \n",
       "4                         0                  1                  0   \n",
       "...                     ...                ...                ...   \n",
       "1455                      0                  1                  0   \n",
       "1456                      0                  1                  0   \n",
       "1457                      0                  1                  0   \n",
       "1458                      0                  1                  0   \n",
       "1459                      0                  1                  0   \n",
       "\n",
       "      MasVnrArea_nan_0  MasVnrArea_nan_1  GarageYrBlt_nan_0  GarageYrBlt_nan_1  \n",
       "0                    1                 0                  1                  0  \n",
       "1                    1                 0                  1                  0  \n",
       "2                    1                 0                  1                  0  \n",
       "3                    1                 0                  1                  0  \n",
       "4                    1                 0                  1                  0  \n",
       "...                ...               ...                ...                ...  \n",
       "1455                 1                 0                  1                  0  \n",
       "1456                 1                 0                  1                  0  \n",
       "1457                 1                 0                  1                  0  \n",
       "1458                 1                 0                  1                  0  \n",
       "1459                 1                 0                  1                  0  \n",
       "\n",
       "[1460 rows x 330 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataframe_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "f8882e9c-6c5d-406e-9b0b-3c0f20a242fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataframe_final.drop('Id',axis=1).to_csv('training_data.csv',index=False)\n",
    "# we drop Id as in this case Id cannot help the model to predict a price"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5fd44c7d-d845-4455-bfc0-56ffa88841ef",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}