03-preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import src.util as util\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder, StandardScaler\n",
    "from src import _Preprocessing_Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "config_data = util.load_config()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Collect Data Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dataset(config_data: dict, config: str) -> pd.DataFrame:\n",
    "    # Load set of data\n",
    "    X_train = util.pickle_load(config_data[\"train_set_eda\"][0])\n",
    "    y_train = util.pickle_load(config_data[\"train_set_eda\"][1])\n",
    "    \n",
    "    X_valid = util.pickle_load(config_data[\"valid_set_eda\"][0])\n",
    "    y_valid = util.pickle_load(config_data[\"valid_set_eda\"][1])\n",
    "    \n",
    "    X_test = util.pickle_load(config_data[\"test_set_eda\"][0])\n",
    "    y_test = util.pickle_load(config_data[\"test_set_eda\"][1])\n",
    "    \n",
    "    dataset_train = pd.concat([X_train[config], y_train[config]], axis=1)\n",
    "    dataset_valid = pd.concat([X_valid[config], y_valid[config]], axis=1)\n",
    "    dataset_test = pd.concat([X_test[config], y_test[config]], axis=1)\n",
    "    \n",
    "    \n",
    "    return dataset_train, dataset_valid, dataset_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_set_rf, valid_set_rf, test_set_rf = load_dataset(config_data, config='rf')\n",
    "train_set_filter, valid_set_filter, test_set_filter = load_dataset(config_data, config='filter')\n",
    "train_set_lasso, valid_set_lasso, test_set_lasso = load_dataset(config_data, config='lasso')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Check Data"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Function Preprocessing data\n",
    "\n",
    "- Split dataset into numerical and categorical data, <br>\n",
    "- Performed **SimpleImputer** in order of missing data <br>\n",
    "- Performed **Encoding** for categorical data such as **LabelEncoder** for ordinal data and **OHE** for non-ordinal data <br>\n",
    "- Suggested to use **OrdinalEncoder** and defined the ranked value of each features <br>\n",
    "- Last, performed **Standardization** using StandardScaler"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preprocessing Data\n",
    "\n",
    "    We are generated dataset with filter method of feature selection, \n",
    "    not the other two (Lasso or Random Forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class _Preprocessing_Data:\n",
    "    \"\"\"\n",
    "    Handling raw dataset, \\n\n",
    "    Performed Imputer data, Label Encoding or OHE, and Standardization.\n",
    "    \"\"\"\n",
    "    def __init__(self):\n",
    "        pass\n",
    "\n",
    "    def _split_numcat(self, data:dict) -> pd.DataFrame:\n",
    "        \"\"\"Split dataset without label into numerical and categorical.\n",
    "\n",
    "        Parameters\n",
    "        -------\n",
    "        data : array-like of shape\n",
    "            train_set without label\n",
    "\n",
    "        Returns\n",
    "        --------\n",
    "        X_numerical : array-like of shape\n",
    "            train_set of numerical features only\n",
    "        \n",
    "        X_category : array-like of shape\n",
    "            train_set of category features only\n",
    "        \"\"\"\n",
    "        numerical_col = data.select_dtypes('float64').columns.to_list()\n",
    "        categorical_col = data.select_dtypes('object').columns.to_list()\n",
    "\n",
    "        self.X_num = data[numerical_col]\n",
    "        self.X_cat = data[categorical_col]\n",
    "\n",
    "        return self.X_num, self.X_cat\n",
    "        \n",
    "    def _split_xy(self, data:dict) -> pd.DataFrame:\n",
    "        \"\"\"Split dataset into Numerical (float64) and Categorical data (object).\n",
    "\n",
    "        Parameters\n",
    "        -------\n",
    "        data : array-like of shape\n",
    "            train_set, valid_set included with label\n",
    "        X : array-like of shape\n",
    "          Predictor array\n",
    "        y : array-like of shape\n",
    "          label array\n",
    "\n",
    "        Return \n",
    "        -------\n",
    "        X_numeric : array-like of shape\n",
    "                predictor for numeric only \n",
    "        X_categoric : array-like of shape\n",
    "                predictor for categoric only\n",
    "        \"\"\"\n",
    "        \n",
    "        X = data.drop(columns = config_data['label'], axis=1)\n",
    "        y = data[config_data['label']]\n",
    "        \n",
    "        self.X = X\n",
    "        self.y = y\n",
    "        \n",
    "        numerical_col = X.select_dtypes('float64').columns.to_list()\n",
    "        categorical_col = X.select_dtypes('object').columns.to_list()\n",
    "\n",
    "        self.X_num = X[numerical_col]\n",
    "        self.X_cat = X[categorical_col]\n",
    "\n",
    "        return self.X_num, self.X_cat\n",
    "    \n",
    "    # Perform sanity check\n",
    "    def _imputer_Num(self, data, imputer=None):\n",
    "        \"\"\"\n",
    "        Handling missing value for numeric if any. \\n\n",
    "        Using median to fill np.NaN by SimpleImputer() from Sklearn Function.\n",
    "\n",
    "        Parameters\n",
    "        ---------\n",
    "        data : pandas.DataFrame\n",
    "            Numeric (int64) dtype only\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "        data_imputed : pandas.DataFrame\n",
    "            Imputed numeric data\n",
    "        \"\"\"\n",
    "        if imputer == None:\n",
    "            imputer = SimpleImputer(missing_values=np.nan,\n",
    "                                    strategy='median')\n",
    "            imputer.fit(data)\n",
    "\n",
    "        data_imputed_num = pd.DataFrame(imputer.transform(data),\n",
    "                                    index = data.index,\n",
    "                                    columns = data.columns)\n",
    "        \n",
    "        data_imputed_num = data_imputed_num.astype('int64')\n",
    "        \n",
    "        self.data_imputed_num = data_imputed_num\n",
    "        \n",
    "        return data_imputed_num\n",
    "\n",
    "    def _imputer_Cat(self, data, imputer = None) -> pd.DataFrame:\n",
    "        \"\"\"\n",
    "        Handling missing value for categorical data. \\n\n",
    "        Using 'most_frequent' strategy from SimpleImputer() of sklearn function\n",
    "\n",
    "        Parameters\n",
    "        --------\n",
    "        data : pandas.DataFrame\n",
    "            categorical (object) dtype only\n",
    "\n",
    "        Return\n",
    "        ----------\n",
    "        imputed_cat : pandas.DataFrame\n",
    "            imputed categorical data\n",
    "        \"\"\"\n",
    "        if imputer == None:\n",
    "            imputer = SimpleImputer(missing_values=np.nan,\n",
    "                                    strategy='most_frequent')\n",
    "            imputer.fit(data)\n",
    "\n",
    "        data_imputed_cat = pd.DataFrame(imputer.transform(data),\n",
    "                                    index=data.index,\n",
    "                                    columns=data.columns\n",
    "                                    )\n",
    "        \n",
    "        self.data_imputed_cat = data_imputed_cat\n",
    "        \n",
    "        return data_imputed_cat\n",
    "    \n",
    "    def _OHE_Cat(self, data, encoder_col = None, encoder = None) -> pd.DataFrame:\n",
    "        \"\"\"\n",
    "        One Hot Encoding using OneHotEncoder() from sklearn.preprocessing \\n\n",
    "        handle_unknown : 'ignore' \\n\n",
    "        drop : 'if binary' \\n\n",
    "        This function for nominal or non-Ordinal categoric data \\n\n",
    "        If encoder_col and encoder == None, function will generate encoder from data.\n",
    "        \n",
    "        Parameters\n",
    "        ------\n",
    "        data : pandas.DataFrame\n",
    "            categorical data non-Ordinal\n",
    "        encoder_col : encoder.get_feature_names_out\n",
    "        encoder : OneHotEncoder()\n",
    "\n",
    "        Returns\n",
    "        ------\n",
    "        data_encoded : pd.DataFrame\n",
    "            OHE encoded data\n",
    "        encoder_cold\n",
    "        encoder\n",
    "        \"\"\"\n",
    "\n",
    "        if encoder == None:\n",
    "            encoder = OneHotEncoder(handle_unknown= 'ignore',\n",
    "                                    drop = 'if_binary')\n",
    "            encoder.fit(data)\n",
    "            encoder_col = encoder.get_feature_names_out(data.columns)\n",
    "\n",
    "        data_encoded = encoder.transform(data).toarray()\n",
    "        data_encoded = pd.DataFrame(data_encoded,\n",
    "                                    index=data.index,\n",
    "                                    columns=encoder_col)\n",
    "        \n",
    "        self.data_encoded = data_encoded\n",
    "        self.encoder = encoder\n",
    "        \n",
    "        util.pickle_dump(encoder, config_data[\"ohe_path\"])\n",
    "        \n",
    "        return self.data_encoded, self.encoder\n",
    "    \n",
    "    def _LE_cat(self, data, encoder = None) -> pd.DataFrame:\n",
    "        \"\"\"\n",
    "        Label Encoder for Ordinal Categoric data using LabelEncoder() from sklearn.preprocessing function \\n\n",
    "        categories parameter is defined as of config file\n",
    "\n",
    "        Parameters\n",
    "        --------\n",
    "        data : pandas.DataFrame\n",
    "            Ordinal data only\n",
    "        encoder : LabelEncoder()\n",
    "\n",
    "        Returns\n",
    "        ---------\n",
    "        data_encoded : pandas.DataFrame\n",
    "                Encoded ordinal data\n",
    "        encoder : LabelEncoder()\n",
    "        \"\"\"\n",
    "        \n",
    "        if encoder == None:\n",
    "            le_encoder = LabelEncoder()\n",
    "                \n",
    "        else:\n",
    "            le_encoder = encoder\n",
    "            \n",
    "        for col in data.columns.to_list():\n",
    "                data[col] = le_encoder.fit_transform(data[col])\n",
    "        \n",
    "        util.pickle_dump(le_encoder, config_data[\"le_encoder_path\"])\n",
    "\n",
    "        self.data_encoded = data\n",
    "        self.encoder = le_encoder\n",
    "        \n",
    "        return self.data_encoded, self.encoder\n",
    "    \n",
    "    def _standardize_Data(self, data, scaler=None) -> pd.DataFrame:\n",
    "        \"\"\"\n",
    "        Standarization or normalization of the predictor value using StandardScaler() from sklearn.preprocessing \\n\n",
    "        Standarization is use (x-mean)/std to get value range from -1 to 1 and gaussian distribution\n",
    "\n",
    "        Paramters\n",
    "        ----------\n",
    "        data : pandas.DataFrame\n",
    "            X_train data\n",
    "        scaler : StandardScaler()\n",
    "\n",
    "        Returns\n",
    "        --------\n",
    "        data_scaled : pandas.DataFrame\n",
    "                standardized data\n",
    "        scaler\n",
    "        \"\"\"\n",
    "        \n",
    "        if scaler == None:\n",
    "            scaler = StandardScaler()\n",
    "            scaler.fit(data)\n",
    "\n",
    "        data_scaled = pd.DataFrame(scaler.transform(data),\n",
    "                                index=data.index,\n",
    "                                columns=data.columns)\n",
    "        \n",
    "        util.pickle_dump(scaler, config_data[\"scaler\"])\n",
    "\n",
    "        self.data_scaled = data_scaled\n",
    "        self.scaler = scaler\n",
    "        \n",
    "        return self.data_scaled, self.scaler\n",
    "    \n",
    "    def _handling_data(self, \n",
    "                       data, \n",
    "                       encoding = 'Label_Encoding',\n",
    "                       encoder = None, \n",
    "                       scaler = None,\n",
    "                       imputer_num = None, \n",
    "                       imputer_cat = None,\n",
    "                       config = \"None\", \n",
    "                       y = True):\n",
    "        \"\"\"\n",
    "        Preprocessed data from dataset (X,y) into cleaned data\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        data : array-like of shape\n",
    "            dataset with predictor and label (opt.)\n",
    "\n",
    "        y : bool. (default = True)\n",
    "            True, data will split into X and y (label)\n",
    "            False, X (predictor) only\n",
    "\n",
    "        imputer : SimpleImputer object. (default = None)\n",
    "\n",
    "        scaler : StandarScaler object. (default = None)\n",
    "\n",
    "        config : str (default = None)\n",
    "            Type of config data to save imputer, encoder, and scaler.\n",
    "\n",
    "            - 'filter' = filter data feature selection\n",
    "            - 'lasso' = lasso data feature selection\n",
    "            - 'random_forest' = random forest data feature selection\n",
    "\n",
    "            If None, will saving none.\n",
    "\n",
    "        Returns\n",
    "        ---------\n",
    "        X : array-like of shape\n",
    "            encoded and scaled data predictor.\n",
    "\n",
    "        y : array-like of shape (if y = True)\n",
    "            label.\n",
    "        \"\"\"\n",
    "\n",
    "        # --- Split into Numeric and Categoric Data --- #\n",
    "        if y == False:\n",
    "            util.print_debug(\"Split Numeric and Categoric Data...\")\n",
    "\n",
    "            self._split_numcat(data)\n",
    "\n",
    "        elif y == True:\n",
    "            self._split_xy(data)\n",
    "\n",
    "        # --- Impute Missing Value --- #\n",
    "        util.print_debug(\"Perform Imputer...\")\n",
    "\n",
    "        self.X_num = self._imputer_Num(self.X_num, imputer_num)\n",
    "        self.X_cat = self._imputer_Cat(self.X_cat, imputer_cat)\n",
    "        \n",
    "        # --- Label Encoding Categoric data --- #\n",
    "\n",
    "        # ---- Label Encoding --- #\n",
    "        if encoding == 'Label_Encoding':\n",
    "            util.print_debug(\"Perform Label Encoding...\")\n",
    "\n",
    "            self._LE_cat(self.X_cat, encoder)\n",
    "            X_train_ = pd.concat([self.X_num, self.data_encoded], axis=1)\n",
    "            \n",
    "        # --- One Hot Encoding --- #\n",
    "        elif encoding == 'One_Hot_Encoding':\n",
    "            util.print_debug(\"Perform One Hot Encoding...\")\n",
    "\n",
    "            self._OHE_Cat(self.X_cat, encoder = encoder)\n",
    "            X_train_ = pd.concat([self.X_num, self.data_encoded], axis=1)\n",
    "        \n",
    "        # --- Both --- #\n",
    "        elif encoding == 'Both':\n",
    "            util.print_debug(\"Perform Both Label Encoding and One Hot Encoding...\")\n",
    "\n",
    "            X_train_ohe, _, encoder_ohe = self._OHE_Cat(self.X_cat)\n",
    "            X_train_le, encoder_le = self._LE_cat(self.X_cat) \n",
    "            \n",
    "            X_train_concat = pd.concat([X_train_ohe, X_train_le], axis=1)\n",
    "            X_train_ = pd.concat([self.X_num, X_train_concat], axis=1)\n",
    "\n",
    "        else:\n",
    "            raise TypeError(\"encoding type is not recognized. Should be Label_Encoding, One_Hot_Encoding, or Both.\")\n",
    "        \n",
    "        # --- Standardize Data --- #\n",
    "        util.print_debug(\"Perform Standardizing....\")\n",
    "\n",
    "        # Reindex data\n",
    "        X_train_ = X_train_.reindex(sorted(X_train_.columns), axis=1)\n",
    "        \n",
    "        # Standardizing data\n",
    "        self._standardize_Data(X_train_, scaler)\n",
    "\n",
    "        util.print_debug(\"Data has been standardized.\")\n",
    "\n",
    "        # --- Dumping/Save data\n",
    "        if config == 'filter':\n",
    "            util.print_debug(\"Dumping encoder and scaler.\")\n",
    "\n",
    "            util.pickle_dump(self.encoder, config_data[\"le_encoder_path_filter\"])\n",
    "            util.pickle_dump(self.scaler, config_data[\"scaler_filter\"])\n",
    "\n",
    "        elif config == 'lasso':\n",
    "            util.print_debug(\"Dumping encoder and scaler.\")\n",
    "\n",
    "            util.pickle_dump(self.encoder, config_data[\"le_encoder_path_lasso\"])\n",
    "            util.pickle_dump(self.scaler, config_data[\"scaler_lasso\"])\n",
    "\n",
    "        elif config == 'random_forest':\n",
    "            util.print_debug(\"Dumping encoder and scaler.\")\n",
    "\n",
    "            util.pickle_dump(self.encoder, config_data[\"le_encoder_path_rf\"])\n",
    "            util.pickle_dump(self.scaler, config_data[\"scaler_rf\"])\n",
    "\n",
    "        else:\n",
    "            pass\n",
    "\n",
    "        util.print_debug(\"Returned scaled data.\")\n",
    "        util.print_debug(\"=\"*40)\n",
    "\n",
    "        if y == True:\n",
    "            return self.data_scaled, self.y\n",
    "        else:\n",
    "            return self.data_scaled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessor = _Preprocessing_Data()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Training dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-07-14 22:43:49.477593 Perform Imputer...\n",
      "2023-07-14 22:43:49.703544 Perform Label Encoding...\n",
      "2023-07-14 22:43:49.845884 Perform Standardizing....\n",
      "2023-07-14 22:43:49.904066 Data has been standardized.\n",
      "2023-07-14 22:43:49.904066 Dumping encoder and scaler.\n",
      "2023-07-14 22:43:49.909995 Returned scaled data.\n",
      "2023-07-14 22:43:49.909995 ========================================\n",
      "2023-07-14 22:43:49.933643 Perform Imputer...\n",
      "2023-07-14 22:43:50.143330 Perform Label Encoding...\n",
      "2023-07-14 22:43:50.288472 Perform Standardizing....\n",
      "2023-07-14 22:43:50.349368 Data has been standardized.\n",
      "2023-07-14 22:43:50.349368 Dumping encoder and scaler.\n",
      "2023-07-14 22:43:50.356440 Returned scaled data.\n",
      "2023-07-14 22:43:50.356440 ========================================\n",
      "2023-07-14 22:43:50.377284 Perform Imputer...\n",
      "2023-07-14 22:43:50.559714 Perform Label Encoding...\n",
      "2023-07-14 22:43:50.706903 Perform Standardizing....\n",
      "2023-07-14 22:43:50.753566 Data has been standardized.\n",
      "2023-07-14 22:43:50.753566 Dumping encoder and scaler.\n",
      "2023-07-14 22:43:50.760157 Returned scaled data.\n",
      "2023-07-14 22:43:50.760157 ========================================\n"
     ]
    }
   ],
   "source": [
    "X_train_rf, y_train_rf = preprocessor._handling_data(\n",
    "                                            data = train_set_rf, \n",
    "                                            encoding='Label_Encoding',\n",
    "                                            config = 'random_forest'\n",
    "                                            )\n",
    "\n",
    "X_train_lasso, y_train_lasso = preprocessor._handling_data(\n",
    "                                                    data = train_set_lasso,\n",
    "                                                    encoding='Label_Encoding',\n",
    "                                                    config = 'lasso'\n",
    "                                                    )\n",
    "\n",
    "X_train_filter, y_train_filter = preprocessor._handling_data(\n",
    "                                                        data = train_set_filter,\n",
    "                                                        encoding='Label_Encoding',\n",
    "                                                        config = 'filter'\n",
    "                                                        )\n",
    "                                                            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = {\n",
    "    \"filter\" : X_train_filter,\n",
    "    \"lasso\" : X_train_lasso,\n",
    "    \"rf\" : X_train_rf\n",
    "    }\n",
    "\n",
    "y_train = {\n",
    "    \"filter\" : y_train_filter,\n",
    "    \"lasso\" : y_train_lasso,\n",
    "    \"rf\" : y_train_rf\n",
    "    }"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Create Function to handle valid and test dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _Concat_Preprocessing_valid(data_filter, data_lasso, data_rf):\n",
    "    \n",
    "    X_rf, y_rf = preprocessor._handling_data(data=data_rf, \n",
    "                                             encoding='Label_Encoding',\n",
    "                                             encoder=util.pickle_load(config_data['le_encoder_path_rf']),\n",
    "                                             scaler=util.pickle_load(config_data['scaler_rf']),\n",
    "                                             y = True\n",
    "                                             )\n",
    "\n",
    "    X_lasso, y_lasso = preprocessor._handling_data(data=data_lasso,\n",
    "                                                    encoding='Label_Encoding',\n",
    "                                                    encoder=util.pickle_load(config_data['le_encoder_path_lasso']),\n",
    "                                                    scaler=util.pickle_load(config_data['scaler_lasso']),\n",
    "                                                    y = True\n",
    "                                                    )\n",
    "\n",
    "    X_filter, y_filter = preprocessor._handling_data(data=data_filter,\n",
    "                                                        encoding='Label_Encoding',\n",
    "                                                        encoder=util.pickle_load(config_data['le_encoder_path_filter']),\n",
    "                                                        scaler=util.pickle_load(config_data['scaler_filter']),\n",
    "                                                        y = True\n",
    "                                                        )\n",
    "    \n",
    "    X = {\n",
    "        \"filter\" : X_filter,\n",
    "        \"lasso\" : X_lasso,\n",
    "        \"rf\" : X_rf\n",
    "    }\n",
    "    \n",
    "    y = {\n",
    "        \"filter\" : y_filter,\n",
    "        \"lasso\" : y_lasso,\n",
    "        \"rf\" : y_rf\n",
    "    }\n",
    "    \n",
    "    return X, y"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Validation dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-07-14 22:43:58.731132 Perform Imputer...\n",
      "2023-07-14 22:43:58.752466 Perform Label Encoding...\n",
      "2023-07-14 22:43:58.774089 Perform Standardizing....\n",
      "2023-07-14 22:43:58.782417 Data has been standardized.\n",
      "2023-07-14 22:43:58.782417 Returned scaled data.\n",
      "2023-07-14 22:43:58.782417 ========================================\n",
      "2023-07-14 22:43:58.791110 Perform Imputer...\n",
      "2023-07-14 22:43:58.831097 Perform Label Encoding...\n",
      "2023-07-14 22:43:58.875653 Perform Standardizing....\n",
      "2023-07-14 22:43:58.887779 Data has been standardized.\n",
      "2023-07-14 22:43:58.887779 Returned scaled data.\n",
      "2023-07-14 22:43:58.887779 ========================================\n",
      "2023-07-14 22:43:58.895777 Perform Imputer...\n",
      "2023-07-14 22:43:58.933977 Perform Label Encoding...\n",
      "2023-07-14 22:43:58.970358 Perform Standardizing....\n",
      "2023-07-14 22:43:58.981496 Data has been standardized.\n",
      "2023-07-14 22:43:58.981496 Returned scaled data.\n",
      "2023-07-14 22:43:58.981496 ========================================\n"
     ]
    }
   ],
   "source": [
    "X_valid, y_valid = _Concat_Preprocessing_valid(data_filter = valid_set_filter,\n",
    "                                         data_lasso = valid_set_lasso,\n",
    "                                         data_rf = valid_set_rf\n",
    "                                         )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-07-14 22:44:00.748262 Perform Imputer...\n",
      "2023-07-14 22:44:00.814322 Perform Label Encoding...\n",
      "2023-07-14 22:44:00.865603 Perform Standardizing....\n",
      "2023-07-14 22:44:00.880577 Data has been standardized.\n",
      "2023-07-14 22:44:00.880577 Returned scaled data.\n",
      "2023-07-14 22:44:00.880577 ========================================\n",
      "2023-07-14 22:44:00.893988 Perform Imputer...\n",
      "2023-07-14 22:44:00.938420 Perform Label Encoding...\n",
      "2023-07-14 22:44:00.983307 Perform Standardizing....\n",
      "2023-07-14 22:44:00.998498 Data has been standardized.\n",
      "2023-07-14 22:44:00.998498 Returned scaled data.\n",
      "2023-07-14 22:44:00.998498 ========================================\n",
      "2023-07-14 22:44:01.007482 Perform Imputer...\n",
      "2023-07-14 22:44:01.052107 Perform Label Encoding...\n",
      "2023-07-14 22:44:01.091579 Perform Standardizing....\n",
      "2023-07-14 22:44:01.102160 Data has been standardized.\n",
      "2023-07-14 22:44:01.102160 Returned scaled data.\n",
      "2023-07-14 22:44:01.102160 ========================================\n"
     ]
    }
   ],
   "source": [
    "X_test, y_test = _Concat_Preprocessing_valid(data_filter = test_set_filter,\n",
    "                                         data_lasso = test_set_lasso,\n",
    "                                         data_rf = test_set_rf\n",
    "                                         )"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dumping and Save Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "util.pickle_dump(X_train, config_data[\"train_set_clean\"][0])\n",
    "util.pickle_dump(y_train, config_data[\"train_set_clean\"][1])\n",
    "\n",
    "util.pickle_dump(X_valid, config_data[\"valid_set_clean\"][0])\n",
    "util.pickle_dump(y_valid, config_data[\"valid_set_clean\"][1])\n",
    "\n",
    "util.pickle_dump(X_test, config_data[\"test_set_clean\"][0])\n",
    "util.pickle_dump(y_test, config_data[\"test_set_clean\"][1])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}