04-modelling.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import src.util as util\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import copy\n",
    "import hashlib\n",
    "import json\n",
    "import warnings\n",
    "\n",
    "from sklearn.neighbors import KNeighborsRegressor, RadiusNeighborsRegressor\n",
    "from sklearn.tree import DecisionTreeRegressor\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.ensemble import RandomForestRegressor, AdaBoostRegressor, GradientBoostingRegressor\n",
    "from xgboost import XGBRegressor\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV, RandomizedSearchCV\n",
    "from sklearn import tree\n",
    "from sklearn.metrics import mean_squared_error, mean_absolute_error, mean_absolute_percentage_error, r2_score\n",
    "\n",
    "from sklearn.model_selection import KFold, cross_val_score\n",
    "\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "config_data = util.load_config()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <b>1. LOAD DATASET</b>\n",
    "-----"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_train_clean(config_data: pd.DataFrame) -> pd.DataFrame:\n",
    "        X = util.pickle_load(config_data[\"train_set_clean\"][0])\n",
    "        y = util.pickle_load(config_data[\"train_set_clean\"][1])\n",
    "\n",
    "        return X, y\n",
    "    \n",
    "def load_valid_clean(config_data: pd.DataFrame) -> pd.DataFrame:\n",
    "        X = util.pickle_load(config_data[\"valid_set_clean\"][0])\n",
    "        y = util.pickle_load(config_data[\"valid_set_clean\"][1])\n",
    "\n",
    "        return X, y\n",
    "\n",
    "def load_test_clean(config_data: pd.DataFrame) -> pd.DataFrame:\n",
    "        X = util.pickle_load(config_data[\"test_set_clean\"][0])\n",
    "        y = util.pickle_load(config_data[\"test_set_clean\"][1])\n",
    "\n",
    "        return X, y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train, y_train = load_train_clean(config_data)\n",
    "X_valid, y_valid = load_valid_clean(config_data)\n",
    "X_test, y_test = load_test_clean(config_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Check for X_valid has same configuration and features as X_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_valid filter has a different shape compared to X_train filter.\n",
      "X_valid lasso has a different shape compared to X_train lasso.\n",
      "X_valid rf has a different shape compared to X_train rf.\n"
     ]
    }
   ],
   "source": [
    "for config in X_train:\n",
    "    if X_train[config].shape == X_valid[config].shape:\n",
    "        if all(X_train[config].columns == X_valid[config].columns):\n",
    "            print(f\"X_valid {config} has the same configuration and features as X_train {config}.\")\n",
    "        else:\n",
    "            print(f\"X_valid {config} has different column names compared to X_train {config}.\")\n",
    "    else:\n",
    "        print(f\"X_valid {config} has a different shape compared to X_train {config}.\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Check does mean of model could be enough to predict data.\n",
    "\n",
    "<b>Reference:</b> <br>\n",
    "[Metrics](https://coderzcolumn.com/tutorials/machine-learning/model-evaluation-scoring-metrics-scikit-learn-sklearn)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <b> 2. Creating Training Log Template</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-07-14 22:45:26.050143 Create training log template\n"
     ]
    }
   ],
   "source": [
    "util.print_debug('Create training log template')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Create training log function\n",
    "def training_log_template() -> dict:\n",
    "    # Debug message\n",
    "    util.print_debug(\"creating training log template\")\n",
    "\n",
    "    # Template for training Log\n",
    "    logger = {\n",
    "        \"model_name\": [],\n",
    "        \"model_uid\": [],\n",
    "        \"training_time\": [],\n",
    "        \"training_date\": [],\n",
    "        \"mse\": [],\n",
    "        \"r2_score\": [],\n",
    "    }\n",
    "\n",
    "    # Debug message\n",
    "    util.print_debug(\"Training log template created\")\n",
    "\n",
    "    return logger"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def training_log_updater(current_log: dict, params: dict) -> list:\n",
    "    # create copy of current log\n",
    "    current_log = copy.deepcopy(current_log)\n",
    "\n",
    "    # Path for training log file\n",
    "    log_path = params[\"training_log_path\"]\n",
    "\n",
    "    # Try to load training log file\n",
    "    try:\n",
    "        with open(log_path, \"r\") as file:\n",
    "            last_log = json.load(file)\n",
    "        file.close()\n",
    "\n",
    "    # If file not found create a new one\n",
    "    except FileNotFoundError as fe:\n",
    "        with open(log_path, \"w\") as file:\n",
    "            file.write(\"[]\")\n",
    "        file.close()\n",
    "\n",
    "        with open(log_path, \"r\") as file:\n",
    "            last_log = json.load(file)\n",
    "        file.close()\n",
    "\n",
    "    # Add current log to previous log\n",
    "    last_log.append(current_log)\n",
    "\n",
    "    # Save updated log\n",
    "    with open(log_path, \"w\") as file:\n",
    "        json.dump(last_log, file)\n",
    "        file.close()\n",
    "\n",
    "    # Return log\n",
    "    return last_log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Create model object of ML model\n",
    "def create_model_object(params: dict) -> list:\n",
    "    # Debug message\n",
    "    util.print_debug(\"Creating model objects.\")\n",
    "\n",
    "    # Create model objects\n",
    "    baseline_knn = KNeighborsRegressor()\n",
    "    baseline_dt = DecisionTreeRegressor()\n",
    "    baseline_lr = LinearRegression()\n",
    "    baseline_rf = RandomForestRegressor()\n",
    "    baseline_ada = AdaBoostRegressor()\n",
    "    baseline_gr = GradientBoostingRegressor()\n",
    "    baseline_xgb = XGBRegressor()\n",
    "\n",
    "    # Create list of model\n",
    "    list_of_model = [\n",
    "        {\"model_name\": baseline_knn.__class__.__name__, \"model_object\": baseline_knn, \"model_uid\": \"\"},\n",
    "        {\"model_name\": baseline_dt.__class__.__name__, \"model_object\": baseline_dt, \"model_uid\": \"\"},\n",
    "        {\"model_name\": baseline_lr.__class__.__name__, \"model_object\": baseline_lr, \"model_uid\": \"\"},\n",
    "        {\"model_name\": baseline_rf.__class__.__name__, \"model_object\": baseline_rf, \"model_uid\": \"\"},\n",
    "        {\"model_name\": baseline_ada.__class__.__name__, \"model_object\": baseline_ada, \"model_uid\": \"\"},\n",
    "        {\"model_name\": baseline_gr.__class__.__name__, \"model_object\": baseline_gr, \"model_uid\": \"\"},\n",
    "        {\"model_name\": baseline_xgb.__class__.__name__, \"model_object\": baseline_xgb, \"model_uid\": \"\"},\n",
    "    ]\n",
    "\n",
    "    # Debug message\n",
    "    util.print_debug(\"Models object created\")\n",
    "\n",
    "    return list_of_model\n",
    "    "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## **3. Creating Baseline Modeling**"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Type I : List of model per config\n",
    "    Use this to Create List of Model per Config (Filter, Lasso, Random Forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_eval(configuration_model: str, params: dict, hyperparams_model: list = None, data: str = 'filter'):\n",
    "\n",
    "    # Variable to store trained models\n",
    "    list_of_trained_model = dict()\n",
    "\n",
    "    # Create training log template\n",
    "    training_log = training_log_template()\n",
    "\n",
    "    # Training for every data configuration\n",
    "    util.print_debug(\"Training model based on current dataset\")\n",
    "\n",
    "    if hyperparams_model == None:\n",
    "        list_of_model = create_model_object(params)\n",
    "    else:\n",
    "        list_of_model = copy.deepcopy(hyperparams_model)\n",
    "\n",
    "    # Variable to store trained model\n",
    "    trained_model = list()\n",
    "\n",
    "    # Train each model by current dataset\n",
    "    for model in list_of_model:\n",
    "        # Debug message\n",
    "        util.print_debug(\"Training model: {}\".format(model[\"model_name\"]))\n",
    "\n",
    "        # Training\n",
    "        training_time = util.time_stamp()\n",
    "        model[\"model_object\"].fit(X_train[data], y_train[data])\n",
    "        training_time = (util.time_stamp() - training_time).total_seconds()\n",
    "\n",
    "        # Debug message\n",
    "        util.print_debug(\"Evaluating model: {}\".format(model[\"model_name\"]))\n",
    "\n",
    "        # Evaluation\n",
    "        y_predict = model[\"model_object\"].predict(X_valid[data])\n",
    "        mse = mean_squared_error(y_valid[data], y_predict, squared=False)\n",
    "        r2 = r2_score(y_valid[data], y_predict)\n",
    "\n",
    "        # Debug message\n",
    "        util.print_debug(\"Logging: {}\".format(model[\"model_name\"]))\n",
    "\n",
    "        # Create UID\n",
    "        uid = hashlib.md5(str(training_time).encode()).hexdigest()\n",
    "\n",
    "        model[\"model_uid\"] = uid\n",
    "\n",
    "        # Create training log data\n",
    "        training_log[\"model_name\"].append(\"{}--{}\".format(configuration_model, model[\"model_name\"]))\n",
    "        training_log[\"model_uid\"].append(uid)\n",
    "        training_log[\"training_time\"].append(training_time)\n",
    "        training_log[\"training_date\"].append(util.time_stamp())\n",
    "        training_log[\"mse\"].append(mse)\n",
    "        training_log[\"r2_score\"].append(r2)\n",
    "\n",
    "        # Collenct current trained model\n",
    "        trained_model.append(copy.deepcopy(model))\n",
    "\n",
    "        # Debug Message\n",
    "        util.print_debug(\"Model {} has been trained\".format(model[\"model_name\"]))\n",
    "\n",
    "    # Debug message\n",
    "    util.print_debug(\"All combination models and data has been trained.\")\n",
    "\n",
    "\n",
    "    return trained_model, training_log"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Type II : List of Model All\n",
    "    Use this function to create model by ignoring config of data,\n",
    "    This notebook use this function instead of above code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_eval(configuration_model: str, params: dict, hyperparams_model: list = None):\n",
    "\n",
    "    # Variable to store trained models\n",
    "    list_of_trained_model = dict()\n",
    "\n",
    "    # Create training log template\n",
    "    training_log = training_log_template()\n",
    "\n",
    "    for config in X_train:\n",
    "        # Debug message\n",
    "        util.print_debug(\"Training model based on configuration data: {}\".format(config))\n",
    "\n",
    "        if hyperparams_model == None:\n",
    "            list_of_model = create_model_object(params)\n",
    "        else:\n",
    "            list_of_model = copy.deepcopy(hyperparams_model)\n",
    "\n",
    "        # Variable to store trained model\n",
    "        trained_model = list()\n",
    "\n",
    "        X_train_data = X_train[config]\n",
    "        y_train_data = y_train[config]\n",
    "        X_valid_data = X_valid[config]\n",
    "        y_valid_data = y_valid[config]\n",
    "\n",
    "        # Train each model by current dataset\n",
    "        for model in list_of_model:\n",
    "            # Debug message\n",
    "            util.print_debug(\"Training model: {}\".format(model[\"model_name\"]))\n",
    "\n",
    "            # Training\n",
    "            training_time = util.time_stamp()\n",
    "            model[\"model_object\"].fit(X_train_data, y_train_data)\n",
    "            training_time = (util.time_stamp() - training_time).total_seconds()\n",
    "\n",
    "            # Debug message\n",
    "            util.print_debug(\"Evaluating model: {}\".format(model[\"model_name\"]))\n",
    "\n",
    "            # Evaluation\n",
    "            y_predict = model[\"model_object\"].predict(X_valid_data)\n",
    "            mse = mean_squared_error(y_valid_data, y_predict, squared=False)\n",
    "            r2 = r2_score(y_valid_data, y_predict)\n",
    "\n",
    "            # Debug message\n",
    "            util.print_debug(\"Logging: {}\".format(model[\"model_name\"]))\n",
    "\n",
    "            # Create UID\n",
    "            uid = hashlib.md5(str(training_time).encode()).hexdigest()\n",
    "\n",
    "            model[\"model_uid\"] = uid\n",
    "\n",
    "            # Create training log data\n",
    "            training_log[\"model_name\"].append(\"{}-{}-{}\".format(configuration_model, config, model[\"model_name\"]))\n",
    "            training_log[\"model_uid\"].append(uid)\n",
    "            training_log[\"training_time\"].append(training_time)\n",
    "            training_log[\"training_date\"].append(util.time_stamp())\n",
    "            training_log[\"mse\"].append(mse)\n",
    "            training_log[\"r2_score\"].append(r2)\n",
    "\n",
    "            # Collenct current trained model\n",
    "            trained_model.append(copy.deepcopy(model))\n",
    "\n",
    "            # Debug Message\n",
    "            util.print_debug(\"Model {} has been trained\".format(model[\"model_name\"]))\n",
    "            util.print_debug(\"-\"*40)\n",
    "\n",
    "        # Collect current trained list of model\n",
    "        list_of_trained_model[config] = copy.deepcopy(trained_model)\n",
    "        util.print_debug(\"=\"*40)\n",
    "\n",
    "    # Debug message\n",
    "    util.print_debug(\"All combination models and data has been trained.\")\n",
    "\n",
    "\n",
    "    return list_of_trained_model, training_log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-07-14 23:11:15.007524 creating training log template\n",
      "2023-07-14 23:11:15.008591 Training log template created\n",
      "2023-07-14 23:11:15.008591 Training model based on configuration data: filter\n",
      "2023-07-14 23:11:15.008591 Creating model objects.\n",
      "2023-07-14 23:11:15.008591 Models object created\n",
      "2023-07-14 23:11:15.008591 Training model: KNeighborsRegressor\n",
      "2023-07-14 23:11:15.017689 Evaluating model: KNeighborsRegressor\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-07-14 23:11:16.012186 Logging: KNeighborsRegressor\n",
      "2023-07-14 23:11:16.015497 Model KNeighborsRegressor has been trained\n",
      "2023-07-14 23:11:16.015497 ----------------------------------------\n",
      "2023-07-14 23:11:16.015497 Training model: DecisionTreeRegressor\n",
      "2023-07-14 23:11:16.551140 Evaluating model: DecisionTreeRegressor\n",
      "2023-07-14 23:11:16.557724 Logging: DecisionTreeRegressor\n",
      "2023-07-14 23:11:16.558752 Model DecisionTreeRegressor has been trained\n",
      "2023-07-14 23:11:16.558752 ----------------------------------------\n",
      "2023-07-14 23:11:16.558752 Training model: LinearRegression\n",
      "2023-07-14 23:11:16.584584 Evaluating model: LinearRegression\n",
      "2023-07-14 23:11:16.589587 Logging: LinearRegression\n",
      "2023-07-14 23:11:16.589587 Model LinearRegression has been trained\n",
      "2023-07-14 23:11:16.589587 ----------------------------------------\n",
      "2023-07-14 23:11:16.589587 Training model: RandomForestRegressor\n",
      "2023-07-14 23:11:50.640981 Evaluating model: RandomForestRegressor\n",
      "2023-07-14 23:11:50.809356 Logging: RandomForestRegressor\n",
      "2023-07-14 23:11:50.838349 Model RandomForestRegressor has been trained\n",
      "2023-07-14 23:11:50.838349 ----------------------------------------\n",
      "2023-07-14 23:11:50.838349 Training model: AdaBoostRegressor\n",
      "2023-07-14 23:11:52.464277 Evaluating model: AdaBoostRegressor\n",
      "2023-07-14 23:11:52.480335 Logging: AdaBoostRegressor\n",
      "2023-07-14 23:11:52.482333 Model AdaBoostRegressor has been trained\n",
      "2023-07-14 23:11:52.482333 ----------------------------------------\n",
      "2023-07-14 23:11:52.482333 Training model: GradientBoostingRegressor\n",
      "2023-07-14 23:12:05.237664 Evaluating model: GradientBoostingRegressor\n",
      "2023-07-14 23:12:05.267138 Logging: GradientBoostingRegressor\n",
      "2023-07-14 23:12:05.277269 Model GradientBoostingRegressor has been trained\n",
      "2023-07-14 23:12:05.277269 ----------------------------------------\n",
      "2023-07-14 23:12:05.277269 Training model: XGBRegressor\n",
      "2023-07-14 23:12:08.303082 Evaluating model: XGBRegressor\n",
      "2023-07-14 23:12:08.321453 Logging: XGBRegressor\n",
      "2023-07-14 23:12:08.336273 Model XGBRegressor has been trained\n",
      "2023-07-14 23:12:08.336273 ----------------------------------------\n",
      "2023-07-14 23:12:08.420267 ========================================\n",
      "2023-07-14 23:12:08.420267 Training model based on configuration data: lasso\n",
      "2023-07-14 23:12:08.420267 Creating model objects.\n",
      "2023-07-14 23:12:08.420267 Models object created\n",
      "2023-07-14 23:12:08.424304 Training model: KNeighborsRegressor\n",
      "2023-07-14 23:12:08.433916 Evaluating model: KNeighborsRegressor\n",
      "2023-07-14 23:12:09.332940 Logging: KNeighborsRegressor\n",
      "2023-07-14 23:12:09.335465 Model KNeighborsRegressor has been trained\n",
      "2023-07-14 23:12:09.335465 ----------------------------------------\n",
      "2023-07-14 23:12:09.335465 Training model: DecisionTreeRegressor\n",
      "2023-07-14 23:12:09.761048 Evaluating model: DecisionTreeRegressor\n",
      "2023-07-14 23:12:09.766779 Logging: DecisionTreeRegressor\n",
      "2023-07-14 23:12:09.767780 Model DecisionTreeRegressor has been trained\n",
      "2023-07-14 23:12:09.767780 ----------------------------------------\n",
      "2023-07-14 23:12:09.767780 Training model: LinearRegression\n",
      "2023-07-14 23:12:09.794823 Evaluating model: LinearRegression\n",
      "2023-07-14 23:12:09.798944 Logging: LinearRegression\n",
      "2023-07-14 23:12:09.798944 Model LinearRegression has been trained\n",
      "2023-07-14 23:12:09.798944 ----------------------------------------\n",
      "2023-07-14 23:12:09.798944 Training model: RandomForestRegressor\n",
      "2023-07-14 23:12:35.812272 Evaluating model: RandomForestRegressor\n",
      "2023-07-14 23:12:36.092113 Logging: RandomForestRegressor\n",
      "2023-07-14 23:12:36.128882 Model RandomForestRegressor has been trained\n",
      "2023-07-14 23:12:36.128882 ----------------------------------------\n",
      "2023-07-14 23:12:36.128882 Training model: AdaBoostRegressor\n",
      "2023-07-14 23:12:36.897335 Evaluating model: AdaBoostRegressor\n",
      "2023-07-14 23:12:36.912336 Logging: AdaBoostRegressor\n",
      "2023-07-14 23:12:36.912951 Model AdaBoostRegressor has been trained\n",
      "2023-07-14 23:12:36.912951 ----------------------------------------\n",
      "2023-07-14 23:12:36.912951 Training model: GradientBoostingRegressor\n",
      "2023-07-14 23:12:47.401348 Evaluating model: GradientBoostingRegressor\n",
      "2023-07-14 23:12:47.435845 Logging: GradientBoostingRegressor\n",
      "2023-07-14 23:12:47.447764 Model GradientBoostingRegressor has been trained\n",
      "2023-07-14 23:12:47.447764 ----------------------------------------\n",
      "2023-07-14 23:12:47.447764 Training model: XGBRegressor\n",
      "2023-07-14 23:12:49.843323 Evaluating model: XGBRegressor\n",
      "2023-07-14 23:12:49.867975 Logging: XGBRegressor\n",
      "2023-07-14 23:12:49.882122 Model XGBRegressor has been trained\n",
      "2023-07-14 23:12:49.882122 ----------------------------------------\n",
      "2023-07-14 23:12:49.968802 ========================================\n",
      "2023-07-14 23:12:49.968802 Training model based on configuration data: rf\n",
      "2023-07-14 23:12:49.968802 Creating model objects.\n",
      "2023-07-14 23:12:49.968802 Models object created\n",
      "2023-07-14 23:12:49.973888 Training model: KNeighborsRegressor\n",
      "2023-07-14 23:12:49.986991 Evaluating model: KNeighborsRegressor\n",
      "2023-07-14 23:12:51.012141 Logging: KNeighborsRegressor\n",
      "2023-07-14 23:12:51.015558 Model KNeighborsRegressor has been trained\n",
      "2023-07-14 23:12:51.015558 ----------------------------------------\n",
      "2023-07-14 23:12:51.015558 Training model: DecisionTreeRegressor\n",
      "2023-07-14 23:12:52.304415 Evaluating model: DecisionTreeRegressor\n",
      "2023-07-14 23:12:52.315251 Logging: DecisionTreeRegressor\n",
      "2023-07-14 23:12:52.320244 Model DecisionTreeRegressor has been trained\n",
      "2023-07-14 23:12:52.320244 ----------------------------------------\n",
      "2023-07-14 23:12:52.320244 Training model: LinearRegression\n",
      "2023-07-14 23:12:52.354761 Evaluating model: LinearRegression\n",
      "2023-07-14 23:12:52.361422 Logging: LinearRegression\n",
      "2023-07-14 23:12:52.361422 Model LinearRegression has been trained\n",
      "2023-07-14 23:12:52.361422 ----------------------------------------\n",
      "2023-07-14 23:12:52.361422 Training model: RandomForestRegressor\n",
      "2023-07-14 23:14:26.518979 Evaluating model: RandomForestRegressor\n",
      "2023-07-14 23:14:26.896254 Logging: RandomForestRegressor\n",
      "2023-07-14 23:14:27.384002 Model RandomForestRegressor has been trained\n",
      "2023-07-14 23:14:27.384526 ----------------------------------------\n",
      "2023-07-14 23:14:27.384526 Training model: AdaBoostRegressor\n",
      "2023-07-14 23:14:28.336085 Evaluating model: AdaBoostRegressor\n",
      "2023-07-14 23:14:28.348330 Logging: AdaBoostRegressor\n",
      "2023-07-14 23:14:28.348330 Model AdaBoostRegressor has been trained\n",
      "2023-07-14 23:14:28.348330 ----------------------------------------\n",
      "2023-07-14 23:14:28.349339 Training model: GradientBoostingRegressor\n",
      "2023-07-14 23:14:40.594801 Evaluating model: GradientBoostingRegressor\n",
      "2023-07-14 23:14:40.635378 Logging: GradientBoostingRegressor\n",
      "2023-07-14 23:14:40.653336 Model GradientBoostingRegressor has been trained\n",
      "2023-07-14 23:14:40.653336 ----------------------------------------\n",
      "2023-07-14 23:14:40.653336 Training model: XGBRegressor\n",
      "2023-07-14 23:14:44.632950 Evaluating model: XGBRegressor\n",
      "2023-07-14 23:14:44.655180 Logging: XGBRegressor\n",
      "2023-07-14 23:14:44.671043 Model XGBRegressor has been trained\n",
      "2023-07-14 23:14:44.671043 ----------------------------------------\n",
      "2023-07-14 23:14:45.189587 ========================================\n",
      "2023-07-14 23:14:45.190586 All combination models and data has been trained.\n"
     ]
    }
   ],
   "source": [
    "list_of_trained_model, training_log = train_eval(\"baseline\", config_data)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <b>4. Save Training Log and Production Model</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_production_model(list_of_model, training_log, params):\n",
    "    # Create copy list of model\n",
    "    list_of_model = copy.deepcopy(list_of_model)\n",
    "    \n",
    "    # Debug message\n",
    "    util.print_debug(\"Choosing model by metrics score.\")\n",
    "\n",
    "    # Create required predefined variabel\n",
    "    curr_production_model = None\n",
    "    prev_production_model = None\n",
    "    production_model_log = None\n",
    "\n",
    "    # Debug message\n",
    "    util.print_debug(\"Converting training log type of data from dict to dataframe.\")\n",
    "\n",
    "    # Convert dictionary to pandas for easy operation\n",
    "    training_log = pd.DataFrame(copy.deepcopy(training_log))\n",
    "\n",
    "    # Debug message\n",
    "    util.print_debug(\"Trying to load previous production model.\")\n",
    "\n",
    "    # Check if there is a previous production model\n",
    "    try:\n",
    "        prev_production_model = util.pickle_load(params[\"production_model_path\"])\n",
    "        util.print_debug(\"Previous production model loaded.\")\n",
    "\n",
    "    except FileNotFoundError as fe:\n",
    "        util.print_debug(\"No previous production model detected, choosing best model only from current trained model.\")\n",
    "\n",
    "    # If previous production model detected:\n",
    "    if prev_production_model != None:\n",
    "        # Debug message\n",
    "        util.print_debug(\"Loading validation data.\")\n",
    "        X_valid['filter'], y_valid['filter']\n",
    "        \n",
    "        # Debug message\n",
    "        util.print_debug(\"Checking compatibilty previous production model's input with current train data's features.\")\n",
    "\n",
    "        # Check list features of previous production model and current dataset\n",
    "        production_model_features = set(prev_production_model[\"model_data\"][\"model_object\"].feature_names_in_)\n",
    "        current_dataset_features = set(X_valid['filter'].columns)\n",
    "        number_of_different_features = len((production_model_features - current_dataset_features) | (current_dataset_features - production_model_features))\n",
    "\n",
    "        # If feature matched:\n",
    "        if number_of_different_features == 0:\n",
    "            # Debug message\n",
    "            util.print_debug(\"Features compatible.\")\n",
    "\n",
    "            # Debug message\n",
    "            util.print_debug(\"Reassesing previous model performance using current validation data.\")\n",
    "\n",
    "            # Re-predict previous production model to provide valid metrics compared to other current models\n",
    "            y_pred = prev_production_model[\"model_data\"][\"model_object\"].predict(X_valid['filter'])\n",
    "\n",
    "            # Re-asses prediction result\n",
    "            eval_res = mean_squared_error(y_valid['filter'], y_pred, squared = False)\n",
    "            eval_r2 = r2_score(y_valid['filter'], y_pred)\n",
    "\n",
    "            # Debug message\n",
    "            util.print_debug(\"Assessing complete.\")\n",
    "\n",
    "            # Debug message\n",
    "            util.print_debug(\"Storing new metrics data to previous model structure.\")\n",
    "\n",
    "            # Update their performance log\n",
    "            prev_production_model[\"model_log\"][\"mse\"] = eval_res\n",
    "            prev_production_model[\"model_log\"][\"r2_score\"] = eval_r2\n",
    "\n",
    "            # Debug message\n",
    "            util.print_debug(\"Adding previous model data to current training log and list of model\")\n",
    "\n",
    "            # Added previous production model log to current logs to compere who has the greatest f1 score\n",
    "            training_log = pd.concat([training_log, pd.DataFrame([prev_production_model[\"model_log\"]])])\n",
    "\n",
    "            # Added previous production model to current list of models to choose from if it has the greatest f1 score\n",
    "            list_of_model[\"prev_production_model\"] = [copy.deepcopy(prev_production_model[\"model_data\"])]\n",
    "        else:\n",
    "            # To indicate that we are not using previous production model\n",
    "            prev_production_model = None\n",
    "\n",
    "            # Debug message\n",
    "            util.print_debug(\"Different features between production model with current dataset is detected, ignoring production dataset.\")\n",
    "\n",
    "    # Debug message\n",
    "    util.print_debug(\"Sorting training log by f1 macro avg and training time.\")\n",
    "\n",
    "    # Sort training log by f1 score macro avg and trining time\n",
    "    best_model_log = training_log.sort_values([\"mse\", \"training_time\"], ascending = [True, True]).iloc[0]\n",
    "    \n",
    "    # Debug message\n",
    "    util.print_debug(\"Searching model data based on sorted training log.\")\n",
    "\n",
    "    # Get model object with greatest f1 score macro avg by using UID\n",
    "    for configuration_data in list_of_model:\n",
    "        for model_data in list_of_model[configuration_data]:\n",
    "            if model_data[\"model_uid\"] == best_model_log[\"model_uid\"]:\n",
    "                curr_production_model = dict()\n",
    "                curr_production_model[\"model_data\"] = copy.deepcopy(model_data)\n",
    "                curr_production_model[\"model_log\"] = copy.deepcopy(best_model_log.to_dict())\n",
    "                curr_production_model[\"model_log\"][\"model_name\"] = \"Filter-{}\".format(curr_production_model[\"model_data\"][\"model_name\"])\n",
    "                curr_production_model[\"model_log\"][\"training_date\"] = str(curr_production_model[\"model_log\"][\"training_date\"])\n",
    "                production_model_log = training_log_updater(curr_production_model[\"model_log\"], params)\n",
    "                break\n",
    "    \n",
    "    # In case UID not found\n",
    "    if curr_production_model == None:\n",
    "        raise RuntimeError(\"The best model not found in your list of model.\")\n",
    "    \n",
    "    # Debug message\n",
    "    util.print_debug(\"Model chosen.\")\n",
    "\n",
    "    # Dump chosen production model\n",
    "    util.pickle_dump(curr_production_model, params[\"production_model_path\"])\n",
    "    \n",
    "    # Return current chosen production model, log of production models and current training log\n",
    "    return curr_production_model, production_model_log, training_log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-07-14 23:14:46.098924 Choosing model by metrics score.\n",
      "2023-07-14 23:14:46.100181 Converting training log type of data from dict to dataframe.\n",
      "2023-07-14 23:14:46.103225 Trying to load previous production model.\n",
      "2023-07-14 23:14:46.192159 Previous production model loaded.\n",
      "2023-07-14 23:14:46.192159 Loading validation data.\n",
      "2023-07-14 23:14:46.192159 Checking compatibilty previous production model's input with current train data's features.\n",
      "2023-07-14 23:14:46.192159 Features compatible.\n",
      "2023-07-14 23:14:46.192159 Reassesing previous model performance using current validation data.\n",
      "2023-07-14 23:14:46.359175 Assessing complete.\n",
      "2023-07-14 23:14:46.360198 Storing new metrics data to previous model structure.\n",
      "2023-07-14 23:14:46.360198 Adding previous model data to current training log and list of model\n",
      "2023-07-14 23:14:46.387205 Sorting training log by f1 macro avg and training time.\n",
      "2023-07-14 23:14:46.389128 Searching model data based on sorted training log.\n",
      "2023-07-14 23:14:46.425722 Model chosen.\n"
     ]
    }
   ],
   "source": [
    "model, production_model_log, training_logs = get_production_model(list_of_trained_model, training_log, config_data)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <b>5. Model Performance</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_name</th>\n",
       "      <th>mse</th>\n",
       "      <th>r2_score</th>\n",
       "      <th>training_time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Filter-RandomForestRegressor</td>\n",
       "      <td>1.008912</td>\n",
       "      <td>0.998880</td>\n",
       "      <td>21.635677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>baseline-filter-RandomForestRegressor</td>\n",
       "      <td>1.031461</td>\n",
       "      <td>0.998829</td>\n",
       "      <td>34.051394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>baseline-lasso-RandomForestRegressor</td>\n",
       "      <td>1.111917</td>\n",
       "      <td>0.998640</td>\n",
       "      <td>26.013328</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>baseline-filter-XGBRegressor</td>\n",
       "      <td>1.266875</td>\n",
       "      <td>0.998234</td>\n",
       "      <td>3.025813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>baseline-lasso-DecisionTreeRegressor</td>\n",
       "      <td>1.308961</td>\n",
       "      <td>0.998115</td>\n",
       "      <td>0.425583</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>baseline-lasso-XGBRegressor</td>\n",
       "      <td>1.378939</td>\n",
       "      <td>0.997908</td>\n",
       "      <td>2.395559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>baseline-filter-DecisionTreeRegressor</td>\n",
       "      <td>1.763415</td>\n",
       "      <td>0.996578</td>\n",
       "      <td>0.535643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>baseline-lasso-GradientBoostingRegressor</td>\n",
       "      <td>19.642598</td>\n",
       "      <td>0.575457</td>\n",
       "      <td>10.488397</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>baseline-filter-GradientBoostingRegressor</td>\n",
       "      <td>20.202004</td>\n",
       "      <td>0.550931</td>\n",
       "      <td>12.755331</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>baseline-filter-KNeighborsRegressor</td>\n",
       "      <td>24.252410</td>\n",
       "      <td>0.352807</td>\n",
       "      <td>0.009098</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   model_name        mse  r2_score  \\\n",
       "0                Filter-RandomForestRegressor   1.008912  0.998880   \n",
       "3       baseline-filter-RandomForestRegressor   1.031461  0.998829   \n",
       "10       baseline-lasso-RandomForestRegressor   1.111917  0.998640   \n",
       "6                baseline-filter-XGBRegressor   1.266875  0.998234   \n",
       "8        baseline-lasso-DecisionTreeRegressor   1.308961  0.998115   \n",
       "13                baseline-lasso-XGBRegressor   1.378939  0.997908   \n",
       "1       baseline-filter-DecisionTreeRegressor   1.763415  0.996578   \n",
       "12   baseline-lasso-GradientBoostingRegressor  19.642598  0.575457   \n",
       "5   baseline-filter-GradientBoostingRegressor  20.202004  0.550931   \n",
       "0         baseline-filter-KNeighborsRegressor  24.252410  0.352807   \n",
       "\n",
       "    training_time  \n",
       "0       21.635677  \n",
       "3       34.051394  \n",
       "10      26.013328  \n",
       "6        3.025813  \n",
       "8        0.425583  \n",
       "13       2.395559  \n",
       "1        0.535643  \n",
       "12      10.488397  \n",
       "5       12.755331  \n",
       "0        0.009098  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.max_colwidth', None)\n",
    "# pd.reset_option('display.max_colwidth')\n",
    "training_logs[[\"model_name\", \"mse\", \"r2_score\", \"training_time\"]].sort_values([\"mse\", \"r2_score\",\"training_time\"], \n",
    "                                                                               ascending=[True, True, True])\\\n",
    "                                                                                   .head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Based on the baseline model evaluation, the <b> <i>Filter method</i> applied on <mark><i>Random Forest Regression</i> appears to be the best model. </mark></b> <br> \n",
    "However, it is worth noting that <b>this model takes <mark>more time for predictions</mark>.</b> If training time is a significant consideration, alternative methods such as <b><i>Lasso Method</i> on <i>Decision Tree Regression</i> or <i>XGBoost Regressor</i> </b> could be viable options.\n",
    "\n",
    "It is important to mention that <b><i>Decision Tree Regression</i> may result in a high variance model, potentially leading to overfitting.</b> To assess the model's performance on the test set, further evaluation should be conducted. Nevertheless, <i>Decision Tree</i> models are relatively easier to interpret due to their inherent structure.\n",
    "\n",
    "On the other hand, if <b>the objective is to minimize error within a <mark>shorter amount of time, <i>XGBoost Regression</i> is the recommended choice</mark></b>. However, it is worth noting that XGBoost models are generally <b>more complex and can be more challenging to interpret.</b>\n",
    "\n",
    "Ultimately, the choice of the model depends on the specific requirements and trade-offs between factors such as <b>accuracy, interpretability, training time, and ease of use.</b>"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <b>6. Cross Validation Score</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-07-14 23:16:03.861509 Creating model objects.\n",
      "2023-07-14 23:16:03.862016 Models object created\n"
     ]
    }
   ],
   "source": [
    "list_of_model = create_model_object(config_data)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Cross Validation score (CVS)</b>\n",
    "----\n",
    "\n",
    "<mark><b>CVS</mark> is performed to understand the distribution of data that we can be sure out model generalises well accross the whole dataset and not just a single portion.</b> <br>\n",
    "\n",
    "    How do we now that single dataset is representative?\n",
    "\n",
    "Cross Val Score train and test our model on <mark><b>multiple folds</b></mark> and give a better understanding of model performance over the whole dataset instead of just a single train/test split. <br>\n",
    "\n",
    "If we see that the metrics for all folds cvs are significant differences between them then this <b>may indicate over-fitting to certain folds.</b> <br>\n",
    "\n",
    "<b>!! Scoring: </b> <br>\n",
    "----\n",
    "\n",
    "<b><i>neg_mean_squared_error</i></b> always <mark><b>return negative (-)</b></mark>, it because cross_val_score function will <b>return maximize value as sign higher is better, the cross_val_score will turn it into negative (-), </b><br>\n",
    "hence, cross_val_score will return the smaller value.\n",
    "\n",
    "<b>As example of,</b> <br>\n",
    "MSE Score 5 is better than 9. <br>\n",
    "Cross val score will return the higher which is 9. <br>\n",
    "As of that, cross_val_score function will turn it into -5 and -9, and <br>\n",
    "cross_val_score will return -5 as the higher value. <br>\n",
    "\n",
    "**Reference:**<br>\n",
    "[cross_val_score](https://scikit-learn.org/stable/modules/model_evaluation.html)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cross validation score for the model KNeighborsRegressor is -25.007 +/- 0.246.\n",
      "cross validation score for the model DecisionTreeRegressor is -1.34 +/- 0.164.\n",
      "cross validation score for the model LinearRegression is -29.54 +/- 0.18.\n",
      "cross validation score for the model RandomForestRegressor is -1.039 +/- 0.138.\n",
      "cross validation score for the model AdaBoostRegressor is -28.665 +/- 0.224.\n",
      "cross validation score for the model GradientBoostingRegressor is -19.941 +/- 0.24.\n",
      "cross validation score for the model XGBRegressor is -1.3 +/- 0.089.\n"
     ]
    }
   ],
   "source": [
    "model_object = []\n",
    "model_name = []\n",
    "\n",
    "for model in list_of_model:\n",
    "    model_object.append(model[\"model_object\"])\n",
    "    model_name.append(model[\"model_name\"])\n",
    "\n",
    "cv = KFold(n_splits=5)\n",
    "\n",
    "for index, model in enumerate(model_object):\n",
    "    cvs = cross_val_score(estimator=model, X=X_train['filter'], \n",
    "                          y=y_train['filter'], \n",
    "                          cv=cv, \n",
    "                          scoring='neg_root_mean_squared_error')\n",
    "    mean = np.round(cvs.mean(), 3)\n",
    "    std = np.round(cvs.std(), 3)\n",
    "    print(f\"cross validation score for the model {model_name[index]} is {np.abs(mean)} +/- {std}.\")\n",
    "\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Cross Val Score show Decision Tree Regressor, Random Forest Regressor, and XGBoost Regressor indicates the dataset is generalised."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <b>7. Perform Hyperparameter for Model</b>"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Param Distribution\n",
    "\n",
    "Define Best parameter for each model: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_dist_params(model_name:str) -> dict:\n",
    "    dist_params_dt = {\n",
    "        \"criterion\" : ['squared_error', 'absolute_error'],\n",
    "        \"min_samples_split\" : np.arange(3,10,1).tolist(),\n",
    "        \"min_samples_leaf\" : np.arange(2,10,1).tolist(),\n",
    "        \"max_depth\" : np.arange(50,501,50).tolist(),\n",
    "        \"random_state\" : [42]\n",
    "    }\n",
    "\n",
    "    dist_params_rf = {\n",
    "        \"n_estimators\" : np.arange(100,500,100).tolist(),\n",
    "        \"criterion\" : [\"squared_error\", \"absolute_error\"],\n",
    "        \"max_depth\" : np.arange(2,11,1).tolist(),\n",
    "        \"min_samples_split\" : np.arange(1,10,1).tolist(),\n",
    "        \"random_state\" : [42]\n",
    "    }\n",
    "\n",
    "    dist_params_xgb = {\n",
    "        \"eta\" : np.arange(0.1,1,0.1).tolist(),\n",
    "        \"max_depth\" : np.arange(1,11,1).tolist(),\n",
    "        \"alpha\" : np.arange(1,10,1).tolist(),\n",
    "        \"random_state\" : [42]\n",
    "    }\n",
    "\n",
    "    dist_params = {\n",
    "        \"DecisionTreeRegressor\" : dist_params_dt,\n",
    "        \"RandomForestRegressor\" : dist_params_rf,\n",
    "        \"XGBRegressor\" : dist_params_xgb\n",
    "    }\n",
    "\n",
    "    return dist_params[model_name]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create Hyper Param Function into one dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def hyper_params_tuning(model: list) -> list:\n",
    "    # Create copy of current best baseline model\n",
    "    model_list = []\n",
    "    trained_model = [DecisionTreeRegressor(),\n",
    "                     RandomForestRegressor(),\n",
    "                     XGBRegressor()]\n",
    "    \n",
    "    for col, mod in list(zip(model, trained_model)):\n",
    "        dist_params = create_dist_params(col)\n",
    "        model_rsc = RandomizedSearchCV(\n",
    "            estimator = mod,\n",
    "            param_distributions = dist_params,\n",
    "            cv = cv,\n",
    "            scoring = 'neg_root_mean_squared_error',\n",
    "            n_jobs=-1\n",
    "        )\n",
    "        \n",
    "        model = {\n",
    "            \"model_name\" : col,\n",
    "            \"model_object\" : model_rsc,\n",
    "            \"model_uid\" : \"\"\n",
    "        }\n",
    "\n",
    "        model_list.append(model.copy())\n",
    "\n",
    "    return model_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'model_name': 'DecisionTreeRegressor',\n",
       "  'model_object': RandomizedSearchCV(cv=KFold(n_splits=5, random_state=None, shuffle=False),\n",
       "                     estimator=DecisionTreeRegressor(), n_jobs=-1,\n",
       "                     param_distributions={'criterion': ['squared_error',\n",
       "                                                        'absolute_error'],\n",
       "                                          'max_depth': [50, 100, 150, 200, 250,\n",
       "                                                        300, 350, 400, 450, 500],\n",
       "                                          'min_samples_leaf': [2, 3, 4, 5, 6, 7,\n",
       "                                                               8, 9],\n",
       "                                          'min_samples_split': [3, 4, 5, 6, 7, 8,\n",
       "                                                                9],\n",
       "                                          'random_state': [42]},\n",
       "                     scoring='neg_root_mean_squared_error'),\n",
       "  'model_uid': ''},\n",
       " {'model_name': 'RandomForestRegressor',\n",
       "  'model_object': RandomizedSearchCV(cv=KFold(n_splits=5, random_state=None, shuffle=False),\n",
       "                     estimator=RandomForestRegressor(), n_jobs=-1,\n",
       "                     param_distributions={'criterion': ['squared_error',\n",
       "                                                        'absolute_error'],\n",
       "                                          'max_depth': [2, 3, 4, 5, 6, 7, 8, 9,\n",
       "                                                        10],\n",
       "                                          'min_samples_split': [1, 2, 3, 4, 5, 6,\n",
       "                                                                7, 8, 9],\n",
       "                                          'n_estimators': [100, 200, 300, 400],\n",
       "                                          'random_state': [42]},\n",
       "                     scoring='neg_root_mean_squared_error'),\n",
       "  'model_uid': ''},\n",
       " {'model_name': 'XGBRegressor',\n",
       "  'model_object': RandomizedSearchCV(cv=KFold(n_splits=5, random_state=None, shuffle=False),\n",
       "                     estimator=XGBRegressor(base_score=None, booster=None,\n",
       "                                            callbacks=None,\n",
       "                                            colsample_bylevel=None,\n",
       "                                            colsample_bynode=None,\n",
       "                                            colsample_bytree=None,\n",
       "                                            early_stopping_rounds=None,\n",
       "                                            enable_categorical=False,\n",
       "                                            eval_metric=None, feature_types=None,\n",
       "                                            gamma=None, gpu_id=None,\n",
       "                                            grow_policy=None,\n",
       "                                            importance_type=...\n",
       "                                            min_child_weight=None, missing=nan,\n",
       "                                            monotone_constraints=None,\n",
       "                                            n_estimators=100, n_jobs=None,\n",
       "                                            num_parallel_tree=None,\n",
       "                                            predictor=None, random_state=None, ...),\n",
       "                     n_jobs=-1,\n",
       "                     param_distributions={'alpha': [1, 2, 3, 4, 5, 6, 7, 8, 9],\n",
       "                                          'eta': [0.1, 0.2, 0.30000000000000004,\n",
       "                                                  0.4, 0.5, 0.6,\n",
       "                                                  0.7000000000000001, 0.8, 0.9],\n",
       "                                          'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9,\n",
       "                                                        10],\n",
       "                                          'random_state': [42]},\n",
       "                     scoring='neg_root_mean_squared_error'),\n",
       "  'model_uid': ''}]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list_model_ = hyper_params_tuning(['DecisionTreeRegressor',\n",
    "                                   'RandomForestRegressor',\n",
    "                                   'XGBRegressor'])\n",
    "\n",
    "list_model_"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Copy list of trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "list__model = copy.deepcopy(list_of_trained_model)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Update list of trained model with hyperparams model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "for config in list__model:\n",
    "    for l in (list_model_[0], list_model_[1], list_model_[2]):\n",
    "        list__model[config].append(l)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNeighborsRegressor\n",
      "DecisionTreeRegressor\n",
      "LinearRegression\n",
      "RandomForestRegressor\n",
      "AdaBoostRegressor\n",
      "GradientBoostingRegressor\n",
      "XGBRegressor\n",
      "DecisionTreeRegressor\n",
      "RandomForestRegressor\n",
      "XGBRegressor\n"
     ]
    }
   ],
   "source": [
    "for model in list__model['filter']:\n",
    "    print(model['model_name'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Check the parameter distribution we added into list__model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'model_name': 'DecisionTreeRegressor',\n",
       "  'model_object': RandomizedSearchCV(cv=KFold(n_splits=5, random_state=None, shuffle=False),\n",
       "                     estimator=DecisionTreeRegressor(), n_jobs=-1,\n",
       "                     param_distributions={'criterion': ['squared_error',\n",
       "                                                        'absolute_error'],\n",
       "                                          'max_depth': [50, 100, 150, 200, 250,\n",
       "                                                        300, 350, 400, 450, 500],\n",
       "                                          'min_samples_leaf': [2, 3, 4, 5, 6, 7,\n",
       "                                                               8, 9],\n",
       "                                          'min_samples_split': [3, 4, 5, 6, 7, 8,\n",
       "                                                                9],\n",
       "                                          'random_state': [42]},\n",
       "                     scoring='neg_root_mean_squared_error'),\n",
       "  'model_uid': ''},\n",
       " {'model_name': 'RandomForestRegressor',\n",
       "  'model_object': RandomizedSearchCV(cv=KFold(n_splits=5, random_state=None, shuffle=False),\n",
       "                     estimator=RandomForestRegressor(), n_jobs=-1,\n",
       "                     param_distributions={'criterion': ['squared_error',\n",
       "                                                        'absolute_error'],\n",
       "                                          'max_depth': [2, 3, 4, 5, 6, 7, 8, 9,\n",
       "                                                        10],\n",
       "                                          'min_samples_split': [1, 2, 3, 4, 5, 6,\n",
       "                                                                7, 8, 9],\n",
       "                                          'n_estimators': [100, 200, 300, 400],\n",
       "                                          'random_state': [42]},\n",
       "                     scoring='neg_root_mean_squared_error'),\n",
       "  'model_uid': ''},\n",
       " {'model_name': 'XGBRegressor',\n",
       "  'model_object': RandomizedSearchCV(cv=KFold(n_splits=5, random_state=None, shuffle=False),\n",
       "                     estimator=XGBRegressor(base_score=None, booster=None,\n",
       "                                            callbacks=None,\n",
       "                                            colsample_bylevel=None,\n",
       "                                            colsample_bynode=None,\n",
       "                                            colsample_bytree=None,\n",
       "                                            early_stopping_rounds=None,\n",
       "                                            enable_categorical=False,\n",
       "                                            eval_metric=None, feature_types=None,\n",
       "                                            gamma=None, gpu_id=None,\n",
       "                                            grow_policy=None,\n",
       "                                            importance_type=...\n",
       "                                            min_child_weight=None, missing=nan,\n",
       "                                            monotone_constraints=None,\n",
       "                                            n_estimators=100, n_jobs=None,\n",
       "                                            num_parallel_tree=None,\n",
       "                                            predictor=None, random_state=None, ...),\n",
       "                     n_jobs=-1,\n",
       "                     param_distributions={'alpha': [1, 2, 3, 4, 5, 6, 7, 8, 9],\n",
       "                                          'eta': [0.1, 0.2, 0.30000000000000004,\n",
       "                                                  0.4, 0.5, 0.6,\n",
       "                                                  0.7000000000000001, 0.8, 0.9],\n",
       "                                          'max_depth': [1, 2, 3, 4, 5, 6, 7, 8, 9,\n",
       "                                                        10],\n",
       "                                          'random_state': [42]},\n",
       "                     scoring='neg_root_mean_squared_error'),\n",
       "  'model_uid': ''}]"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list__model['filter'][-3:]"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### <b> Run Hyperparamater Model </b>\n",
    "\n",
    "<b><mark>!! Be Aware this function takes long times to finish </b><mark>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_eval_hyper(configuration_model: str, params: dict, hyperparams_model: list = None, log_path: str = None):\n",
    "\n",
    "    # Variable to store trained models\n",
    "    list_of_trained_model = dict()\n",
    "\n",
    "    # Create training log template\n",
    "    training_log = training_log_template()\n",
    "\n",
    "    for config in X_train:\n",
    "        # Debug message\n",
    "        util.print_debug(\"Training model based on configuration data: {}\".format(config))\n",
    "\n",
    "        if hyperparams_model == None:\n",
    "            list_of_model = create_model_object(params)\n",
    "        else:\n",
    "            ## list of hyperparams is three models behind (look list__model[config])\n",
    "            list_of_model = copy.deepcopy(hyperparams_model[config][-3:])\n",
    "\n",
    "        # Variable to store trained model\n",
    "        trained_model = list()\n",
    "\n",
    "        X_train_data = X_train[config]\n",
    "        y_train_data = y_train[config]\n",
    "        X_valid_data = X_valid[config]\n",
    "        y_valid_data = y_valid[config]\n",
    "\n",
    "        # Train each model by current dataset\n",
    "        for model in list_of_model:\n",
    "            # Debug message\n",
    "            util.print_debug(\"Training model: {}\".format(model[\"model_name\"]))\n",
    "\n",
    "            # Training\n",
    "            training_time = util.time_stamp()\n",
    "            model[\"model_object\"].fit(X_train_data, y_train_data)\n",
    "            training_time = (util.time_stamp() - training_time).total_seconds()\n",
    "\n",
    "            # Debug message\n",
    "            util.print_debug(\"Evaluating model: {}\".format(model[\"model_name\"]))\n",
    "\n",
    "            # Evaluation\n",
    "            y_predict = model[\"model_object\"].predict(X_valid_data)\n",
    "            mse = mean_squared_error(y_valid_data, y_predict, squared=False)\n",
    "            r2 = r2_score(y_valid_data, y_predict)\n",
    "\n",
    "            # Debug message\n",
    "            util.print_debug(\"Logging: {}\".format(model[\"model_name\"]))\n",
    "\n",
    "            # Create UID\n",
    "            uid = hashlib.md5(str(training_time).encode()).hexdigest()\n",
    "\n",
    "            model[\"model_uid\"] = uid\n",
    "\n",
    "            # Create training log data\n",
    "            training_log[\"model_name\"].append(\"{}-{}-{}\".format(configuration_model, config, model[\"model_name\"]))\n",
    "            training_log[\"model_uid\"].append(uid)\n",
    "            training_log[\"training_time\"].append(training_time)\n",
    "            training_log[\"training_date\"].append(util.time_stamp())\n",
    "            training_log[\"mse\"].append(mse)\n",
    "            training_log[\"r2_score\"].append(r2)\n",
    "\n",
    "            # Collenct current trained model\n",
    "            trained_model.append(copy.deepcopy(model))\n",
    "\n",
    "            # Debug Message\n",
    "            util.print_debug(\"Model {} has been trained\".format(model[\"model_name\"]))\n",
    "\n",
    "        # Collect current trained list of model\n",
    "        list_of_trained_model[config] = copy.deepcopy(trained_model)\n",
    "\n",
    "    # Debug message\n",
    "    util.print_debug(\"All combination models and data has been trained.\")\n",
    "\n",
    "    if log_path == None:\n",
    "        training_log_ = training_log\n",
    "    else:\n",
    "        training_log_ = training_log_updater(training_log, log_path)\n",
    "\n",
    "    return list_of_trained_model, training_log_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-07-15 00:20:37.162309 creating training log template\n",
      "2023-07-15 00:20:37.162309 Training log template created\n",
      "2023-07-15 00:20:37.162309 Training model based on configuration data: filter\n",
      "2023-07-15 00:20:37.163818 Training model: DecisionTreeRegressor\n",
      "2023-07-15 00:29:44.269418 Evaluating model: DecisionTreeRegressor\n",
      "2023-07-15 00:29:44.272417 Logging: DecisionTreeRegressor\n",
      "2023-07-15 00:29:44.274496 Model DecisionTreeRegressor has been trained\n",
      "2023-07-15 00:29:44.274496 Training model: RandomForestRegressor\n"
     ]
    }
   ],
   "source": [
    "list_of_hyperparam_model, training_log_ = train_eval_hyper(\"hyperparam\", \n",
    "                                                           config_data, \n",
    "                                                           list__model, \n",
    "                                                           config_data['training_log_path'])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <b>8. Try the MODEL to Test Dataset</b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_eval(configuration_model: str, params: dict, hyperparams_model: list = None,\n",
    "               X_t: pd.DataFrame = pd.DataFrame(), y_t: pd.DataFrame = pd.DataFrame()):\n",
    "\n",
    "    # Variable to store trained models\n",
    "    list_of_trained_model = dict()\n",
    "\n",
    "    # Create training log template\n",
    "    training_log = training_log_template()\n",
    "\n",
    "    for config in X_train:\n",
    "        # Debug message\n",
    "        util.print_debug(\"Training model based on configuration data: {}\".format(config))\n",
    "\n",
    "        if hyperparams_model == None:\n",
    "            list_of_model = create_model_object(params)\n",
    "        else:\n",
    "            list_of_model = copy.deepcopy(hyperparams_model)\n",
    "\n",
    "        # Variable to store trained model\n",
    "        trained_model = list()\n",
    "\n",
    "        X_train_data = X_train[config]\n",
    "        y_train_data = y_train[config]\n",
    "        X_valid_data = X_t[config]\n",
    "        y_valid_data = y_t[config]\n",
    "\n",
    "        # Train each model by current dataset\n",
    "        for model in list_of_model:\n",
    "            # Debug message\n",
    "            util.print_debug(\"Training model: {}\".format(model[\"model_name\"]))\n",
    "\n",
    "            # Training\n",
    "            training_time = util.time_stamp()\n",
    "            model[\"model_object\"].fit(X_train_data, y_train_data)\n",
    "            training_time = (util.time_stamp() - training_time).total_seconds()\n",
    "\n",
    "            # Debug message\n",
    "            util.print_debug(\"Evaluating model: {}\".format(model[\"model_name\"]))\n",
    "\n",
    "            # Evaluation\n",
    "            y_predict = model[\"model_object\"].predict(X_valid_data)\n",
    "            mse = mean_squared_error(y_valid_data, y_predict, squared=False)\n",
    "            r2 = r2_score(y_valid_data, y_predict)\n",
    "\n",
    "            # Debug message\n",
    "            util.print_debug(\"Logging: {}\".format(model[\"model_name\"]))\n",
    "\n",
    "            # Create UID\n",
    "            uid = hashlib.md5(str(training_time).encode()).hexdigest()\n",
    "\n",
    "            model[\"model_uid\"] = uid\n",
    "\n",
    "            # Create training log data\n",
    "            training_log[\"model_name\"].append(\"{}-{}-{}\".format(configuration_model, config, model[\"model_name\"]))\n",
    "            training_log[\"model_uid\"].append(uid)\n",
    "            training_log[\"training_time\"].append(training_time)\n",
    "            training_log[\"training_date\"].append(util.time_stamp())\n",
    "            training_log[\"mse\"].append(mse)\n",
    "            training_log[\"r2_score\"].append(r2)\n",
    "\n",
    "            # Collenct current trained model\n",
    "            trained_model.append(copy.deepcopy(model))\n",
    "\n",
    "            # Debug Message\n",
    "            util.print_debug(\"Model {} has been trained\".format(model[\"model_name\"]))\n",
    "            util.print_debug(\"-\"*40)\n",
    "\n",
    "        # Collect current trained list of model\n",
    "        list_of_trained_model[config] = copy.deepcopy(trained_model)\n",
    "        util.print_debug(\"=\"*40)\n",
    "\n",
    "    # Debug message\n",
    "    util.print_debug(\"All combination models and data has been trained.\")\n",
    "\n",
    "\n",
    "    return list_of_trained_model, training_log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2023-07-14 23:28:14.217449 creating training log template\n",
      "2023-07-14 23:28:14.217449 Training log template created\n",
      "2023-07-14 23:28:14.217449 Training model based on configuration data: filter\n",
      "2023-07-14 23:28:14.217449 Creating model objects.\n",
      "2023-07-14 23:28:14.217449 Models object created\n",
      "2023-07-14 23:28:14.217449 Training model: KNeighborsRegressor\n",
      "2023-07-14 23:28:14.221955 Evaluating model: KNeighborsRegressor\n",
      "2023-07-14 23:28:15.085797 Logging: KNeighborsRegressor\n",
      "2023-07-14 23:28:15.089801 Model KNeighborsRegressor has been trained\n",
      "2023-07-14 23:28:15.089801 Training model: DecisionTreeRegressor\n",
      "2023-07-14 23:28:15.682975 Evaluating model: DecisionTreeRegressor\n",
      "2023-07-14 23:28:15.692243 Logging: DecisionTreeRegressor\n",
      "2023-07-14 23:28:15.692753 Model DecisionTreeRegressor has been trained\n",
      "2023-07-14 23:28:15.692753 Training model: LinearRegression\n",
      "2023-07-14 23:28:15.723696 Evaluating model: LinearRegression\n",
      "2023-07-14 23:28:15.730756 Logging: LinearRegression\n",
      "2023-07-14 23:28:15.730756 Model LinearRegression has been trained\n",
      "2023-07-14 23:28:15.730756 Training model: RandomForestRegressor\n",
      "2023-07-14 23:28:53.108462 Evaluating model: RandomForestRegressor\n",
      "2023-07-14 23:28:53.313770 Logging: RandomForestRegressor\n",
      "2023-07-14 23:28:53.346994 Model RandomForestRegressor has been trained\n",
      "2023-07-14 23:28:53.346994 Training model: AdaBoostRegressor\n",
      "2023-07-14 23:28:54.328543 Evaluating model: AdaBoostRegressor\n",
      "2023-07-14 23:28:54.340729 Logging: AdaBoostRegressor\n",
      "2023-07-14 23:28:54.342949 Model AdaBoostRegressor has been trained\n",
      "2023-07-14 23:28:54.342949 Training model: GradientBoostingRegressor\n",
      "2023-07-14 23:29:07.832247 Evaluating model: GradientBoostingRegressor\n",
      "2023-07-14 23:29:07.869377 Logging: GradientBoostingRegressor\n",
      "2023-07-14 23:29:07.880864 Model GradientBoostingRegressor has been trained\n",
      "2023-07-14 23:29:07.880864 Training model: XGBRegressor\n",
      "2023-07-14 23:29:11.122684 Evaluating model: XGBRegressor\n",
      "2023-07-14 23:29:11.142399 Logging: XGBRegressor\n",
      "2023-07-14 23:29:11.153965 Model XGBRegressor has been trained\n",
      "2023-07-14 23:29:11.226168 Training model based on configuration data: lasso\n",
      "2023-07-14 23:29:11.226168 Creating model objects.\n",
      "2023-07-14 23:29:11.226908 Models object created\n",
      "2023-07-14 23:29:11.230538 Training model: KNeighborsRegressor\n",
      "2023-07-14 23:29:11.244767 Evaluating model: KNeighborsRegressor\n",
      "2023-07-14 23:29:12.109053 Logging: KNeighborsRegressor\n",
      "2023-07-14 23:29:12.111190 Model KNeighborsRegressor has been trained\n",
      "2023-07-14 23:29:12.111190 Training model: DecisionTreeRegressor\n",
      "2023-07-14 23:29:12.575973 Evaluating model: DecisionTreeRegressor\n",
      "2023-07-14 23:29:12.585466 Logging: DecisionTreeRegressor\n",
      "2023-07-14 23:29:12.586464 Model DecisionTreeRegressor has been trained\n",
      "2023-07-14 23:29:12.586464 Training model: LinearRegression\n",
      "2023-07-14 23:29:12.617031 Evaluating model: LinearRegression\n",
      "2023-07-14 23:29:12.622122 Logging: LinearRegression\n",
      "2023-07-14 23:29:12.622122 Model LinearRegression has been trained\n",
      "2023-07-14 23:29:12.622122 Training model: RandomForestRegressor\n",
      "2023-07-14 23:29:41.660321 Evaluating model: RandomForestRegressor\n",
      "2023-07-14 23:29:41.868529 Logging: RandomForestRegressor\n",
      "2023-07-14 23:29:41.896566 Model RandomForestRegressor has been trained\n",
      "2023-07-14 23:29:41.897573 Training model: AdaBoostRegressor\n",
      "2023-07-14 23:29:43.242223 Evaluating model: AdaBoostRegressor\n",
      "2023-07-14 23:29:43.258727 Logging: AdaBoostRegressor\n",
      "2023-07-14 23:29:43.260725 Model AdaBoostRegressor has been trained\n",
      "2023-07-14 23:29:43.260725 Training model: GradientBoostingRegressor\n",
      "2023-07-14 23:29:52.837981 Evaluating model: GradientBoostingRegressor\n",
      "2023-07-14 23:29:52.865065 Logging: GradientBoostingRegressor\n",
      "2023-07-14 23:29:52.874096 Model GradientBoostingRegressor has been trained\n",
      "2023-07-14 23:29:52.874096 Training model: XGBRegressor\n",
      "2023-07-14 23:29:55.004555 Evaluating model: XGBRegressor\n",
      "2023-07-14 23:29:55.020525 Logging: XGBRegressor\n",
      "2023-07-14 23:29:55.033610 Model XGBRegressor has been trained\n",
      "2023-07-14 23:29:55.103966 Training model based on configuration data: rf\n",
      "2023-07-14 23:29:55.103966 Creating model objects.\n",
      "2023-07-14 23:29:55.103966 Models object created\n",
      "2023-07-14 23:29:55.106965 Training model: KNeighborsRegressor\n",
      "2023-07-14 23:29:55.118284 Evaluating model: KNeighborsRegressor\n",
      "2023-07-14 23:29:55.979113 Logging: KNeighborsRegressor\n",
      "2023-07-14 23:29:55.982113 Model KNeighborsRegressor has been trained\n",
      "2023-07-14 23:29:55.982113 Training model: DecisionTreeRegressor\n",
      "2023-07-14 23:29:57.329010 Evaluating model: DecisionTreeRegressor\n",
      "2023-07-14 23:29:57.338341 Logging: DecisionTreeRegressor\n",
      "2023-07-14 23:29:57.343406 Model DecisionTreeRegressor has been trained\n",
      "2023-07-14 23:29:57.343406 Training model: LinearRegression\n",
      "2023-07-14 23:29:57.373774 Evaluating model: LinearRegression\n",
      "2023-07-14 23:29:57.378336 Logging: LinearRegression\n",
      "2023-07-14 23:29:57.378336 Model LinearRegression has been trained\n",
      "2023-07-14 23:29:57.378336 Training model: RandomForestRegressor\n",
      "2023-07-14 23:31:33.355917 Evaluating model: RandomForestRegressor\n",
      "2023-07-14 23:31:33.942870 Logging: RandomForestRegressor\n",
      "2023-07-14 23:31:34.555110 Model RandomForestRegressor has been trained\n",
      "2023-07-14 23:31:34.556133 Training model: AdaBoostRegressor\n",
      "2023-07-14 23:31:36.250586 Evaluating model: AdaBoostRegressor\n",
      "2023-07-14 23:31:36.266779 Logging: AdaBoostRegressor\n",
      "2023-07-14 23:31:36.267780 Model AdaBoostRegressor has been trained\n",
      "2023-07-14 23:31:36.268777 Training model: GradientBoostingRegressor\n",
      "2023-07-14 23:31:52.716509 Evaluating model: GradientBoostingRegressor\n",
      "2023-07-14 23:31:52.742726 Logging: GradientBoostingRegressor\n",
      "2023-07-14 23:31:52.752795 Model GradientBoostingRegressor has been trained\n",
      "2023-07-14 23:31:52.752795 Training model: XGBRegressor\n",
      "2023-07-14 23:31:55.922656 Evaluating model: XGBRegressor\n",
      "2023-07-14 23:31:55.943032 Logging: XGBRegressor\n",
      "2023-07-14 23:31:55.957077 Model XGBRegressor has been trained\n",
      "2023-07-14 23:31:56.429542 All combination models and data has been trained.\n"
     ]
    }
   ],
   "source": [
    "list_of_test_model, testing_log = train_eval(\"baseline\", config_data, \n",
    "                                             X_t=X_test, \n",
    "                                             y_t=y_test)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### <b>Show Best Performance Model </b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>model_name</th>\n",
       "      <th>training_time</th>\n",
       "      <th>mse</th>\n",
       "      <th>r2_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>baseline-lasso-RandomForestRegressor</td>\n",
       "      <td>29.038199</td>\n",
       "      <td>0.817855</td>\n",
       "      <td>0.999256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>baseline-filter-RandomForestRegressor</td>\n",
       "      <td>37.377706</td>\n",
       "      <td>0.962922</td>\n",
       "      <td>0.998969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>baseline-lasso-DecisionTreeRegressor</td>\n",
       "      <td>0.463786</td>\n",
       "      <td>0.968400</td>\n",
       "      <td>0.998957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>baseline-lasso-XGBRegressor</td>\n",
       "      <td>2.130459</td>\n",
       "      <td>1.115612</td>\n",
       "      <td>0.998616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>baseline-filter-XGBRegressor</td>\n",
       "      <td>3.241820</td>\n",
       "      <td>1.168490</td>\n",
       "      <td>0.998482</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               model_name  training_time       mse  r2_score\n",
       "10   baseline-lasso-RandomForestRegressor      29.038199  0.817855  0.999256\n",
       "3   baseline-filter-RandomForestRegressor      37.377706  0.962922  0.998969\n",
       "8    baseline-lasso-DecisionTreeRegressor       0.463786  0.968400  0.998957\n",
       "13            baseline-lasso-XGBRegressor       2.130459  1.115612  0.998616\n",
       "6            baseline-filter-XGBRegressor       3.241820  1.168490  0.998482"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(testing_log)[['model_name',\n",
    "                           'training_time',\n",
    "                           'mse',\n",
    "                           'r2_score']]\\\n",
    ".sort_values(['mse', 'r2_score', 'training_time'],\n",
    "             ascending=[True,True,True])\\\n",
    ".head(5)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <b> Check Final Model </b>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'model_data': {'model_name': 'RandomForestRegressor',\n",
       "  'model_object': RandomForestRegressor(),\n",
       "  'model_uid': '7fb91c0e0bf0ac7694a7dfbf32a743ae'},\n",
       " 'model_log': {'model_name': 'Filter-RandomForestRegressor',\n",
       "  'model_uid': '7fb91c0e0bf0ac7694a7dfbf32a743ae',\n",
       "  'training_time': 21.635677,\n",
       "  'training_date': '2023-07-14 22:49:32.996386',\n",
       "  'mse': 1.0089119124699573,\n",
       "  'r2_score': 0.9988799674630775,\n",
       "  'rmse': 1.0179032471237865}}"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "final_model = util.pickle_load(config_data[\"production_model_path\"])\n",
    "\n",
    "final_model"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "<b>Best Model</b><br>\n",
    "-----------\n",
    "\n",
    "Best model performance based on <b>validation data is <mark><i>Random Forest Regressor</i></mark> on <i>Filter Data Configuration</i>,</b><br>\n",
    "Show up with <b>MSE Score = 1.023</b> and <b>R2_Score = 0.998</b> <br>\n",
    "However, it defent on <b>training time: 47.97s</b>\n",
    "\n",
    "If you prefer more fast training time with nearly score, you can choose:\n",
    "<b><i>Random Forest Regressor</i> on <i>Lasso Data Configuration</i></b><br>"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----------"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}