rf-hypertune-2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c6db2cc3",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2024-01-24T07:43:54.284781Z",
     "iopub.status.busy": "2024-01-24T07:43:54.284041Z",
     "iopub.status.idle": "2024-01-24T07:43:56.076539Z",
     "shell.execute_reply": "2024-01-24T07:43:56.075604Z"
    },
    "papermill": {
     "duration": 1.800987,
     "end_time": "2024-01-24T07:43:56.079266",
     "exception": false,
     "start_time": "2024-01-24T07:43:54.278279",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, ShuffleSplit\n",
    "from sklearn.ensemble import RandomForestRegressor, ExtraTreesRegressor\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.multioutput import MultiOutputRegressor\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.neural_network import MLPRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ae82a176",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-24T07:43:56.088759Z",
     "iopub.status.busy": "2024-01-24T07:43:56.088270Z",
     "iopub.status.idle": "2024-01-24T07:43:56.092964Z",
     "shell.execute_reply": "2024-01-24T07:43:56.091793Z"
    },
    "papermill": {
     "duration": 0.011718,
     "end_time": "2024-01-24T07:43:56.095223",
     "exception": false,
     "start_time": "2024-01-24T07:43:56.083505",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "path_eval = \"/kaggle/input/dataprojet/evaluation.csv\"\n",
    "path_clean = \"/kaggle/input/df-no-out-norms/development_noNoise_noRMS_noOutliers.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "5dd45b39",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-24T07:43:56.104185Z",
     "iopub.status.busy": "2024-01-24T07:43:56.103826Z",
     "iopub.status.idle": "2024-01-24T07:43:56.112032Z",
     "shell.execute_reply": "2024-01-24T07:43:56.111169Z"
    },
    "papermill": {
     "duration": 0.014891,
     "end_time": "2024-01-24T07:43:56.114002",
     "exception": false,
     "start_time": "2024-01-24T07:43:56.099111",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def prediction(eval_path, results_path, model):\n",
    "    df_test = pd.read_csv(eval_path)\n",
    "    ## Remove noise columns\n",
    "    ix = 0,7,12,15,16,17\n",
    "    noise = [f\"rms[{i}]\" for i in range(18)]\n",
    "    noise += [f\"pmax[{i}]\" for i in ix]\n",
    "    noise += [f\"negpmax[{i}]\" for i in ix]\n",
    "    noise += [f\"tmax[{i}]\" for i in ix]\n",
    "    noise += [f\"area[{i}]\" for i in ix]\n",
    "    df_test = df_test.drop(columns = noise)\n",
    "    y_pred_eval = model.predict(df_test.drop(columns = [\"Id\"]))\n",
    "    df_test[[\"x\",\"y\"]] =  y_pred_eval\n",
    "    df_test['Predicted'] = df_test['x'].astype(str) + '|' + df_test['y'].astype(str)\n",
    "    df_test[['Id', 'Predicted']].to_csv(results_path, index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "bd4d8873",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-24T07:43:56.122775Z",
     "iopub.status.busy": "2024-01-24T07:43:56.122282Z",
     "iopub.status.idle": "2024-01-24T07:43:56.126856Z",
     "shell.execute_reply": "2024-01-24T07:43:56.126152Z"
    },
    "papermill": {
     "duration": 0.011057,
     "end_time": "2024-01-24T07:43:56.128740",
     "exception": false,
     "start_time": "2024-01-24T07:43:56.117683",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "## Metric to be used to evaluate the performances of models \n",
    "def distance(y_true, y_pred):\n",
    "    n = len(y_true)\n",
    "    d = np.sum(np.sqrt(np.sum((y_true - y_pred)**2, axis=1))) / n\n",
    "    return d\n",
    "scorer = make_scorer(distance)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "41ff8fb0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-24T07:43:56.137433Z",
     "iopub.status.busy": "2024-01-24T07:43:56.136827Z",
     "iopub.status.idle": "2024-01-24T07:43:56.146837Z",
     "shell.execute_reply": "2024-01-24T07:43:56.145946Z"
    },
    "papermill": {
     "duration": 0.016756,
     "end_time": "2024-01-24T07:43:56.148986",
     "exception": false,
     "start_time": "2024-01-24T07:43:56.132230",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.linear_model import Ridge\n",
    "def multi_pipeline_train(df_path, model, tuning_params=None, train_size=0.7):\n",
    "    # Load the data\n",
    "    df = pd.read_csv(df_path)\n",
    "    np.random.seed(124)\n",
    "    print(\"Start performing a hypeparameter Tunning for multi output regresor using RF\")\n",
    "    ## Split the dataset\n",
    "    X = df.drop(columns=[\"x\", \"y\"])\n",
    "    y = df[[\"x\", \"y\"]]\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X, y, train_size=train_size, random_state=124, shuffle = True)\n",
    "\n",
    "    print(\"Data splited\")\n",
    "    # Multi-output regressor without hyperparameter tuning\n",
    "    if tuning_params is None:\n",
    "        print(\"Trainig the base model\")\n",
    "        multi_output_reg = MultiOutputRegressor(model)\n",
    "        multi_output_reg.fit(X_train, y_train)\n",
    "        y_pred = multi_output_reg.predict(X_test)\n",
    "        distance_metric = distance(y_test, y_pred)\n",
    "        print(f\"Base Multi-output regressor using RF distance on test dataset: {distance_metric}\")\n",
    "        return multi_output_reg\n",
    "\n",
    "    # Multi-output regressor with hyperparameter tuning\n",
    "    else:\n",
    "        print(\"Hyperparamter Tunning Performing start\")\n",
    "        multi_output_reg = MultiOutputRegressor(model)\n",
    "        cvparam = ShuffleSplit(test_size=0.3, n_splits=1, random_state=124)\n",
    "        grid_search = GridSearchCV(estimator=multi_output_reg, param_grid=tuning_params, cv=cvparam,\n",
    "                                   scoring=scorer, n_jobs=-1, verbose = 4)\n",
    "        grid_search.fit(X, y)\n",
    "        best_model = grid_search.best_estimator_\n",
    "        #distance_metric_tuned = distance(y_test, y_pred_best)\n",
    "       # print(f\"Tuned Multi-output regressor using RF distance on test dataset: {distance_metric_tuned}\")\n",
    "        print(\"Best Hyperparameters found:\", grid_search.best_params_)\n",
    "        return best_model, grid_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f1ba4cc2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-24T07:43:56.157452Z",
     "iopub.status.busy": "2024-01-24T07:43:56.157087Z",
     "iopub.status.idle": "2024-01-24T08:37:38.424674Z",
     "shell.execute_reply": "2024-01-24T08:37:38.423555Z"
    },
    "papermill": {
     "duration": 3222.278264,
     "end_time": "2024-01-24T08:37:38.430856",
     "exception": false,
     "start_time": "2024-01-24T07:43:56.152592",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start performing a hypeparameter Tunning for multi output regresor using RF\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data splited\n",
      "Trainig the base model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Base Multi-output regressor using RF distance on test dataset: 4.689793757257982\n"
     ]
    }
   ],
   "source": [
    "rf_regressor = RandomForestRegressor(random_state = 42)\n",
    "model_base = multi_pipeline_train(path_clean, rf_regressor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "39dbbc61",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-24T08:37:38.441549Z",
     "iopub.status.busy": "2024-01-24T08:37:38.440610Z",
     "iopub.status.idle": "2024-01-24T08:37:55.505615Z",
     "shell.execute_reply": "2024-01-24T08:37:55.504436Z"
    },
    "papermill": {
     "duration": 17.072995,
     "end_time": "2024-01-24T08:37:55.508137",
     "exception": false,
     "start_time": "2024-01-24T08:37:38.435142",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Base model evaluation saved on /kaggle/working/rf_base_eval.csv\n"
     ]
    }
   ],
   "source": [
    "result_path = \"/kaggle/working/rf_base_eval.csv\"\n",
    "prediction(path_eval, result_path, model_base)\n",
    "print(\"Base model evaluation saved on\", result_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6f688c5b",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-24T08:37:55.518877Z",
     "iopub.status.busy": "2024-01-24T08:37:55.518470Z"
    },
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": false,
     "start_time": "2024-01-24T08:37:55.512624",
     "status": "running"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start performing a hypeparameter Tunning for multi output regresor using RF\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data splited\n",
      "Hyperparamter Tunning Performing start\n",
      "Fitting 1 folds for each of 9 candidates, totalling 9 fits\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/1] END estimator__max_depth=None, estimator__n_estimators=300;, score=4.658 total time=244.8min\n",
      "[CV 1/1] END estimator__max_depth=20, estimator__n_estimators=200;, score=4.718 total time=163.8min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/joblib/externals/loky/process_executor.py:752: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/1] END estimator__max_depth=None, estimator__n_estimators=500;, score=4.651 total time=410.5min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/1] END estimator__max_depth=None, estimator__n_estimators=200;, score=4.667 total time=164.9min\n",
      "[CV 1/1] END estimator__max_depth=10, estimator__n_estimators=500;, score=8.673 total time=267.8min\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/1] END estimator__max_depth=10, estimator__n_estimators=200;, score=8.680 total time=109.5min\n",
      "[CV 1/1] END estimator__max_depth=10, estimator__n_estimators=300;, score=8.677 total time=161.6min\n",
      "[CV 1/1] END estimator__max_depth=20, estimator__n_estimators=300;, score=4.710 total time=219.5min\n"
     ]
    }
   ],
   "source": [
    "rf_param_grid = {\n",
    "    'estimator__n_estimators': [200, 300, 500],\n",
    "    'estimator__max_depth': [None, 10, 20],\n",
    "}\n",
    "\n",
    "best_model_rf, hyperparameters_rf = multi_pipeline_train(path_clean, rf_regressor, tuning_params=rf_param_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89aac02c",
   "metadata": {
    "papermill": {
     "duration": null,
     "end_time": null,
     "exception": null,
     "start_time": null,
     "status": "pending"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "result_path = \"/kaggle/working/rf_best_eval.csv\"\n",
    "prediction(path_eval, result_path, best_model_rf)\n",
    "print(\"Base model evaluation saved on\", result_path)"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [
    {
     "datasetId": 4296863,
     "sourceId": 7391419,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 4326069,
     "sourceId": 7433727,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 4327700,
     "sourceId": 7436028,
     "sourceType": "datasetVersion"
    }
   ],
   "dockerImageVersionId": 30635,
   "isGpuEnabled": false,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": null,
   "end_time": null,
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-01-24T07:43:51.080549",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}