mlp-hypertun-3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0070770e",
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5",
    "execution": {
     "iopub.execute_input": "2024-01-25T15:57:12.357779Z",
     "iopub.status.busy": "2024-01-25T15:57:12.357301Z",
     "iopub.status.idle": "2024-01-25T15:57:14.437888Z",
     "shell.execute_reply": "2024-01-25T15:57:14.436578Z"
    },
    "papermill": {
     "duration": 2.090432,
     "end_time": "2024-01-25T15:57:14.440985",
     "exception": false,
     "start_time": "2024-01-25T15:57:12.350553",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, ShuffleSplit\n",
    "from sklearn.neighbors import KNeighborsRegressor\n",
    "from sklearn.multioutput import MultiOutputRegressor\n",
    "from sklearn.metrics import make_scorer\n",
    "from sklearn.neural_network import MLPRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "0151be47",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-25T15:57:14.452477Z",
     "iopub.status.busy": "2024-01-25T15:57:14.451097Z",
     "iopub.status.idle": "2024-01-25T15:57:14.457541Z",
     "shell.execute_reply": "2024-01-25T15:57:14.456162Z"
    },
    "papermill": {
     "duration": 0.014711,
     "end_time": "2024-01-25T15:57:14.460181",
     "exception": false,
     "start_time": "2024-01-25T15:57:14.445470",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "path_eval = \"/kaggle/input/dataprojet/evaluation.csv\"\n",
    "path_clean = \"/kaggle/input/df-no-out-norms/development_noNoise_noRMS_noOutliers.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d0c01324",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-25T15:57:14.470593Z",
     "iopub.status.busy": "2024-01-25T15:57:14.470133Z",
     "iopub.status.idle": "2024-01-25T15:57:14.481320Z",
     "shell.execute_reply": "2024-01-25T15:57:14.479987Z"
    },
    "papermill": {
     "duration": 0.019507,
     "end_time": "2024-01-25T15:57:14.483895",
     "exception": false,
     "start_time": "2024-01-25T15:57:14.464388",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.preprocessing import MinMaxScaler\n",
    "\n",
    "def prediction(eval_path, results_path, model):\n",
    "    df_test = pd.read_csv(eval_path)\n",
    "\n",
    "    # Remove noise columns\n",
    "    ix = [0, 7, 12, 15, 16, 17]\n",
    "    noise = [f\"rms[{i}]\" for i in range(18)]\n",
    "    noise += [f\"pmax[{i}]\" for i in ix]\n",
    "    noise += [f\"negpmax[{i}]\" for i in ix]\n",
    "    noise += [f\"tmax[{i}]\" for i in ix]\n",
    "    noise += [f\"area[{i}]\" for i in ix]\n",
    "    df_test = df_test.drop(columns=noise)\n",
    "\n",
    "    X_eval = df_test.drop(columns=[\"Id\"])\n",
    "    scaler = MinMaxScaler()\n",
    "    X_eval_scaled = scaler.fit_transform(X_eval)\n",
    "    y_pred_eval = model.predict(X_eval_scaled)\n",
    "    df_test[[\"x\", \"y\"]] = y_pred_eval\n",
    "    df_test['Predicted'] = df_test['x'].astype(str) + '|' + df_test['y'].astype(str)\n",
    "    df_test[['Id', 'Predicted']].to_csv(results_path, index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "047d9eda",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-25T15:57:14.494283Z",
     "iopub.status.busy": "2024-01-25T15:57:14.493786Z",
     "iopub.status.idle": "2024-01-25T15:57:14.500607Z",
     "shell.execute_reply": "2024-01-25T15:57:14.499344Z"
    },
    "papermill": {
     "duration": 0.015196,
     "end_time": "2024-01-25T15:57:14.503417",
     "exception": false,
     "start_time": "2024-01-25T15:57:14.488221",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "## Metric to be used to evaluate the performances of models \n",
    "def distance(y_true, y_pred):\n",
    "    n = len(y_true)\n",
    "    d = np.sum(np.sqrt(np.sum((y_true - y_pred)**2, axis=1))) / n\n",
    "    return d\n",
    "scorer = make_scorer(distance, greater_is_better=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "25ad4f8c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-25T15:57:14.513821Z",
     "iopub.status.busy": "2024-01-25T15:57:14.513359Z",
     "iopub.status.idle": "2024-01-25T15:57:14.526022Z",
     "shell.execute_reply": "2024-01-25T15:57:14.524961Z"
    },
    "papermill": {
     "duration": 0.020694,
     "end_time": "2024-01-25T15:57:14.528367",
     "exception": false,
     "start_time": "2024-01-25T15:57:14.507673",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def multi_pipeline_train(df_path, model, tuning_params=None, train_size=0.7):\n",
    "    # Load the data\n",
    "    df = pd.read_csv(df_path)\n",
    "    np.random.seed(124)\n",
    "    print(\"Start performing hyperparameter tuning for Multi-Output Regressor using MLP Model\")\n",
    "    \n",
    "    # Split the dataset\n",
    "    X = df.drop(columns=[\"x\", \"y\"])\n",
    "    y = df[[\"x\", \"y\"]]\n",
    "    scaler = MinMaxScaler()\n",
    "    X_scaled = scaler.fit_transform(X)\n",
    "\n",
    "    X_train, X_test, y_train, y_test = train_test_split(X_scaled, y, train_size=train_size, random_state=124, shuffle=True)\n",
    "\n",
    "    print(\"Data split\")\n",
    "\n",
    "    # Multi-output regressor without hyperparameter tuning\n",
    "    if tuning_params is None:\n",
    "        print(\"Training the base model\")\n",
    "        multi_output_reg = MultiOutputRegressor(model)\n",
    "        multi_output_reg.fit(X_train, y_train)\n",
    "        y_pred = multi_output_reg.predict(X_test)\n",
    "        # Assuming 'distance' function calculates distance metric\n",
    "        distance_metric = distance(y_test, y_pred)\n",
    "        print(f\"Base Multi-Output Regressor using MLP distance on test dataset: {distance_metric}\")\n",
    "        return multi_output_reg\n",
    "\n",
    "    # Multi-output regressor with hyperparameter tuning\n",
    "    else:\n",
    "        print(\"Hyperparameter Tuning Performeing start\")\n",
    "        multi_output_reg = MultiOutputRegressor(model)\n",
    "        cvparam = ShuffleSplit(test_size=0.3, n_splits=1, random_state=0)\n",
    "        grid_search = GridSearchCV(estimator=multi_output_reg, param_grid=tuning_params, cv=cvparam,\n",
    "                                   scoring=scorer, n_jobs=-1, verbose=4)\n",
    "        grid_search.fit(X, y)\n",
    "        best_model = grid_search.best_estimator_\n",
    "        #y_pred_best = best_model.predict(X_test)\n",
    "        # Assuming 'distance' function calculates distance metric\n",
    "        #distance_metric_tuned = distance(y_test, y_pred_best)\n",
    "        #print(f\"Tuned Multi-Output Regressor using MLP distance on test dataset: {distance_metric_tuned}\")\n",
    "        print(\"Best Hyperparameters found:\", grid_search.best_params_)\n",
    "        return best_model, grid_search.best_params_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "5e26687f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-25T15:57:14.539002Z",
     "iopub.status.busy": "2024-01-25T15:57:14.538575Z",
     "iopub.status.idle": "2024-01-25T15:57:14.543496Z",
     "shell.execute_reply": "2024-01-25T15:57:14.542157Z"
    },
    "papermill": {
     "duration": 0.013171,
     "end_time": "2024-01-25T15:57:14.545957",
     "exception": false,
     "start_time": "2024-01-25T15:57:14.532786",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "mlp_regressor = MLPRegressor()\n",
    "#model_base_mlp = multi_pipeline_train(path_clean, mlp_regressor)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8d9cf8f7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-25T15:57:14.556208Z",
     "iopub.status.busy": "2024-01-25T15:57:14.555715Z",
     "iopub.status.idle": "2024-01-25T15:57:14.560913Z",
     "shell.execute_reply": "2024-01-25T15:57:14.559766Z"
    },
    "papermill": {
     "duration": 0.013169,
     "end_time": "2024-01-25T15:57:14.563376",
     "exception": false,
     "start_time": "2024-01-25T15:57:14.550207",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "#result_path = \"/kaggle/working/mlp_base_eval.csv\"\n",
    "#prediction(path_eval, result_path, model_base_mlp)\n",
    "#print(\"Base model evaluation saved on\", result_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "963049f1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-25T15:57:14.573671Z",
     "iopub.status.busy": "2024-01-25T15:57:14.573185Z",
     "iopub.status.idle": "2024-01-25T20:03:37.423908Z",
     "shell.execute_reply": "2024-01-25T20:03:37.421592Z"
    },
    "papermill": {
     "duration": 14782.880331,
     "end_time": "2024-01-25T20:03:37.447943",
     "exception": false,
     "start_time": "2024-01-25T15:57:14.567612",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start performing hyperparameter tuning for Multi-Output Regressor using MLP Model\n",
      "Data split\n",
      "Hyperparameter Tuning Performeing start\n",
      "Fitting 1 folds for each of 9 candidates, totalling 9 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:686: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:686: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:686: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (300) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:686: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (300) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:686: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (300) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:686: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (200) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/1] END estimator__alpha=0.0001, estimator__max_iter=300;, score=-6.188 total time=55.0min\n",
      "[CV 1/1] END estimator__alpha=0.01, estimator__max_iter=200;, score=-6.127 total time=28.3min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:686: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (300) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/1] END estimator__alpha=0.0001, estimator__max_iter=200;, score=-6.109 total time=26.6min\n",
      "[CV 1/1] END estimator__alpha=0.001, estimator__max_iter=500;, score=-6.027 total time=61.7min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:686: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (300) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/1] END estimator__alpha=0.001, estimator__max_iter=200;, score=-6.248 total time=20.4min\n",
      "[CV 1/1] END estimator__alpha=0.001, estimator__max_iter=300;, score=-5.898 total time=42.4min\n",
      "[CV 1/1] END estimator__alpha=0.01, estimator__max_iter=300;, score=-6.016 total time=45.2min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/sklearn/neural_network/_multilayer_perceptron.py:686: ConvergenceWarning: Stochastic Optimizer: Maximum iterations (500) reached and the optimization hasn't converged yet.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CV 1/1] END estimator__alpha=0.0001, estimator__max_iter=500;, score=-5.695 total time=70.9min\n",
      "[CV 1/1] END estimator__alpha=0.01, estimator__max_iter=500;, score=-5.925 total time=73.7min\n",
      "Best Hyperparameters found: {'estimator__alpha': 0.0001, 'estimator__max_iter': 500}\n"
     ]
    }
   ],
   "source": [
    "mlp_param_grid = {\n",
    "    'estimator__alpha': [0.0001, 0.001, 0.01],\n",
    "    'estimator__max_iter': [200, 300, 500]\n",
    "}\n",
    "best_model_mlp, hyperparameters_mlp = multi_pipeline_train(path_clean, MLPRegressor(), tuning_params=mlp_param_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c175dead",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-01-25T20:03:37.479816Z",
     "iopub.status.busy": "2024-01-25T20:03:37.478005Z",
     "iopub.status.idle": "2024-01-25T20:03:46.341378Z",
     "shell.execute_reply": "2024-01-25T20:03:46.340049Z"
    },
    "papermill": {
     "duration": 8.883792,
     "end_time": "2024-01-25T20:03:46.344223",
     "exception": false,
     "start_time": "2024-01-25T20:03:37.460431",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.10/site-packages/sklearn/base.py:439: UserWarning: X does not have valid feature names, but MLPRegressor was fitted with feature names\n",
      "  warnings.warn(\n",
      "/opt/conda/lib/python3.10/site-packages/sklearn/base.py:439: UserWarning: X does not have valid feature names, but MLPRegressor was fitted with feature names\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Base model evaluation saved on /kaggle/working/mlp_best_eval.csv\n"
     ]
    }
   ],
   "source": [
    "result_path = \"/kaggle/working/mlp_best_eval.csv\"\n",
    "prediction(path_eval, result_path, best_model_mlp)\n",
    "print(\"Base model evaluation saved on\", result_path)"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "none",
   "dataSources": [
    {
     "datasetId": 4296863,
     "sourceId": 7391419,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 4326069,
     "sourceId": 7433727,
     "sourceType": "datasetVersion"
    },
    {
     "datasetId": 4327700,
     "sourceId": 7436028,
     "sourceType": "datasetVersion"
    }
   ],
   "dockerImageVersionId": 30635,
   "isGpuEnabled": false,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 14800.314846,
   "end_time": "2024-01-25T20:03:49.021642",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-01-25T15:57:08.706796",
   "version": "2.4.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}