notebooks/3_tp_training_ranking_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "notebook_start_time = time.time()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Set up environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "⛳️ Local environment\n",
      "Adding the following directory to the PYTHONPATH: /Users/pauliusztin/Documents/01_projects/hopsworks_recsys/hands-on-recommender-system\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "from pathlib import Path\n",
    "\n",
    "\n",
    "def is_google_colab() -> bool:\n",
    "    if \"google.colab\" in str(get_ipython()):\n",
    "        return True\n",
    "    return False\n",
    "\n",
    "\n",
    "def clone_repository() -> None:\n",
    "    !git clone https://github.com/decodingml/hands-on-recommender-system.git\n",
    "    %cd hands-on-recommender-system/\n",
    "\n",
    "\n",
    "def install_dependencies() -> None:\n",
    "    !pip install --upgrade uv\n",
    "    !uv pip install --all-extras --system --requirement pyproject.toml\n",
    "\n",
    "\n",
    "if is_google_colab():\n",
    "    clone_repository()\n",
    "    install_dependencies()\n",
    "\n",
    "    root_dir = str(Path().absolute())\n",
    "    print(\"⛳️ Google Colab environment\")\n",
    "else:\n",
    "    root_dir = str(Path().absolute().parent)\n",
    "    print(\"⛳️ Local environment\")\n",
    "\n",
    "# Add the root directory to the `PYTHONPATH` to use the `recsys` Python module from the notebook.\n",
    "if root_dir not in sys.path:\n",
    "    print(f\"Adding the following directory to the PYTHONPATH: {root_dir}\")\n",
    "    sys.path.append(root_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 🧬 Training pipeline: Training ranking model </span>\n",
    "\n",
    "In this notebook, you will train a ranking model using gradient boosted trees. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 📝 Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2\n",
    "\n",
    "import warnings\n",
    "\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "from loguru import logger\n",
    "\n",
    "from recsys import hopsworks_integration, training\n",
    "from recsys.config import settings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Constants"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'CUSTOMER_DATA_SIZE': <CustomerDatasetSize.SMALL: 'SMALL'>,\n",
      " 'CUSTOM_HOPSWORKS_INFERENCE_ENV': 'custom_env_name',\n",
      " 'FEATURES_EMBEDDING_MODEL_ID': 'all-MiniLM-L6-v2',\n",
      " 'HOPSWORKS_API_KEY': SecretStr('**********'),\n",
      " 'OPENAI_API_KEY': SecretStr('**********'),\n",
      " 'OPENAI_MODEL_ID': 'gpt-4o-mini',\n",
      " 'RANKING_DATASET_VALIDATON_SPLIT_SIZE': 0.1,\n",
      " 'RANKING_EARLY_STOPPING_ROUNDS': 5,\n",
      " 'RANKING_ITERATIONS': 100,\n",
      " 'RANKING_LEARNING_RATE': 0.2,\n",
      " 'RANKING_MODEL_TYPE': 'ranking',\n",
      " 'RANKING_SCALE_POS_WEIGHT': 10,\n",
      " 'RECSYS_DIR': PosixPath('/Users/pauliusztin/Documents/01_projects/hopsworks_recsys/hands-on-recommender-system/recsys'),\n",
      " 'TWO_TOWER_DATASET_TEST_SPLIT_SIZE': 0.1,\n",
      " 'TWO_TOWER_DATASET_VALIDATON_SPLIT_SIZE': 0.1,\n",
      " 'TWO_TOWER_LEARNING_RATE': 0.01,\n",
      " 'TWO_TOWER_MODEL_BATCH_SIZE': 2048,\n",
      " 'TWO_TOWER_MODEL_EMBEDDING_SIZE': 16,\n",
      " 'TWO_TOWER_NUM_EPOCHS': 10,\n",
      " 'TWO_TOWER_WEIGHT_DECAY': 0.001}\n"
     ]
    }
   ],
   "source": [
    "from pprint import pprint\n",
    "\n",
    "pprint(dict(settings))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <span style=\"color:#ff5f27\">🔮 Connect to Hopsworks Feature Store </span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2024-12-24 13:02:45.188\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mrecsys.hopsworks_integration.feature_store\u001b[0m:\u001b[36mget_feature_store\u001b[0m:\u001b[36m13\u001b[0m - \u001b[1mLoging to Hopsworks using HOPSWORKS_API_KEY env var.\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2024-12-24 13:02:45,189 INFO: Initializing external client\n",
      "2024-12-24 13:02:45,189 INFO: Base URL: https://c.app.hopsworks.ai:443\n",
      "2024-12-24 13:02:46,594 INFO: Python Engine initialized.\n",
      "\n",
      "Logged in to project, explore it here https://c.app.hopsworks.ai:443/p/1192098\n"
     ]
    }
   ],
   "source": [
    "project, fs = hopsworks_integration.get_feature_store()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting the training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_view_ranking = hopsworks_integration.feature_store.create_ranking_feature_views(\n",
    "    fs\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished: Reading data from Hopsworks, using Hopsworks Feature Query Service (5.54s) \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "VersionWarning: Incremented version to `2`.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age</th>\n",
       "      <th>product_type_name</th>\n",
       "      <th>product_group_name</th>\n",
       "      <th>graphical_appearance_name</th>\n",
       "      <th>colour_group_name</th>\n",
       "      <th>perceived_colour_value_name</th>\n",
       "      <th>perceived_colour_master_name</th>\n",
       "      <th>department_name</th>\n",
       "      <th>index_name</th>\n",
       "      <th>index_group_name</th>\n",
       "      <th>section_name</th>\n",
       "      <th>garment_group_name</th>\n",
       "      <th>month_sin</th>\n",
       "      <th>month_cos</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>23.0</td>\n",
       "      <td>Leggings/Tights</td>\n",
       "      <td>Garment Lower body</td>\n",
       "      <td>Check</td>\n",
       "      <td>Grey</td>\n",
       "      <td>Medium Dusty</td>\n",
       "      <td>Grey</td>\n",
       "      <td>Jersey fancy</td>\n",
       "      <td>Ladieswear</td>\n",
       "      <td>Ladieswear</td>\n",
       "      <td>Womens Everyday Collection</td>\n",
       "      <td>Jersey Fancy</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>-8.660254e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>50.0</td>\n",
       "      <td>Dress</td>\n",
       "      <td>Garment Full body</td>\n",
       "      <td>Solid</td>\n",
       "      <td>Dark Beige</td>\n",
       "      <td>Dark</td>\n",
       "      <td>Beige</td>\n",
       "      <td>Knitwear</td>\n",
       "      <td>Ladieswear</td>\n",
       "      <td>Ladieswear</td>\n",
       "      <td>Womens Everyday Collection</td>\n",
       "      <td>Knitwear</td>\n",
       "      <td>-0.5</td>\n",
       "      <td>8.660254e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>26.0</td>\n",
       "      <td>Top</td>\n",
       "      <td>Garment Upper body</td>\n",
       "      <td>Solid</td>\n",
       "      <td>White</td>\n",
       "      <td>Light</td>\n",
       "      <td>White</td>\n",
       "      <td>Jersey</td>\n",
       "      <td>Ladieswear</td>\n",
       "      <td>Ladieswear</td>\n",
       "      <td>Womens Tailoring</td>\n",
       "      <td>Jersey Fancy</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>-1.836970e-16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    age product_type_name  product_group_name graphical_appearance_name  \\\n",
       "0  23.0   Leggings/Tights  Garment Lower body                     Check   \n",
       "1  50.0             Dress   Garment Full body                     Solid   \n",
       "2  26.0               Top  Garment Upper body                     Solid   \n",
       "\n",
       "  colour_group_name perceived_colour_value_name perceived_colour_master_name  \\\n",
       "0              Grey                Medium Dusty                         Grey   \n",
       "1        Dark Beige                        Dark                        Beige   \n",
       "2             White                       Light                        White   \n",
       "\n",
       "  department_name  index_name index_group_name                section_name  \\\n",
       "0    Jersey fancy  Ladieswear       Ladieswear  Womens Everyday Collection   \n",
       "1        Knitwear  Ladieswear       Ladieswear  Womens Everyday Collection   \n",
       "2          Jersey  Ladieswear       Ladieswear            Womens Tailoring   \n",
       "\n",
       "  garment_group_name  month_sin     month_cos  \n",
       "0       Jersey Fancy       -0.5 -8.660254e-01  \n",
       "1           Knitwear       -0.5  8.660254e-01  \n",
       "2       Jersey Fancy       -1.0 -1.836970e-16  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train, X_val, y_train, y_val = feature_view_ranking.train_test_split(\n",
    "    test_size=settings.RANKING_DATASET_VALIDATON_SPLIT_SIZE,\n",
    "    description=\"Ranking training dataset\",\n",
    ")\n",
    "X_train.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   label\n",
       "0      1\n",
       "1      1\n",
       "2      1"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train.head(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Training the ranking model\n",
    "\n",
    "Let's train the ranking model:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "model = training.ranking.RankingModelFactory.build()\n",
    "trainer = training.ranking.RankingModelTrainer(\n",
    "    model=model, train_dataset=(X_train, y_train), eval_dataset=(X_val, y_val)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0:\tlearn: 0.5142740\ttest: 0.5142148\tbest: 0.5142148 (0)\ttotal: 93.3ms\tremaining: 9.23s\n",
      "1:\tlearn: 0.3941888\ttest: 0.3940829\tbest: 0.3940829 (1)\ttotal: 134ms\tremaining: 6.55s\n",
      "2:\tlearn: 0.3081133\ttest: 0.3079641\tbest: 0.3079641 (2)\ttotal: 148ms\tremaining: 4.8s\n",
      "3:\tlearn: 0.2441452\ttest: 0.2439562\tbest: 0.2439562 (3)\ttotal: 190ms\tremaining: 4.56s\n",
      "4:\tlearn: 0.1954138\ttest: 0.1951864\tbest: 0.1951864 (4)\ttotal: 218ms\tremaining: 4.13s\n",
      "5:\tlearn: 0.1576688\ttest: 0.1574069\tbest: 0.1574069 (5)\ttotal: 232ms\tremaining: 3.63s\n",
      "6:\tlearn: 0.1281129\ttest: 0.1278174\tbest: 0.1278174 (6)\ttotal: 259ms\tremaining: 3.45s\n",
      "7:\tlearn: 0.1047526\ttest: 0.1044253\tbest: 0.1044253 (7)\ttotal: 273ms\tremaining: 3.14s\n",
      "8:\tlearn: 0.0861819\ttest: 0.0858238\tbest: 0.0858238 (8)\ttotal: 288ms\tremaining: 2.91s\n",
      "9:\tlearn: 0.0714014\ttest: 0.0710115\tbest: 0.0710115 (9)\ttotal: 336ms\tremaining: 3.03s\n",
      "10:\tlearn: 0.0595149\ttest: 0.0590966\tbest: 0.0590966 (10)\ttotal: 365ms\tremaining: 2.95s\n",
      "11:\tlearn: 0.0499656\ttest: 0.0495185\tbest: 0.0495185 (11)\ttotal: 393ms\tremaining: 2.88s\n",
      "12:\tlearn: 0.0422800\ttest: 0.0418059\tbest: 0.0418059 (12)\ttotal: 407ms\tremaining: 2.73s\n",
      "13:\tlearn: 0.0360981\ttest: 0.0355975\tbest: 0.0355975 (13)\ttotal: 436ms\tremaining: 2.68s\n",
      "14:\tlearn: 0.0311194\ttest: 0.0305938\tbest: 0.0305938 (14)\ttotal: 459ms\tremaining: 2.6s\n",
      "15:\tlearn: 0.0271224\ttest: 0.0265726\tbest: 0.0265726 (15)\ttotal: 498ms\tremaining: 2.62s\n",
      "16:\tlearn: 0.0239110\ttest: 0.0233345\tbest: 0.0233345 (16)\ttotal: 534ms\tremaining: 2.6s\n",
      "17:\tlearn: 0.0213093\ttest: 0.0207109\tbest: 0.0207109 (17)\ttotal: 552ms\tremaining: 2.52s\n",
      "18:\tlearn: 0.0192203\ttest: 0.0186003\tbest: 0.0186003 (18)\ttotal: 576ms\tremaining: 2.46s\n",
      "19:\tlearn: 0.0175479\ttest: 0.0169074\tbest: 0.0169074 (19)\ttotal: 604ms\tremaining: 2.42s\n",
      "20:\tlearn: 0.0162033\ttest: 0.0155451\tbest: 0.0155451 (20)\ttotal: 623ms\tremaining: 2.34s\n",
      "21:\tlearn: 0.0151326\ttest: 0.0144588\tbest: 0.0144588 (21)\ttotal: 663ms\tremaining: 2.35s\n",
      "22:\tlearn: 0.0142733\ttest: 0.0135859\tbest: 0.0135859 (22)\ttotal: 689ms\tremaining: 2.31s\n",
      "23:\tlearn: 0.0135849\ttest: 0.0128867\tbest: 0.0128867 (23)\ttotal: 729ms\tremaining: 2.31s\n",
      "24:\tlearn: 0.0130384\ttest: 0.0123307\tbest: 0.0123307 (24)\ttotal: 758ms\tremaining: 2.27s\n",
      "25:\tlearn: 0.0125961\ttest: 0.0118788\tbest: 0.0118788 (25)\ttotal: 800ms\tremaining: 2.28s\n",
      "26:\tlearn: 0.0122379\ttest: 0.0115119\tbest: 0.0115119 (26)\ttotal: 835ms\tremaining: 2.26s\n",
      "27:\tlearn: 0.0119509\ttest: 0.0112170\tbest: 0.0112170 (27)\ttotal: 858ms\tremaining: 2.21s\n",
      "28:\tlearn: 0.0117196\ttest: 0.0109794\tbest: 0.0109794 (28)\ttotal: 888ms\tremaining: 2.17s\n",
      "29:\tlearn: 0.0115791\ttest: 0.0108520\tbest: 0.0108520 (29)\ttotal: 931ms\tremaining: 2.17s\n",
      "30:\tlearn: 0.0114187\ttest: 0.0106866\tbest: 0.0106866 (30)\ttotal: 959ms\tremaining: 2.13s\n",
      "31:\tlearn: 0.0112893\ttest: 0.0105529\tbest: 0.0105529 (31)\ttotal: 980ms\tremaining: 2.08s\n",
      "32:\tlearn: 0.0112310\ttest: 0.0105056\tbest: 0.0105056 (32)\ttotal: 1.03s\tremaining: 2.09s\n",
      "33:\tlearn: 0.0111381\ttest: 0.0104098\tbest: 0.0104098 (33)\ttotal: 1.05s\tremaining: 2.04s\n",
      "34:\tlearn: 0.0110626\ttest: 0.0103347\tbest: 0.0103347 (34)\ttotal: 1.07s\tremaining: 1.99s\n",
      "35:\tlearn: 0.0109976\ttest: 0.0102764\tbest: 0.0102764 (35)\ttotal: 1.11s\tremaining: 1.98s\n",
      "36:\tlearn: 0.0109955\ttest: 0.0102758\tbest: 0.0102758 (36)\ttotal: 1.14s\tremaining: 1.95s\n",
      "37:\tlearn: 0.0109625\ttest: 0.0102348\tbest: 0.0102348 (37)\ttotal: 1.16s\tremaining: 1.89s\n",
      "38:\tlearn: 0.0109149\ttest: 0.0101833\tbest: 0.0101833 (38)\ttotal: 1.19s\tremaining: 1.86s\n",
      "39:\tlearn: 0.0108808\ttest: 0.0101482\tbest: 0.0101482 (39)\ttotal: 1.23s\tremaining: 1.85s\n",
      "40:\tlearn: 0.0108672\ttest: 0.0101325\tbest: 0.0101325 (40)\ttotal: 1.26s\tremaining: 1.81s\n",
      "41:\tlearn: 0.0108322\ttest: 0.0101076\tbest: 0.0101076 (41)\ttotal: 1.3s\tremaining: 1.8s\n",
      "42:\tlearn: 0.0108076\ttest: 0.0100813\tbest: 0.0100813 (42)\ttotal: 1.33s\tremaining: 1.77s\n",
      "43:\tlearn: 0.0107862\ttest: 0.0100580\tbest: 0.0100580 (43)\ttotal: 1.38s\tremaining: 1.76s\n",
      "44:\tlearn: 0.0107691\ttest: 0.0100401\tbest: 0.0100401 (44)\ttotal: 1.41s\tremaining: 1.73s\n",
      "45:\tlearn: 0.0107651\ttest: 0.0100323\tbest: 0.0100323 (45)\ttotal: 1.44s\tremaining: 1.69s\n",
      "46:\tlearn: 0.0107646\ttest: 0.0100339\tbest: 0.0100323 (45)\ttotal: 1.48s\tremaining: 1.67s\n",
      "47:\tlearn: 0.0107620\ttest: 0.0100366\tbest: 0.0100323 (45)\ttotal: 1.5s\tremaining: 1.63s\n",
      "48:\tlearn: 0.0107620\ttest: 0.0100369\tbest: 0.0100323 (45)\ttotal: 1.52s\tremaining: 1.59s\n",
      "49:\tlearn: 0.0107610\ttest: 0.0100396\tbest: 0.0100323 (45)\ttotal: 1.56s\tremaining: 1.56s\n",
      "50:\tlearn: 0.0107477\ttest: 0.0100254\tbest: 0.0100254 (50)\ttotal: 1.57s\tremaining: 1.51s\n",
      "51:\tlearn: 0.0107402\ttest: 0.0100245\tbest: 0.0100245 (51)\ttotal: 1.6s\tremaining: 1.48s\n",
      "52:\tlearn: 0.0107390\ttest: 0.0100238\tbest: 0.0100238 (52)\ttotal: 1.63s\tremaining: 1.45s\n",
      "53:\tlearn: 0.0107271\ttest: 0.0100240\tbest: 0.0100238 (52)\ttotal: 1.68s\tremaining: 1.43s\n",
      "54:\tlearn: 0.0107263\ttest: 0.0100252\tbest: 0.0100238 (52)\ttotal: 1.71s\tremaining: 1.4s\n",
      "55:\tlearn: 0.0107162\ttest: 0.0100140\tbest: 0.0100140 (55)\ttotal: 1.75s\tremaining: 1.37s\n",
      "56:\tlearn: 0.0107097\ttest: 0.0100052\tbest: 0.0100052 (56)\ttotal: 1.79s\tremaining: 1.35s\n",
      "57:\tlearn: 0.0107096\ttest: 0.0100051\tbest: 0.0100051 (57)\ttotal: 1.83s\tremaining: 1.32s\n",
      "58:\tlearn: 0.0106994\ttest: 0.0099976\tbest: 0.0099976 (58)\ttotal: 1.87s\tremaining: 1.3s\n",
      "59:\tlearn: 0.0106972\ttest: 0.0099970\tbest: 0.0099970 (59)\ttotal: 1.91s\tremaining: 1.27s\n",
      "60:\tlearn: 0.0106911\ttest: 0.0099903\tbest: 0.0099903 (60)\ttotal: 1.94s\tremaining: 1.24s\n",
      "61:\tlearn: 0.0106911\ttest: 0.0099905\tbest: 0.0099903 (60)\ttotal: 1.96s\tremaining: 1.2s\n",
      "62:\tlearn: 0.0106908\ttest: 0.0099900\tbest: 0.0099900 (62)\ttotal: 1.98s\tremaining: 1.16s\n",
      "63:\tlearn: 0.0106908\ttest: 0.0099901\tbest: 0.0099900 (62)\ttotal: 2s\tremaining: 1.13s\n",
      "64:\tlearn: 0.0106841\ttest: 0.0099835\tbest: 0.0099835 (64)\ttotal: 2.04s\tremaining: 1.1s\n",
      "65:\tlearn: 0.0106833\ttest: 0.0099832\tbest: 0.0099832 (65)\ttotal: 2.06s\tremaining: 1.06s\n",
      "66:\tlearn: 0.0106793\ttest: 0.0099786\tbest: 0.0099786 (66)\ttotal: 2.1s\tremaining: 1.03s\n",
      "67:\tlearn: 0.0106789\ttest: 0.0099788\tbest: 0.0099786 (66)\ttotal: 2.12s\tremaining: 996ms\n",
      "68:\tlearn: 0.0106777\ttest: 0.0099797\tbest: 0.0099786 (66)\ttotal: 2.15s\tremaining: 967ms\n",
      "69:\tlearn: 0.0106777\ttest: 0.0099799\tbest: 0.0099786 (66)\ttotal: 2.2s\tremaining: 942ms\n",
      "70:\tlearn: 0.0106774\ttest: 0.0099791\tbest: 0.0099786 (66)\ttotal: 2.23s\tremaining: 913ms\n",
      "71:\tlearn: 0.0106680\ttest: 0.0099738\tbest: 0.0099738 (71)\ttotal: 2.28s\tremaining: 886ms\n",
      "72:\tlearn: 0.0106620\ttest: 0.0099661\tbest: 0.0099661 (72)\ttotal: 2.33s\tremaining: 862ms\n",
      "73:\tlearn: 0.0106619\ttest: 0.0099653\tbest: 0.0099653 (73)\ttotal: 2.37s\tremaining: 831ms\n",
      "74:\tlearn: 0.0106618\ttest: 0.0099648\tbest: 0.0099648 (74)\ttotal: 2.39s\tremaining: 796ms\n",
      "75:\tlearn: 0.0106613\ttest: 0.0099643\tbest: 0.0099643 (75)\ttotal: 2.42s\tremaining: 763ms\n",
      "76:\tlearn: 0.0106613\ttest: 0.0099644\tbest: 0.0099643 (75)\ttotal: 2.44s\tremaining: 727ms\n",
      "77:\tlearn: 0.0106613\ttest: 0.0099641\tbest: 0.0099641 (77)\ttotal: 2.46s\tremaining: 693ms\n",
      "78:\tlearn: 0.0106613\ttest: 0.0099641\tbest: 0.0099641 (77)\ttotal: 2.48s\tremaining: 660ms\n",
      "79:\tlearn: 0.0106520\ttest: 0.0099584\tbest: 0.0099584 (79)\ttotal: 2.53s\tremaining: 632ms\n",
      "80:\tlearn: 0.0106520\ttest: 0.0099585\tbest: 0.0099584 (79)\ttotal: 2.55s\tremaining: 598ms\n",
      "81:\tlearn: 0.0106520\ttest: 0.0099589\tbest: 0.0099584 (79)\ttotal: 2.57s\tremaining: 565ms\n",
      "82:\tlearn: 0.0106425\ttest: 0.0099500\tbest: 0.0099500 (82)\ttotal: 2.61s\tremaining: 535ms\n",
      "83:\tlearn: 0.0106425\ttest: 0.0099502\tbest: 0.0099500 (82)\ttotal: 2.65s\tremaining: 505ms\n",
      "84:\tlearn: 0.0106424\ttest: 0.0099502\tbest: 0.0099500 (82)\ttotal: 2.68s\tremaining: 473ms\n",
      "85:\tlearn: 0.0106424\ttest: 0.0099503\tbest: 0.0099500 (82)\ttotal: 2.71s\tremaining: 441ms\n",
      "86:\tlearn: 0.0106423\ttest: 0.0099508\tbest: 0.0099500 (82)\ttotal: 2.76s\tremaining: 412ms\n",
      "87:\tlearn: 0.0106423\ttest: 0.0099508\tbest: 0.0099500 (82)\ttotal: 2.78s\tremaining: 379ms\n",
      "Stopped by overfitting detector  (5 iterations wait)\n",
      "\n",
      "bestTest = 0.009949956215\n",
      "bestIteration = 82\n",
      "\n",
      "Shrink model to first 83 iterations.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<catboost.core.CatBoostClassifier at 0x354cf7150>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.fit()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating the ranking model\n",
    "\n",
    "Next, you'll evaluate how well the model performs on the validation data using metrics for classification such as precision, recall and f1-score:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2024-12-24 13:03:06.983\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mrecsys.training.ranking\u001b[0m:\u001b[36mevaluate\u001b[0m:\u001b[36m62\u001b[0m - \u001b[1m              precision    recall  f1-score   support\n",
      "\n",
      "           0       1.00      1.00      1.00     38778\n",
      "           1       0.96      1.00      0.98      1942\n",
      "\n",
      "    accuracy                           1.00     40720\n",
      "   macro avg       0.98      1.00      0.99     40720\n",
      "weighted avg       1.00      1.00      1.00     40720\n",
      "\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "metrics = trainer.evaluate(log=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It can be seen that the model has a low F1-score on the positive class (higher is better). The performance could potentially be improved by adding more features to the dataset, e.g. image embeddings.\n",
    "\n",
    "Let's see which features your model considers important."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'month_cos': 58.88502448246417,\n",
       " 'month_sin': 33.568428970468496,\n",
       " 'product_type_name': 1.5060291283122975,\n",
       " 'age': 1.4510136344766673,\n",
       " 'perceived_colour_value_name': 0.9090543083951164,\n",
       " 'index_group_name': 0.905068938543685,\n",
       " 'product_group_name': 0.8481770590684912,\n",
       " 'graphical_appearance_name': 0.47203813718737087,\n",
       " 'garment_group_name': 0.45948435511105373,\n",
       " 'department_name': 0.40798195143449595,\n",
       " 'index_name': 0.3400126867120175,\n",
       " 'perceived_colour_master_name': 0.15406431986826075,\n",
       " 'section_name': 0.09209849465645363,\n",
       " 'colour_group_name': 0.001523533301427642}"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.get_feature_importance()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <span style=\"color:#ff5f27\">  Uploading the model to Hopsworks model registry </span>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "mr = project.get_model_registry()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Uploading: 100.000%|██████████| 618847/618847 elapsed<00:02 remaining<00:00  1.18it/s]\n",
      "Uploading: 100.000%|██████████| 448/448 elapsed<00:02 remaining<00:00<00:07,  1.95s/it]\n",
      "Model export complete: 100%|██████████| 6/6 [00:12<00:00,  2.04s/it]                   "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model created, explore it at https://c.app.hopsworks.ai:443/p/1192098/models/ranking_model/2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "ranking_module = hopsworks_integration.ranking_serving.HopsworksRankingModel(\n",
    "    model=model\n",
    ")\n",
    "ranking_module.register(mr, feature_view_ranking, X_train, metrics)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## <span style=\"color:#ff5f27\"> Inspecting the model in the Hopsworks model registry </span>\n",
    "\n",
    "View results in [Hopsworks Serverless](https://rebrand.ly/serverless-github): **Data Science → Model Registry**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2024-12-24 13:03:20.163\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36m__main__\u001b[0m:\u001b[36m<module>\u001b[0m:\u001b[36m4\u001b[0m - \u001b[1m⌛️ Notebook Execution time: 39.05 seconds ~ 0.65 minutes\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "notebook_end_time = time.time()\n",
    "notebook_execution_time = notebook_end_time - notebook_start_time\n",
    "\n",
    "logger.info(\n",
    "    f\"⌛️ Notebook Execution time: {notebook_execution_time:.2f} seconds ~ {notebook_execution_time / 60:.2f} minutes\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# <span style=\"color:#ff5f27\">→ Next Steps </span>\n",
    "\n",
    "In the next notebook, you will compute embeddings for all the items, populate a vector index with them (as a feature group) and create an online feature view which will allow you to retrieve candidates, for each user, with very low latency."
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}