preprocessing_and_lgbm.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.metrics import roc_auc_score, classification_report\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from scripts.preprocessing import NanHandler, TrendHandler, OkvedHandler, ColumnRemover, InfoHandler"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_parquet('data/train.parquet')\n",
    "test_df = pd.read_parquet('data/test.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "### Preprocessing config ###\n",
    "\n",
    "# Running mode \n",
    "split_train = True # whether 'train_df' will be splitted on train and val during current run. Should not be splitted if submittion is expected futher.  \n",
    "\n",
    "# Nan handling \n",
    "del_high_miss_cols = True # Whether to delete columns with high percentage of missing values\n",
    "impute_strategy = None  # None - do not impute, 'avg' - simple imputer (implemented), 'kmeans' - kmeans imputer (not implemented), 'iterative' - iteratieve imputer (not implemented)\n",
    "update_nans = True # Whether to translate values from NanHandler.convert_to_na_dict to np.nan i.e. replace -999 in rko_start_months to np.nan \n",
    "\n",
    "# Trend generator\n",
    "apply_trend_gen = True # Whether to apply trend handler to data (if False, options below will be ignored) -> generates columns '{x}_trend' for x in [unique prefixes of columns that end with _1m, _3m, _1y ...] \n",
    "drop_old_trend_cols = True  # Whether to drop columns out of which trend was calculated i.e. sum_deb_e_oper_1m, cnt_deb_e_oper_1m \n",
    "\n",
    "# Okved handling\n",
    "apply_okved_transform = True # Whether to apply okved handler to data (if False, options below will be ignored) -> generates column 'okved_groups'\n",
    "drop_old_okved = True # Whether to drop original 'okved' column\n",
    "apply_okved_ohe = False # Whether to apply one hot encoding to new transformed column 'okved_groups'\n",
    "\n",
    "\n",
    "col_remover = ColumnRemover()\n",
    "coltype_handler = InfoHandler()\n",
    "nan_handler = NanHandler(del_high_miss_cols, impute_strategy, update_nans)\n",
    "trend_handler = TrendHandler(apply_handler=apply_trend_gen, drop_old_cols=drop_old_trend_cols)\n",
    "okved_handler = OkvedHandler(drop_old_col=drop_old_okved, apply_ohe=apply_okved_ohe, apply_handler=apply_okved_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ColumnRemover] Dropped columns ['id']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[TrendHandler] Calculating m3 trends...: 100%|██████████| 32/32 [00:01<00:00, 31.50it/s]\n",
      "[TrendHandler] Calculating m1 trends...: 100%|██████████| 1/1 [00:00<00:00, 18.52it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TrendHandler] Dropped old columns: ['sum_a_oper_1m', 'cnt_a_oper_1m', 'sum_b_oper_1m', 'cnt_b_oper_1m', 'sum_c_oper_1m', 'cnt_c_oper_1m', 'sum_deb_d_oper_1m', 'cnt_deb_d_oper_1m', 'sum_deb_e_oper_1m', 'cnt_deb_e_oper_1m', 'cnt_days_deb_e_oper_1m', 'sum_cred_e_oper_1m', 'cnt_cred_e_oper_1m', 'cnt_days_cred_e_oper_1m', 'sum_deb_f_oper_1m', 'cnt_deb_f_oper_1m', 'cnt_days_deb_f_oper_1m', 'sum_cred_f_oper_1m', 'cnt_cred_f_oper_1m', 'cnt_days_cred_f_oper_1m', 'sum_deb_g_oper_1m', 'cnt_deb_g_oper_1m', 'cnt_days_deb_g_oper_1m', 'sum_cred_g_oper_1m', 'cnt_cred_g_oper_1m', 'cnt_days_cred_g_oper_1m', 'sum_deb_h_oper_1m', 'cnt_deb_h_oper_1m', 'cnt_days_deb_h_oper_1m', 'sum_cred_h_oper_1m', 'cnt_cred_h_oper_1m', 'cnt_days_cred_h_oper_1m', 'sum_a_oper_3m', 'cnt_a_oper_3m', 'sum_b_oper_3m', 'cnt_b_oper_3m', 'sum_c_oper_3m', 'cnt_c_oper_3m', 'sum_deb_d_oper_3m', 'cnt_deb_d_oper_3m', 'sum_deb_e_oper_3m', 'cnt_deb_e_oper_3m', 'cnt_days_deb_e_oper_3m', 'sum_cred_e_oper_3m', 'cnt_cred_e_oper_3m', 'cnt_days_cred_e_oper_3m', 'sum_deb_f_oper_3m', 'cnt_deb_f_oper_3m', 'cnt_days_deb_f_oper_3m', 'sum_cred_f_oper_3m', 'cnt_cred_f_oper_3m', 'cnt_days_cred_f_oper_3m', 'sum_deb_g_oper_3m', 'cnt_deb_g_oper_3m', 'cnt_days_deb_g_oper_3m', 'sum_cred_g_oper_3m', 'cnt_cred_g_oper_3m', 'cnt_days_cred_g_oper_3m', 'sum_deb_h_oper_3m', 'cnt_deb_h_oper_3m', 'cnt_days_deb_h_oper_3m', 'sum_cred_h_oper_3m', 'cnt_cred_h_oper_3m', 'cnt_days_cred_h_oper_3m', 'sum_of_paym_2m', 'sum_of_paym_6m', 'sum_of_paym_1y']\n",
      "[OkvedHandler] Applying okved -> okved_groups transform...\n",
      "[OkvedHandler] Dropped old column 'okved'\n",
      "[ColumnRemover] Dropped columns ['id']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[TrendHandler] Calculating m3 trends...: 100%|██████████| 32/32 [00:00<00:00, 135.02it/s]\n",
      "[TrendHandler] Calculating m1 trends...: 100%|██████████| 1/1 [00:00<00:00, 83.33it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[TrendHandler] Dropped old columns: ['sum_a_oper_1m', 'cnt_a_oper_1m', 'sum_b_oper_1m', 'cnt_b_oper_1m', 'sum_c_oper_1m', 'cnt_c_oper_1m', 'sum_deb_d_oper_1m', 'cnt_deb_d_oper_1m', 'sum_deb_e_oper_1m', 'cnt_deb_e_oper_1m', 'cnt_days_deb_e_oper_1m', 'sum_cred_e_oper_1m', 'cnt_cred_e_oper_1m', 'cnt_days_cred_e_oper_1m', 'sum_deb_f_oper_1m', 'cnt_deb_f_oper_1m', 'cnt_days_deb_f_oper_1m', 'sum_cred_f_oper_1m', 'cnt_cred_f_oper_1m', 'cnt_days_cred_f_oper_1m', 'sum_deb_g_oper_1m', 'cnt_deb_g_oper_1m', 'cnt_days_deb_g_oper_1m', 'sum_cred_g_oper_1m', 'cnt_cred_g_oper_1m', 'cnt_days_cred_g_oper_1m', 'sum_deb_h_oper_1m', 'cnt_deb_h_oper_1m', 'cnt_days_deb_h_oper_1m', 'sum_cred_h_oper_1m', 'cnt_cred_h_oper_1m', 'cnt_days_cred_h_oper_1m', 'sum_a_oper_3m', 'cnt_a_oper_3m', 'sum_b_oper_3m', 'cnt_b_oper_3m', 'sum_c_oper_3m', 'cnt_c_oper_3m', 'sum_deb_d_oper_3m', 'cnt_deb_d_oper_3m', 'sum_deb_e_oper_3m', 'cnt_deb_e_oper_3m', 'cnt_days_deb_e_oper_3m', 'sum_cred_e_oper_3m', 'cnt_cred_e_oper_3m', 'cnt_days_cred_e_oper_3m', 'sum_deb_f_oper_3m', 'cnt_deb_f_oper_3m', 'cnt_days_deb_f_oper_3m', 'sum_cred_f_oper_3m', 'cnt_cred_f_oper_3m', 'cnt_days_cred_f_oper_3m', 'sum_deb_g_oper_3m', 'cnt_deb_g_oper_3m', 'cnt_days_deb_g_oper_3m', 'sum_cred_g_oper_3m', 'cnt_cred_g_oper_3m', 'cnt_days_cred_g_oper_3m', 'sum_deb_h_oper_3m', 'cnt_deb_h_oper_3m', 'cnt_days_deb_h_oper_3m', 'sum_cred_h_oper_3m', 'cnt_cred_h_oper_3m', 'cnt_days_cred_h_oper_3m', 'sum_of_paym_2m', 'sum_of_paym_6m', 'sum_of_paym_1y']\n",
      "[OkvedHandler] Applying okved -> okved_groups transform...\n",
      "[OkvedHandler] Dropped old column 'okved'\n"
     ]
    }
   ],
   "source": [
    "col_remover.process(train_df)\n",
    "trend_handler.process(train_df)\n",
    "okved_handler.process(train_df)\n",
    "\n",
    "col_remover.process(test_df)\n",
    "trend_handler.process(test_df)\n",
    "okved_handler.process(test_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "if split_train:\n",
    "    train_indices, val_indices = train_test_split(list(train_df.index), stratify=train_df['total_target'], test_size=0.2, random_state=42)\n",
    "    val_df = train_df.iloc[val_indices]\n",
    "    train_df = train_df.iloc[train_indices]\n",
    "    X_train, y1_train, y2_train, y_train = train_df.drop(['target_1', 'target_2', 'total_target'], axis=1), train_df['target_1'], train_df['target_2'], train_df['total_target']\n",
    "    X_val, y1_val, y2_val, y_val = val_df.drop(['target_1', 'target_2', 'total_target'], axis=1), val_df['target_1'], val_df['target_2'], val_df['total_target']\n",
    "else:\n",
    "    X_train, y1_train, y2_train, y_train = train_df.drop(['target_1', 'target_2', 'total_target'], axis=1), train_df['target_1'], train_df['target_2'], train_df['total_target']\n",
    "\n",
    "X_test = test_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[NanHandler] Deleting high percentage nan columns: ['max_end_plan_non_fin_deals', 'max_start_non_fin_deals', 'min_start_non_fin_deals', 'min_end_plan_non_fin_deals', 'max_start_fin_deals', 'min_end_fact_fin_deals', 'min_start_fin_deals', 'max_end_fact_fin_deals']\n",
      "[NanHandler] Converted value -999 from column rko_start_months to np.nan\n",
      "[NanHandler] Deleting high percentage nan columns: ['max_end_plan_non_fin_deals', 'max_start_non_fin_deals', 'min_start_non_fin_deals', 'min_end_plan_non_fin_deals', 'max_start_fin_deals', 'min_end_fact_fin_deals', 'min_start_fin_deals', 'max_end_fact_fin_deals']\n",
      "[NanHandler] Converted value -999 from column rko_start_months to np.nan\n",
      "[NanHandler] Deleting high percentage nan columns: ['max_end_plan_non_fin_deals', 'max_start_non_fin_deals', 'min_start_non_fin_deals', 'min_end_plan_non_fin_deals', 'max_start_fin_deals', 'min_end_fact_fin_deals', 'min_start_fin_deals', 'max_end_fact_fin_deals']\n",
      "[NanHandler] Converted value -999 from column rko_start_months to np.nan\n"
     ]
    }
   ],
   "source": [
    "nan_handler.process(X_train, is_train=True)\n",
    "nan_handler.process(X_val, is_train=False)\n",
    "nan_handler.process(X_test, is_train=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "col_types = InfoHandler.get_cols_info(X_train)\n",
    "categorical_columns = col_types['categorical_cols']\n",
    "numerical_columns = col_types['numerical_cols']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train[categorical_columns] = X_train[categorical_columns].astype(\"category\")\n",
    "X_val[categorical_columns] = X_val[categorical_columns].astype(\"category\")\n",
    "X_test[categorical_columns] = X_test[categorical_columns].astype(\"category\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ROC-AUC: 0.8821342177202827\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.99      0.97     66937\n",
      "           1       0.63      0.24      0.35      5063\n",
      "\n",
      "    accuracy                           0.94     72000\n",
      "   macro avg       0.79      0.62      0.66     72000\n",
      "weighted avg       0.92      0.94      0.92     72000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "model = LGBMClassifier(verbose=-1)\n",
    "model.fit(X_train, y_train)\n",
    "y_val_pred = model.predict(X_val)\n",
    "y_val_proba = model.predict_proba(X_val)[:, 1]\n",
    "print(f\"ROC-AUC: {roc_auc_score(y_val, y_val_proba)}\")\n",
    "print(classification_report(y_val, y_val_pred))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}