4-Modelling_VIPs.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Capstone Hotel reservation prediction\n",
    "\n",
    "![Logo](img/hotel-logo.png)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Modelling solvent guest\n",
    "#### (VIP)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import required libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:06:38.519947Z",
     "start_time": "2020-08-24T21:06:37.403410Z"
    }
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "pd.set_option(\"display.max_columns\", None)\n",
    "\n",
    "#modelling\n",
    "from sklearn import preprocessing\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.ensemble import AdaBoostClassifier, RandomForestClassifier\n",
    "from sklearn.inspection import permutation_importance\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score, classification_report, confusion_matrix, f1_score, precision_score, recall_score, roc_auc_score, roc_curve\n",
    "from sklearn.model_selection import cross_val_score, cross_validate, cross_val_predict, RandomizedSearchCV, train_test_split\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.utils import resample\n",
    "\n",
    "#visualisations\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "sns.set(style=\"whitegrid\")\n",
    "\n",
    "import visuals as vs\n",
    "\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Loading cleaned dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:06:40.047904Z",
     "start_time": "2020-08-24T21:06:38.520944Z"
    }
   },
   "outputs": [],
   "source": [
    "df = pd.read_csv(\"data/dataset_w_feature.csv\", index_col=0, encoding=\"iso-8859-15\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:06:40.189033Z",
     "start_time": "2020-08-24T21:06:40.049899Z"
    }
   },
   "outputs": [],
   "source": [
    "df[\"buchungsdatum\"]      = pd.to_datetime(df[\"buchungsdatum\"])\n",
    "df[\"anreisedatum\"]       = pd.to_datetime(df[\"anreisedatum\"])\n",
    "df[\"abreisedatum\"]       = pd.to_datetime(df[\"abreisedatum\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:06:40.234910Z",
     "start_time": "2020-08-24T21:06:40.190649Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>buchungsdatum</th>\n",
       "      <th>anreisedatum</th>\n",
       "      <th>abreisedatum</th>\n",
       "      <th>naechte_insgesamt</th>\n",
       "      <th>gebuchte_naechte</th>\n",
       "      <th>personen_anzahl</th>\n",
       "      <th>gesamtpreis</th>\n",
       "      <th>saison</th>\n",
       "      <th>buchung_telefon</th>\n",
       "      <th>buchung_online</th>\n",
       "      <th>buchung_reisebuero</th>\n",
       "      <th>zimmerarten</th>\n",
       "      <th>reise_adventure</th>\n",
       "      <th>reise_relax</th>\n",
       "      <th>reise_standard</th>\n",
       "      <th>reise_genuss</th>\n",
       "      <th>kind</th>\n",
       "      <th>entfernung</th>\n",
       "      <th>zahler</th>\n",
       "      <th>kunden_id</th>\n",
       "      <th>buchungsnr</th>\n",
       "      <th>buchungsnr_reisebuero</th>\n",
       "      <th>menue_code</th>\n",
       "      <th>zusatzleistung</th>\n",
       "      <th>destination</th>\n",
       "      <th>anrede</th>\n",
       "      <th>plz</th>\n",
       "      <th>ort</th>\n",
       "      <th>kundenclub</th>\n",
       "      <th>jahrgangsalter_klassiert</th>\n",
       "      <th>bundesland</th>\n",
       "      <th>einwohner_adjusted_plz</th>\n",
       "      <th>bev_m_plz</th>\n",
       "      <th>bev_w_plz</th>\n",
       "      <th>bev_insgesamt_ort</th>\n",
       "      <th>bev_weiblich_ort</th>\n",
       "      <th>bev_maennlich_ort</th>\n",
       "      <th>qkm_plz</th>\n",
       "      <th>flaeche_ort</th>\n",
       "      <th>einwohnerdichte_prokm2_plz</th>\n",
       "      <th>haushalte_regiog_2017_plz</th>\n",
       "      <th>kaufkraft_einwohner_in_eur_regio</th>\n",
       "      <th>kk_einwohner_index_regiog_2017</th>\n",
       "      <th>kk_hh_index_regiog_2017</th>\n",
       "      <th>gemeindetyp_regiostar_2016</th>\n",
       "      <th>wirtschaftskraftmr1</th>\n",
       "      <th>wirtschaftskraftmr2</th>\n",
       "      <th>buchungsjahr</th>\n",
       "      <th>buchungsquartal</th>\n",
       "      <th>buchungsmonat</th>\n",
       "      <th>buchungswoche</th>\n",
       "      <th>buchungstag</th>\n",
       "      <th>buchung_wochentag</th>\n",
       "      <th>anreisejahr</th>\n",
       "      <th>anreisequartal</th>\n",
       "      <th>anreisemonat</th>\n",
       "      <th>anreisewoche</th>\n",
       "      <th>anreisetag</th>\n",
       "      <th>anreise_wochentag</th>\n",
       "      <th>abreisejahr</th>\n",
       "      <th>abreisequartal</th>\n",
       "      <th>abreisemonat</th>\n",
       "      <th>abreisewoche</th>\n",
       "      <th>abreisetag</th>\n",
       "      <th>abreise_wochentag</th>\n",
       "      <th>buchungsmonat-jahr</th>\n",
       "      <th>buchungsquartal-jahr</th>\n",
       "      <th>anreisemonat-jahr</th>\n",
       "      <th>anreisequartal-jahr</th>\n",
       "      <th>abreisemonat-jahr</th>\n",
       "      <th>abreisequartal-jahr</th>\n",
       "      <th>buchungskanal</th>\n",
       "      <th>werbe_ban</th>\n",
       "      <th>anz_werbe_ban</th>\n",
       "      <th>aufenthalt</th>\n",
       "      <th>naechte_ausserhalb</th>\n",
       "      <th>anz_buch</th>\n",
       "      <th>buchungsvorlauf</th>\n",
       "      <th>preis_per</th>\n",
       "      <th>preis_nacht</th>\n",
       "      <th>preis_nacht_per</th>\n",
       "      <th>VIP</th>\n",
       "      <th>buch_zeilen</th>\n",
       "      <th>erstbuchung</th>\n",
       "      <th>folgebuchung</th>\n",
       "      <th>repeater</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2010-01-29</td>\n",
       "      <td>2010-04-02</td>\n",
       "      <td>2010-04-10</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>2</td>\n",
       "      <td>2982</td>\n",
       "      <td>2010</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>classic</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>535.48</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1151</td>\n",
       "      <td>456</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>linz</td>\n",
       "      <td>Herr</td>\n",
       "      <td>29664</td>\n",
       "      <td>Walsrode</td>\n",
       "      <td>0</td>\n",
       "      <td>55-64</td>\n",
       "      <td>Niedersachsen</td>\n",
       "      <td>25173</td>\n",
       "      <td>12540</td>\n",
       "      <td>12633</td>\n",
       "      <td>25173</td>\n",
       "      <td>11577</td>\n",
       "      <td>11491</td>\n",
       "      <td>344.326464</td>\n",
       "      <td>344.326464</td>\n",
       "      <td>73.108</td>\n",
       "      <td>11453</td>\n",
       "      <td>20641.89</td>\n",
       "      <td>92.819</td>\n",
       "      <td>98.104</td>\n",
       "      <td>Landregion - Zentrale Stadt</td>\n",
       "      <td>-0.333392</td>\n",
       "      <td>-0.388738</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>29</td>\n",
       "      <td>Freitag</td>\n",
       "      <td>2010</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>13</td>\n",
       "      <td>2</td>\n",
       "      <td>Freitag</td>\n",
       "      <td>2010</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>Samstag</td>\n",
       "      <td>20101</td>\n",
       "      <td>20101</td>\n",
       "      <td>20104</td>\n",
       "      <td>20102</td>\n",
       "      <td>20104</td>\n",
       "      <td>20102</td>\n",
       "      <td>reisebuero</td>\n",
       "      <td>keine</td>\n",
       "      <td>0</td>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>63</td>\n",
       "      <td>1491.00</td>\n",
       "      <td>372.75</td>\n",
       "      <td>186.38</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2010-01-14</td>\n",
       "      <td>2010-04-05</td>\n",
       "      <td>2010-04-10</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>1437</td>\n",
       "      <td>2010</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>classic</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>544.72</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>40</td>\n",
       "      <td>6413</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>linz</td>\n",
       "      <td>Frau</td>\n",
       "      <td>50226</td>\n",
       "      <td>Frechen</td>\n",
       "      <td>0</td>\n",
       "      <td>55-64</td>\n",
       "      <td>Nordrhein-Westfalen</td>\n",
       "      <td>52473</td>\n",
       "      <td>25645</td>\n",
       "      <td>26828</td>\n",
       "      <td>52473</td>\n",
       "      <td>26828</td>\n",
       "      <td>25645</td>\n",
       "      <td>45.118960</td>\n",
       "      <td>45.118960</td>\n",
       "      <td>1162.992</td>\n",
       "      <td>26227</td>\n",
       "      <td>25268.58</td>\n",
       "      <td>113.623</td>\n",
       "      <td>111.379</td>\n",
       "      <td>Landregion - Zentrale Stadt</td>\n",
       "      <td>0.853802</td>\n",
       "      <td>-0.345112</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>14</td>\n",
       "      <td>Donnerstag</td>\n",
       "      <td>2010</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>5</td>\n",
       "      <td>Montag</td>\n",
       "      <td>2010</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>Samstag</td>\n",
       "      <td>20101</td>\n",
       "      <td>20101</td>\n",
       "      <td>20104</td>\n",
       "      <td>20102</td>\n",
       "      <td>20104</td>\n",
       "      <td>20102</td>\n",
       "      <td>reisebuero</td>\n",
       "      <td>generell</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>81</td>\n",
       "      <td>718.50</td>\n",
       "      <td>287.40</td>\n",
       "      <td>143.70</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2010-03-01</td>\n",
       "      <td>2010-04-05</td>\n",
       "      <td>2010-04-10</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>2</td>\n",
       "      <td>1737</td>\n",
       "      <td>2010</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>classic</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>606.63</td>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>2134</td>\n",
       "      <td>4322</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>linz</td>\n",
       "      <td>Herr</td>\n",
       "      <td>23970</td>\n",
       "      <td>Wismar</td>\n",
       "      <td>0</td>\n",
       "      <td>75-84</td>\n",
       "      <td>Mecklenburg-Vorpommern</td>\n",
       "      <td>1423</td>\n",
       "      <td>692</td>\n",
       "      <td>731</td>\n",
       "      <td>46442</td>\n",
       "      <td>21844</td>\n",
       "      <td>20706</td>\n",
       "      <td>50.563027</td>\n",
       "      <td>83.131346</td>\n",
       "      <td>17.117</td>\n",
       "      <td>6355</td>\n",
       "      <td>18697.65</td>\n",
       "      <td>84.076</td>\n",
       "      <td>73.665</td>\n",
       "      <td>Stadtregion - Kleinstädtischer, dörflicher Raum</td>\n",
       "      <td>-1.012536</td>\n",
       "      <td>-1.012536</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>1</td>\n",
       "      <td>Montag</td>\n",
       "      <td>2010</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>5</td>\n",
       "      <td>Montag</td>\n",
       "      <td>2010</td>\n",
       "      <td>2</td>\n",
       "      <td>4</td>\n",
       "      <td>14</td>\n",
       "      <td>10</td>\n",
       "      <td>Samstag</td>\n",
       "      <td>20103</td>\n",
       "      <td>20101</td>\n",
       "      <td>20104</td>\n",
       "      <td>20102</td>\n",
       "      <td>20104</td>\n",
       "      <td>20102</td>\n",
       "      <td>reisebuero</td>\n",
       "      <td>keine</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>35</td>\n",
       "      <td>868.50</td>\n",
       "      <td>347.40</td>\n",
       "      <td>173.70</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2010-01-25</td>\n",
       "      <td>2010-03-21</td>\n",
       "      <td>2010-03-25</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1047</td>\n",
       "      <td>2010</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>classic</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>421.59</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>776</td>\n",
       "      <td>6414</td>\n",
       "      <td>69</td>\n",
       "      <td>9</td>\n",
       "      <td>linz</td>\n",
       "      <td>Frau</td>\n",
       "      <td>15732</td>\n",
       "      <td>Eichwalde</td>\n",
       "      <td>0</td>\n",
       "      <td>65-74</td>\n",
       "      <td>Brandenburg</td>\n",
       "      <td>14690</td>\n",
       "      <td>7302</td>\n",
       "      <td>7388</td>\n",
       "      <td>14690</td>\n",
       "      <td>4135</td>\n",
       "      <td>4087</td>\n",
       "      <td>11.965427</td>\n",
       "      <td>11.965427</td>\n",
       "      <td>1227.704</td>\n",
       "      <td>7159</td>\n",
       "      <td>23442.06</td>\n",
       "      <td>105.410</td>\n",
       "      <td>104.087</td>\n",
       "      <td>Landregion - Städtischer Raum</td>\n",
       "      <td>0.417226</td>\n",
       "      <td>0.417226</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>25</td>\n",
       "      <td>Montag</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>11</td>\n",
       "      <td>21</td>\n",
       "      <td>Sonntag</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "      <td>25</td>\n",
       "      <td>Donnerstag</td>\n",
       "      <td>20101</td>\n",
       "      <td>20101</td>\n",
       "      <td>20103</td>\n",
       "      <td>20101</td>\n",
       "      <td>20103</td>\n",
       "      <td>20101</td>\n",
       "      <td>reisebuero</td>\n",
       "      <td>keine</td>\n",
       "      <td>0</td>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>55</td>\n",
       "      <td>523.50</td>\n",
       "      <td>261.75</td>\n",
       "      <td>130.88</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2010-01-26</td>\n",
       "      <td>2010-03-07</td>\n",
       "      <td>2010-03-09</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>20</td>\n",
       "      <td>5279</td>\n",
       "      <td>2010</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>classic</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>37.88</td>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>813</td>\n",
       "      <td>2938</td>\n",
       "      <td>113</td>\n",
       "      <td>2</td>\n",
       "      <td>duesseldorf</td>\n",
       "      <td>Frau</td>\n",
       "      <td>53881</td>\n",
       "      <td>Euskirchen</td>\n",
       "      <td>1</td>\n",
       "      <td>45-54</td>\n",
       "      <td>Nordrhein-Westfalen</td>\n",
       "      <td>28531</td>\n",
       "      <td>13975</td>\n",
       "      <td>14556</td>\n",
       "      <td>57975</td>\n",
       "      <td>29578</td>\n",
       "      <td>28397</td>\n",
       "      <td>127.679129</td>\n",
       "      <td>139.430913</td>\n",
       "      <td>204.625</td>\n",
       "      <td>11683</td>\n",
       "      <td>23328.82</td>\n",
       "      <td>104.901</td>\n",
       "      <td>120.110</td>\n",
       "      <td>Stadtregion - Kleinstädtischer, dörflicher Raum</td>\n",
       "      <td>0.212335</td>\n",
       "      <td>-0.579173</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>4</td>\n",
       "      <td>26</td>\n",
       "      <td>Dienstag</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>Sonntag</td>\n",
       "      <td>2010</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>Dienstag</td>\n",
       "      <td>20101</td>\n",
       "      <td>20101</td>\n",
       "      <td>20103</td>\n",
       "      <td>20101</td>\n",
       "      <td>20103</td>\n",
       "      <td>20101</td>\n",
       "      <td>reisebuero</td>\n",
       "      <td>keine</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>40</td>\n",
       "      <td>263.95</td>\n",
       "      <td>2639.50</td>\n",
       "      <td>131.98</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  buchungsdatum anreisedatum abreisedatum  naechte_insgesamt  \\\n",
       "0    2010-01-29   2010-04-02   2010-04-10                  8   \n",
       "1    2010-01-14   2010-04-05   2010-04-10                  5   \n",
       "2    2010-03-01   2010-04-05   2010-04-10                  5   \n",
       "3    2010-01-25   2010-03-21   2010-03-25                  4   \n",
       "4    2010-01-26   2010-03-07   2010-03-09                  2   \n",
       "\n",
       "   gebuchte_naechte  personen_anzahl  gesamtpreis  saison  buchung_telefon  \\\n",
       "0                 8                2         2982    2010                0   \n",
       "1                 5                2         1437    2010                0   \n",
       "2                 5                2         1737    2010                0   \n",
       "3                 4                2         1047    2010                0   \n",
       "4                 2               20         5279    2010                0   \n",
       "\n",
       "   buchung_online  buchung_reisebuero zimmerarten  reise_adventure  \\\n",
       "0               0                   1     classic                0   \n",
       "1               0                   1     classic                0   \n",
       "2               0                   1     classic                0   \n",
       "3               0                   1     classic                0   \n",
       "4               0                   1     classic                1   \n",
       "\n",
       "   reise_relax  reise_standard  reise_genuss  kind  entfernung  zahler  \\\n",
       "0            0               0             1     0      535.48       1   \n",
       "1            0               1             0     0      544.72       0   \n",
       "2            0               1             0     0      606.63       0   \n",
       "3            1               0             0     0      421.59       1   \n",
       "4            0               0             0     0       37.88       0   \n",
       "\n",
       "   kunden_id  buchungsnr  buchungsnr_reisebuero  menue_code  zusatzleistung  \\\n",
       "0          1        1151                    456           5               5   \n",
       "1          2          40                   6413           5               5   \n",
       "2          3        2134                   4322           5               5   \n",
       "3          4         776                   6414          69               9   \n",
       "4          5         813                   2938         113               2   \n",
       "\n",
       "   destination anrede    plz         ort  kundenclub jahrgangsalter_klassiert  \\\n",
       "0         linz   Herr  29664    Walsrode           0                    55-64   \n",
       "1         linz   Frau  50226     Frechen           0                    55-64   \n",
       "2         linz   Herr  23970      Wismar           0                    75-84   \n",
       "3         linz   Frau  15732   Eichwalde           0                    65-74   \n",
       "4  duesseldorf   Frau  53881  Euskirchen           1                    45-54   \n",
       "\n",
       "               bundesland  einwohner_adjusted_plz  bev_m_plz  bev_w_plz  \\\n",
       "0           Niedersachsen                   25173      12540      12633   \n",
       "1     Nordrhein-Westfalen                   52473      25645      26828   \n",
       "2  Mecklenburg-Vorpommern                    1423        692        731   \n",
       "3             Brandenburg                   14690       7302       7388   \n",
       "4     Nordrhein-Westfalen                   28531      13975      14556   \n",
       "\n",
       "   bev_insgesamt_ort  bev_weiblich_ort  bev_maennlich_ort     qkm_plz  \\\n",
       "0              25173             11577              11491  344.326464   \n",
       "1              52473             26828              25645   45.118960   \n",
       "2              46442             21844              20706   50.563027   \n",
       "3              14690              4135               4087   11.965427   \n",
       "4              57975             29578              28397  127.679129   \n",
       "\n",
       "   flaeche_ort  einwohnerdichte_prokm2_plz  haushalte_regiog_2017_plz  \\\n",
       "0   344.326464                      73.108                      11453   \n",
       "1    45.118960                    1162.992                      26227   \n",
       "2    83.131346                      17.117                       6355   \n",
       "3    11.965427                    1227.704                       7159   \n",
       "4   139.430913                     204.625                      11683   \n",
       "\n",
       "   kaufkraft_einwohner_in_eur_regio  kk_einwohner_index_regiog_2017  \\\n",
       "0                          20641.89                          92.819   \n",
       "1                          25268.58                         113.623   \n",
       "2                          18697.65                          84.076   \n",
       "3                          23442.06                         105.410   \n",
       "4                          23328.82                         104.901   \n",
       "\n",
       "   kk_hh_index_regiog_2017                       gemeindetyp_regiostar_2016  \\\n",
       "0                   98.104                      Landregion - Zentrale Stadt   \n",
       "1                  111.379                      Landregion - Zentrale Stadt   \n",
       "2                   73.665  Stadtregion - Kleinstädtischer, dörflicher Raum   \n",
       "3                  104.087                    Landregion - Städtischer Raum   \n",
       "4                  120.110  Stadtregion - Kleinstädtischer, dörflicher Raum   \n",
       "\n",
       "   wirtschaftskraftmr1  wirtschaftskraftmr2  buchungsjahr  buchungsquartal  \\\n",
       "0            -0.333392            -0.388738          2010                1   \n",
       "1             0.853802            -0.345112          2010                1   \n",
       "2            -1.012536            -1.012536          2010                1   \n",
       "3             0.417226             0.417226          2010                1   \n",
       "4             0.212335            -0.579173          2010                1   \n",
       "\n",
       "   buchungsmonat  buchungswoche  buchungstag buchung_wochentag  anreisejahr  \\\n",
       "0              1              4           29           Freitag         2010   \n",
       "1              1              2           14        Donnerstag         2010   \n",
       "2              3              9            1            Montag         2010   \n",
       "3              1              4           25            Montag         2010   \n",
       "4              1              4           26          Dienstag         2010   \n",
       "\n",
       "   anreisequartal  anreisemonat  anreisewoche  anreisetag anreise_wochentag  \\\n",
       "0               2             4            13           2           Freitag   \n",
       "1               2             4            14           5            Montag   \n",
       "2               2             4            14           5            Montag   \n",
       "3               1             3            11          21           Sonntag   \n",
       "4               1             3             9           7           Sonntag   \n",
       "\n",
       "   abreisejahr  abreisequartal  abreisemonat  abreisewoche  abreisetag  \\\n",
       "0         2010               2             4            14          10   \n",
       "1         2010               2             4            14          10   \n",
       "2         2010               2             4            14          10   \n",
       "3         2010               1             3            12          25   \n",
       "4         2010               1             3            10           9   \n",
       "\n",
       "  abreise_wochentag  buchungsmonat-jahr  buchungsquartal-jahr  \\\n",
       "0           Samstag               20101                 20101   \n",
       "1           Samstag               20101                 20101   \n",
       "2           Samstag               20103                 20101   \n",
       "3        Donnerstag               20101                 20101   \n",
       "4          Dienstag               20101                 20101   \n",
       "\n",
       "   anreisemonat-jahr  anreisequartal-jahr  abreisemonat-jahr  \\\n",
       "0              20104                20102              20104   \n",
       "1              20104                20102              20104   \n",
       "2              20104                20102              20104   \n",
       "3              20103                20101              20103   \n",
       "4              20103                20101              20103   \n",
       "\n",
       "   abreisequartal-jahr buchungskanal werbe_ban  anz_werbe_ban  aufenthalt  \\\n",
       "0                20102    reisebuero     keine              0           8   \n",
       "1                20102    reisebuero  generell              5           5   \n",
       "2                20102    reisebuero     keine              0           5   \n",
       "3                20101    reisebuero     keine              0           4   \n",
       "4                20101    reisebuero     keine              0           2   \n",
       "\n",
       "   naechte_ausserhalb  anz_buch  buchungsvorlauf  preis_per  preis_nacht  \\\n",
       "0                   0         1               63    1491.00       372.75   \n",
       "1                   0         1               81     718.50       287.40   \n",
       "2                   0         1               35     868.50       347.40   \n",
       "3                   0         1               55     523.50       261.75   \n",
       "4                   0         1               40     263.95      2639.50   \n",
       "\n",
       "   preis_nacht_per  VIP  buch_zeilen  erstbuchung  folgebuchung  repeater  \n",
       "0           186.38    0            2            1             0         0  \n",
       "1           143.70    0            2            1             0         0  \n",
       "2           173.70    0            2            1             0         0  \n",
       "3           130.88    0            1            1             0         0  \n",
       "4           131.98    0            7            1             0         1  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Dropping redundant variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:06:40.328659Z",
     "start_time": "2020-08-24T21:06:40.235908Z"
    }
   },
   "outputs": [],
   "source": [
    "df.drop(\"buchungskanal\", axis=1, inplace=True)\n",
    "df.drop(\"werbe_ban\", axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Grouping variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:06:40.392489Z",
     "start_time": "2020-08-24T21:06:40.329657Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categorial = list(df.columns[df.dtypes==object])\n",
    "datetime64 = ['buchungsdatum', 'anreisedatum', 'abreisedatum']\n",
    "numeric    = list(df.select_dtypes(include=['int64', 'float64']).columns)\n",
    "\n",
    "for entry in ['VIP', 'preis_nacht', 'preis_nacht_per', 'preis_per', 'gesamtpreis', 'naechte_insgesamt', 'aufenthalt', 'gebuchte_naechte', 'reise_standard', 'reise_relax', 'menue_code', 'zusatzleistung']: \n",
    "    numeric.remove(entry)\n",
    "    \n",
    "for entry in ['menue_code', 'zusatzleistung']: \n",
    "    categorial.append(entry)    \n",
    "\n",
    "categorial.remove('ort')\n",
    "\n",
    "# 11 = VIP, preis_nacht_per, preis_per, gesamtpreis, naechte_insgesamt, aufenthalt, gebuchte_naechte, reise_standard, reise_relax, ort\n",
    "df.shape[1] == len(categorial) + len(datetime64) + len(numeric) + 11"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### renaming destinations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:06:40.438366Z",
     "start_time": "2020-08-24T21:06:40.394485Z"
    }
   },
   "outputs": [],
   "source": [
    "aenderung = { \"saint etienne\": \"saint-etienne\",\n",
    "              \"viana do castelo\": \"viana-do-castelo\",\n",
    "            }\n",
    "\n",
    "df.destination.replace(aenderung, inplace=True)\n",
    "\n",
    "aenderung2 = {\n",
    "\"Landregion - Kleinstädtischer, dörflicher Raum\": \"Kleinstädt_(Land)\",\n",
    "\"Landregion - Städtischer Raum\": \"Stadt_(Land)\",\n",
    "\"Landregion - Zentrale Stadt\": \"zen_Stadt_(Land)\",\n",
    "\"Stadtregion - Kleinstädtischer, dörflicher Raum\": \"Kleinstädt_(Stadt)\",\n",
    "\"Stadtregion - Metropole\": \"Metropole\",\n",
    "\"Stadtregion - Mittelstadt, städtischer Raum\": \"Mittelstadt\",\n",
    "\"Stadtregion -Regiopole und Großstadt\": \"Regiopole\"\n",
    "    }\n",
    "\n",
    "df.gemeindetyp_regiostar_2016.replace(aenderung2, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:06:40.660783Z",
     "start_time": "2020-08-24T21:06:40.441359Z"
    }
   },
   "outputs": [],
   "source": [
    "df.menue_code     = df.menue_code.astype(\"str\")\n",
    "df.zusatzleistung = df.zusatzleistung.astype(\"str\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normalisation and dummy encoding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:06:41.306070Z",
     "start_time": "2020-08-24T21:06:40.661780Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(196989, 283)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = df.reset_index()\n",
    "x = df[numeric]\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "x_scaled = min_max_scaler.fit_transform(x)\n",
    "columns = df[numeric].columns\n",
    "df_num = pd.DataFrame(x_scaled)\n",
    "df_num.columns = columns\n",
    "\n",
    "df_dum = pd.get_dummies(df[categorial])\n",
    "df_dum.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Logistic Regression\n",
    "Basic (only numeric features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:06:41.322027Z",
     "start_time": "2020-08-24T21:06:41.307068Z"
    }
   },
   "outputs": [],
   "source": [
    "model, acc, pre, rec, f1, roc = [], [], [], [], [], []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:07:43.156337Z",
     "start_time": "2020-08-24T21:06:41.323025Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   54.6s remaining:  1.4min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  1.0min finished\n"
     ]
    }
   ],
   "source": [
    "X = df[numeric]\n",
    "y = df.VIP\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10)\n",
    "\n",
    "lr1 = LogisticRegression(solver='liblinear', max_iter=10_000, random_state=1337)\n",
    "\n",
    "y_pred1 = cross_val_predict(lr1, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"Basic Logistic Regression\")\n",
    "acc.append(accuracy_score(y, y_pred1)*100)\n",
    "pre.append(precision_score(y, y_pred1)*100)\n",
    "rec.append(recall_score(y, y_pred1)*100)\n",
    "f1.append(f1_score(y, y_pred1)*100)\n",
    "#roc1 = roc_curve(y, y_pred1)*100\n",
    "roc.append(roc_auc_score(y, y_pred1)*100) #ändern test oder train?\n",
    "\n",
    "conf_matrix1 = confusion_matrix(y, y_pred1)\n",
    "class_rep1 = classification_report(y, y_pred1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-09T18:09:45.250801Z",
     "start_time": "2020-08-09T18:09:16.781424Z"
    }
   },
   "source": [
    "only numeric features but normalised"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:07:58.566735Z",
     "start_time": "2020-08-24T21:07:43.157334Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   14.1s remaining:   21.2s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   14.4s finished\n"
     ]
    }
   ],
   "source": [
    "X = df_num\n",
    "y = df.VIP\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10, shuffle = True, stratify = y)\n",
    "\n",
    "lr2 = LogisticRegression(solver='liblinear', max_iter=10_000, random_state=1337)\n",
    "\n",
    "y_pred2 = cross_val_predict(lr2, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"Normalised Logistic Regression\")\n",
    "acc.append(accuracy_score(y, y_pred2)*100)\n",
    "pre.append(precision_score(y, y_pred2)*100)\n",
    "rec.append(recall_score(y, y_pred2)*100)\n",
    "f1.append(f1_score(y, y_pred2)*100)\n",
    "#roc2 = roc_curve(y, y_pred2)*100\n",
    "roc.append(roc_auc_score(y, y_pred2)*100)\n",
    "\n",
    "conf_matrix2 = confusion_matrix(y, y_pred2)\n",
    "class_rep2 = classification_report(y, y_pred2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### extended Logistic Regression\n",
    "(categorial variables as dummies)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:08:26.061106Z",
     "start_time": "2020-08-24T21:07:58.567734Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   24.8s remaining:   37.3s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   26.4s finished\n"
     ]
    }
   ],
   "source": [
    "X = pd.concat([df_num, df_dum], axis=1)\n",
    "y = df.VIP\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10)\n",
    "\n",
    "lr3 = LogisticRegression(solver='liblinear', max_iter=10_000, random_state=1337)\n",
    "\n",
    "y_pred3 = cross_val_predict(lr3, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"Logistic Regression (w. dummies)\")\n",
    "acc.append(accuracy_score(y, y_pred3)*100)\n",
    "pre.append(precision_score(y, y_pred3)*100)\n",
    "rec.append(recall_score(y, y_pred3)*100)\n",
    "f1.append(f1_score(y, y_pred3)*100)\n",
    "#roc3 = roc_curve(y, y_pred3)*100\n",
    "roc.append(roc_auc_score(y, y_pred3)*100)\n",
    "\n",
    "conf_matrix3 = confusion_matrix(y, y_pred3)\n",
    "class_rep3 = classification_report(y, y_pred3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Dealing with imbalace \n",
    "in target feature folgebuchung by resampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:08:26.077064Z",
     "start_time": "2020-08-24T21:08:26.062104Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    147855\n",
       "1     49134\n",
       "Name: VIP, dtype: int64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.VIP.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "New train/test split without normalisation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:08:26.155853Z",
     "start_time": "2020-08-24T21:08:26.078062Z"
    }
   },
   "outputs": [],
   "source": [
    "X = pd.concat([df[numeric], df_dum], axis=1)\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10, random_state=1337)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Upsampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:08:26.596707Z",
     "start_time": "2020-08-24T21:08:26.156851Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    147855\n",
       "0    147855\n",
       "Name: VIP, dtype: int64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# concatenate our training data back together\n",
    "X = pd.concat([X, y], axis=1)\n",
    "\n",
    "# separate minority and majority classes\n",
    "k_VIP  = X[X.VIP==0]\n",
    "VIP    = X[X.VIP==1]\n",
    "\n",
    "# upsample minority\n",
    "VIP_upsampled = resample(VIP, replace=True, n_samples=len(k_VIP), random_state=1337) \n",
    "\n",
    "# combine majority and upsampled minority\n",
    "upsampled = pd.concat([k_VIP, VIP_upsampled])\n",
    "\n",
    "upsampled.VIP.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Downsampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:08:26.704926Z",
     "start_time": "2020-08-24T21:08:26.597705Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    49134\n",
       "0    49134\n",
       "Name: VIP, dtype: int64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# downsample majority\n",
    "k_VIP_downsampled = resample(k_VIP, replace=False, n_samples=len(VIP), random_state=1337) \n",
    "\n",
    "# combine majority and upsampled minority\n",
    "downsampled = pd.concat([VIP, k_VIP_downsampled])\n",
    "\n",
    "downsampled.VIP.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### balanced Logistic Regression\n",
    "with upsampled data and additional dummy variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:10:08.976748Z",
     "start_time": "2020-08-24T21:08:26.705924Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  1.5min remaining:  2.3min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  1.7min finished\n"
     ]
    }
   ],
   "source": [
    "y = upsampled.VIP\n",
    "X = upsampled.drop('VIP', axis=1)\n",
    "\n",
    "lr4 = LogisticRegression(solver='liblinear', max_iter=10_000, n_jobs=-1, random_state=1337)\n",
    "\n",
    "y_pred4 = cross_val_predict(lr4, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"Balanced Logistic Regression (upsampled)\")\n",
    "acc.append(accuracy_score(y, y_pred4)*100)\n",
    "pre.append(precision_score(y, y_pred4)*100)\n",
    "rec.append(recall_score(y, y_pred4)*100)\n",
    "f1.append(f1_score(y, y_pred4)*100)\n",
    "#roc4 = roc_curve(y, y_pred4)*100\n",
    "roc.append(roc_auc_score(y, y_pred4)*100)\n",
    "\n",
    "conf_matrix4 = confusion_matrix(y, y_pred4)\n",
    "class_rep4 = classification_report(y, y_pred4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2nd balanced Logistic Regression\n",
    "downsampled"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:10:43.802637Z",
     "start_time": "2020-08-24T21:10:08.977747Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   29.6s remaining:   44.4s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   34.2s finished\n"
     ]
    }
   ],
   "source": [
    "y = downsampled.VIP\n",
    "X = downsampled.drop('VIP', axis=1)\n",
    "\n",
    "lr5 = LogisticRegression(solver='liblinear', max_iter=10_000, n_jobs=-1, random_state=1337)\n",
    "\n",
    "y_pred5 = cross_val_predict(lr5, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"Balanced Logistic Regression (downsampled)\")\n",
    "acc.append(accuracy_score(y, y_pred5)*100)\n",
    "pre.append(precision_score(y, y_pred5)*100)\n",
    "rec.append(recall_score(y, y_pred5)*100)\n",
    "f1.append(f1_score(y, y_pred5)*100)\n",
    "#roc5 = roc_curve(y, y_pred5)*100\n",
    "roc.append(roc_auc_score(y, y_pred5)*100)\n",
    "\n",
    "conf_matrix5 = confusion_matrix(y, y_pred5)\n",
    "class_rep5 = classification_report(y, y_pred5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Advanced models\n",
    "## Random forest\n",
    "fixed settings after reandom search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:30:20.324964Z",
     "start_time": "2020-08-24T21:10:43.803634Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed: 18.8min remaining: 28.2min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed: 19.6min finished\n"
     ]
    }
   ],
   "source": [
    "X = pd.concat([df[numeric], df_dum], axis=1)\n",
    "y = df.VIP\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10, random_state=1337)\n",
    "\n",
    "lr6 = RandomForestClassifier(n_estimators= 120, min_samples_split= 5,\n",
    "                             min_samples_leaf= 6, max_features= None,\n",
    "                             max_depth= None, random_state= 1337)\n",
    "\n",
    "y_pred6 = cross_val_predict(lr6, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"Random Forest\")\n",
    "acc.append(accuracy_score(y, y_pred6)*100)\n",
    "pre.append(precision_score(y, y_pred6)*100)\n",
    "rec.append(recall_score(y, y_pred6)*100)\n",
    "f1.append(f1_score(y, y_pred6)*100)\n",
    "#roc6 = roc_curve(y, y_pred6)*100\n",
    "roc.append(roc_auc_score(y, y_pred6)*100)\n",
    "\n",
    "conf_matrix6 = confusion_matrix(y, y_pred6)\n",
    "class_rep6 = classification_report(y, y_pred6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### GaussianNB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:30:24.583578Z",
     "start_time": "2020-08-24T21:30:20.325962Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:    3.3s remaining:    4.9s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:    3.4s finished\n"
     ]
    }
   ],
   "source": [
    "X = pd.concat([df[numeric], df_dum], axis=1)\n",
    "y = df.VIP\n",
    "\n",
    "#X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.10, random_state=1337)\n",
    "\n",
    "lr7 = GaussianNB()\n",
    "\n",
    "y_pred7 = cross_val_predict(lr7, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"GaussianNB\")\n",
    "acc.append(accuracy_score(y, y_pred7)*100)\n",
    "pre.append(precision_score(y, y_pred7)*100)\n",
    "rec.append(recall_score(y, y_pred7)*100)\n",
    "f1.append(f1_score(y, y_pred7)*100)\n",
    "#roc7 = roc_curve(y, y_pred7)*100\n",
    "roc.append(roc_auc_score(y, y_pred7)*100)\n",
    "\n",
    "conf_matrix7 = confusion_matrix(y, y_pred7)\n",
    "class_rep7 = classification_report(y, y_pred7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### KNeighborsClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:31:55.276099Z",
     "start_time": "2020-08-24T21:30:24.584576Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:  1.4min remaining:  2.1min\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:  1.5min finished\n"
     ]
    }
   ],
   "source": [
    "lr8 = KNeighborsClassifier(n_neighbors=3)\n",
    "\n",
    "y_pred8 = cross_val_predict(lr8, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"KNeighborsClassifier\")\n",
    "acc.append(accuracy_score(y, y_pred8)*100)\n",
    "pre.append(precision_score(y, y_pred8)*100)\n",
    "rec.append(recall_score(y, y_pred8)*100)\n",
    "f1.append(f1_score(y, y_pred8)*100)\n",
    "#roc8 = roc_curve(y, y_pred8)*100\n",
    "roc.append(roc_auc_score(y, y_pred8)*100)\n",
    "\n",
    "conf_matrix8 = confusion_matrix(y, y_pred8)\n",
    "class_rep8 = classification_report(y, y_pred8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### DecisionTreeClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:32:11.041947Z",
     "start_time": "2020-08-24T21:31:55.278095Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   14.6s remaining:   22.0s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   15.0s finished\n"
     ]
    }
   ],
   "source": [
    "lr9 = DecisionTreeClassifier(random_state=1337)\n",
    "\n",
    "y_pred9 = cross_val_predict(lr9, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"DecisionTreeClassifier\")\n",
    "acc.append(accuracy_score(y, y_pred9)*100)\n",
    "pre.append(precision_score(y, y_pred9)*100)\n",
    "rec.append(recall_score(y, y_pred9)*100)\n",
    "f1.append(f1_score(y, y_pred9)*100)\n",
    "#roc9 = roc_curve(y, y_pred9)*100\n",
    "roc.append(roc_auc_score(y, y_pred9)*100)\n",
    "\n",
    "conf_matrix9 = confusion_matrix(y, y_pred9)\n",
    "class_rep9 = classification_report(y, y_pred9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AdaBoost"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:32:28.366627Z",
     "start_time": "2020-08-24T21:32:11.043943Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Using backend LokyBackend with 8 concurrent workers.\n",
      "[Parallel(n_jobs=-1)]: Done   2 out of   5 | elapsed:   15.9s remaining:   24.0s\n",
      "[Parallel(n_jobs=-1)]: Done   5 out of   5 | elapsed:   16.5s finished\n"
     ]
    }
   ],
   "source": [
    "lr10 = AdaBoostClassifier(DecisionTreeClassifier(random_state=1337))\n",
    "\n",
    "y_pred10 = cross_val_predict(lr10, X, y, cv=5, n_jobs=-1, verbose=5)\n",
    "\n",
    "model.append(\"AdaBoost\")\n",
    "acc.append(accuracy_score(y, y_pred10)*100)\n",
    "pre.append(precision_score(y, y_pred10)*100)\n",
    "rec.append(recall_score(y, y_pred10)*100)\n",
    "f1.append(f1_score(y, y_pred10)*100)\n",
    "#roc10 = roc_curve(y, y_pred10)*100\n",
    "roc.append(roc_auc_score(y, y_pred10)*100)\n",
    "\n",
    "conf_matrix10 = confusion_matrix(y, y_pred10)\n",
    "class_rep10 = classification_report(y, y_pred10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model comparsion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Naive Predictor Performace\n",
    "\n",
    "What if we chose a model that always predicted an individual as repeater, what would that model's accuracy, precision, recall and F-score be on this dataset?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:32:28.553129Z",
     "start_time": "2020-08-24T21:32:28.367625Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Naive Predictor:\n",
      "\tAccuracy score:\t24.94\n",
      "\tPrecision score:24.94\n",
      "\tRecall score:\t100.00\n",
      "\tF-score:\t29.35\n"
     ]
    }
   ],
   "source": [
    "# Calculate accuracy\n",
    "VIP_c = df.query(\"VIP == 1\").count()[0]\n",
    "n_VIP_c = df.query(\"VIP == 0\").count()[0]\n",
    "accuracy = VIP_c / df.shape[0]\n",
    "\n",
    "# Calculating precision\n",
    "precision = VIP_c / (VIP_c + n_VIP_c)\n",
    "\n",
    "#Calculating recall\n",
    "recall = VIP_c / (VIP_c + 0)\n",
    "\n",
    "# Calculate F-score using the formula above for beta = 0.5\n",
    "fscore =  (1  + (0.5*0.5)) * ( precision * recall / (( 0.5*0.5 * (precision))+ recall))\n",
    "\n",
    "# Print the results \n",
    "print (f\"Naive Predictor:\\n\\tAccuracy score:\\t{accuracy*100:.2f}\\n\\tPrecision score:{precision*100:.2f}\\n\\tRecall score:\\t{recall*100:.2f}\\n\\tF-score:\\t{fscore*100:.2f}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Confusion Matrix & Classification Report\n",
    "<img src=\"img/conf_matrix.png\" alt=\"Drawing\" style=\"width: 400px;\"/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:32:28.569086Z",
     "start_time": "2020-08-24T21:32:28.554127Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model1 Basic Logistic Regression:\n",
      "\tAccuracy:\t76.28\n",
      "\tPrecision:\t54.04\n",
      "\tRecall:\t\t32.77\n",
      "\tF1:\t\t40.8\n",
      "\tROC AUC:\t61.76\n",
      "\n",
      "\n",
      "Model2 Normalised Logistic Regression:\n",
      "\tAccuracy:\t79.06\n",
      "\tPrecision:\t60.62\n",
      "\tRecall:\t\t45.87\n",
      "\tF1:\t\t52.22\n",
      "\tROC AUC:\t67.98\n",
      "\n",
      "\n",
      "Model3 Logistic Regression (w. dummies):\n",
      "\tAccuracy:\t84.11\n",
      "\tPrecision:\t68.02\n",
      "\tRecall:\t\t68.47\n",
      "\tF1:\t\t68.24\n",
      "\tROC AUC:\t78.88\n",
      "\n",
      "\n",
      "Model4 Balanced Logistic Regression (upsampled):\n",
      "\tAccuracy:\t69.87\n",
      "\tPrecision:\t66.76\n",
      "\tRecall:\t\t79.13\n",
      "\tF1:\t\t72.42\n",
      "\tROC AUC:\t69.87\n",
      "\n",
      "\n",
      "Model5 Balanced Logistic Regression (downsampled):\n",
      "\tAccuracy:\t69.36\n",
      "\tPrecision:\t70.83\n",
      "\tRecall:\t\t65.81\n",
      "\tF1:\t\t68.23\n",
      "\tROC AUC:\t69.36\n",
      "\n",
      "\n",
      "Model6 Random Forest:\n",
      "\tAccuracy:\t64.08\n",
      "\tPrecision:\t33.68\n",
      "\tRecall:\t\t45.4\n",
      "\tF1:\t\t38.67\n",
      "\tROC AUC:\t57.85\n",
      "\n",
      "\n",
      "Model7 GaussianNB:\n",
      "\tAccuracy:\t68.39\n",
      "\tPrecision:\t40.95\n",
      "\tRecall:\t\t60.44\n",
      "\tF1:\t\t48.82\n",
      "\tROC AUC:\t65.74\n",
      "\n",
      "\n",
      "Model8 KNeighborsClassifier:\n",
      "\tAccuracy:\t53.09\n",
      "\tPrecision:\t22.79\n",
      "\tRecall:\t\t36.89\n",
      "\tF1:\t\t28.18\n",
      "\tROC AUC:\t47.68\n",
      "\n",
      "\n",
      "Model9 DecisionTreeClassifier:\n",
      "\tAccuracy:\t63.12\n",
      "\tPrecision:\t33.29\n",
      "\tRecall:\t\t47.68\n",
      "\tF1:\t\t39.21\n",
      "\tROC AUC:\t57.96\n",
      "\n",
      "\n",
      "Model10 AdaBoost:\n",
      "\tAccuracy:\t63.11\n",
      "\tPrecision:\t33.27\n",
      "\tRecall:\t\t47.61\n",
      "\tF1:\t\t39.17\n",
      "\tROC AUC:\t57.94\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for entry in range(0,len(acc)):\n",
    "    print(f\"Model{entry+1} {model[entry]}:\\n\\tAccuracy:\\t{round(acc[entry],2)}\")\n",
    "    print(f\"\\tPrecision:\\t{round(pre[entry],2)}\")\n",
    "    print(f\"\\tRecall:\\t\\t{round(rec[entry],2)}\")\n",
    "    print(f\"\\tF1:\\t\\t{round(f1[entry],2)}\")\n",
    "    print(f\"\\tROC AUC:\\t{round(roc[entry],2)}\\n\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Best results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-08-24T21:38:35.312802Z",
     "start_time": "2020-08-24T21:38:35.295847Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Logistic Regression (w. dummies)\n",
      "\n",
      "[[132039  15816]\n",
      " [ 15494  33640]]\n",
      "\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.89      0.89      0.89    147855\n",
      "           1       0.68      0.68      0.68     49134\n",
      "\n",
      "    accuracy                           0.84    196989\n",
      "   macro avg       0.79      0.79      0.79    196989\n",
      "weighted avg       0.84      0.84      0.84    196989\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"Logistic Regression (w. dummies)\\n\")\n",
    "print(conf_matrix3)\n",
    "print(f\"\\n{class_rep3}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There first four most imporant features have a cumulative feature weight of 40%, 30 more features each add 2 more percent."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}