Churn Model-Keras (Sequential Deep Learning)-Git.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ed1f6b7a",
   "metadata": {},
   "source": [
    "\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.compose import ColumnTransformer\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from scipy.stats import randint\n",
    "from sklearn.model_selection import ShuffleSplit\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.linear_model import LogisticRegressionCV\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "from sklearn.model_selection import cross_validate\n",
    "from sklearn.metrics import recall_score\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.metrics import ConfusionMatrixDisplay\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn import svm\n",
    "from sklearn.compose import make_column_transformer\n",
    "from scipy.stats import randint\n",
    "import seaborn as sns\n",
    "from sklearn.decomposition import KernelPCA\n",
    "from sklearn.decomposition import PCA\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.datasets import load_digits\n",
    "from sklearn.model_selection import learning_curve\n",
    "#from xgboost import XGBClassifier\n",
    "#from xgboost import XGBRegressor\n",
    "#from mlxtend.preprocessing import DenseTransformer\n",
    "from sklearn.base import BaseEstimator, TransformerMixin\n",
    "from sklearn.metrics import classification_report\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from imblearn.over_sampling import SMOTENC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "0a7b2a20",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f03c2275",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "df=pd.read_csv(r'D:\\Python Tutorial\\Churn_Modelling.csv')\n",
    "df = df.groupby('Exited').apply(lambda x: x.sample(frac=1))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9bd4d71b",
   "metadata": {},
   "source": [
    "# EDA- Exploratory data analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "75c189f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df=df.drop(columns=['RowNumber','CustomerId'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "7aa43ca3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remove rows with na\n",
    "df=df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "920542e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "##Convert all features with unique values less than 5 to object\n",
    "def convert_to_object(data):\n",
    "    for i in data.columns:\n",
    "        if data[i].nunique()<5:\n",
    "            data[i]=data[i].astype('str')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "46e2e6f0",
   "metadata": {},
   "outputs": [],
   "source": [
    "convert_to_object(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "7a4f0702",
   "metadata": {},
   "outputs": [],
   "source": [
    "##Create a function to drop a variable if it is object but nunique>20,\n",
    "##There is something called as \"Pass by value\" in python ad doing df= in the last statement will not alter df\n",
    "##while the same df= in for loop will work\n",
    "#refer https://stackoverflow.com/questions/38895768/python-pandas-dataframe-is-it-pass-by-value-or-pass-by-reference/38925257#38925257\n",
    "#https://stackoverflow.com/questions/70902808/for-loop-inside-a-def-function-not-working/70902856#70902856\n",
    "def drop_object_nunique(data):\n",
    "    for i in data.columns:\n",
    "        if data[i].dtype=='object' and data[i].nunique()>20:\n",
    "            data.drop(columns=[i],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2f68c193",
   "metadata": {},
   "outputs": [],
   "source": [
    "drop_object_nunique(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b279636f",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install imbalanced-learn\n",
    "#from imblearn.over_sampling import SMOTE"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "53d8a060",
   "metadata": {},
   "source": [
    "for i in df.columns:\n",
    "    if df[i].dtype=='object' and df[i].nunique()>20:\n",
    "        df=df.drop(columns=[i],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "6fb6140e",
   "metadata": {},
   "outputs": [],
   "source": [
    "X=df.drop(columns=['Exited'],axis=1)\n",
    "y=df['Exited']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "60bfe6e3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    7963\n",
       "1    2037\n",
       "Name: Exited, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##There is imbalance in the class of the prediction variable\n",
    "y.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f3606cc7",
   "metadata": {},
   "source": [
    "Use SMOTENC to solve the problem of imbalance class where X has categorical variables + numerical variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ca0b631",
   "metadata": {},
   "outputs": [],
   "source": [
    "# define function to select and pass only cat variables index location to SMOTEC without the loc of y variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "ffe48b7b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[1, 2, 6, 7, 8]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def SMOTEC_Cat(data):\n",
    "    cat_bool=list([data.dtypes==object][0])\n",
    "    loc=[]\n",
    "    for i in range(len(cat_bool)):\n",
    "        if cat_bool[i]==True:\n",
    "            loc.append(i)\n",
    "    return loc[:len(loc)-1]\n",
    "#This will return the index location of all cat vars except the last y var, since we dont want it for SMOTEC function\n",
    "SMOTEC_Cat(df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b9c73b32",
   "metadata": {},
   "outputs": [],
   "source": [
    "#smote = SMOTENC(categorical_features =[df.dtypes==object],sampling_strategy='minority',random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0b2aa6a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "smote = SMOTENC(categorical_features =SMOTEC_Cat(df),sampling_strategy='minority',random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7dc10aae",
   "metadata": {},
   "outputs": [],
   "source": [
    "#sm = SMOTENC(random_state=42, categorical_features=loc)\n",
    "X, y = smote.fit_resample(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "74b7e4ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(15926, 10) , (15926,)\n"
     ]
    }
   ],
   "source": [
    "print(X.shape,\",\",y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ab4e9bbc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#train Test split\n",
    "#X_train_full,X_test_full,y_train,y_test=train_test_split(X,y,test_size=0.2,random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0ef7e5a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(X_train_full.shape,y_train.shape)\n",
    "#print(X_test_full.shape,y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "d18d78fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "##Lets create a function to select only numerical variables\n",
    "numerical_variables=[]\n",
    "def select_num_variables(data,numerical_variables):\n",
    "    for i in data.columns:\n",
    "        if data[i].dtype in ['float64','int64']:\n",
    "            numerical_variables.append(i)\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "fae669bb",
   "metadata": {},
   "outputs": [],
   "source": [
    "select_num_variables(X,numerical_variables)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f218687c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CreditScore', 'Age', 'Tenure', 'Balance', 'EstimatedSalary']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "numerical_variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "0c298484",
   "metadata": {},
   "outputs": [],
   "source": [
    "##Lets create a function to select only categorical variables\n",
    "categorical_variables=[]\n",
    "def select_cat_variables(data,categorical_variables):\n",
    "    for i in data.columns:\n",
    "        if data[i].dtype=='object':\n",
    "            categorical_variables.append(i)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "006e95aa",
   "metadata": {},
   "outputs": [],
   "source": [
    "select_cat_variables(X,categorical_variables)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c0080337",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Geography', 'Gender', 'NumOfProducts', 'HasCrCard', 'IsActiveMember']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "categorical_variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "ff67afe3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CreditScore',\n",
       " 'Age',\n",
       " 'Tenure',\n",
       " 'Balance',\n",
       " 'EstimatedSalary',\n",
       " 'Geography',\n",
       " 'Gender',\n",
       " 'NumOfProducts',\n",
       " 'HasCrCard',\n",
       " 'IsActiveMember']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##Keep Selected Columns\n",
    "my_cols=numerical_variables+categorical_variables\n",
    "my_cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "efda3980",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train,X_test,y_train,y_test=train_test_split(X,y,test_size=0.2,random_state=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "34ebbfb6",
   "metadata": {},
   "source": [
    "# Preprocessing transformation of numerical and categorical data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "46f9ef61",
   "metadata": {},
   "outputs": [],
   "source": [
    "# numerical transformation\n",
    "numerical_transformer=Pipeline(steps=[\n",
    "    ('imputer', SimpleImputer(strategy=\"constant\")),\n",
    "    ('scaler',  StandardScaler())\n",
    "     ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "c28cb374",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Categorical transformation\n",
    "categorical_transformer=Pipeline(steps=[\n",
    "    ('imputer',SimpleImputer(strategy=\"constant\")),\n",
    "    ('encoeder',OneHotEncoder(handle_unknown=\"ignore\"))\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "f3cd56dc",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Bundle preprocessing of numerical and categorical data using ColumnTransformer\n",
    "preprocessor=ColumnTransformer(transformers=[\n",
    "                                             ('num',numerical_transformer,numerical_variables),\n",
    "                                             ('cat',categorical_transformer,categorical_variables)\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2c65370",
   "metadata": {},
   "source": [
    "# Preprocess the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3f9e236",
   "metadata": {},
   "outputs": [],
   "source": [
    "#preprocessor.fit_transform(X_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "957cca34",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(12740, 10)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ee5ab1b4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(12740, 18)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##This gives me numpy array\n",
    "X_train_transformed=preprocessor.fit_transform(X_train)\n",
    "X_train_transformed.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "ae5867a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Convert it to DataFrame to extract and rename features\n",
    "X_df=pd.DataFrame(X_train_transformed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "a3966f28",
   "metadata": {},
   "outputs": [],
   "source": [
    "#This will give me the categorical variables transformed\n",
    "cat_names=preprocessor.transformers_[1][1]['encoeder']\\\n",
    ".get_feature_names(categorical_variables)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "6098992d",
   "metadata": {},
   "outputs": [],
   "source": [
    "##Rename cat columns\n",
    "for i,j in zip(range(5,18), range(len(cat_names))):\n",
    "    X_df=X_df.rename(columns={X_df.columns[i]:cat_names[j]})\n",
    "    \n",
    "##Rename numerical columns\n",
    "for i,j in zip(range(0,5), range(len(numerical_variables))):\n",
    "    X_df=X_df.rename(columns={X_df.columns[i]:numerical_variables[j]})"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "838ac0e3",
   "metadata": {},
   "source": [
    "# Feature selection"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9fdf7bd7",
   "metadata": {},
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "lsvc = LinearSVC(C=0.01, penalty=\"l1\", dual=False).fit(X_train_transformed, y_train)\n",
    "model_lsvc = SelectFromModel(lsvc, prefit=True)\n",
    "X_new = model_lsvc.transform(X_train_transformed)\n",
    "X_new.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "e7a05eae",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "14"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import LinearSVC\n",
    "lsvc = LinearSVC(C=0.01, penalty=\"l1\", dual=False).fit(X_df, y_train)\n",
    "model_lsvc = SelectFromModel(lsvc, prefit=True)\n",
    "X_new = model_lsvc.transform(X_df)\n",
    "X_new.shape[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "892d425a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#This will give the position of selected features in fporm of list [0, 1, 2, 3, 4, 6, 9, 10, 11, 12, 17]\n",
    "select_var_True=list(model_lsvc.get_support([0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "ce973f76",
   "metadata": {},
   "outputs": [],
   "source": [
    "#This will subset the X_df dataset to select only the importnat features\n",
    "X_df_selected_features=X_df.iloc[:,select_var_True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "a3e254c5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CreditScore</th>\n",
       "      <th>Age</th>\n",
       "      <th>Tenure</th>\n",
       "      <th>Balance</th>\n",
       "      <th>EstimatedSalary</th>\n",
       "      <th>Geography_Germany</th>\n",
       "      <th>Geography_Spain</th>\n",
       "      <th>Gender_Female</th>\n",
       "      <th>Gender_Male</th>\n",
       "      <th>NumOfProducts_2</th>\n",
       "      <th>NumOfProducts_3</th>\n",
       "      <th>HasCrCard_1</th>\n",
       "      <th>IsActiveMember_0</th>\n",
       "      <th>IsActiveMember_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.429726</td>\n",
       "      <td>-0.403114</td>\n",
       "      <td>-0.306699</td>\n",
       "      <td>-0.069754</td>\n",
       "      <td>0.339267</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.702982</td>\n",
       "      <td>0.598733</td>\n",
       "      <td>-1.412969</td>\n",
       "      <td>0.218925</td>\n",
       "      <td>1.687104</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.194845</td>\n",
       "      <td>0.698918</td>\n",
       "      <td>-0.306699</td>\n",
       "      <td>0.238759</td>\n",
       "      <td>0.759504</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.145538</td>\n",
       "      <td>-1.104407</td>\n",
       "      <td>1.537085</td>\n",
       "      <td>-1.347315</td>\n",
       "      <td>0.998791</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.189259</td>\n",
       "      <td>-0.002375</td>\n",
       "      <td>-1.044213</td>\n",
       "      <td>-1.347315</td>\n",
       "      <td>1.432141</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   CreditScore       Age    Tenure   Balance  EstimatedSalary  \\\n",
       "0     0.429726 -0.403114 -0.306699 -0.069754         0.339267   \n",
       "1     0.702982  0.598733 -1.412969  0.218925         1.687104   \n",
       "2     1.194845  0.698918 -0.306699  0.238759         0.759504   \n",
       "3     0.145538 -1.104407  1.537085 -1.347315         0.998791   \n",
       "4     0.189259 -0.002375 -1.044213 -1.347315         1.432141   \n",
       "\n",
       "   Geography_Germany  Geography_Spain  Gender_Female  Gender_Male  \\\n",
       "0                1.0              0.0            1.0          0.0   \n",
       "1                1.0              0.0            0.0          1.0   \n",
       "2                1.0              0.0            1.0          0.0   \n",
       "3                0.0              1.0            1.0          0.0   \n",
       "4                0.0              0.0            1.0          0.0   \n",
       "\n",
       "   NumOfProducts_2  NumOfProducts_3  HasCrCard_1  IsActiveMember_0  \\\n",
       "0              0.0              0.0          1.0               1.0   \n",
       "1              0.0              0.0          1.0               1.0   \n",
       "2              0.0              0.0          1.0               1.0   \n",
       "3              1.0              0.0          0.0               0.0   \n",
       "4              0.0              0.0          1.0               1.0   \n",
       "\n",
       "   IsActiveMember_1  \n",
       "0               0.0  \n",
       "1               0.0  \n",
       "2               0.0  \n",
       "3               1.0  \n",
       "4               0.0  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_df_selected_features.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "582c5c76",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CreditScore',\n",
       " 'Age',\n",
       " 'Tenure',\n",
       " 'Balance',\n",
       " 'EstimatedSalary',\n",
       " 'Geography_Germany',\n",
       " 'Geography_Spain',\n",
       " 'Gender_Female',\n",
       " 'Gender_Male',\n",
       " 'NumOfProducts_2',\n",
       " 'NumOfProducts_3',\n",
       " 'HasCrCard_1',\n",
       " 'IsActiveMember_0',\n",
       " 'IsActiveMember_1']"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_features=list(X_df_selected_features.columns)\n",
    "selected_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "ddc76c3a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "14 , 14\n"
     ]
    }
   ],
   "source": [
    "print(X_new.shape[1],\",\",X_df_selected_features.shape[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c22fe66",
   "metadata": {},
   "source": [
    "# Keras Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "id": "229bd1a8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to create model, required for KerasClassifier\n",
    "def create_model(optimizer='rmsprop', init='glorot_uniform'):\n",
    "    # create model\n",
    "    model = Sequential()\n",
    "    model.add(Dense(14, input_dim=X_new.shape[1], kernel_initializer=init, activation='relu'))\n",
    "    keras.layers.Dropout(0.5)\n",
    "    model.add(Dense(8, kernel_initializer=init, activation='relu'))\n",
    "    keras.layers.Dropout(0.5)\n",
    "    model.add(Dense(1, kernel_initializer=init, activation='sigmoid'))\n",
    "    # Compile model\n",
    "    model.compile(loss='binary_crossentropy', optimizer=optimizer, metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "id": "5ad5583f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create model\n",
    "model = KerasClassifier(build_fn=create_model,verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "81884885",
   "metadata": {},
   "outputs": [],
   "source": [
    "##Define Gridsearch parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "id": "6fb1734d",
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid={'optimizer': ['adam','rmsprop'],\n",
    "            'init':['uniform','glorot_uniform'],\n",
    "            'epochs':[150,200],\n",
    "            'batch_size':[50,100,500]}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "id": "f5f95e7a",
   "metadata": {},
   "outputs": [],
   "source": [
    "grid=GridSearchCV(estimator=model, param_grid=param_grid, cv=3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "id": "b0036794",
   "metadata": {},
   "outputs": [],
   "source": [
    "grid_result=grid.fit(X_df_selected_features,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "88f1b305",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Best: 0.834850 using {'batch_size': 50, 'epochs': 150, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n"
     ]
    }
   ],
   "source": [
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "id": "ef63c959",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.829513 (0.002507) with: {'batch_size': 50, 'epochs': 150, 'init': 'uniform', 'optimizer': 'adam'}\n",
      "0.828022 (0.002963) with: {'batch_size': 50, 'epochs': 150, 'init': 'uniform', 'optimizer': 'rmsprop'}\n",
      "0.834850 (0.004929) with: {'batch_size': 50, 'epochs': 150, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "0.832653 (0.002458) with: {'batch_size': 50, 'epochs': 150, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}\n",
      "0.831632 (0.004557) with: {'batch_size': 50, 'epochs': 200, 'init': 'uniform', 'optimizer': 'adam'}\n",
      "0.826766 (0.004117) with: {'batch_size': 50, 'epochs': 200, 'init': 'uniform', 'optimizer': 'rmsprop'}\n",
      "0.832810 (0.001180) with: {'batch_size': 50, 'epochs': 200, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "0.832103 (0.003184) with: {'batch_size': 50, 'epochs': 200, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}\n",
      "0.828885 (0.002036) with: {'batch_size': 100, 'epochs': 150, 'init': 'uniform', 'optimizer': 'adam'}\n",
      "0.829356 (0.000241) with: {'batch_size': 100, 'epochs': 150, 'init': 'uniform', 'optimizer': 'rmsprop'}\n",
      "0.829199 (0.002919) with: {'batch_size': 100, 'epochs': 150, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "0.832025 (0.004762) with: {'batch_size': 100, 'epochs': 150, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}\n",
      "0.831005 (0.000762) with: {'batch_size': 100, 'epochs': 200, 'init': 'uniform', 'optimizer': 'adam'}\n",
      "0.831868 (0.002714) with: {'batch_size': 100, 'epochs': 200, 'init': 'uniform', 'optimizer': 'rmsprop'}\n",
      "0.830298 (0.000658) with: {'batch_size': 100, 'epochs': 200, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "0.833281 (0.002358) with: {'batch_size': 100, 'epochs': 200, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}\n",
      "0.826452 (0.001777) with: {'batch_size': 500, 'epochs': 150, 'init': 'uniform', 'optimizer': 'adam'}\n",
      "0.825981 (0.003577) with: {'batch_size': 500, 'epochs': 150, 'init': 'uniform', 'optimizer': 'rmsprop'}\n",
      "0.827943 (0.002305) with: {'batch_size': 500, 'epochs': 150, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "0.831083 (0.002642) with: {'batch_size': 500, 'epochs': 150, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}\n",
      "0.827943 (0.001676) with: {'batch_size': 500, 'epochs': 200, 'init': 'uniform', 'optimizer': 'adam'}\n",
      "0.826295 (0.002180) with: {'batch_size': 500, 'epochs': 200, 'init': 'uniform', 'optimizer': 'rmsprop'}\n",
      "0.831083 (0.002766) with: {'batch_size': 500, 'epochs': 200, 'init': 'glorot_uniform', 'optimizer': 'adam'}\n",
      "0.831004 (0.004607) with: {'batch_size': 500, 'epochs': 200, 'init': 'glorot_uniform', 'optimizer': 'rmsprop'}\n"
     ]
    }
   ],
   "source": [
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f1d1e25",
   "metadata": {},
   "source": [
    "#Linear SVC method of feature selection\n",
    "from sklearn.svm import LinearSVC\n",
    "my_pipeline=Pipeline(steps=[('preprocessor',preprocessor),\n",
    "                            ('feature selection',SelectFromModel(LinearSVC(penalty=\"l1\",dual=False))),\n",
    "                             ('model',grid)])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64f6ba34",
   "metadata": {},
   "source": [
    "# Test Data Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "a4c177a5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CreditScore</th>\n",
       "      <th>Geography</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Age</th>\n",
       "      <th>Tenure</th>\n",
       "      <th>Balance</th>\n",
       "      <th>NumOfProducts</th>\n",
       "      <th>HasCrCard</th>\n",
       "      <th>IsActiveMember</th>\n",
       "      <th>EstimatedSalary</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>10878</th>\n",
       "      <td>746</td>\n",
       "      <td>France</td>\n",
       "      <td>Male</td>\n",
       "      <td>34</td>\n",
       "      <td>5</td>\n",
       "      <td>149871.000675</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>179675.337123</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13564</th>\n",
       "      <td>700</td>\n",
       "      <td>Germany</td>\n",
       "      <td>Male</td>\n",
       "      <td>45</td>\n",
       "      <td>4</td>\n",
       "      <td>118707.729240</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>65434.024057</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3959</th>\n",
       "      <td>616</td>\n",
       "      <td>France</td>\n",
       "      <td>Male</td>\n",
       "      <td>29</td>\n",
       "      <td>9</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>166984.440000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12402</th>\n",
       "      <td>665</td>\n",
       "      <td>France</td>\n",
       "      <td>Female</td>\n",
       "      <td>37</td>\n",
       "      <td>8</td>\n",
       "      <td>186564.781205</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>184262.859420</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14151</th>\n",
       "      <td>748</td>\n",
       "      <td>Germany</td>\n",
       "      <td>Male</td>\n",
       "      <td>45</td>\n",
       "      <td>8</td>\n",
       "      <td>117795.503364</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>102002.383520</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       CreditScore Geography  Gender  Age  Tenure        Balance  \\\n",
       "10878          746    France    Male   34       5  149871.000675   \n",
       "13564          700   Germany    Male   45       4  118707.729240   \n",
       "3959           616    France    Male   29       9       0.000000   \n",
       "12402          665    France  Female   37       8  186564.781205   \n",
       "14151          748   Germany    Male   45       8  117795.503364   \n",
       "\n",
       "      NumOfProducts HasCrCard IsActiveMember  EstimatedSalary  \n",
       "10878             1         1              0    179675.337123  \n",
       "13564             1         1              0     65434.024057  \n",
       "3959              1         1              1    166984.440000  \n",
       "12402             1         1              0    184262.859420  \n",
       "14151             1         1              0    102002.383520  "
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aaa02b2d",
   "metadata": {},
   "source": [
    "# Preprocess the test data¶\n",
    "In the last section I have compressed all the test- preprocess steps into one function (Below codes will help from learning purpose in term sof steps involved to match the shape of test and train data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "id": "aa77efcb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3186, 18)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "##This gives me numpy array\n",
    "X_test_transformed=preprocessor.fit_transform(X_test)\n",
    "X_test_transformed.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "3c9c96f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Convert it to DataFrame to extract and rename features\n",
    "X_test_df=pd.DataFrame(X_test_transformed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "id": "8e63c5da",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3186, 18)"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_df.head()\n",
    "X_test_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "6c967ae0",
   "metadata": {},
   "outputs": [],
   "source": [
    "#This will give me the categorical variables transformed\n",
    "cat_names=preprocessor.transformers_[1][1]['encoeder']\\\n",
    ".get_feature_names(categorical_variables)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "8c609148",
   "metadata": {},
   "outputs": [],
   "source": [
    "##Rename cat columns\n",
    "for i,j in zip(range(5,18), range(len(cat_names))):\n",
    "    X_test_df=X_test_df.rename(columns={X_test_df.columns[i]:cat_names[j]})\n",
    "    \n",
    "##Rename numerical columns\n",
    "for i,j in zip(range(0,5), range(len(numerical_variables))):\n",
    "    X_test_df=X_test_df.rename(columns={X_test_df.columns[i]:numerical_variables[j]})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "id": "6a3dfa02",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3186, 18)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_df.head()\n",
    "X_test_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ed1aebf1",
   "metadata": {},
   "outputs": [],
   "source": [
    "##Now select the same features as that in training data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "49d309cf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['CreditScore',\n",
       " 'Age',\n",
       " 'Tenure',\n",
       " 'Balance',\n",
       " 'EstimatedSalary',\n",
       " 'Geography_Germany',\n",
       " 'Geography_Spain',\n",
       " 'Gender_Female',\n",
       " 'Gender_Male',\n",
       " 'NumOfProducts_2',\n",
       " 'NumOfProducts_3',\n",
       " 'HasCrCard_1',\n",
       " 'IsActiveMember_0',\n",
       " 'IsActiveMember_1']"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_features=list(X_df_selected_features.columns)\n",
    "selected_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "459a4d5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_test_processed=X_test_df[selected_features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "872f6c91",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3186, 14)"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_processed.head()\n",
    "X_test_processed.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "64b3a412",
   "metadata": {},
   "outputs": [],
   "source": [
    "help(X_test_processed.reindex)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a5fb4c64",
   "metadata": {},
   "source": [
    "A test dataset can have less classes for a categorical variable, hence there could be a possibility that the selected features in the train dataset might not be available in test dataset,\n",
    "\n",
    "In that case we will have to create those features in test dataset with value=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "2855b460",
   "metadata": {},
   "outputs": [],
   "source": [
    "#we canuse reindex function to get the missing colums in test datatset with fill_value=0\n",
    "#help(X_test_processed.reindex) for more details\n",
    "X_test_processed=X_test_processed.reindex(columns=X_df_selected_features.columns,fill_value=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "1ee3d7cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>CreditScore</th>\n",
       "      <th>Age</th>\n",
       "      <th>Tenure</th>\n",
       "      <th>Balance</th>\n",
       "      <th>EstimatedSalary</th>\n",
       "      <th>Geography_Germany</th>\n",
       "      <th>Geography_Spain</th>\n",
       "      <th>Gender_Female</th>\n",
       "      <th>Gender_Male</th>\n",
       "      <th>NumOfProducts_2</th>\n",
       "      <th>NumOfProducts_3</th>\n",
       "      <th>HasCrCard_1</th>\n",
       "      <th>IsActiveMember_0</th>\n",
       "      <th>IsActiveMember_1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.047415</td>\n",
       "      <td>-0.693467</td>\n",
       "      <td>0.068500</td>\n",
       "      <td>1.120198</td>\n",
       "      <td>1.320767</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.549147</td>\n",
       "      <td>0.415612</td>\n",
       "      <td>-0.292824</td>\n",
       "      <td>0.612140</td>\n",
       "      <td>-0.645129</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-0.360734</td>\n",
       "      <td>-1.197593</td>\n",
       "      <td>1.513796</td>\n",
       "      <td>-1.323168</td>\n",
       "      <td>1.102379</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.170030</td>\n",
       "      <td>-0.390991</td>\n",
       "      <td>1.152472</td>\n",
       "      <td>1.718422</td>\n",
       "      <td>1.399711</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.069078</td>\n",
       "      <td>0.415612</td>\n",
       "      <td>1.152472</td>\n",
       "      <td>0.597267</td>\n",
       "      <td>-0.015851</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   CreditScore       Age    Tenure   Balance  EstimatedSalary  \\\n",
       "0     1.047415 -0.693467  0.068500  1.120198         1.320767   \n",
       "1     0.549147  0.415612 -0.292824  0.612140        -0.645129   \n",
       "2    -0.360734 -1.197593  1.513796 -1.323168         1.102379   \n",
       "3     0.170030 -0.390991  1.152472  1.718422         1.399711   \n",
       "4     1.069078  0.415612  1.152472  0.597267        -0.015851   \n",
       "\n",
       "   Geography_Germany  Geography_Spain  Gender_Female  Gender_Male  \\\n",
       "0                0.0              0.0            0.0          1.0   \n",
       "1                1.0              0.0            0.0          1.0   \n",
       "2                0.0              0.0            0.0          1.0   \n",
       "3                0.0              0.0            1.0          0.0   \n",
       "4                1.0              0.0            0.0          1.0   \n",
       "\n",
       "   NumOfProducts_2  NumOfProducts_3  HasCrCard_1  IsActiveMember_0  \\\n",
       "0              0.0              0.0          1.0               1.0   \n",
       "1              0.0              0.0          1.0               1.0   \n",
       "2              0.0              0.0          1.0               0.0   \n",
       "3              0.0              0.0          1.0               1.0   \n",
       "4              0.0              0.0          1.0               1.0   \n",
       "\n",
       "   IsActiveMember_1  \n",
       "0               0.0  \n",
       "1               0.0  \n",
       "2               1.0  \n",
       "3               0.0  \n",
       "4               0.0  "
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test_processed.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "817073f6",
   "metadata": {},
   "outputs": [],
   "source": [
    "preds=grid.predict(X_test_processed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "c721c20e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3186, 1)"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "preds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "653e8abc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           1       0.83      0.82      0.83      1608\n",
      "           0       0.82      0.83      0.83      1578\n",
      "\n",
      "   micro avg       0.83      0.83      0.83      3186\n",
      "   macro avg       0.83      0.83      0.83      3186\n",
      "weighted avg       0.83      0.83      0.83      3186\n",
      "\n"
     ]
    }
   ],
   "source": [
    "matrix=classification_report(y_test,preds,labels=[1,0])\n",
    "print('Classification report: \\n',matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "39598a25",
   "metadata": {},
   "source": [
    "# Create a function to preprocess and predict on test data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5a1e1589",
   "metadata": {},
   "source": [
    "Above steps written in an efficient way looped under a function to directly give the test prediction results are preprocessing, feature reconciliation and fitting the Keras model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "id": "bae5e375",
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_test(test_data):\n",
    "    X_test_df=pd.DataFrame(preprocessor.fit_transform(X_test))\n",
    "    \n",
    "    cat_names=preprocessor.transformers_[1][1]['encoeder']\\\n",
    "    .get_feature_names(categorical_variables)\n",
    "    \n",
    "    ##Rename cat columns\n",
    "    for i,j in zip(range(5,18), range(len(cat_names))):\n",
    "        X_test_df=X_test_df.rename(columns={X_test_df.columns[i]:cat_names[j]})\n",
    "    \n",
    "    ##Rename numerical columns\n",
    "    for i,j in zip(range(0,5), range(len(numerical_variables))):\n",
    "        X_test_df=X_test_df.rename(columns={X_test_df.columns[i]:numerical_variables[j]})\n",
    "    \n",
    "    preds=grid.predict(X_test_df[selected_features])\n",
    "    matrix=classification_report(y_test,preds,labels=[1,0])\n",
    "    print('Classification report: \\n',matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "id": "1f79acea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Classification report: \n",
      "               precision    recall  f1-score   support\n",
      "\n",
      "           1       0.85      0.83      0.84      1608\n",
      "           0       0.83      0.85      0.84      1578\n",
      "\n",
      "   micro avg       0.84      0.84      0.84      3186\n",
      "   macro avg       0.84      0.84      0.84      3186\n",
      "weighted avg       0.84      0.84      0.84      3186\n",
      "\n"
     ]
    }
   ],
   "source": [
    "predict_test(X_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}