schoolPaper_attention.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>LIMIT_BAL</th>\n",
       "      <th>SEX</th>\n",
       "      <th>EDUCATION</th>\n",
       "      <th>MARRIAGE</th>\n",
       "      <th>AGE</th>\n",
       "      <th>PAY_0</th>\n",
       "      <th>PAY_2</th>\n",
       "      <th>PAY_3</th>\n",
       "      <th>PAY_4</th>\n",
       "      <th>PAY_5</th>\n",
       "      <th>PAY_6</th>\n",
       "      <th>BILL_AMT1</th>\n",
       "      <th>BILL_AMT2</th>\n",
       "      <th>BILL_AMT3</th>\n",
       "      <th>BILL_AMT4</th>\n",
       "      <th>BILL_AMT5</th>\n",
       "      <th>BILL_AMT6</th>\n",
       "      <th>PAY_AMT1</th>\n",
       "      <th>PAY_AMT2</th>\n",
       "      <th>PAY_AMT3</th>\n",
       "      <th>PAY_AMT4</th>\n",
       "      <th>PAY_AMT5</th>\n",
       "      <th>PAY_AMT6</th>\n",
       "      <th>default.payment.next.month</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>20000.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-2</td>\n",
       "      <td>-2</td>\n",
       "      <td>3913.0</td>\n",
       "      <td>3102.0</td>\n",
       "      <td>689.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>689.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>120000.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>26</td>\n",
       "      <td>-1</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>2682.0</td>\n",
       "      <td>1725.0</td>\n",
       "      <td>2682.0</td>\n",
       "      <td>3272.0</td>\n",
       "      <td>3455.0</td>\n",
       "      <td>3261.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>90000.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>34</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>29239.0</td>\n",
       "      <td>14027.0</td>\n",
       "      <td>13559.0</td>\n",
       "      <td>14331.0</td>\n",
       "      <td>14948.0</td>\n",
       "      <td>15549.0</td>\n",
       "      <td>1518.0</td>\n",
       "      <td>1500.0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>5000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>50000.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>37</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>46990.0</td>\n",
       "      <td>48233.0</td>\n",
       "      <td>49291.0</td>\n",
       "      <td>28314.0</td>\n",
       "      <td>28959.0</td>\n",
       "      <td>29547.0</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>1200.0</td>\n",
       "      <td>1100.0</td>\n",
       "      <td>1069.0</td>\n",
       "      <td>1000.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>50000.0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>57</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>8617.0</td>\n",
       "      <td>5670.0</td>\n",
       "      <td>35835.0</td>\n",
       "      <td>20940.0</td>\n",
       "      <td>19146.0</td>\n",
       "      <td>19131.0</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>36681.0</td>\n",
       "      <td>10000.0</td>\n",
       "      <td>9000.0</td>\n",
       "      <td>689.0</td>\n",
       "      <td>679.0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   ID  LIMIT_BAL  SEX  EDUCATION  MARRIAGE  AGE  PAY_0  PAY_2  PAY_3  PAY_4  PAY_5  PAY_6  \\\n",
       "0   1    20000.0    2          2         1   24      2      2     -1     -1     -2     -2   \n",
       "1   2   120000.0    2          2         2   26     -1      2      0      0      0      2   \n",
       "2   3    90000.0    2          2         2   34      0      0      0      0      0      0   \n",
       "3   4    50000.0    2          2         1   37      0      0      0      0      0      0   \n",
       "4   5    50000.0    1          2         1   57     -1      0     -1      0      0      0   \n",
       "\n",
       "   BILL_AMT1  BILL_AMT2  BILL_AMT3  BILL_AMT4  BILL_AMT5  BILL_AMT6  PAY_AMT1  PAY_AMT2  PAY_AMT3  \\\n",
       "0     3913.0     3102.0      689.0        0.0        0.0        0.0       0.0     689.0       0.0   \n",
       "1     2682.0     1725.0     2682.0     3272.0     3455.0     3261.0       0.0    1000.0    1000.0   \n",
       "2    29239.0    14027.0    13559.0    14331.0    14948.0    15549.0    1518.0    1500.0    1000.0   \n",
       "3    46990.0    48233.0    49291.0    28314.0    28959.0    29547.0    2000.0    2019.0    1200.0   \n",
       "4     8617.0     5670.0    35835.0    20940.0    19146.0    19131.0    2000.0   36681.0   10000.0   \n",
       "\n",
       "   PAY_AMT4  PAY_AMT5  PAY_AMT6  default.payment.next.month  \n",
       "0       0.0       0.0       0.0                           1  \n",
       "1    1000.0       0.0    2000.0                           1  \n",
       "2    1000.0    1000.0    5000.0                           0  \n",
       "3    1100.0    1069.0    1000.0                           0  \n",
       "4    9000.0     689.0     679.0                           0  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt # this is used for the plot the graph \n",
    "import seaborn as sns # used for plot interactive graph. \n",
    "from pandas import set_option\n",
    "plt.style.use('ggplot') # nice plots\n",
    "pd.set_option('display.width', 100)\n",
    "pd.set_option('display.max_columns', None)\n",
    "\n",
    "dataset = pd.read_csv('/Users/nic/Documents/学校论文/代码/datasets/UCI_Credit_Card.csv')\n",
    "dataset.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "import done!\n"
     ]
    }
   ],
   "source": [
    "X = dataset.drop(['default.payment.next.month'],axis=1)\n",
    "y = dataset['default.payment.next.month']\n",
    "\n",
    "import tensorflow as tf\n",
    "%matplotlib inline\n",
    "from keras.layers import Input, Dense, LSTM\n",
    "from keras.models import Model\n",
    "\n",
    "from keras.layers import *\n",
    "from keras.models import *\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras import optimizers\n",
    "from sklearn.metrics import  accuracy_score, f1_score, precision_score, recall_score, roc_auc_score\n",
    "print('import done!')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(24000, 6, 2) (24000,) (6000, 6, 2) (6000,)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import MinMaxScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "scaler = MinMaxScaler(feature_range=(-1, 1))\n",
    "\n",
    "output_dim = 1\n",
    "batch_size = 256 #每轮训练模型时，样本的数量\n",
    "epochs = 60 #训练60轮次\n",
    "seq_len = 6 # 每3个为一组时间窗口\n",
    "hidden_size = 128\n",
    "\n",
    "TIME_STEPS = 6\n",
    "INPUT_DIM = 2\n",
    "\n",
    "lstm_units = 64\n",
    "dataset_flow = dataset[['PAY_0','PAY_2','PAY_3','PAY_4','PAY_5','PAY_6',\n",
    "#                         'BILL_AMT1','BILL_AMT2','BILL_AMT3','BILL_AMT4','BILL_AMT5','BILL_AMT6',\n",
    "                        'PAY_AMT1','PAY_AMT2','PAY_AMT3','PAY_AMT4','PAY_AMT5','PAY_AMT6'\n",
    "                       ]]\n",
    "scaler.fit(dataset_flow)\n",
    "X_train, X_test, y_train, y_test = train_test_split(dataset_flow, y, test_size = 0.2, random_state = 1)\n",
    "# 归一化\n",
    "X_train = scaler.transform(X_train)\n",
    "X_test = scaler.transform(X_test)\n",
    "train_num = X_train.shape[0]\n",
    "test_num = X_test.shape[0]\n",
    "X_train = X_train.reshape((train_num, seq_len, INPUT_DIM))\n",
    "y_train = y_train.values\n",
    "X_test = X_test.reshape((test_num, seq_len, INPUT_DIM))\n",
    "y_test = y_test.values\n",
    "# X_train = np.array([data_train[i : i + seq_len, :] for i in range(data_train.shape[0] - seq_len)])\n",
    "# y_train = np.array([data_train[i + seq_len, 0] for i in range(data_train.shape[0]- seq_len)])\n",
    "# X_test = np.array([data_test[i : i + seq_len, :] for i in range(data_test.shape[0]- seq_len)])\n",
    "# y_test = np.array([data_test[i + seq_len, 0] for i in range(data_test.shape[0] - seq_len)])\n",
    "\n",
    "print(X_train.shape, y_train.shape, X_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Dense, Lambda, Dot, Activation, Concatenate\n",
    "from tensorflow.keras.layers import Layer\n",
    "\n",
    "\n",
    "class Attention_(Layer):\n",
    "\n",
    "    def __init__(self, units=128, **kwargs):\n",
    "        self.units = units\n",
    "        super().__init__(**kwargs)\n",
    "\n",
    "    def __call__(self, inputs):\n",
    "        \"\"\"\n",
    "        Many-to-one attention mechanism for Keras.\n",
    "        @param inputs: 3D tensor with shape (batch_size, time_steps, input_dim).\n",
    "        @return: 2D tensor with shape (batch_size, 128)\n",
    "        @author: felixhao28, philipperemy.\n",
    "        \"\"\"\n",
    "        hidden_states = inputs\n",
    "        hidden_size = int(hidden_states.shape[2])\n",
    "        # Inside dense layer\n",
    "        #              hidden_states            dot               W            =>           score_first_part\n",
    "        # (batch_size, time_steps, hidden_size) dot (hidden_size, hidden_size) => (batch_size, time_steps, hidden_size)\n",
    "        # W is the trainable weight matrix of attention Luong's multiplicative style score\n",
    "        score_first_part = Dense(hidden_size, use_bias=False, name='attention_score_vec')(hidden_states)\n",
    "        #            score_first_part           dot        last_hidden_state     => attention_weights\n",
    "        # (batch_size, time_steps, hidden_size) dot   (batch_size, hidden_size)  => (batch_size, time_steps)\n",
    "        h_t = Lambda(lambda x: x[:, -1, :], output_shape=(hidden_size,), name='last_hidden_state')(hidden_states)\n",
    "        score = Dot(axes=[1, 2], name='attention_score')([h_t, score_first_part])\n",
    "        attention_weights = Activation('softmax', name='attention_weight')(score)\n",
    "        # (batch_size, time_steps, hidden_size) dot (batch_size, time_steps) => (batch_size, hidden_size)\n",
    "        context_vector = Dot(axes=[1, 1], name='context_vector')([hidden_states, attention_weights])\n",
    "        pre_activation = Concatenate(name='attention_output')([context_vector, h_t])\n",
    "        attention_vector = Dense(self.units, use_bias=False, activation='tanh', name='attention_vector')(pre_activation)\n",
    "        return attention_vector\n",
    "\n",
    "    def get_config(self):\n",
    "        return {'units': self.units}\n",
    "\n",
    "    @classmethod\n",
    "    def from_config(cls, config):\n",
    "        return cls(**config)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"model_17\"\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_18 (InputLayer)           [(None, 6, 2)]       0                                            \n",
      "__________________________________________________________________________________________________\n",
      "lstm_13 (LSTM)                  (None, 6, 32)        4480        input_18[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "last_hidden_state (Lambda)      (None, 32)           0           lstm_13[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "attention_score_vec (Dense)     (None, 6, 32)        1024        lstm_13[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "attention_score (Dot)           (None, 6)            0           last_hidden_state[0][0]          \n",
      "                                                                 attention_score_vec[0][0]        \n",
      "__________________________________________________________________________________________________\n",
      "attention_weight (Activation)   (None, 6)            0           attention_score[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "context_vector (Dot)            (None, 32)           0           lstm_13[0][0]                    \n",
      "                                                                 attention_weight[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "attention_output (Concatenate)  (None, 64)           0           context_vector[0][0]             \n",
      "                                                                 last_hidden_state[0][0]          \n",
      "__________________________________________________________________________________________________\n",
      "attention_vector (Dense)        (None, 32)           2048        attention_output[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "dense_17 (Dense)                (None, 1)            33          attention_vector[0][0]           \n",
      "==================================================================================================\n",
      "Total params: 7,585\n",
      "Trainable params: 7,585\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model_input = Input(shape=(TIME_STEPS, INPUT_DIM))\n",
    "# model_input = Input(shape=[X_train.shape[1], X_train.shape[2],])\n",
    "x = LSTM(32, return_sequences=True)(model_input)\n",
    "# print(x.shape, type(x))\n",
    "x = Attention_(32)(x)\n",
    "x = Dense(1)(x)\n",
    "model = Model(model_input, x)\n",
    "print(model.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/nic/Library/Python/3.7/lib/python/site-packages/keras/optimizer_v2/optimizer_v2.py:356: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
      "  \"The `lr` argument is deprecated, use `learning_rate` instead.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/200\n",
      "94/94 [==============================] - 2s 4ms/step - loss: 0.6565\n",
      "Epoch 2/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6560\n",
      "Epoch 3/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6559\n",
      "Epoch 4/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6558\n",
      "Epoch 5/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6557\n",
      "Epoch 6/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6557\n",
      "Epoch 7/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6556\n",
      "Epoch 8/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6556\n",
      "Epoch 9/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6555\n",
      "Epoch 10/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6554\n",
      "Epoch 11/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6554\n",
      "Epoch 12/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6553\n",
      "Epoch 13/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6553\n",
      "Epoch 14/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6552\n",
      "Epoch 15/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6552\n",
      "Epoch 16/200\n",
      "94/94 [==============================] - 0s 5ms/step - loss: 0.6551\n",
      "Epoch 17/200\n",
      "94/94 [==============================] - 1s 5ms/step - loss: 0.6551\n",
      "Epoch 18/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6551\n",
      "Epoch 19/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6550\n",
      "Epoch 20/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6550\n",
      "Epoch 21/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6549\n",
      "Epoch 22/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6549\n",
      "Epoch 23/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6548\n",
      "Epoch 24/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6548\n",
      "Epoch 25/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6548\n",
      "Epoch 26/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6547\n",
      "Epoch 27/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6547\n",
      "Epoch 28/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6546\n",
      "Epoch 29/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6546\n",
      "Epoch 30/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6546\n",
      "Epoch 31/200\n",
      "94/94 [==============================] - 0s 5ms/step - loss: 0.6546\n",
      "Epoch 32/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6545\n",
      "Epoch 33/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6545\n",
      "Epoch 34/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6545\n",
      "Epoch 35/200\n",
      "94/94 [==============================] - 0s 5ms/step - loss: 0.6544\n",
      "Epoch 36/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6544\n",
      "Epoch 37/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6543\n",
      "Epoch 38/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6543\n",
      "Epoch 39/200\n",
      "94/94 [==============================] - 0s 5ms/step - loss: 0.6543\n",
      "Epoch 40/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6542\n",
      "Epoch 41/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6542\n",
      "Epoch 42/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6542\n",
      "Epoch 43/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6542\n",
      "Epoch 44/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6541\n",
      "Epoch 45/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6541\n",
      "Epoch 46/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6541\n",
      "Epoch 47/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6540\n",
      "Epoch 48/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6540\n",
      "Epoch 49/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6540\n",
      "Epoch 50/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6540\n",
      "Epoch 51/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6539\n",
      "Epoch 52/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6539\n",
      "Epoch 53/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6539\n",
      "Epoch 54/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6538\n",
      "Epoch 55/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6538\n",
      "Epoch 56/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6538\n",
      "Epoch 57/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6538\n",
      "Epoch 58/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6537\n",
      "Epoch 59/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6537\n",
      "Epoch 60/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6537\n",
      "Epoch 61/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6536\n",
      "Epoch 62/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6536\n",
      "Epoch 63/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6536\n",
      "Epoch 64/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6535\n",
      "Epoch 65/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6535\n",
      "Epoch 66/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6535\n",
      "Epoch 67/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6535\n",
      "Epoch 68/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6535\n",
      "Epoch 69/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6534\n",
      "Epoch 70/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6534\n",
      "Epoch 71/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6533\n",
      "Epoch 72/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6533\n",
      "Epoch 73/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6533\n",
      "Epoch 74/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6533\n",
      "Epoch 75/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6533\n",
      "Epoch 76/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6532\n",
      "Epoch 77/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6532\n",
      "Epoch 78/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6531\n",
      "Epoch 79/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6531\n",
      "Epoch 80/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6531\n",
      "Epoch 81/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6531\n",
      "Epoch 82/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6530\n",
      "Epoch 83/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6530\n",
      "Epoch 84/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6530\n",
      "Epoch 85/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6529\n",
      "Epoch 86/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6529\n",
      "Epoch 87/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6529\n",
      "Epoch 88/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6528\n",
      "Epoch 89/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6528\n",
      "Epoch 90/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6528\n",
      "Epoch 91/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6528\n",
      "Epoch 92/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6527\n",
      "Epoch 93/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6527\n",
      "Epoch 94/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6526\n",
      "Epoch 95/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6526\n",
      "Epoch 96/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6526\n",
      "Epoch 97/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6525\n",
      "Epoch 98/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6525\n",
      "Epoch 99/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6525\n",
      "Epoch 100/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6524\n",
      "Epoch 101/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6524\n",
      "Epoch 102/200\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6524\n",
      "Epoch 103/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6523\n",
      "Epoch 104/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6523\n",
      "Epoch 105/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6522\n",
      "Epoch 106/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6522\n",
      "Epoch 107/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6522\n",
      "Epoch 108/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6521\n",
      "Epoch 109/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6521\n",
      "Epoch 110/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6520\n",
      "Epoch 111/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6520\n",
      "Epoch 112/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6520\n",
      "Epoch 113/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6519\n",
      "Epoch 114/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6519\n",
      "Epoch 115/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6519\n",
      "Epoch 116/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6518\n",
      "Epoch 117/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6518\n",
      "Epoch 118/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6517\n",
      "Epoch 119/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6517\n",
      "Epoch 120/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6516\n",
      "Epoch 121/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6516\n",
      "Epoch 122/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6516\n",
      "Epoch 123/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6515\n",
      "Epoch 124/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6515\n",
      "Epoch 125/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6514\n",
      "Epoch 126/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6514\n",
      "Epoch 127/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6513\n",
      "Epoch 128/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6513\n",
      "Epoch 129/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6512\n",
      "Epoch 130/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6512\n",
      "Epoch 131/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6512\n",
      "Epoch 132/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6511\n",
      "Epoch 133/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6511\n",
      "Epoch 134/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6510\n",
      "Epoch 135/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6510\n",
      "Epoch 136/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6510\n",
      "Epoch 137/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6509\n",
      "Epoch 138/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6509\n",
      "Epoch 139/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6508\n",
      "Epoch 140/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6508\n",
      "Epoch 141/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6508\n",
      "Epoch 142/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6507\n",
      "Epoch 143/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6507\n",
      "Epoch 144/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6506\n",
      "Epoch 145/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6506\n",
      "Epoch 146/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6505\n",
      "Epoch 147/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6505\n",
      "Epoch 148/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6505\n",
      "Epoch 149/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6504\n",
      "Epoch 150/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6504\n",
      "Epoch 151/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6503\n",
      "Epoch 152/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6503\n",
      "Epoch 153/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6503\n",
      "Epoch 154/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6502\n",
      "Epoch 155/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6502\n",
      "Epoch 156/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6501\n",
      "Epoch 157/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6501\n",
      "Epoch 158/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6501\n",
      "Epoch 159/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6500\n",
      "Epoch 160/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6500\n",
      "Epoch 161/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6499\n",
      "Epoch 162/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6499\n",
      "Epoch 163/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6499\n",
      "Epoch 164/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6498\n",
      "Epoch 165/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6498\n",
      "Epoch 166/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6498\n",
      "Epoch 167/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6497\n",
      "Epoch 168/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6497\n",
      "Epoch 169/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6496\n",
      "Epoch 170/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6496\n",
      "Epoch 171/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6496\n",
      "Epoch 172/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6495\n",
      "Epoch 173/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6495\n",
      "Epoch 174/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6494\n",
      "Epoch 175/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6494\n",
      "Epoch 176/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6493\n",
      "Epoch 177/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6493\n",
      "Epoch 178/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6492\n",
      "Epoch 179/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6492\n",
      "Epoch 180/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6492\n",
      "Epoch 181/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6491\n",
      "Epoch 182/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6491\n",
      "Epoch 183/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6490\n",
      "Epoch 184/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6490\n",
      "Epoch 185/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6489\n",
      "Epoch 186/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6489\n",
      "Epoch 187/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6489\n",
      "Epoch 188/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6488\n",
      "Epoch 189/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6488\n",
      "Epoch 190/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6487\n",
      "Epoch 191/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6487\n",
      "Epoch 192/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6486\n",
      "Epoch 193/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6486\n",
      "Epoch 194/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6485\n",
      "Epoch 195/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6485\n",
      "Epoch 196/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6485\n",
      "Epoch 197/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6484\n",
      "Epoch 198/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6484\n",
      "Epoch 199/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6483\n",
      "Epoch 200/200\n",
      "94/94 [==============================] - 0s 4ms/step - loss: 0.6483\n",
      "94/94 [==============================] - 1s 2ms/step - loss: 0.4733\n",
      "Train loss: 0.4732588231563568\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "24/24 [==============================] - 0s 2ms/step - loss: 0.4910\n",
      "Test loss: 0.4909914433956146\n",
      "ROC-AUC: 0.7550705108453362\n"
     ]
    }
   ],
   "source": [
    "sgd = optimizers.SGD(lr=0.02, decay=1e-6, momentum=0.9, nesterov=True)\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam')\n",
    "cw = {0:1, 1:2}\n",
    "model.fit(X_train, y_train, epochs=200, batch_size=256, shuffle=False, class_weight=cw)\n",
    "y_pred = model.predict(X_test)\n",
    "print('Train loss:', model.evaluate(X_train, y_train, batch_size=batch_size))\n",
    "print('Test loss:', model.evaluate(X_test, y_test, batch_size=batch_size))\n",
    "print('ROC-AUC:', roc_auc_score(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1 0 ... 0 0 0] \n",
      " [[0.19445398]\n",
      " [0.5939353 ]\n",
      " [0.26893526]\n",
      " ...\n",
      " [0.20419982]\n",
      " [0.24588855]\n",
      " [0.28427085]]\n",
      "6000\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.86      0.88      0.87      4663\n",
      "           1       0.55      0.49      0.52      1337\n",
      "\n",
      "    accuracy                           0.80      6000\n",
      "   macro avg       0.70      0.69      0.69      6000\n",
      "weighted avg       0.79      0.80      0.79      6000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "y_pred_flatten = y_pred.flatten()\n",
    "print(y_test, '\\n',y_pred)\n",
    "print(len([y_p for y_p in y_pred if y_p!=0]))\n",
    "y_pred_new = [0 if y_p<0.5 else 1 for y_p in y_pred_flatten]\n",
    "print(classification_report(y_test, y_pred_new))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7550705108453362 0.7955 0.5459098497495827 0.48915482423335827 0.5159763313609468\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import  accuracy_score, f1_score, precision_score, recall_score, roc_auc_score\n",
    "roc=roc_auc_score(y_test, y_pred)\n",
    "acc = accuracy_score(y_test, y_pred_new)\n",
    "prec = precision_score(y_test, y_pred_new)\n",
    "rec = recall_score(y_test, y_pred_new)\n",
    "f1 = f1_score(y_test, y_pred_new)\n",
    "print(roc,acc,prec,rec,f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}