3-ANN-Classification-Attack.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5bd1c278-13ec-4bad-96f6-34c41beb48da",
   "metadata": {},
   "source": [
    "# Classification Artificial Neural Networks (Attack)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6553d7e0-0212-41f8-a9f1-2a9930fca755",
   "metadata": {},
   "source": [
    "### Importing libraries and reading data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b63032dd-c7f5-4ec0-baad-9db247f2d5c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import datetime\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import pytz\n",
    "import datetime\n",
    "import os\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import Dense, Activation, Dropout\n",
    "from tensorflow.keras.optimizers import Adam\n",
    "from tensorflow.keras.callbacks import EarlyStopping\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score, classification_report\n",
    "import tensorflow as tf\n",
    "from sklearn.model_selection import KFold"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0c1e66a5-ed66-4439-bb9d-489095061841",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train = pd.read_csv('df_train_attack.csv')\n",
    "df_test = pd.read_csv('df_test_attack.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "43fbb7f5-87c5-4cf3-8383-3da12ecefa46",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = df_train.drop('dist_BM_RA', axis=1)\n",
    "y_train = df_train['dist_BM_RA']\n",
    "X_test = df_test.drop(['dist_BM_RA'], axis=1)\n",
    "y_test = df_test['dist_BM_RA']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "394a9523-26d1-456c-ab93-7ac6a6b5cd2c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.851"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "round(1-y_train.value_counts()[1] / y_train.value_counts()[0],3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "833276b7-ffcb-4015-9c2b-d1388b13410e",
   "metadata": {},
   "source": [
    "### Scaling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "856c88e3-9d68-4ed1-9654-3ef540a9b4e3",
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler = StandardScaler()\n",
    "#scaler = MinMaxScaler()\n",
    "\n",
    "scaler.fit(X_train)\n",
    "\n",
    "X_train = scaler.transform(X_train)\n",
    "X_test = scaler.transform(X_test)\n",
    "\n",
    "y_train = y_train.values\n",
    "y_test = y_test.values"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5f984a90-1e89-4408-887b-4890dd23a6a2",
   "metadata": {},
   "source": [
    "### Training the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "e6c28d3b-39d8-422e-b92a-7ed4816e924b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "------------------------------------------------------------------------\n",
      "Training for fold 1 ...\n",
      "Epoch 1/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.3555 - accuracy: 0.8672 - precision_13: 0.3682 - recall_13: 0.0447 - val_loss: 0.2053 - val_accuracy: 0.9012 - val_precision_13: 0.5811 - val_recall_13: 0.7963\n",
      "Epoch 2/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.2290 - accuracy: 0.9031 - precision_13: 0.6867 - recall_13: 0.4481 - val_loss: 0.1998 - val_accuracy: 0.9194 - val_precision_13: 0.7005 - val_recall_13: 0.6389\n",
      "Epoch 3/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.2004 - accuracy: 0.9138 - precision_13: 0.7379 - recall_13: 0.5596 - val_loss: 0.1829 - val_accuracy: 0.9200 - val_precision_13: 0.7326 - val_recall_13: 0.5833\n",
      "Epoch 4/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1897 - accuracy: 0.9205 - precision_13: 0.7371 - recall_13: 0.6346 - val_loss: 0.1754 - val_accuracy: 0.9265 - val_precision_13: 0.7136 - val_recall_13: 0.7037\n",
      "Epoch 5/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1875 - accuracy: 0.9286 - precision_13: 0.7885 - recall_13: 0.6387 - val_loss: 0.1785 - val_accuracy: 0.9306 - val_precision_13: 0.7663 - val_recall_13: 0.6528\n",
      "Epoch 6/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1644 - accuracy: 0.9277 - precision_13: 0.7432 - recall_13: 0.6282 - val_loss: 0.1694 - val_accuracy: 0.9265 - val_precision_13: 0.6842 - val_recall_13: 0.7824\n",
      "Epoch 7/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1691 - accuracy: 0.9251 - precision_13: 0.7515 - recall_13: 0.6435 - val_loss: 0.1667 - val_accuracy: 0.9318 - val_precision_13: 0.7427 - val_recall_13: 0.7083\n",
      "Epoch 8/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1642 - accuracy: 0.9323 - precision_13: 0.7967 - recall_13: 0.6535 - val_loss: 0.1873 - val_accuracy: 0.9265 - val_precision_13: 0.8370 - val_recall_13: 0.5231\n",
      "Epoch 9/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1676 - accuracy: 0.9333 - precision_13: 0.7775 - recall_13: 0.6413 - val_loss: 0.1606 - val_accuracy: 0.9318 - val_precision_13: 0.7212 - val_recall_13: 0.7546\n",
      "Epoch 10/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1632 - accuracy: 0.9327 - precision_13: 0.7663 - recall_13: 0.6562 - val_loss: 0.1624 - val_accuracy: 0.9335 - val_precision_13: 0.7441 - val_recall_13: 0.7269\n",
      "Epoch 11/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1493 - accuracy: 0.9416 - precision_13: 0.7922 - recall_13: 0.7356 - val_loss: 0.1948 - val_accuracy: 0.9265 - val_precision_13: 0.7040 - val_recall_13: 0.7269\n",
      "Epoch 12/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1611 - accuracy: 0.9375 - precision_13: 0.8071 - recall_13: 0.6956 - val_loss: 0.1736 - val_accuracy: 0.9224 - val_precision_13: 0.6981 - val_recall_13: 0.6852\n",
      "Epoch 13/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1517 - accuracy: 0.9379 - precision_13: 0.7810 - recall_13: 0.7103 - val_loss: 0.1743 - val_accuracy: 0.9282 - val_precision_13: 0.7448 - val_recall_13: 0.6620\n",
      "Epoch 14/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1411 - accuracy: 0.9470 - precision_13: 0.8464 - recall_13: 0.7379 - val_loss: 0.1643 - val_accuracy: 0.9347 - val_precision_13: 0.8035 - val_recall_13: 0.6435\n",
      "Epoch 15/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1505 - accuracy: 0.9417 - precision_13: 0.8526 - recall_13: 0.6709 - val_loss: 0.1635 - val_accuracy: 0.9376 - val_precision_13: 0.7957 - val_recall_13: 0.6852\n",
      "Epoch 16/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1396 - accuracy: 0.9452 - precision_13: 0.8105 - recall_13: 0.7323 - val_loss: 0.1733 - val_accuracy: 0.9306 - val_precision_13: 0.7227 - val_recall_13: 0.7361\n",
      "Epoch 17/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1431 - accuracy: 0.9445 - precision_13: 0.8057 - recall_13: 0.7380 - val_loss: 0.1684 - val_accuracy: 0.9388 - val_precision_13: 0.7718 - val_recall_13: 0.7361\n",
      "Epoch 18/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1363 - accuracy: 0.9459 - precision_13: 0.8503 - recall_13: 0.7179 - val_loss: 0.1718 - val_accuracy: 0.9324 - val_precision_13: 0.7488 - val_recall_13: 0.7037\n",
      "Epoch 19/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1361 - accuracy: 0.9464 - precision_13: 0.8252 - recall_13: 0.7472 - val_loss: 0.2562 - val_accuracy: 0.9335 - val_precision_13: 0.7562 - val_recall_13: 0.7037\n",
      "Epoch 20/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1437 - accuracy: 0.9429 - precision_13: 0.8059 - recall_13: 0.7280 - val_loss: 0.1640 - val_accuracy: 0.9347 - val_precision_13: 0.7749 - val_recall_13: 0.6852\n",
      "Epoch 21/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1419 - accuracy: 0.9430 - precision_13: 0.8355 - recall_13: 0.6912 - val_loss: 0.1624 - val_accuracy: 0.9318 - val_precision_13: 0.7358 - val_recall_13: 0.7222\n",
      "Epoch 22/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1358 - accuracy: 0.9517 - precision_13: 0.8335 - recall_13: 0.7884 - val_loss: 0.1700 - val_accuracy: 0.9247 - val_precision_13: 0.6746 - val_recall_13: 0.7870\n",
      "Epoch 23/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1367 - accuracy: 0.9500 - precision_13: 0.8105 - recall_13: 0.8184 - val_loss: 0.1984 - val_accuracy: 0.9335 - val_precision_13: 0.7352 - val_recall_13: 0.7454\n",
      "Epoch 24/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1352 - accuracy: 0.9484 - precision_13: 0.8403 - recall_13: 0.7712 - val_loss: 0.1835 - val_accuracy: 0.9324 - val_precision_13: 0.7644 - val_recall_13: 0.6759\n",
      "Epoch 25/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1272 - accuracy: 0.9514 - precision_13: 0.8522 - recall_13: 0.7705 - val_loss: 0.1787 - val_accuracy: 0.9306 - val_precision_13: 0.7290 - val_recall_13: 0.7222\n",
      "Epoch 26/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1345 - accuracy: 0.9492 - precision_13: 0.8320 - recall_13: 0.7534 - val_loss: 0.1851 - val_accuracy: 0.9347 - val_precision_13: 0.7612 - val_recall_13: 0.7083\n",
      "Epoch 27/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1441 - accuracy: 0.9456 - precision_13: 0.8128 - recall_13: 0.7549 - val_loss: 0.1638 - val_accuracy: 0.9276 - val_precision_13: 0.7085 - val_recall_13: 0.7315\n",
      "Epoch 28/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1352 - accuracy: 0.9475 - precision_13: 0.8235 - recall_13: 0.7515 - val_loss: 0.2032 - val_accuracy: 0.9353 - val_precision_13: 0.7524 - val_recall_13: 0.7315\n",
      "Epoch 29/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1250 - accuracy: 0.9516 - precision_13: 0.8212 - recall_13: 0.7848 - val_loss: 0.1654 - val_accuracy: 0.9318 - val_precision_13: 0.7404 - val_recall_13: 0.7130\n",
      "Epoch 30/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1212 - accuracy: 0.9544 - precision_13: 0.8420 - recall_13: 0.8031 - val_loss: 0.1885 - val_accuracy: 0.9247 - val_precision_13: 0.6803 - val_recall_13: 0.7685\n",
      "Epoch 31/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1284 - accuracy: 0.9484 - precision_13: 0.7917 - recall_13: 0.8009 - val_loss: 0.2112 - val_accuracy: 0.9312 - val_precision_13: 0.7346 - val_recall_13: 0.7176\n",
      "Epoch 32/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1317 - accuracy: 0.9449 - precision_13: 0.8164 - recall_13: 0.7373 - val_loss: 0.1781 - val_accuracy: 0.9229 - val_precision_13: 0.6809 - val_recall_13: 0.7407\n",
      "Epoch 33/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1295 - accuracy: 0.9458 - precision_13: 0.8074 - recall_13: 0.7611 - val_loss: 0.1721 - val_accuracy: 0.9312 - val_precision_13: 0.7260 - val_recall_13: 0.7361\n",
      "Epoch 34/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1180 - accuracy: 0.9530 - precision_13: 0.8241 - recall_13: 0.7878 - val_loss: 0.1759 - val_accuracy: 0.9341 - val_precision_13: 0.7385 - val_recall_13: 0.7454\n",
      "Epoch 00034: early stopping\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.95      0.95      1312\n",
      "           1       0.68      0.71      0.69       188\n",
      "\n",
      "    accuracy                           0.92      1500\n",
      "   macro avg       0.82      0.83      0.82      1500\n",
      "weighted avg       0.92      0.92      0.92      1500\n",
      "\n",
      "------------------------------------------------------------------------\n",
      "Training for fold 2 ...\n",
      "Epoch 1/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.3765 - accuracy: 0.8616 - precision_14: 0.1010 - recall_14: 0.0063 - val_loss: 0.2187 - val_accuracy: 0.8706 - val_precision_14: 0.0000e+00 - val_recall_14: 0.0000e+00\n",
      "Epoch 2/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.2522 - accuracy: 0.8740 - precision_14: 0.0000e+00 - recall_14: 0.0000e+00 - val_loss: 0.2080 - val_accuracy: 0.8706 - val_precision_14: 0.0000e+00 - val_recall_14: 0.0000e+00\n",
      "Epoch 3/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.2260 - accuracy: 0.8761 - precision_14: 0.0000e+00 - recall_14: 0.0000e+00 - val_loss: 0.2025 - val_accuracy: 0.8706 - val_precision_14: 0.0000e+00 - val_recall_14: 0.0000e+00\n",
      "Epoch 4/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.2207 - accuracy: 0.8731 - precision_14: 0.0000e+00 - recall_14: 0.0000e+00 - val_loss: 0.2021 - val_accuracy: 0.8706 - val_precision_14: 0.0000e+00 - val_recall_14: 0.0000e+00\n",
      "Epoch 5/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.2266 - accuracy: 0.8783 - precision_14: 0.3861 - recall_14: 0.1559 - val_loss: 0.1891 - val_accuracy: 0.9153 - val_precision_14: 0.6473 - val_recall_14: 0.7591\n",
      "Epoch 6/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1908 - accuracy: 0.9199 - precision_14: 0.6964 - recall_14: 0.7344 - val_loss: 0.1801 - val_accuracy: 0.9200 - val_precision_14: 0.7442 - val_recall_14: 0.5818\n",
      "Epoch 7/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1836 - accuracy: 0.9184 - precision_14: 0.7360 - recall_14: 0.6005 - val_loss: 0.1839 - val_accuracy: 0.9253 - val_precision_14: 0.8121 - val_recall_14: 0.5500\n",
      "Epoch 8/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1694 - accuracy: 0.9262 - precision_14: 0.7681 - recall_14: 0.6074 - val_loss: 0.1659 - val_accuracy: 0.9265 - val_precision_14: 0.7714 - val_recall_14: 0.6136\n",
      "Epoch 9/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1718 - accuracy: 0.9313 - precision_14: 0.7683 - recall_14: 0.6822 - val_loss: 0.1693 - val_accuracy: 0.9253 - val_precision_14: 0.7360 - val_recall_14: 0.6591\n",
      "Epoch 10/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1678 - accuracy: 0.9326 - precision_14: 0.7777 - recall_14: 0.6845 - val_loss: 0.1648 - val_accuracy: 0.9294 - val_precision_14: 0.7941 - val_recall_14: 0.6136\n",
      "Epoch 11/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1679 - accuracy: 0.9291 - precision_14: 0.7934 - recall_14: 0.6064 - val_loss: 0.1595 - val_accuracy: 0.9271 - val_precision_14: 0.7759 - val_recall_14: 0.6136\n",
      "Epoch 12/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1571 - accuracy: 0.9338 - precision_14: 0.7941 - recall_14: 0.6756 - val_loss: 0.1625 - val_accuracy: 0.9271 - val_precision_14: 0.7759 - val_recall_14: 0.6136\n",
      "Epoch 13/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1460 - accuracy: 0.9422 - precision_14: 0.8006 - recall_14: 0.7130 - val_loss: 0.1747 - val_accuracy: 0.9265 - val_precision_14: 0.7879 - val_recall_14: 0.5909\n",
      "Epoch 14/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1437 - accuracy: 0.9412 - precision_14: 0.8109 - recall_14: 0.7103 - val_loss: 0.1857 - val_accuracy: 0.9341 - val_precision_14: 0.7647 - val_recall_14: 0.7091\n",
      "Epoch 15/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1532 - accuracy: 0.9339 - precision_14: 0.7809 - recall_14: 0.6911 - val_loss: 0.1730 - val_accuracy: 0.9324 - val_precision_14: 0.7901 - val_recall_14: 0.6500\n",
      "Epoch 16/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1638 - accuracy: 0.9340 - precision_14: 0.7733 - recall_14: 0.6790 - val_loss: 0.1902 - val_accuracy: 0.9288 - val_precision_14: 0.7797 - val_recall_14: 0.6273\n",
      "Epoch 17/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1389 - accuracy: 0.9435 - precision_14: 0.8004 - recall_14: 0.7366 - val_loss: 0.1729 - val_accuracy: 0.9235 - val_precision_14: 0.7064 - val_recall_14: 0.7000\n",
      "Epoch 18/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1519 - accuracy: 0.9378 - precision_14: 0.7669 - recall_14: 0.7440 - val_loss: 0.2143 - val_accuracy: 0.9324 - val_precision_14: 0.8000 - val_recall_14: 0.6364\n",
      "Epoch 19/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1440 - accuracy: 0.9344 - precision_14: 0.7928 - recall_14: 0.6840 - val_loss: 0.1924 - val_accuracy: 0.9224 - val_precision_14: 0.6964 - val_recall_14: 0.7091\n",
      "Epoch 20/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1379 - accuracy: 0.9386 - precision_14: 0.7755 - recall_14: 0.7244 - val_loss: 0.2095 - val_accuracy: 0.9241 - val_precision_14: 0.7407 - val_recall_14: 0.6364\n",
      "Epoch 21/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1353 - accuracy: 0.9361 - precision_14: 0.7688 - recall_14: 0.7045 - val_loss: 0.1853 - val_accuracy: 0.9271 - val_precision_14: 0.7330 - val_recall_14: 0.6864\n",
      "Epoch 22/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1456 - accuracy: 0.9441 - precision_14: 0.8092 - recall_14: 0.7209 - val_loss: 0.1854 - val_accuracy: 0.9294 - val_precision_14: 0.7404 - val_recall_14: 0.7000\n",
      "Epoch 23/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1332 - accuracy: 0.9419 - precision_14: 0.7983 - recall_14: 0.7295 - val_loss: 0.2158 - val_accuracy: 0.9318 - val_precision_14: 0.7889 - val_recall_14: 0.6455\n",
      "Epoch 24/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1257 - accuracy: 0.9453 - precision_14: 0.8286 - recall_14: 0.7101 - val_loss: 0.1768 - val_accuracy: 0.9312 - val_precision_14: 0.7614 - val_recall_14: 0.6818\n",
      "Epoch 25/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1272 - accuracy: 0.9461 - precision_14: 0.8073 - recall_14: 0.7685 - val_loss: 0.1982 - val_accuracy: 0.9312 - val_precision_14: 0.7373 - val_recall_14: 0.7273\n",
      "Epoch 26/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1228 - accuracy: 0.9460 - precision_14: 0.7968 - recall_14: 0.7712 - val_loss: 0.2079 - val_accuracy: 0.9294 - val_precision_14: 0.7841 - val_recall_14: 0.6273\n",
      "Epoch 27/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1295 - accuracy: 0.9453 - precision_14: 0.7990 - recall_14: 0.7529 - val_loss: 0.1839 - val_accuracy: 0.9276 - val_precision_14: 0.7939 - val_recall_14: 0.5955\n",
      "Epoch 28/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1313 - accuracy: 0.9382 - precision_14: 0.7873 - recall_14: 0.7254 - val_loss: 0.2242 - val_accuracy: 0.9288 - val_precision_14: 0.7565 - val_recall_14: 0.6636\n",
      "Epoch 29/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1402 - accuracy: 0.9471 - precision_14: 0.8474 - recall_14: 0.7285 - val_loss: 0.1970 - val_accuracy: 0.9276 - val_precision_14: 0.7389 - val_recall_14: 0.6818\n",
      "Epoch 30/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1253 - accuracy: 0.9487 - precision_14: 0.8100 - recall_14: 0.7719 - val_loss: 0.1901 - val_accuracy: 0.9341 - val_precision_14: 0.7872 - val_recall_14: 0.6727\n",
      "Epoch 31/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1261 - accuracy: 0.9466 - precision_14: 0.8161 - recall_14: 0.7251 - val_loss: 0.1656 - val_accuracy: 0.9341 - val_precision_14: 0.7935 - val_recall_14: 0.6636\n",
      "Epoch 32/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1263 - accuracy: 0.9411 - precision_14: 0.8268 - recall_14: 0.6863 - val_loss: 0.1921 - val_accuracy: 0.9247 - val_precision_14: 0.7170 - val_recall_14: 0.6909\n",
      "Epoch 33/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1284 - accuracy: 0.9526 - precision_14: 0.8375 - recall_14: 0.7947 - val_loss: 0.2054 - val_accuracy: 0.9312 - val_precision_14: 0.7754 - val_recall_14: 0.6591\n",
      "Epoch 34/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1314 - accuracy: 0.9437 - precision_14: 0.8038 - recall_14: 0.7671 - val_loss: 0.2194 - val_accuracy: 0.9282 - val_precision_14: 0.7269 - val_recall_14: 0.7136\n",
      "Epoch 35/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1331 - accuracy: 0.9467 - precision_14: 0.7944 - recall_14: 0.7644 - val_loss: 0.1754 - val_accuracy: 0.9265 - val_precision_14: 0.7093 - val_recall_14: 0.7318\n",
      "Epoch 36/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1264 - accuracy: 0.9462 - precision_14: 0.8105 - recall_14: 0.7808 - val_loss: 0.2079 - val_accuracy: 0.9300 - val_precision_14: 0.7440 - val_recall_14: 0.7000\n",
      "Epoch 00036: early stopping\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.96      0.96      0.96      1312\n",
      "           1       0.72      0.71      0.72       188\n",
      "\n",
      "    accuracy                           0.93      1500\n",
      "   macro avg       0.84      0.84      0.84      1500\n",
      "weighted avg       0.93      0.93      0.93      1500\n",
      "\n",
      "------------------------------------------------------------------------\n",
      "Training for fold 3 ...\n",
      "Epoch 1/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.3510 - accuracy: 0.8643 - precision_15: 0.0000e+00 - recall_15: 0.0000e+00 - val_loss: 0.2445 - val_accuracy: 0.8647 - val_precision_15: 0.0000e+00 - val_recall_15: 0.0000e+00\n",
      "Epoch 2/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.2473 - accuracy: 0.8809 - precision_15: 0.4511 - recall_15: 0.1192 - val_loss: 0.1961 - val_accuracy: 0.9124 - val_precision_15: 0.6816 - val_recall_15: 0.6609\n",
      "Epoch 3/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.2004 - accuracy: 0.9023 - precision_15: 0.6539 - recall_15: 0.5193 - val_loss: 0.1913 - val_accuracy: 0.9229 - val_precision_15: 0.7797 - val_recall_15: 0.6000\n",
      "Epoch 4/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1872 - accuracy: 0.9169 - precision_15: 0.7063 - recall_15: 0.5809 - val_loss: 0.1804 - val_accuracy: 0.9259 - val_precision_15: 0.7653 - val_recall_15: 0.6522\n",
      "Epoch 5/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1875 - accuracy: 0.9145 - precision_15: 0.7079 - recall_15: 0.6086 - val_loss: 0.1791 - val_accuracy: 0.9294 - val_precision_15: 0.7391 - val_recall_15: 0.7391\n",
      "Epoch 6/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1672 - accuracy: 0.9263 - precision_15: 0.7259 - recall_15: 0.6770 - val_loss: 0.1886 - val_accuracy: 0.9265 - val_precision_15: 0.8261 - val_recall_15: 0.5783\n",
      "Epoch 7/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1701 - accuracy: 0.9328 - precision_15: 0.7775 - recall_15: 0.6575 - val_loss: 0.2036 - val_accuracy: 0.9218 - val_precision_15: 0.7709 - val_recall_15: 0.6000\n",
      "Epoch 8/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1710 - accuracy: 0.9350 - precision_15: 0.7881 - recall_15: 0.6916 - val_loss: 0.1814 - val_accuracy: 0.9241 - val_precision_15: 0.7853 - val_recall_15: 0.6043\n",
      "Epoch 9/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1698 - accuracy: 0.9273 - precision_15: 0.7726 - recall_15: 0.6520 - val_loss: 0.1760 - val_accuracy: 0.9229 - val_precision_15: 0.7895 - val_recall_15: 0.5870\n",
      "Epoch 10/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1515 - accuracy: 0.9405 - precision_15: 0.7942 - recall_15: 0.6987 - val_loss: 0.1689 - val_accuracy: 0.9300 - val_precision_15: 0.8033 - val_recall_15: 0.6391\n",
      "Epoch 11/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1598 - accuracy: 0.9348 - precision_15: 0.7849 - recall_15: 0.6766 - val_loss: 0.1778 - val_accuracy: 0.9300 - val_precision_15: 0.7906 - val_recall_15: 0.6565\n",
      "Epoch 12/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1551 - accuracy: 0.9366 - precision_15: 0.7655 - recall_15: 0.7060 - val_loss: 0.2276 - val_accuracy: 0.9253 - val_precision_15: 0.7877 - val_recall_15: 0.6130\n",
      "Epoch 13/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1416 - accuracy: 0.9444 - precision_15: 0.7933 - recall_15: 0.7575 - val_loss: 0.1803 - val_accuracy: 0.9282 - val_precision_15: 0.7596 - val_recall_15: 0.6870\n",
      "Epoch 14/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1518 - accuracy: 0.9413 - precision_15: 0.8053 - recall_15: 0.7145 - val_loss: 0.2058 - val_accuracy: 0.9271 - val_precision_15: 0.7454 - val_recall_15: 0.7000\n",
      "Epoch 15/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1466 - accuracy: 0.9496 - precision_15: 0.8110 - recall_15: 0.7981 - val_loss: 0.1807 - val_accuracy: 0.9212 - val_precision_15: 0.7791 - val_recall_15: 0.5826\n",
      "Epoch 16/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1413 - accuracy: 0.9428 - precision_15: 0.8026 - recall_15: 0.7314 - val_loss: 0.1798 - val_accuracy: 0.9288 - val_precision_15: 0.7444 - val_recall_15: 0.7217\n",
      "Epoch 17/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1399 - accuracy: 0.9456 - precision_15: 0.7849 - recall_15: 0.7754 - val_loss: 0.1795 - val_accuracy: 0.9265 - val_precision_15: 0.7442 - val_recall_15: 0.6957\n",
      "Epoch 18/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1418 - accuracy: 0.9443 - precision_15: 0.7798 - recall_15: 0.7633 - val_loss: 0.1885 - val_accuracy: 0.9259 - val_precision_15: 0.7149 - val_recall_15: 0.7522\n",
      "Epoch 19/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1327 - accuracy: 0.9470 - precision_15: 0.7768 - recall_15: 0.8207 - val_loss: 0.2124 - val_accuracy: 0.9259 - val_precision_15: 0.7653 - val_recall_15: 0.6522\n",
      "Epoch 20/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1494 - accuracy: 0.9374 - precision_15: 0.7558 - recall_15: 0.7449 - val_loss: 0.2243 - val_accuracy: 0.9265 - val_precision_15: 0.8000 - val_recall_15: 0.6087\n",
      "Epoch 21/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1365 - accuracy: 0.9420 - precision_15: 0.7918 - recall_15: 0.7652 - val_loss: 0.2583 - val_accuracy: 0.9265 - val_precision_15: 0.7665 - val_recall_15: 0.6565\n",
      "Epoch 22/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1358 - accuracy: 0.9438 - precision_15: 0.7707 - recall_15: 0.8032 - val_loss: 0.2009 - val_accuracy: 0.9259 - val_precision_15: 0.7626 - val_recall_15: 0.6565\n",
      "Epoch 23/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1238 - accuracy: 0.9503 - precision_15: 0.7731 - recall_15: 0.8105 - val_loss: 0.2294 - val_accuracy: 0.9212 - val_precision_15: 0.7034 - val_recall_15: 0.7217\n",
      "Epoch 24/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1344 - accuracy: 0.9434 - precision_15: 0.7491 - recall_15: 0.8272 - val_loss: 0.1836 - val_accuracy: 0.9206 - val_precision_15: 0.6820 - val_recall_15: 0.7739\n",
      "Epoch 25/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1356 - accuracy: 0.9381 - precision_15: 0.7348 - recall_15: 0.8152 - val_loss: 0.2124 - val_accuracy: 0.9229 - val_precision_15: 0.7391 - val_recall_15: 0.6652\n",
      "Epoch 26/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1280 - accuracy: 0.9489 - precision_15: 0.7912 - recall_15: 0.7925 - val_loss: 0.1986 - val_accuracy: 0.9229 - val_precision_15: 0.6941 - val_recall_15: 0.7696\n",
      "Epoch 27/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1284 - accuracy: 0.9506 - precision_15: 0.7880 - recall_15: 0.8513 - val_loss: 0.2071 - val_accuracy: 0.9241 - val_precision_15: 0.7167 - val_recall_15: 0.7261\n",
      "Epoch 28/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1171 - accuracy: 0.9527 - precision_15: 0.7813 - recall_15: 0.8581 - val_loss: 0.1981 - val_accuracy: 0.9047 - val_precision_15: 0.6062 - val_recall_15: 0.8435\n",
      "Epoch 29/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1367 - accuracy: 0.9470 - precision_15: 0.7734 - recall_15: 0.8456 - val_loss: 0.2136 - val_accuracy: 0.9229 - val_precision_15: 0.7220 - val_recall_15: 0.7000\n",
      "Epoch 30/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1237 - accuracy: 0.9523 - precision_15: 0.7960 - recall_15: 0.8522 - val_loss: 0.2376 - val_accuracy: 0.9235 - val_precision_15: 0.7066 - val_recall_15: 0.7435\n",
      "Epoch 31/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1309 - accuracy: 0.9479 - precision_15: 0.7693 - recall_15: 0.8558 - val_loss: 0.2517 - val_accuracy: 0.9212 - val_precision_15: 0.7222 - val_recall_15: 0.6783\n",
      "Epoch 32/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1189 - accuracy: 0.9536 - precision_15: 0.7909 - recall_15: 0.8575 - val_loss: 0.2094 - val_accuracy: 0.9182 - val_precision_15: 0.6798 - val_recall_15: 0.7478\n",
      "Epoch 33/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1295 - accuracy: 0.9491 - precision_15: 0.7734 - recall_15: 0.8546 - val_loss: 0.2548 - val_accuracy: 0.9200 - val_precision_15: 0.7080 - val_recall_15: 0.6957\n",
      "Epoch 34/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1296 - accuracy: 0.9506 - precision_15: 0.7954 - recall_15: 0.8409 - val_loss: 0.2223 - val_accuracy: 0.9200 - val_precision_15: 0.7098 - val_recall_15: 0.6913\n",
      "Epoch 35/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1296 - accuracy: 0.9488 - precision_15: 0.7718 - recall_15: 0.8357 - val_loss: 0.1889 - val_accuracy: 0.9024 - val_precision_15: 0.5970 - val_recall_15: 0.8565\n",
      "Epoch 00035: early stopping\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.99      0.90      0.94      1312\n",
      "           1       0.57      0.92      0.70       188\n",
      "\n",
      "    accuracy                           0.90      1500\n",
      "   macro avg       0.78      0.91      0.82      1500\n",
      "weighted avg       0.93      0.90      0.91      1500\n",
      "\n",
      "------------------------------------------------------------------------\n",
      "Training for fold 4 ...\n",
      "Epoch 1/1000\n",
      "6800/6800 [==============================] - 13s 2ms/step - loss: 0.3915 - accuracy: 0.8487 - precision_16: 0.1128 - recall_16: 0.0207 - val_loss: 0.2241 - val_accuracy: 0.8712 - val_precision_16: 0.0000e+00 - val_recall_16: 0.0000e+00\n",
      "Epoch 2/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.2382 - accuracy: 0.8679 - precision_16: 0.3963 - recall_16: 0.0678 - val_loss: 0.2021 - val_accuracy: 0.9118 - val_precision_16: 0.6605 - val_recall_16: 0.6484\n",
      "Epoch 3/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.2054 - accuracy: 0.9055 - precision_16: 0.6827 - recall_16: 0.5672 - val_loss: 0.2124 - val_accuracy: 0.9100 - val_precision_16: 0.7230 - val_recall_16: 0.4886\n",
      "Epoch 4/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1921 - accuracy: 0.9086 - precision_16: 0.7249 - recall_16: 0.4936 - val_loss: 0.2103 - val_accuracy: 0.9141 - val_precision_16: 0.6454 - val_recall_16: 0.7397\n",
      "Epoch 5/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1765 - accuracy: 0.9105 - precision_16: 0.7189 - recall_16: 0.5260 - val_loss: 0.1917 - val_accuracy: 0.9241 - val_precision_16: 0.7473 - val_recall_16: 0.6210\n",
      "Epoch 6/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1690 - accuracy: 0.9196 - precision_16: 0.7604 - recall_16: 0.5418 - val_loss: 0.2153 - val_accuracy: 0.9212 - val_precision_16: 0.7457 - val_recall_16: 0.5890\n",
      "Epoch 7/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1678 - accuracy: 0.9236 - precision_16: 0.7989 - recall_16: 0.5759 - val_loss: 0.1809 - val_accuracy: 0.9194 - val_precision_16: 0.7733 - val_recall_16: 0.5297\n",
      "Epoch 8/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1578 - accuracy: 0.9283 - precision_16: 0.8148 - recall_16: 0.5706 - val_loss: 0.1761 - val_accuracy: 0.9200 - val_precision_16: 0.6912 - val_recall_16: 0.6849\n",
      "Epoch 9/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1669 - accuracy: 0.9338 - precision_16: 0.7771 - recall_16: 0.6948 - val_loss: 0.1856 - val_accuracy: 0.9247 - val_precision_16: 0.7407 - val_recall_16: 0.6393\n",
      "Epoch 10/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1587 - accuracy: 0.9413 - precision_16: 0.7975 - recall_16: 0.7211 - val_loss: 0.2052 - val_accuracy: 0.9247 - val_precision_16: 0.8054 - val_recall_16: 0.5479\n",
      "Epoch 11/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1584 - accuracy: 0.9329 - precision_16: 0.7753 - recall_16: 0.6945 - val_loss: 0.1703 - val_accuracy: 0.9224 - val_precision_16: 0.7514 - val_recall_16: 0.5936\n",
      "Epoch 12/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1664 - accuracy: 0.9316 - precision_16: 0.7987 - recall_16: 0.6779 - val_loss: 0.2002 - val_accuracy: 0.9276 - val_precision_16: 0.8117 - val_recall_16: 0.5708\n",
      "Epoch 13/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1476 - accuracy: 0.9428 - precision_16: 0.8029 - recall_16: 0.7470 - val_loss: 0.1705 - val_accuracy: 0.9229 - val_precision_16: 0.6897 - val_recall_16: 0.7306\n",
      "Epoch 14/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1403 - accuracy: 0.9423 - precision_16: 0.7747 - recall_16: 0.7585 - val_loss: 0.1773 - val_accuracy: 0.9288 - val_precision_16: 0.7722 - val_recall_16: 0.6347\n",
      "Epoch 15/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1503 - accuracy: 0.9399 - precision_16: 0.8035 - recall_16: 0.7381 - val_loss: 0.1813 - val_accuracy: 0.9265 - val_precision_16: 0.7554 - val_recall_16: 0.6347\n",
      "Epoch 16/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1501 - accuracy: 0.9390 - precision_16: 0.7633 - recall_16: 0.7620 - val_loss: 0.1777 - val_accuracy: 0.9241 - val_precision_16: 0.7103 - val_recall_16: 0.6941\n",
      "Epoch 17/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1514 - accuracy: 0.9410 - precision_16: 0.7698 - recall_16: 0.7756 - val_loss: 0.1806 - val_accuracy: 0.9271 - val_precision_16: 0.7130 - val_recall_16: 0.7260\n",
      "Epoch 18/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1326 - accuracy: 0.9466 - precision_16: 0.7774 - recall_16: 0.8151 - val_loss: 0.2305 - val_accuracy: 0.9294 - val_precision_16: 0.7765 - val_recall_16: 0.6347\n",
      "Epoch 19/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1400 - accuracy: 0.9457 - precision_16: 0.7989 - recall_16: 0.7872 - val_loss: 0.1779 - val_accuracy: 0.9294 - val_precision_16: 0.7538 - val_recall_16: 0.6712\n",
      "Epoch 20/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1420 - accuracy: 0.9475 - precision_16: 0.8013 - recall_16: 0.8088 - val_loss: 0.1765 - val_accuracy: 0.9241 - val_precision_16: 0.7616 - val_recall_16: 0.5982\n",
      "Epoch 21/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1325 - accuracy: 0.9470 - precision_16: 0.8220 - recall_16: 0.7493 - val_loss: 0.2937 - val_accuracy: 0.9276 - val_precision_16: 0.7963 - val_recall_16: 0.5890\n",
      "Epoch 22/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1395 - accuracy: 0.9499 - precision_16: 0.8348 - recall_16: 0.7777 - val_loss: 0.2256 - val_accuracy: 0.9241 - val_precision_16: 0.7500 - val_recall_16: 0.6164\n",
      "Epoch 23/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1354 - accuracy: 0.9487 - precision_16: 0.8263 - recall_16: 0.7665 - val_loss: 0.1717 - val_accuracy: 0.9212 - val_precision_16: 0.7457 - val_recall_16: 0.5890\n",
      "Epoch 24/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1388 - accuracy: 0.9385 - precision_16: 0.8078 - recall_16: 0.7149 - val_loss: 0.2443 - val_accuracy: 0.9259 - val_precision_16: 0.7751 - val_recall_16: 0.5982\n",
      "Epoch 25/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1233 - accuracy: 0.9506 - precision_16: 0.8213 - recall_16: 0.8083 - val_loss: 0.2427 - val_accuracy: 0.9229 - val_precision_16: 0.7444 - val_recall_16: 0.6119\n",
      "Epoch 26/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1215 - accuracy: 0.9531 - precision_16: 0.8314 - recall_16: 0.7967 - val_loss: 0.1828 - val_accuracy: 0.9188 - val_precision_16: 0.7263 - val_recall_16: 0.5936\n",
      "Epoch 27/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1217 - accuracy: 0.9542 - precision_16: 0.8367 - recall_16: 0.8048 - val_loss: 0.2139 - val_accuracy: 0.9194 - val_precision_16: 0.7228 - val_recall_16: 0.6073\n",
      "Epoch 28/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1226 - accuracy: 0.9527 - precision_16: 0.8328 - recall_16: 0.7950 - val_loss: 0.2635 - val_accuracy: 0.9212 - val_precision_16: 0.7348 - val_recall_16: 0.6073\n",
      "Epoch 29/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1464 - accuracy: 0.9475 - precision_16: 0.8068 - recall_16: 0.7715 - val_loss: 0.2133 - val_accuracy: 0.9241 - val_precision_16: 0.7296 - val_recall_16: 0.6530\n",
      "Epoch 30/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1241 - accuracy: 0.9507 - precision_16: 0.8183 - recall_16: 0.7891 - val_loss: 0.1864 - val_accuracy: 0.9229 - val_precision_16: 0.7095 - val_recall_16: 0.6804\n",
      "Epoch 31/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1227 - accuracy: 0.9517 - precision_16: 0.8139 - recall_16: 0.8036 - val_loss: 0.2174 - val_accuracy: 0.9200 - val_precision_16: 0.7107 - val_recall_16: 0.6393\n",
      "Epoch 32/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1362 - accuracy: 0.9404 - precision_16: 0.8024 - recall_16: 0.7604 - val_loss: 0.2810 - val_accuracy: 0.9229 - val_precision_16: 0.7340 - val_recall_16: 0.6301\n",
      "Epoch 33/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1168 - accuracy: 0.9553 - precision_16: 0.8357 - recall_16: 0.8169 - val_loss: 0.2227 - val_accuracy: 0.9247 - val_precision_16: 0.7486 - val_recall_16: 0.6256\n",
      "Epoch 34/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1279 - accuracy: 0.9439 - precision_16: 0.7837 - recall_16: 0.7873 - val_loss: 0.2059 - val_accuracy: 0.9229 - val_precision_16: 0.7292 - val_recall_16: 0.6393\n",
      "Epoch 35/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1223 - accuracy: 0.9536 - precision_16: 0.8204 - recall_16: 0.8223 - val_loss: 0.2346 - val_accuracy: 0.9212 - val_precision_16: 0.7094 - val_recall_16: 0.6575\n",
      "Epoch 36/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.1217 - accuracy: 0.9499 - precision_16: 0.8197 - recall_16: 0.7988 - val_loss: 0.1856 - val_accuracy: 0.9194 - val_precision_16: 0.7531 - val_recall_16: 0.5571\n",
      "Epoch 00036: early stopping\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.97      0.96      1312\n",
      "           1       0.77      0.66      0.71       188\n",
      "\n",
      "    accuracy                           0.93      1500\n",
      "   macro avg       0.86      0.82      0.84      1500\n",
      "weighted avg       0.93      0.93      0.93      1500\n",
      "\n",
      "------------------------------------------------------------------------\n",
      "Training for fold 5 ...\n",
      "Epoch 1/1000\n",
      "6800/6800 [==============================] - 12s 2ms/step - loss: 0.4086 - accuracy: 0.8221 - precision_17: 0.1320 - recall_17: 0.0688 - val_loss: 0.2069 - val_accuracy: 0.8729 - val_precision_17: 0.0000e+00 - val_recall_17: 0.0000e+00\n",
      "Epoch 2/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.2564 - accuracy: 0.8613 - precision_17: 0.0000e+00 - recall_17: 0.0000e+00 - val_loss: 0.1979 - val_accuracy: 0.8729 - val_precision_17: 0.0000e+00 - val_recall_17: 0.0000e+00\n",
      "Epoch 3/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.2288 - accuracy: 0.8724 - precision_17: 0.3660 - recall_17: 0.1622 - val_loss: 0.1843 - val_accuracy: 0.9218 - val_precision_17: 0.7196 - val_recall_17: 0.6296\n",
      "Epoch 4/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.2026 - accuracy: 0.9093 - precision_17: 0.6427 - recall_17: 0.6614 - val_loss: 0.1838 - val_accuracy: 0.9141 - val_precision_17: 0.6306 - val_recall_17: 0.7824\n",
      "Epoch 5/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1987 - accuracy: 0.9166 - precision_17: 0.6475 - recall_17: 0.7945 - val_loss: 0.1967 - val_accuracy: 0.9253 - val_precision_17: 0.7214 - val_recall_17: 0.6713\n",
      "Epoch 6/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1857 - accuracy: 0.9259 - precision_17: 0.6900 - recall_17: 0.7503 - val_loss: 0.2018 - val_accuracy: 0.9076 - val_precision_17: 0.5891 - val_recall_17: 0.9028\n",
      "Epoch 7/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1835 - accuracy: 0.9255 - precision_17: 0.6823 - recall_17: 0.8137 - val_loss: 0.1773 - val_accuracy: 0.9300 - val_precision_17: 0.6902 - val_recall_17: 0.8148\n",
      "Epoch 8/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1824 - accuracy: 0.9244 - precision_17: 0.6956 - recall_17: 0.8103 - val_loss: 0.1717 - val_accuracy: 0.9241 - val_precision_17: 0.6790 - val_recall_17: 0.7639\n",
      "Epoch 9/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1509 - accuracy: 0.9387 - precision_17: 0.7178 - recall_17: 0.8213 - val_loss: 0.2323 - val_accuracy: 0.9300 - val_precision_17: 0.7343 - val_recall_17: 0.7037\n",
      "Epoch 10/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1654 - accuracy: 0.9346 - precision_17: 0.7321 - recall_17: 0.8179 - val_loss: 0.1794 - val_accuracy: 0.9265 - val_precision_17: 0.6857 - val_recall_17: 0.7778\n",
      "Epoch 11/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1562 - accuracy: 0.9329 - precision_17: 0.7123 - recall_17: 0.8110 - val_loss: 0.1814 - val_accuracy: 0.9229 - val_precision_17: 0.6523 - val_recall_17: 0.8426\n",
      "Epoch 12/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1578 - accuracy: 0.9310 - precision_17: 0.7028 - recall_17: 0.8262 - val_loss: 0.1746 - val_accuracy: 0.9206 - val_precision_17: 0.6517 - val_recall_17: 0.8056\n",
      "Epoch 13/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1627 - accuracy: 0.9363 - precision_17: 0.7143 - recall_17: 0.8263 - val_loss: 0.1852 - val_accuracy: 0.9229 - val_precision_17: 0.6592 - val_recall_17: 0.8148\n",
      "Epoch 14/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1512 - accuracy: 0.9342 - precision_17: 0.7077 - recall_17: 0.8521 - val_loss: 0.1726 - val_accuracy: 0.9194 - val_precision_17: 0.6426 - val_recall_17: 0.8241\n",
      "Epoch 15/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1485 - accuracy: 0.9323 - precision_17: 0.7020 - recall_17: 0.8058 - val_loss: 0.1722 - val_accuracy: 0.9224 - val_precision_17: 0.6736 - val_recall_17: 0.7546\n",
      "Epoch 16/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1563 - accuracy: 0.9302 - precision_17: 0.6912 - recall_17: 0.8101 - val_loss: 0.1858 - val_accuracy: 0.9259 - val_precision_17: 0.6758 - val_recall_17: 0.8009\n",
      "Epoch 17/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1528 - accuracy: 0.9322 - precision_17: 0.7030 - recall_17: 0.8550 - val_loss: 0.1824 - val_accuracy: 0.9141 - val_precision_17: 0.6144 - val_recall_17: 0.8704\n",
      "Epoch 18/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1438 - accuracy: 0.9338 - precision_17: 0.7015 - recall_17: 0.8375 - val_loss: 0.1811 - val_accuracy: 0.9259 - val_precision_17: 0.6758 - val_recall_17: 0.8009\n",
      "Epoch 19/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1522 - accuracy: 0.9306 - precision_17: 0.6964 - recall_17: 0.8141 - val_loss: 0.2091 - val_accuracy: 0.9218 - val_precision_17: 0.6615 - val_recall_17: 0.7870\n",
      "Epoch 20/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1493 - accuracy: 0.9392 - precision_17: 0.7350 - recall_17: 0.8487 - val_loss: 0.1870 - val_accuracy: 0.9024 - val_precision_17: 0.5740 - val_recall_17: 0.8981\n",
      "Epoch 21/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1569 - accuracy: 0.9339 - precision_17: 0.7119 - recall_17: 0.8550 - val_loss: 0.1787 - val_accuracy: 0.9188 - val_precision_17: 0.6336 - val_recall_17: 0.8565\n",
      "Epoch 22/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1487 - accuracy: 0.9380 - precision_17: 0.7308 - recall_17: 0.8538 - val_loss: 0.1969 - val_accuracy: 0.9206 - val_precision_17: 0.6552 - val_recall_17: 0.7917\n",
      "Epoch 23/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1296 - accuracy: 0.9430 - precision_17: 0.7455 - recall_17: 0.8435 - val_loss: 0.2375 - val_accuracy: 0.9271 - val_precision_17: 0.7212 - val_recall_17: 0.6944\n",
      "Epoch 24/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1505 - accuracy: 0.9377 - precision_17: 0.7304 - recall_17: 0.8592 - val_loss: 0.2589 - val_accuracy: 0.9276 - val_precision_17: 0.7225 - val_recall_17: 0.6991\n",
      "Epoch 25/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1442 - accuracy: 0.9394 - precision_17: 0.7322 - recall_17: 0.8539 - val_loss: 0.1917 - val_accuracy: 0.9194 - val_precision_17: 0.6561 - val_recall_17: 0.7685\n",
      "Epoch 26/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1334 - accuracy: 0.9463 - precision_17: 0.7580 - recall_17: 0.8806 - val_loss: 0.1855 - val_accuracy: 0.9182 - val_precision_17: 0.6442 - val_recall_17: 0.7963\n",
      "Epoch 27/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1432 - accuracy: 0.9387 - precision_17: 0.7350 - recall_17: 0.8641 - val_loss: 0.2057 - val_accuracy: 0.9247 - val_precision_17: 0.6833 - val_recall_17: 0.7593\n",
      "Epoch 28/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1326 - accuracy: 0.9444 - precision_17: 0.7387 - recall_17: 0.8820 - val_loss: 0.1864 - val_accuracy: 0.9024 - val_precision_17: 0.5740 - val_recall_17: 0.8981\n",
      "Epoch 29/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1284 - accuracy: 0.9451 - precision_17: 0.7281 - recall_17: 0.8851 - val_loss: 0.2918 - val_accuracy: 0.9253 - val_precision_17: 0.7192 - val_recall_17: 0.6759\n",
      "Epoch 30/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1270 - accuracy: 0.9475 - precision_17: 0.7411 - recall_17: 0.8773 - val_loss: 0.2687 - val_accuracy: 0.9194 - val_precision_17: 0.6710 - val_recall_17: 0.7176\n",
      "Epoch 31/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1286 - accuracy: 0.9429 - precision_17: 0.7354 - recall_17: 0.8894 - val_loss: 0.2735 - val_accuracy: 0.9229 - val_precision_17: 0.6641 - val_recall_17: 0.7963\n",
      "Epoch 32/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1376 - accuracy: 0.9459 - precision_17: 0.7396 - recall_17: 0.8873 - val_loss: 0.1837 - val_accuracy: 0.9094 - val_precision_17: 0.6013 - val_recall_17: 0.8519\n",
      "Epoch 33/1000\n",
      "6800/6800 [==============================] - 11s 2ms/step - loss: 0.1388 - accuracy: 0.9425 - precision_17: 0.7287 - recall_17: 0.8857 - val_loss: 0.1847 - val_accuracy: 0.9171 - val_precision_17: 0.6316 - val_recall_17: 0.8333\n",
      "Epoch 00033: early stopping\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.98      0.94      0.96      1312\n",
      "           1       0.65      0.85      0.74       188\n",
      "\n",
      "    accuracy                           0.92      1500\n",
      "   macro avg       0.81      0.89      0.85      1500\n",
      "weighted avg       0.94      0.92      0.93      1500\n",
      "\n"
     ]
    }
   ],
   "source": [
    "metrics = pd.DataFrame(columns=['fold', 'accuracy', 'precision', 'recall', 'f1_score'])\n",
    "\n",
    "# Define the K-fold Cross Validator\n",
    "kfold = KFold(n_splits=5, shuffle=True, random_state=42)\n",
    "\n",
    "# K-fold Cross Validation model evaluation\n",
    "fold_no = 1\n",
    "for train, val in kfold.split(X_train, y_train):\n",
    "\n",
    "    # Define the model architecture\n",
    "    model = Sequential()\n",
    "\n",
    "    model.add(Dense(units=36,activation='relu'))\n",
    "    model.add(Dropout(0.2))\n",
    "\n",
    "    #hidden layers\n",
    "    model.add(Dense(units=18,activation='relu'))\n",
    "    model.add(Dropout(0.2))\n",
    "\n",
    "    model.add(Dense(units=9,activation='relu'))\n",
    "    model.add(Dropout(0.2))\n",
    "\n",
    "    model.add(Dense(units=1,activation='sigmoid'))\n",
    "\n",
    "    early_stop = EarlyStopping(monitor='val_loss', mode='min', verbose=1, patience=25)\n",
    "\n",
    "    # Compile the model\n",
    "    # For a binary classification problem\n",
    "    model.compile(loss='binary_crossentropy', optimizer='adam',metrics=['accuracy', tf.keras.metrics.Precision(), tf.keras.metrics.Recall()])\n",
    "\n",
    "    # Generate a print\n",
    "    print('------------------------------------------------------------------------')\n",
    "    print(f'Training for fold {fold_no} ...')\n",
    "\n",
    "    # Fit data to model\n",
    "    history = model.fit(X_train[train], y_train[train],\n",
    "          epochs=1000,\n",
    "          batch_size=1,\n",
    "          validation_data=(X_train[val], y_train[val]), \n",
    "          callbacks=[early_stop])\n",
    "\n",
    "    # Generate generalization metrics\n",
    "    y_pred = (model.predict(X_test) > 0.5).astype(\"int32\")\n",
    "\n",
    "    scores = [fold_no,accuracy_score(y_test, y_pred), precision_score(y_test, y_pred), \n",
    "              recall_score(y_test, y_pred), f1_score(y_test, y_pred)]\n",
    "\n",
    "    metrics.loc[len(metrics)] = scores\n",
    "\n",
    "    print(classification_report(y_test,y_pred))\n",
    "    # Increase fold number\n",
    "    fold_no = fold_no + 1\n",
    "\n",
    "\n",
    "metrics.loc[len(metrics)] = ['mean', metrics['accuracy'].mean(), metrics['precision'].mean(), \n",
    "                             metrics['recall'].mean(), metrics['f1_score'].mean()]\n",
    "metrics.loc[len(metrics)] = ['std', metrics['accuracy'].iloc[:-1].std(), metrics['precision'].iloc[:-1].std(), \n",
    "                             metrics['recall'].iloc[:-1].std(), metrics['f1_score'].iloc[:-1].std()]  \n",
    "metrics = metrics.set_index('fold')\n",
    "\n",
    "tz = pytz.timezone('Brazil/East')\n",
    "metrics.to_csv( f'attack_{datetime.datetime.now(tz).strftime(\"%d-%m-%Y-%H:%M\")}.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "e99bb3b5-8799-420a-bfd9-4f63581ea007",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accuracy</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1_score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>fold</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1.0</th>\n",
       "      <td>0.921333</td>\n",
       "      <td>0.678571</td>\n",
       "      <td>0.707447</td>\n",
       "      <td>0.692708</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2.0</th>\n",
       "      <td>0.929333</td>\n",
       "      <td>0.720430</td>\n",
       "      <td>0.712766</td>\n",
       "      <td>0.716578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3.0</th>\n",
       "      <td>0.901333</td>\n",
       "      <td>0.565359</td>\n",
       "      <td>0.920213</td>\n",
       "      <td>0.700405</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4.0</th>\n",
       "      <td>0.933333</td>\n",
       "      <td>0.771605</td>\n",
       "      <td>0.664894</td>\n",
       "      <td>0.714286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5.0</th>\n",
       "      <td>0.924000</td>\n",
       "      <td>0.651639</td>\n",
       "      <td>0.845745</td>\n",
       "      <td>0.736111</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.921867</td>\n",
       "      <td>0.677521</td>\n",
       "      <td>0.770213</td>\n",
       "      <td>0.712018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.012386</td>\n",
       "      <td>0.077369</td>\n",
       "      <td>0.107862</td>\n",
       "      <td>0.016689</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      accuracy  precision    recall  f1_score\n",
       "fold                                         \n",
       "1.0   0.921333   0.678571  0.707447  0.692708\n",
       "2.0   0.929333   0.720430  0.712766  0.716578\n",
       "3.0   0.901333   0.565359  0.920213  0.700405\n",
       "4.0   0.933333   0.771605  0.664894  0.714286\n",
       "5.0   0.924000   0.651639  0.845745  0.736111\n",
       "mean  0.921867   0.677521  0.770213  0.712018\n",
       "std   0.012386   0.077369  0.107862  0.016689"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:work-asa]",
   "language": "python",
   "name": "conda-env-work-asa-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}