Kyrkos_TA_4th_Assignment.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/konkyrkos/textAnalyticsThirdAssignment/blob/master/Galanos_Kyrkos_Saltos_TA_3rd_Assignment.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "liMSglMZkfVw"
   },
   "source": [
    "# 4th Assignment: Sentiment Analysis with RNNs\n",
    "\n",
    "---\n",
    ">Konstantinos Kyrkos <br>\n",
    ">Registration Number p3351907 <br>\n",
    ">kkyrkos@aueb.gr"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4M05ONsfmKnZ"
   },
   "source": [
    "This notebook accompanied with the report given provides our solution to the 2nd assignment of the Text Analytics course at Data Science MSc at AUEB. <br>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "UG28TqlMaMBz"
   },
   "source": [
    "#### Install required packages\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ux-swtfpaOb7"
   },
   "outputs": [],
   "source": [
    "%%capture\n",
    "!pip install numpy\n",
    "!pip install scikit-learn\n",
    "!pip install pandas\n",
    "!pip install matplotlib\n",
    "!pip install --upgrade tensorflow\n",
    "!pip install emoji --upgrade"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "THw-KcjLlAS1"
   },
   "source": [
    "#### Mount drive to colab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 121
    },
    "colab_type": "code",
    "id": "oZOvYUN2k_Zp",
    "outputId": "0294b22a-41cb-4dfb-f157-352eaef0f7ac"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
      "\n",
      "Enter your authorization code:\n",
      "··········\n",
      "Mounted at /content/gdrive\n"
     ]
    }
   ],
   "source": [
    "from google.colab import drive\n",
    "drive.mount('/content/gdrive')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TmxhOk-MRdmh"
   },
   "source": [
    "Check the GPU allocated"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 302
    },
    "colab_type": "code",
    "id": "mVhKho8DP3By",
    "outputId": "910b5195-ac23-4f9e-8fa0-58e463674608"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sat Jun  6 13:21:46 2020       \n",
      "+-----------------------------------------------------------------------------+\n",
      "| NVIDIA-SMI 440.82       Driver Version: 418.67       CUDA Version: 10.1     |\n",
      "|-------------------------------+----------------------+----------------------+\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
      "|===============================+======================+======================|\n",
      "|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |\n",
      "| N/A   73C    P8    36W / 149W |      0MiB / 11441MiB |      0%      Default |\n",
      "+-------------------------------+----------------------+----------------------+\n",
      "                                                                               \n",
      "+-----------------------------------------------------------------------------+\n",
      "| Processes:                                                       GPU Memory |\n",
      "|  GPU       PID   Type   Process name                             Usage      |\n",
      "|=============================================================================|\n",
      "|  No running processes found                                                 |\n",
      "+-----------------------------------------------------------------------------+\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "utS1TYaaa29j"
   },
   "source": [
    "#### Import all libraries for later use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "whMA-02VaRWF",
    "outputId": "6ac3b1ad-f01b-427a-e7b3-3eb4a7cda90d"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import matplotlib.pyplot as plt\n",
    "import nltk\n",
    "import re\n",
    "import itertools\n",
    "import emoji\n",
    "import time\n",
    "\n",
    "from nltk.tokenize import WordPunctTokenizer\n",
    "from bs4 import BeautifulSoup \n",
    "from nltk.stem import WordNetLemmatizer\n",
    "from gensim.models.wrappers import FastText\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.dummy import DummyClassifier\n",
    "from tabulate import tabulate\n",
    "from sklearn.metrics import auc\n",
    "\n",
    "import warnings\n",
    "import sklearn.exceptions\n",
    "warnings.filterwarnings(\"ignore\", category=sklearn.exceptions.UndefinedMetricWarning)\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout\n",
    "from keras.optimizers import Adam\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "kigSUaGmFwxj",
    "outputId": "1861cc8c-2e9f-449b-d32f-ef0d1fcfcdbb"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 5,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "nltk.download('wordnet',quiet=True)\n",
    "#nltk.download('stopwords',quiet=True)\n",
    "nltk.download('punkt',quiet=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "FjaVFJ6suKY9"
   },
   "source": [
    "#### Download, unzip & load fasttext word embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 202
    },
    "colab_type": "code",
    "id": "3yniNlDyZng2",
    "outputId": "7c8baf3f-34b1-4a0f-fdda-b97481ce3d70"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2020-06-06 13:22:14--  https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.en.300.bin.gz\n",
      "Resolving dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)... 104.22.75.142, 172.67.9.4, 104.22.74.142, ...\n",
      "Connecting to dl.fbaipublicfiles.com (dl.fbaipublicfiles.com)|104.22.75.142|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 4503593528 (4.2G) [application/octet-stream]\n",
      "Saving to: ‘cc.en.300.bin.gz’\n",
      "\n",
      "cc.en.300.bin.gz    100%[===================>]   4.19G  12.0MB/s    in 5m 58s  \n",
      "\n",
      "2020-06-06 13:28:12 (12.0 MB/s) - ‘cc.en.300.bin.gz’ saved [4503593528/4503593528]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!wget https://dl.fbaipublicfiles.com/fasttext/vectors-crawl/cc.en.300.bin.gz\n",
    "!gunzip -d cc.en.300.bin.gz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "lZvkvm98yx4c",
    "outputId": "3212beb8-03a3-4d3a-ca9f-8f040900bdf9"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<gensim.models.deprecated.fasttext_wrapper.FastText at 0x7f05f39fd128>"
      ]
     },
     "execution_count": 7,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "fasttext = FastText.load_fasttext_format('cc.en.300.bin')\n",
    "fasttext"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TnIYxnYcbHKH"
   },
   "source": [
    "#### Download a sentiment analysis dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 319
    },
    "colab_type": "code",
    "id": "LlOAA9oo8tqL",
    "outputId": "d559cf59-bf42-496d-af83-bd9eb2fd8889"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--2020-06-06 13:39:14--  http://cs.stanford.edu/people/alecmgo/trainingandtestdata.zip\n",
      "Resolving cs.stanford.edu (cs.stanford.edu)... 171.64.64.64\n",
      "Connecting to cs.stanford.edu (cs.stanford.edu)|171.64.64.64|:80... connected.\n",
      "HTTP request sent, awaiting response... 301 Moved Permanently\n",
      "Location: https://cs.stanford.edu/people/alecmgo/trainingandtestdata.zip [following]\n",
      "--2020-06-06 13:39:14--  https://cs.stanford.edu/people/alecmgo/trainingandtestdata.zip\n",
      "Connecting to cs.stanford.edu (cs.stanford.edu)|171.64.64.64|:443... connected.\n",
      "HTTP request sent, awaiting response... 200 OK\n",
      "Length: 81363704 (78M) [application/zip]\n",
      "Saving to: ‘trainingandtestdata.zip’\n",
      "\n",
      "trainingandtestdata 100%[===================>]  77.59M  21.0MB/s    in 4.5s    \n",
      "\n",
      "2020-06-06 13:39:19 (17.2 MB/s) - ‘trainingandtestdata.zip’ saved [81363704/81363704]\n",
      "\n",
      "Archive:  trainingandtestdata.zip\n",
      "  inflating: testdata.manual.2009.06.14.csv  \n",
      "  inflating: training.1600000.processed.noemoticon.csv  \n"
     ]
    }
   ],
   "source": [
    "!wget http://cs.stanford.edu/people/alecmgo/trainingandtestdata.zip\n",
    "!unzip \"trainingandtestdata.zip\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RzYXzMsBtInq"
   },
   "source": [
    "#### Load Data from csv file & remove useless columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 195
    },
    "colab_type": "code",
    "id": "CSjcKvmraReg",
    "outputId": "84a90e91-00ae-4413-ac85-3ed57ec1c8b4"
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>@switchfoot http://twitpic.com/2y1zl - Awww, that's a bummer.  You shoulda got David Carr of Third Day to do it. ;D</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>is upset that he can't update his Facebook by texting it... and might cry as a result  School today also. Blah!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>@Kenichan I dived many times for the ball. Managed to save 50%  The rest go out of bounds</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>my whole body feels itchy and like its on fire</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>@nationwideclass no, it's not behaving at all. i'm mad. why am i here? because I can't see you all over there.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   sentiment                                                                                                                 text\n",
       "0          0  @switchfoot http://twitpic.com/2y1zl - Awww, that's a bummer.  You shoulda got David Carr of Third Day to do it. ;D\n",
       "1          0      is upset that he can't update his Facebook by texting it... and might cry as a result  School today also. Blah!\n",
       "2          0                            @Kenichan I dived many times for the ball. Managed to save 50%  The rest go out of bounds\n",
       "3          0                                                                      my whole body feels itchy and like its on fire \n",
       "4          0      @nationwideclass no, it's not behaving at all. i'm mad. why am i here? because I can't see you all over there. "
      ]
     },
     "execution_count": 9,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('max_colwidth', 800)\n",
    "cols = ['sentiment','id','date','query_string','user','text']\n",
    "tweets_final = pd.read_csv(\"/content/training.1600000.processed.noemoticon.csv\", encoding = \"ISO-8859-1\", header=None, names=cols) #import data to dataframe\n",
    "\n",
    "tweets_final.drop(['id','date','query_string','user'],axis=1,inplace=True) # drop useless columns\n",
    "tweets_final.loc[tweets_final['sentiment'] == 4, \"sentiment\"] = 1 #change labels\n",
    "\n",
    "tweets_final.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "sR8jBj_Z3QIX"
   },
   "source": [
    "#### Delete unnecessary files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9E63yG5U3BnY"
   },
   "outputs": [],
   "source": [
    "!rm testdata.manual.2009.06.14.csv\n",
    "!rm cc.en.300.bin\n",
    "!rm training.1600000.processed.noemoticon.csv\n",
    "!rm trainingandtestdata.zip"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "99sc24JrtV8V"
   },
   "source": [
    "#### Get a count of data to see that there are no neutral tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 67
    },
    "colab_type": "code",
    "id": "fzufFrszmGSu",
    "outputId": "38634f2b-e1b4-499f-ebb5-8605cd68c5a2"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    800000\n",
       "0    800000\n",
       "Name: sentiment, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tweets_final.sentiment.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "6wh97Gnx36Wg"
   },
   "source": [
    "### Data preprocessing/ cleaning/ preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3DUeficR4Gs4"
   },
   "source": [
    "##### Create a dictionary with contractions and smileyfaces\n",
    "https://github.com/charlesmalafosse/FastText-sentiment-analysis-for-tweets/blob/master/betsentiment_sentiment_analysis_fasttext.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "1I04k3PM3-GP"
   },
   "outputs": [],
   "source": [
    "def load_dict_smileys():\n",
    "    \n",
    "    return {\n",
    "        \":-)\":\"smiley\",\n",
    "        \":-]\":\"smiley\",\n",
    "        \":-3\":\"smiley\",\n",
    "        \":->\":\"smiley\",\n",
    "        \"8-)\":\"smiley\",\n",
    "        \":-}\":\"smiley\",\n",
    "        \":)\":\"smiley\",\n",
    "        \":]\":\"smiley\",\n",
    "        \":3\":\"smiley\",\n",
    "        \":>\":\"smiley\",\n",
    "        \"8)\":\"smiley\",\n",
    "        \":}\":\"smiley\",\n",
    "        \":o)\":\"smiley\",\n",
    "        \":c)\":\"smiley\",\n",
    "        \":^)\":\"smiley\",\n",
    "        \"=]\":\"smiley\",\n",
    "        \"=)\":\"smiley\",\n",
    "        \":-))\":\"smiley\",\n",
    "        \":‑D\":\"smiley\",\n",
    "        \"8‑D\":\"smiley\",\n",
    "        \"x‑D\":\"smiley\",\n",
    "        \"X‑D\":\"smiley\",\n",
    "        \":D\":\"smiley\",\n",
    "        \"8D\":\"smiley\",\n",
    "        \"xD\":\"smiley\",\n",
    "        \"XD\":\"smiley\",\n",
    "        \":‑(\":\"sad\",\n",
    "        \":‑c\":\"sad\",\n",
    "        \":‑<\":\"sad\",\n",
    "        \":‑[\":\"sad\",\n",
    "        \":(\":\"sad\",\n",
    "        \":c\":\"sad\",\n",
    "        \":<\":\"sad\",\n",
    "        \":[\":\"sad\",\n",
    "        \":-||\":\"sad\",\n",
    "        \">:[\":\"sad\",\n",
    "        \":{\":\"sad\",\n",
    "        \":@\":\"sad\",\n",
    "        \">:(\":\"sad\",\n",
    "        \":'‑(\":\"sad\",\n",
    "        \":'(\":\"sad\",\n",
    "        \":‑P\":\"playful\",\n",
    "        \"X‑P\":\"playful\",\n",
    "        \"x‑p\":\"playful\",\n",
    "        \":‑p\":\"playful\",\n",
    "        \":‑Þ\":\"playful\",\n",
    "        \":‑þ\":\"playful\",\n",
    "        \":‑b\":\"playful\",\n",
    "        \":P\":\"playful\",\n",
    "        \"XP\":\"playful\",\n",
    "        \"xp\":\"playful\",\n",
    "        \":p\":\"playful\",\n",
    "        \":Þ\":\"playful\",\n",
    "        \":þ\":\"playful\",\n",
    "        \":b\":\"playful\",\n",
    "        \"<3\":\"love\"\n",
    "        }\n",
    "\n",
    "\n",
    "def load_dict_contractions():\n",
    "    \n",
    "    return {\n",
    "        \"ain't\":\"is not\",\n",
    "        \"amn't\":\"am not\",\n",
    "        \"aren't\":\"are not\",\n",
    "        \"can't\":\"cannot\",\n",
    "        \"'cause\":\"because\",\n",
    "        \"couldn't\":\"could not\",\n",
    "        \"couldn't've\":\"could not have\",\n",
    "        \"could've\":\"could have\",\n",
    "        \"daren't\":\"dare not\",\n",
    "        \"daresn't\":\"dare not\",\n",
    "        \"dasn't\":\"dare not\",\n",
    "        \"didn't\":\"did not\",\n",
    "        \"doesn't\":\"does not\",\n",
    "        \"don't\":\"do not\",\n",
    "        \"e'er\":\"ever\",\n",
    "        \"em\":\"them\",\n",
    "        \"everyone's\":\"everyone is\",\n",
    "        \"finna\":\"fixing to\",\n",
    "        \"gimme\":\"give me\",\n",
    "        \"gonna\":\"going to\",\n",
    "        \"gon't\":\"go not\",\n",
    "        \"gotta\":\"got to\",\n",
    "        \"hadn't\":\"had not\",\n",
    "        \"hasn't\":\"has not\",\n",
    "        \"haven't\":\"have not\",\n",
    "        \"he'd\":\"he would\",\n",
    "        \"he'll\":\"he will\",\n",
    "        \"he's\":\"he is\",\n",
    "        \"he've\":\"he have\",\n",
    "        \"how'd\":\"how would\",\n",
    "        \"how'll\":\"how will\",\n",
    "        \"how're\":\"how are\",\n",
    "        \"how's\":\"how is\",\n",
    "        \"I'd\":\"I would\",\n",
    "        \"I'll\":\"I will\",\n",
    "        \"I'm\":\"I am\",\n",
    "        \"I'm'a\":\"I am about to\",\n",
    "        \"I'm'o\":\"I am going to\",\n",
    "        \"isn't\":\"is not\",\n",
    "        \"it'd\":\"it would\",\n",
    "        \"it'll\":\"it will\",\n",
    "        \"it's\":\"it is\",\n",
    "        \"I've\":\"I have\",\n",
    "        \"kinda\":\"kind of\",\n",
    "        \"let's\":\"let us\",\n",
    "        \"mayn't\":\"may not\",\n",
    "        \"may've\":\"may have\",\n",
    "        \"mightn't\":\"might not\",\n",
    "        \"might've\":\"might have\",\n",
    "        \"mustn't\":\"must not\",\n",
    "        \"mustn't've\":\"must not have\",\n",
    "        \"must've\":\"must have\",\n",
    "        \"needn't\":\"need not\",\n",
    "        \"ne'er\":\"never\",\n",
    "        \"o'\":\"of\",\n",
    "        \"o'er\":\"over\",\n",
    "        \"ol'\":\"old\",\n",
    "        \"oughtn't\":\"ought not\",\n",
    "        \"shalln't\":\"shall not\",\n",
    "        \"shan't\":\"shall not\",\n",
    "        \"she'd\":\"she would\",\n",
    "        \"she'll\":\"she will\",\n",
    "        \"she's\":\"she is\",\n",
    "        \"shouldn't\":\"should not\",\n",
    "        \"shouldn't've\":\"should not have\",\n",
    "        \"should've\":\"should have\",\n",
    "        \"somebody's\":\"somebody is\",\n",
    "        \"someone's\":\"someone is\",\n",
    "        \"something's\":\"something is\",\n",
    "        \"that'd\":\"that would\",\n",
    "        \"that'll\":\"that will\",\n",
    "        \"that're\":\"that are\",\n",
    "        \"that's\":\"that is\",\n",
    "        \"there'd\":\"there would\",\n",
    "        \"there'll\":\"there will\",\n",
    "        \"there're\":\"there are\",\n",
    "        \"there's\":\"there is\",\n",
    "        \"these're\":\"these are\",\n",
    "        \"they'd\":\"they would\",\n",
    "        \"they'll\":\"they will\",\n",
    "        \"they're\":\"they are\",\n",
    "        \"they've\":\"they have\",\n",
    "        \"this's\":\"this is\",\n",
    "        \"those're\":\"those are\",\n",
    "        \"'tis\":\"it is\",\n",
    "        \"'twas\":\"it was\",\n",
    "        \"wanna\":\"want to\",\n",
    "        \"wasn't\":\"was not\",\n",
    "        \"we'd\":\"we would\",\n",
    "        \"we'd've\":\"we would have\",\n",
    "        \"we'll\":\"we will\",\n",
    "        \"we're\":\"we are\",\n",
    "        \"weren't\":\"were not\",\n",
    "        \"we've\":\"we have\",\n",
    "        \"what'd\":\"what did\",\n",
    "        \"what'll\":\"what will\",\n",
    "        \"what're\":\"what are\",\n",
    "        \"what's\":\"what is\",\n",
    "        \"what've\":\"what have\",\n",
    "        \"when's\":\"when is\",\n",
    "        \"where'd\":\"where did\",\n",
    "        \"where're\":\"where are\",\n",
    "        \"where's\":\"where is\",\n",
    "        \"where've\":\"where have\",\n",
    "        \"which's\":\"which is\",\n",
    "        \"who'd\":\"who would\",\n",
    "        \"who'd've\":\"who would have\",\n",
    "        \"who'll\":\"who will\",\n",
    "        \"who're\":\"who are\",\n",
    "        \"who's\":\"who is\",\n",
    "        \"who've\":\"who have\",\n",
    "        \"why'd\":\"why did\",\n",
    "        \"why're\":\"why are\",\n",
    "        \"why's\":\"why is\",\n",
    "        \"won't\":\"will not\",\n",
    "        \"wouldn't\":\"would not\",\n",
    "        \"would've\":\"would have\",\n",
    "        \"y'all\":\"you all\",\n",
    "        \"you'd\":\"you would\",\n",
    "        \"you'll\":\"you will\",\n",
    "        \"you're\":\"you are\",\n",
    "        \"you've\":\"you have\",\n",
    "        \"Whatcha\":\"What are you\",\n",
    "        \"luv\":\"love\",\n",
    "        \"sux\":\"sucks\"\n",
    "        }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "XeUA8F3I4ewo"
   },
   "source": [
    "##### Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "8usHgE57H00j",
    "outputId": "6707ad26-ab4c-401f-df57-636bcda15ed8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "lot of rock thumb up fell fromm the sky smiley he will consider taken an umbrella love\n"
     ]
    }
   ],
   "source": [
    "lemmatizer = WordNetLemmatizer()\n",
    "\n",
    "def tweet_cleaner(tweet):\n",
    "\n",
    "    #remove references\n",
    "    pat1 = r'@[A-Za-z0-9]+'\n",
    "    #remove links\n",
    "    pat2 = r\"(\\w+:\\/\\/\\S+)\"\n",
    "    combined_pat1 = r'|'.join((pat1, pat2))\n",
    "\n",
    "    #remove hashtags, numbers\n",
    "    pat4 = \"(#[A-Za-z0-9]+)\" #r'\\^[a-zA-Z!]\\s+' \n",
    "    # Substituting multiple spaces with single space\n",
    "    pat5 = r'\\s+'\n",
    "    combined_pat2 = r'|'.join((pat4, pat5))\n",
    "\n",
    "    #remove html related tags\n",
    "    tweet = BeautifulSoup(tweet, 'lxml')\n",
    "    tweet = tweet.get_text()\n",
    "    \n",
    "    tweet = re.sub(combined_pat1, '', tweet)\n",
    "    tweet = re.sub(combined_pat2, ' ', tweet)\n",
    "\n",
    "    try:\n",
    "        #replace possibe faulty characters\n",
    "        tweet = tweet.replace(u\"\\ufffd\", \"?\")\n",
    "    except:\n",
    "        tweet = tweet\n",
    "    tweet = tweet.lower()\n",
    "    \n",
    "    #CONTRACTIONS source: https://en.wikipedia.org/wiki/Contraction_%28grammar%29\n",
    "    CONTRACTIONS = load_dict_contractions()\n",
    "    #Deal with smileys\n",
    "    #source: https://en.wikipedia.org/wiki/List_of_emoticons\n",
    "    SMILEY = load_dict_smileys()  \n",
    "\n",
    "    tweet = tweet.replace(\"’\",\"'\")\n",
    "    words = tweet.split()\n",
    "    reformed = [CONTRACTIONS[word] if word in CONTRACTIONS else word for word in words] #replace contractions\n",
    "    reformed2 = [SMILEY[word] if word in SMILEY else word for word in reformed] #replace smileys\n",
    "    reformed2 = [lemmatizer.lemmatize(word) for word in reformed2] #lemmatize\n",
    "    tweet = \" \".join(reformed2)\n",
    "    # Standardizing words\n",
    "    tweet = ''.join(''.join(s)[:2] for _, s in itertools.groupby(tweet))\n",
    "\n",
    "    #Deal with emojis\n",
    "    tweet = emoji.demojize(tweet)\n",
    "\n",
    "    #Removal of Punctuation\n",
    "    tweet = ' '.join(re.sub(\"[\\.\\,\\!\\?\\:\\;\\-\\=]\", \" \", tweet).split())\n",
    "    #keep only letters\n",
    "    tweet = re.sub(\"[^a-zA-Z!]\", \" \", tweet)\n",
    "    #lemmatize\n",
    "    words = tweet.split()\n",
    "    reformed = [lemmatizer.lemmatize(word) for word in words]\n",
    "    tweet = \" \".join(reformed)\n",
    "    # Substituting multiple spaces with single space\n",
    "    tweet = ' '.join(re.sub(\"\\s+\", \" \", tweet).split())\n",
    "    return tweet\n",
    "\n",
    "#test cleaner\n",
    "print(tweet_cleaner(\"@someone Lots of rocks 👍 fell frommmmm the skies.... :-) He'll consider taken an umbrella <3 ?! 95345!! #rocksfell  https://www.fsdf.gr http://asdfsdf/w\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "fNK-iSgvtksK"
   },
   "source": [
    "#### Clean dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 182
    },
    "colab_type": "code",
    "id": "QWIX2AuSTVIe",
    "outputId": "1b81d469-9ffd-4dd2-d450-8577bd9e0ce8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time passed 65 seconds\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiment</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>200475</th>\n",
       "      <td>0</td>\n",
       "      <td>trouble is unless you have a piece of paper that say look i told you i can do this already they do not believe you</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1270958</th>\n",
       "      <td>1</td>\n",
       "      <td>raqu l wond r omgi want travis to call me so bad lmao im inlove with a singer lolz</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1138680</th>\n",
       "      <td>1</td>\n",
       "      <td>across the universe wa great i have new love for the beatles and janis joplin hey jude</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>160994</th>\n",
       "      <td>0</td>\n",
       "      <td>sorry is only in spanish for the moment</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         sentiment                                                                                                                text\n",
       "200475           0  trouble is unless you have a piece of paper that say look i told you i can do this already they do not believe you\n",
       "1270958          1                                  raqu l wond r omgi want travis to call me so bad lmao im inlove with a singer lolz\n",
       "1138680          1                              across the universe wa great i have new love for the beatles and janis joplin hey jude\n",
       "160994           0                                                                             sorry is only in spanish for the moment"
      ]
     },
     "execution_count": 13,
     "metadata": {
      "tags": []
     },
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# set N for sample size\n",
    "N = 100000\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "# shuffle\n",
    "tweets_final = tweets_final.sample(frac=1)\n",
    "\n",
    "# get a subspace of the original dataset and preproccess it\n",
    "tweets_final = tweets_final[:N]\n",
    "tweets_final['text'] = tweets_final['text'].apply(lambda x : tweet_cleaner(x))\n",
    "\n",
    "elapsed_time = time.time() - start_time\n",
    "\n",
    "print('Time passed %0.0f seconds' % elapsed_time)\n",
    "\n",
    "# drop empty sentences\n",
    "tweets_final = tweets_final[(tweets_final['text']!='')]\n",
    "tweets_final.head(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "PkvYbMWMl_4J",
    "outputId": "9d533c87-6201-4dcb-8f52-7ae4f3c3863b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Time passed 6 seconds\n"
     ]
    }
   ],
   "source": [
    "tok = WordPunctTokenizer()\n",
    "def text_centroid(text, model):\n",
    "    \n",
    "  text_vec =[]\n",
    "  counter = 0\n",
    "  sent_tokenized = tok.tokenize(text)\n",
    "  for word in sent_tokenized:\n",
    "    try:\n",
    "      if counter == 0:\n",
    "        text_vec = model[word]\n",
    "        counter+=1\n",
    "      else:\n",
    "        text_vec = np.add(text_vec, model[word])\n",
    "        counter+=1\n",
    "    except:\n",
    "      pass\n",
    "    \n",
    "  return np.asarray(text_vec) / counter\n",
    "\n",
    "start_time = time.time()\n",
    "\n",
    "tweets_final['centroid'] = tweets_final.text.apply(lambda x : text_centroid(x, fasttext))\n",
    "\n",
    "elapsed_time = time.time() - start_time\n",
    "\n",
    "print('Time passed %0.0f seconds' % elapsed_time)\n",
    "\n",
    "# drop empty centroids\n",
    "tweets_final = tweets_final[tweets_final['centroid'].map(lambda d: len(d)) > 0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "49FqdzrsFweT"
   },
   "source": [
    "#### Train, Dev & Test split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 339
    },
    "colab_type": "code",
    "id": "5QLpSty20Uhb",
    "outputId": "2cb2e584-37f0-45c7-81a3-9f98271b4b91"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         sentiment  ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         centroid\n",
      "200475           0  ...  [0.0048201387, -0.035242263, 0.023707423, 0.039374247, -0.09091509, -0.04399461, -0.05705161, 0.014566073, 0.04219777, -0.037388302, -0.027243381, -0.013947438, 0.002187863, 0.020463515, -0.012725016, 0.07605736, 0.05734066, 0.0022523613, -0.038361948, 0.10499069, -0.0044409763, 0.015155567, -0.0009873111, -0.05729038, -0.0730489, -0.043622047, -0.02576198, 0.026332831, -0.020547047, 0.18379179, -0.025962934, -0.02988856, -0.006434019, -0.020436754, -0.038263325, -0.031284016, 0.024433015, -1.9742549e-05, 0.00089153915, -0.053463306, -0.004212988, 0.004301491, -0.026110694, 0.057323553, 0.10125736, 0.06211611, 0.005657443, 0.10572687, -0.012081973, 0.013327395, -0.018527277, -0.041432213, 0.000688071, -0.028665608, -0.018158391, 0.00015911579, -0.041188683, -0.0076642106, -0.13373595, ...\n",
      "1270958          1  ...  [0.02984836, -0.1390044, -0.037042145, 0.019688928, -0.054950174, -0.047423545, -0.09479018, 0.043963205, -0.0173926, 0.015328444, -0.01995413, -0.015323907, 0.041687116, -0.004500644, 0.027487552, 0.050778307, 0.036578953, 0.03280434, -0.08164487, 0.08806493, 0.008936938, 0.031593695, 0.016841508, -0.05753726, -0.0063472074, -0.013144794, -0.035412133, 0.0061906883, -0.003290804, 0.25960827, -0.025810305, 0.0071802964, 0.009943835, 0.078638434, -0.0010176428, -0.045314908, 0.01959818, 0.029071739, 0.000607342, 0.004343068, -0.010555145, -0.03979797, -0.06448771, 0.03320943, 0.08913084, 0.042134173, -0.014958327, 0.085603416, -0.08749925, -0.0066542705, 0.07262399, -0.029396668, 0.004498319, -0.00024914625, -0.04814646, -0.0230333, 0.013274662, -0.011208435, -0.11590318, -0.013247515, ...\n",
      "1138680          1  ...  [-0.010380622, -0.0018741769, -0.01283384, 0.054609578, 0.0038031396, 0.017758623, -0.011192824, 0.01950224, -0.00038711223, 0.016679473, -0.07391881, -0.041477546, -0.015663648, 0.008745396, 0.03801816, 0.011693052, 0.0106466785, 0.021544475, -0.055704772, 0.032253683, 0.001652915, 0.014388462, 0.0065553077, -0.045880694, -0.011356252, -0.025123578, -0.010493555, -0.015169177, -0.017341005, 0.15115923, -0.0070497263, -0.023249311, -0.026373664, -0.012898496, -0.023877854, -0.018218156, 0.0048150606, 0.03225839, -0.039402824, -0.0014810238, -0.010531512, -0.03972026, -0.026637657, 0.0052149706, 0.02287252, 0.05216702, -0.023291301, 0.03167071, -0.07666989, -0.0011003609, 0.03571951, 0.004715663, 0.04611613, -0.009450783, -0.02439247, 0.011561286, 0.045968946, 0.011043705, -0.018742636,...\n",
      "\n",
      "[3 rows x 3 columns]\n",
      "         sentiment  ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         centroid\n",
      "172279           0  ...  [-0.028644022, 0.07817834, 0.065696016, 0.13501939, -0.09242058, 0.0132116955, 0.037746225, -0.011786341, 0.017453013, -0.02574324, -0.008967593, -0.0072139776, 0.0018795987, 0.018182317, 0.010460839, 0.04280637, 0.045317166, 0.030080097, -0.029501857, 0.009109659, -0.03027052, 0.047544118, 0.0065365573, -0.035328772, 0.05890577, -0.042247795, -0.031069683, 0.030255092, -0.01584414, 0.076532654, -0.006236664, -0.03523601, -0.030074695, -0.05394536, -0.04511692, 0.022831272, 0.0024226515, 0.111022905, -0.047573376, -0.008339738, -0.035109162, -0.005616356, 0.0071749263, 0.03256968, -0.0070693726, 0.0038617165, -0.046563577, 0.081483595, -0.0142211, 0.00068059616, 0.0025905392, 0.02224976, -0.007881162, -0.029988203, 0.005487114, 0.024370717, 0.030151537, -0.051566046, -0.11488517, -0.00...\n",
      "587264           0  ...  [0.01834451, -0.016342321, -0.079036504, 0.07601115, -0.027410746, -0.0015870543, -0.06947294, 0.04100589, 0.01579482, -0.041617747, -0.037372645, -0.027885031, -3.476143e-05, 0.015623354, 0.071353406, 0.08553041, 0.05947843, 0.016920153, -0.07617659, 0.13837054, 0.024436908, 0.058068097, -0.025157651, -0.13269383, -0.052841663, -0.074058816, -0.024099626, 0.04174931, -0.045160897, 0.23184226, -0.014798558, -0.059044283, -0.04028854, 0.016882496, -0.009226329, -0.03627271, -0.07828102, 0.09415047, -0.024004472, -0.030400507, -0.0021520096, -0.018977525, -0.003541173, 0.049434092, 0.10407828, 0.119979754, 0.038695883, 0.10650578, -0.038553238, 0.0067567006, 0.05568344, -0.048233084, 0.061451722, -0.014496998, -0.056426622, -0.004522359, 0.043447383, 0.06400021, -0.14020357, -0.022902358...\n",
      "1129675          1  ...  [0.018196346, -0.056593027, 0.02597624, 0.10080394, -0.05876688, 0.0077349576, 0.01935848, 0.0024084412, -0.024229957, -0.024671739, -0.017663173, 0.005231684, -0.020070044, 0.013355976, 0.022426637, 0.03184813, -0.006794977, -0.008717971, -0.041022792, 0.043328933, -0.023480421, 0.039570507, -0.002777733, -0.00029262595, -0.056431618, -0.044189863, 0.008144097, -0.0070852246, -0.08248782, 0.10934439, -0.049760338, -0.0065964176, -0.012626443, -0.026609838, -0.016667869, -0.0062841596, -0.055098534, 0.10921193, -0.035143495, -0.018266022, -0.0073503037, -0.0035782517, -0.040703032, 0.076911755, 0.0022439444, 0.056697294, -0.0005612793, 0.038192734, 0.0069638477, 0.039976906, 0.021322595, -0.020962747, -0.016284548, 0.002552531, -0.024963414, -0.04543173, -0.009340901, -0.002317156, -0....\n",
      "\n",
      "[3 rows x 3 columns]\n",
      "         sentiment  ...                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         centroid\n",
      "381950           0  ...  [0.041267317, 0.009630367, -0.019573301, 0.0756495, 0.0005779192, -0.056862917, -0.045729354, 0.046152513, -0.007501947, 0.028337032, -0.031400137, -0.04449618, 0.022720855, -0.024535209, 0.03506376, 0.100270644, 0.027247548, 0.037264954, -0.097988755, 0.033664867, 0.023455806, 0.063001305, 0.008836258, -0.03792664, 0.013357983, -0.031775903, -0.031166315, 0.01509179, -0.01376581, 0.26156065, 0.008346307, -0.034245685, -0.025186667, 0.024025291, 0.017173836, -0.039033785, 0.03507827, 0.0064556994, -0.016775798, -0.025040923, 0.008805489, -0.0018809094, -0.04573147, 0.051185463, 0.044631775, 0.062198307, 0.030664444, 0.046859268, -0.13894993, 0.011696409, 0.07732642, -0.06414942, 0.030124359, 0.0359479, -0.08356395, -0.039910868, -0.059737284, 0.081369884, -0.11125304, -0.0053134058, 0....\n",
      "396423           0  ...  [0.014874392, 0.017885774, -0.0043059653, 0.027103025, -0.03196812, 0.021375598, 0.060578503, 0.025014859, -0.006897486, 0.013195171, 0.01804048, 0.024050597, -0.0005397359, -0.0020624462, 0.013749573, 0.06105618, 0.013444008, 0.010725909, -0.009831703, 0.02051026, -0.007059648, 0.03143451, 0.011566153, -0.04677634, -0.0427905, -0.047096573, -0.0011231744, 0.007823335, -0.031019483, 0.122751676, -0.05632799, -0.011820248, -0.034035917, 0.008855717, 0.0069531244, 0.02523768, -0.014345685, 0.038641598, 0.00020621487, 0.012876431, 0.04410496, -0.030494936, -0.021267058, 0.073927455, -0.004312218, 0.08072232, 0.010182077, 0.02523788, -0.03361257, 0.022056827, 0.028044976, -0.034914568, 0.0140631, -0.024156569, -0.031545807, -0.028163463, 0.05725702, -0.037680242, -0.027149407, -0.040488522...\n",
      "1098727          1  ...  [-0.017995533, -0.019904787, -0.061386798, 0.082716845, 0.03009932, 0.003929644, 0.0638953, -0.002326971, 0.034454796, -0.038500838, -0.07238326, -0.049264528, -0.029062346, -0.06313533, -0.01417714, 0.03946137, -0.01439834, 0.02206072, -0.031094834, -0.030160902, 0.05313082, 0.024066336, -0.038374238, 0.023363184, 0.013449699, -0.042299412, -0.04046086, -0.010488631, -0.04385644, 0.08798188, -0.019556953, 0.0043891435, -0.040388014, -0.007164701, -0.03926041, -0.039649937, 0.035504706, 0.025042843, 0.04432695, 0.068397, 0.083489135, 0.035667453, 0.031896636, 0.014701295, -0.116335414, 0.020295931, -0.015666516, 0.0117256, -0.020615842, -0.022656903, -0.052231144, -0.006086834, 0.03416138, -0.0408011, -0.06577577, 0.062421452, 0.04443557, 0.0036323697, -0.033697776, 0.0043765884, 0.010...\n",
      "\n",
      "[3 rows x 3 columns]\n"
     ]
    }
   ],
   "source": [
    "total_length = len(tweets_final)\n",
    "train_set = tweets_final[0:int((total_length*8)/10)]\n",
    "print(train_set[0:3])\n",
    "dev_set = tweets_final[int((total_length*8)/10)+1:int((total_length*9)/10)]\n",
    "print(dev_set[0:3])\n",
    "test_set = tweets_final[int((total_length*9)/10)+1:total_length]\n",
    "print(test_set[0:3])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3kL7jlmA0a4z"
   },
   "source": [
    "### Some statistics on the dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 101
    },
    "colab_type": "code",
    "id": "7-J-LLGr0cbJ",
    "outputId": "6c668c0c-b0c9-4a6b-fa1b-7be6e5c87130"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Average document length per sentiment is: \n",
      "sentiment\n",
      "0    13.527859\n",
      "1    12.345976\n",
      "Name: text, dtype: float64\n"
     ]
    }
   ],
   "source": [
    "avg_doc_len = tweets_final.groupby('sentiment').text.apply(lambda x: x.str.split().str.len().mean())\n",
    "print(\"Average document length per sentiment is: \")\n",
    "print(avg_doc_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "v4y6bZAX3xa1",
    "outputId": "f2af4cf6-3ebc-4c2b-8661-f80c6361899b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The total number of documents is:  99764 .\n"
     ]
    }
   ],
   "source": [
    "print(\"The total number of documents is: \", total_length, \".\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 67
    },
    "colab_type": "code",
    "id": "4HsYGM6e37d2",
    "outputId": "8aa797ac-fbf4-44e8-ebcd-79e598dbe4a6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training set contains:  79811  documents.\n",
      "Dev set contains:  9975  documents.\n",
      "Test set contains:  9976  documents.\n"
     ]
    }
   ],
   "source": [
    "print(\"Training set contains: \", len(train_set),\" documents.\")\n",
    "print(\"Dev set contains: \", len(dev_set),\" documents.\")\n",
    "print(\"Test set contains: \", len(test_set),\" documents.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "x6_7yYn7nrrZ"
   },
   "outputs": [],
   "source": [
    "X_train = np.stack(train_set.centroid, axis=0)\n",
    "y_train = train_set.sentiment.tolist()\n",
    "\n",
    "X_dev = np.stack(dev_set.centroid, axis=0)\n",
    "y_dev = dev_set.sentiment.tolist()\n",
    "\n",
    "X_test = np.stack(test_set.centroid, axis=0)\n",
    "y_test = test_set.sentiment.tolist()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "DTLFdDd0EeMS"
   },
   "source": [
    "### Macro Average - Precision, Recall & F1 scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "pbD2s-j8IIFi"
   },
   "outputs": [],
   "source": [
    "def get_scores(y_true, y_scores, threshold):\n",
    "  \"\"\"\n",
    "  input:\n",
    "    y_true: array with gound truth\n",
    "    y_scores: array-like, shape = [n_samples, n_classes], the probability of the sample for each class in the model\n",
    "    threshold: classification threshold in order to classify to positive class\n",
    "\n",
    "  output:\n",
    "    precision: dictionary with scores for all classes + macro precision\n",
    "    recall: dictionary with scores for all classes + macro recall\n",
    "    f1: dictionary with scores for all classes + macro f1\n",
    "\n",
    "  \"\"\"\n",
    "  N_labels = y_scores.shape[1] # number of classes\n",
    "  \n",
    "  precision = {}\n",
    "  recall = {}\n",
    "  f1 = {}\n",
    "\n",
    "  for label in range(N_labels):\n",
    "    \n",
    "    # create array with predictions based on which class is considered positive and a threshold\n",
    "    y_pred = np.array([int(label if elem >= threshold else 1-label) for elem in y_scores[:,label]])\n",
    "  \n",
    "    # calculate True Positive, False Positive, True Negative, False Negative\n",
    "    TP = 0\n",
    "    FP = 0\n",
    "    TN = 0\n",
    "    FN = 0\n",
    "\n",
    "    for i in range(len(y_pred)): \n",
    "      if y_true[i]==y_pred[i]==label:\n",
    "        TP += 1\n",
    "      if y_pred[i]==label and y_true[i]!=y_pred[i]:\n",
    "        FP += 1\n",
    "      if y_pred[i]==1-label and y_true[i]!=y_pred[i]:\n",
    "        FN += 1\n",
    "    precision[label] = 0 if (TP + FP) == 0 else TP / (TP + FP) \n",
    "    recall[label] = 0 if (TP + FN) == 0 else TP / (TP + FN)\n",
    "    f1[label] = 0 if (precision[label] + recall[label]) ==0 else 2 * (precision[label] * recall[label]) / (precision[label] + recall[label]) \n",
    "\n",
    "  precision['macro'] = sum(precision.values())/N_labels\n",
    "  recall['macro'] = sum(recall.values())/N_labels\n",
    "  f1['macro'] = sum(f1.values())/N_labels\n",
    "  \n",
    "  return precision, recall, f1\n",
    "\n",
    "def print_scores(estimator, X, y_true, threshold):\n",
    "  y_pred = estimator.predict_proba(X)\n",
    "  prec_dict, rec_dict, f1_dict = get_scores(y_true, y_pred, threshold)\n",
    "  scores_df = pd.DataFrame.from_dict({'precision': prec_dict, 'recall': rec_dict, 'f1': f1_dict})\n",
    "  print(tabulate(scores_df.round(3), headers='keys', tablefmt='psql'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ojY9bXhTBzPw"
   },
   "source": [
    "### Most probable classifier as Baseline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 454
    },
    "colab_type": "code",
    "id": "xLbSHGizBzWS",
    "outputId": "e14f9177-b8a3-4426-fa10-efc61d4f382b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Baseline- Train scores:\n",
      "+-------+-------------+----------+-------+\n",
      "|       |   precision |   recall |    f1 |\n",
      "|-------+-------------+----------+-------|\n",
      "| 0     |       0     |      0   | 0     |\n",
      "| 1     |       0.503 |      1   | 0.67  |\n",
      "| macro |       0.252 |      0.5 | 0.335 |\n",
      "+-------+-------------+----------+-------+\n",
      "---------------------------------------------------------\n",
      "Baseline- Dev scores:\n",
      "+-------+-------------+----------+-------+\n",
      "|       |   precision |   recall |    f1 |\n",
      "|-------+-------------+----------+-------|\n",
      "| 0     |       0     |      0   | 0     |\n",
      "| 1     |       0.498 |      1   | 0.665 |\n",
      "| macro |       0.249 |      0.5 | 0.332 |\n",
      "+-------+-------------+----------+-------+\n",
      "---------------------------------------------------------\n",
      "Baseline- Test scores:\n",
      "+-------+-------------+----------+-------+\n",
      "|       |   precision |   recall |    f1 |\n",
      "|-------+-------------+----------+-------|\n",
      "| 0     |       0     |      0   | 0     |\n",
      "| 1     |       0.493 |      1   | 0.661 |\n",
      "| macro |       0.247 |      0.5 | 0.33  |\n",
      "+-------+-------------+----------+-------+\n"
     ]
    }
   ],
   "source": [
    "def print_classifier_scores(estimator, est_name, X_train, y_train\n",
    "                            ,X_dev, y_dev\n",
    "                            ,X_test,y_test\n",
    "                            , thres):\n",
    "\n",
    "  print(\"{}- Train scores:\".format(est_name))\n",
    "  print_scores(estimator, X_train, y_train, thres)\n",
    "\n",
    "  print(\"---------------------------------------------------------\")\n",
    "  print(\"{}- Dev scores:\".format(est_name))\n",
    "  print_scores(estimator, X_dev ,y_dev, thres)\n",
    "\n",
    "  print(\"---------------------------------------------------------\")\n",
    "  print(\"{}- Test scores:\".format(est_name))\n",
    "  print_scores(estimator, X_test, y_test, thres)\n",
    "\n",
    "# fit on training set and print scores\n",
    "dummy = DummyClassifier(strategy=\"most_frequent\")\n",
    "dummy.fit(X_train, y_train)\n",
    "\n",
    "print_classifier_scores(dummy, 'Baseline', X_train, y_train\n",
    "  , X_dev, y_dev\n",
    "  , X_test, y_test,0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "H5gMZDaFBzeY"
   },
   "source": [
    "### Logistic Regression Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 454
    },
    "colab_type": "code",
    "id": "NRLNmZwtBzkW",
    "outputId": "a05764a9-31f4-4306-f2bc-662520cb5759"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SGD classifier- Train scores:\n",
      "+-------+-------------+----------+-------+\n",
      "|       |   precision |   recall |    f1 |\n",
      "|-------+-------------+----------+-------|\n",
      "| 0     |       0.737 |    0.745 | 0.741 |\n",
      "| 1     |       0.745 |    0.738 | 0.742 |\n",
      "| macro |       0.741 |    0.741 | 0.741 |\n",
      "+-------+-------------+----------+-------+\n",
      "---------------------------------------------------------\n",
      "SGD classifier- Dev scores:\n",
      "+-------+-------------+----------+-------+\n",
      "|       |   precision |   recall |    f1 |\n",
      "|-------+-------------+----------+-------|\n",
      "| 0     |       0.747 |    0.739 | 0.743 |\n",
      "| 1     |       0.739 |    0.747 | 0.743 |\n",
      "| macro |       0.743 |    0.743 | 0.743 |\n",
      "+-------+-------------+----------+-------+\n",
      "---------------------------------------------------------\n",
      "SGD classifier- Test scores:\n",
      "+-------+-------------+----------+-------+\n",
      "|       |   precision |   recall |    f1 |\n",
      "|-------+-------------+----------+-------|\n",
      "| 0     |       0.753 |    0.751 | 0.752 |\n",
      "| 1     |       0.745 |    0.748 | 0.746 |\n",
      "| macro |       0.749 |    0.749 | 0.749 |\n",
      "+-------+-------------+----------+-------+\n"
     ]
    }
   ],
   "source": [
    "sgd = SGDClassifier(loss=\"log\", max_iter=5000, penalty= 'l2', alpha= 0.0001)\n",
    "sgd.fit(X_train, y_train)\n",
    "\n",
    "print_classifier_scores(sgd, 'SGD classifier', X_train, y_train\n",
    "  , X_dev, y_dev\n",
    "  , X_test, y_test,0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "3TVgYKjp5YZn"
   },
   "source": [
    "### Keras MLP Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZVIPLH3-w1dS"
   },
   "outputs": [],
   "source": [
    "def recall(y_true, y_pred):\n",
    "    \n",
    "    \"\"\"\n",
    "    Recall metric.\n",
    "    Only computes a batch-wise average of recall.\n",
    "    Computes the recall, a metric for multi-label classification of\n",
    "    how many relevant items are selected.\n",
    "    \"\"\"\n",
    "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "    possible_positives = K.sum(K.round(K.clip(y_true, 0, 1)))\n",
    "    recall = true_positives / (possible_positives + K.epsilon())\n",
    "    return recall\n",
    "\n",
    "\n",
    "def precision(y_true, y_pred):\n",
    "    \n",
    "    \"\"\"\n",
    "    Precision metric.\n",
    "    Only computes a batch-wise average of precision.\n",
    "    Computes the precision, a metric for multi-label classification of\n",
    "    how many selected items are relevant.\n",
    "    Source\n",
    "    ------\n",
    "    https://github.com/fchollet/keras/issues/5400#issuecomment-314747992\n",
    "    \"\"\"\n",
    "    true_positives = K.sum(K.round(K.clip(y_true * y_pred, 0, 1)))\n",
    "    predicted_positives = K.sum(K.round(K.clip(y_pred, 0, 1)))\n",
    "    precision = true_positives / (predicted_positives + K.epsilon())\n",
    "    return precision\n",
    "\n",
    "\n",
    "def f1(y_true, y_pred):\n",
    "    \n",
    "    \"\"\"Calculate the F1 score.\"\"\"\n",
    "    p = precision(y_true, y_pred)\n",
    "    r = recall(y_true, y_pred)\n",
    "    return 2 * ((p * r) / (p + r))\n",
    "\n",
    "\n",
    "def accuracy(y_true, y_pred):\n",
    "    return K.mean(K.equal(y_true, K.round(y_pred)), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "_AlEXNVn1b4k"
   },
   "outputs": [],
   "source": [
    "# make lists to arrays\n",
    "y_train = np.array(y_train)\n",
    "y_dev = np.array(y_dev)\n",
    "y_test = np.array(y_test)\n",
    "\n",
    "\"\"\"\n",
    " prepare two-dim y vectors for keras fit\n",
    " in order to have 2 values for predict_proba we have DENSE 2 output in keras\n",
    " thus it expects a two dimensional y vector. We create it stacking vertically the \n",
    " suplement of y\n",
    "\"\"\"\n",
    "y_train2 = np.vstack((1-y_train, y_train)).T\n",
    "y_dev2 = np.vstack(( 1-y_dev, y_dev)).T\n",
    "y_test2 = np.vstack((1-y_test, y_test )).T"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "JVJa6Oh1Y3Da"
   },
   "source": [
    "#### Define Keras MLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 370
    },
    "colab_type": "code",
    "id": "vyOGOjdMwPLp",
    "outputId": "66169397-c6dc-4abe-dcd3-8931d63384ed"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential_2\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "dense_5 (Dense)              (None, 256)               77056     \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 256)               65792     \n",
      "_________________________________________________________________\n",
      "dropout_4 (Dropout)          (None, 256)               0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 128)               32896     \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 2)                 258       \n",
      "=================================================================\n",
      "Total params: 176,002\n",
      "Trainable params: 176,002\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(256, input_dim=X_train.shape[1] , activation='relu'))\n",
    "model.add(Dropout(0.5))\n",
    "model.add(Dense(256,  activation='relu'))\n",
    "model.add(Dropout(0.3))\n",
    "model.add(Dense(128,  activation='relu'))\n",
    "model.add(Dense(2,  activation='softmax'))\n",
    "\n",
    "print(model.summary())\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer=Adam(lr=0.0001),\n",
    "                  metrics=[precision, recall, f1, accuracy])\n",
    "\n",
    "my_callbacks = [\n",
    "    EarlyStopping(patience=5),\n",
    "    ModelCheckpoint('keras_fastetxt_centroids_model2', monitor='val_f1', verbose=1, save_best_only=True, mode='max')\n",
    "]\n",
    "\n",
    "#model.load_weights(\"/content/gdrive/My Drive/keras_checkpoints/sentiment_keras_weights.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "yN-t4IYTYoRA",
    "outputId": "fc60914c-e975-4af0-cf4b-7bc9042acc02"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 79811 samples, validate on 9975 samples\n",
      "Epoch 1/30\n",
      " - 14s - loss: 0.5859 - precision: 0.6860 - recall: 0.6860 - f1: 0.6860 - accuracy: 0.6859 - val_loss: 0.5224 - val_precision: 0.7427 - val_recall: 0.7427 - val_f1: 0.7427 - val_accuracy: 0.7426\n",
      "\n",
      "Epoch 00001: val_f1 improved from -inf to 0.74267, saving model to keras_fastetxt_centroids_model2\n",
      "Epoch 2/30\n",
      " - 14s - loss: 0.5312 - precision: 0.7341 - recall: 0.7341 - f1: 0.7341 - accuracy: 0.7342 - val_loss: 0.5056 - val_precision: 0.7553 - val_recall: 0.7553 - val_f1: 0.7553 - val_accuracy: 0.7552\n",
      "\n",
      "Epoch 00002: val_f1 improved from 0.74267 to 0.75533, saving model to keras_fastetxt_centroids_model2\n",
      "Epoch 3/30\n",
      " - 14s - loss: 0.5182 - precision: 0.7429 - recall: 0.7429 - f1: 0.7429 - accuracy: 0.7429 - val_loss: 0.4992 - val_precision: 0.7570 - val_recall: 0.7570 - val_f1: 0.7570 - val_accuracy: 0.7569\n",
      "\n",
      "Epoch 00003: val_f1 improved from 0.75533 to 0.75695, saving model to keras_fastetxt_centroids_model2\n",
      "Epoch 4/30\n",
      " - 14s - loss: 0.5106 - precision: 0.7481 - recall: 0.7481 - f1: 0.7481 - accuracy: 0.7481 - val_loss: 0.4947 - val_precision: 0.7611 - val_recall: 0.7611 - val_f1: 0.7611 - val_accuracy: 0.7610\n",
      "\n",
      "Epoch 00004: val_f1 improved from 0.75695 to 0.76110, saving model to keras_fastetxt_centroids_model2\n",
      "Epoch 5/30\n",
      " - 14s - loss: 0.5027 - precision: 0.7534 - recall: 0.7534 - f1: 0.7534 - accuracy: 0.7533 - val_loss: 0.4918 - val_precision: 0.7634 - val_recall: 0.7634 - val_f1: 0.7634 - val_accuracy: 0.7633\n",
      "\n",
      "Epoch 00005: val_f1 improved from 0.76110 to 0.76340, saving model to keras_fastetxt_centroids_model2\n",
      "Epoch 6/30\n",
      " - 15s - loss: 0.4973 - precision: 0.7574 - recall: 0.7574 - f1: 0.7574 - accuracy: 0.7574 - val_loss: 0.4888 - val_precision: 0.7632 - val_recall: 0.7632 - val_f1: 0.7632 - val_accuracy: 0.7631\n",
      "\n",
      "Epoch 00006: val_f1 did not improve from 0.76340\n",
      "Epoch 7/30\n",
      " - 14s - loss: 0.4931 - precision: 0.7579 - recall: 0.7579 - f1: 0.7579 - accuracy: 0.7581 - val_loss: 0.4846 - val_precision: 0.7667 - val_recall: 0.7667 - val_f1: 0.7667 - val_accuracy: 0.7666\n",
      "\n",
      "Epoch 00007: val_f1 improved from 0.76340 to 0.76671, saving model to keras_fastetxt_centroids_model2\n",
      "Epoch 8/30\n",
      " - 14s - loss: 0.4902 - precision: 0.7615 - recall: 0.7615 - f1: 0.7615 - accuracy: 0.7615 - val_loss: 0.4831 - val_precision: 0.7697 - val_recall: 0.7697 - val_f1: 0.7697 - val_accuracy: 0.7696\n",
      "\n",
      "Epoch 00008: val_f1 improved from 0.76671 to 0.76971, saving model to keras_fastetxt_centroids_model2\n",
      "Epoch 9/30\n",
      " - 14s - loss: 0.4866 - precision: 0.7629 - recall: 0.7629 - f1: nan - accuracy: 0.7632 - val_loss: 0.4821 - val_precision: 0.7684 - val_recall: 0.7684 - val_f1: 0.7684 - val_accuracy: 0.7683\n",
      "\n",
      "Epoch 00009: val_f1 did not improve from 0.76971\n",
      "Epoch 10/30\n",
      " - 14s - loss: 0.4829 - precision: 0.7651 - recall: 0.7651 - f1: 0.7651 - accuracy: 0.7652 - val_loss: 0.4811 - val_precision: 0.7708 - val_recall: 0.7708 - val_f1: 0.7708 - val_accuracy: 0.7707\n",
      "\n",
      "Epoch 00010: val_f1 improved from 0.76971 to 0.77082, saving model to keras_fastetxt_centroids_model2\n",
      "Epoch 11/30\n",
      " - 14s - loss: 0.4807 - precision: 0.7664 - recall: 0.7664 - f1: 0.7664 - accuracy: 0.7665 - val_loss: 0.4779 - val_precision: 0.7686 - val_recall: 0.7686 - val_f1: 0.7686 - val_accuracy: 0.7685\n",
      "\n",
      "Epoch 00011: val_f1 did not improve from 0.77082\n",
      "Epoch 12/30\n",
      " - 14s - loss: 0.4772 - precision: 0.7683 - recall: 0.7683 - f1: 0.7683 - accuracy: 0.7683 - val_loss: 0.4774 - val_precision: 0.7702 - val_recall: 0.7702 - val_f1: 0.7702 - val_accuracy: 0.7700\n",
      "\n",
      "Epoch 00012: val_f1 did not improve from 0.77082\n",
      "Epoch 13/30\n",
      " - 14s - loss: 0.4740 - precision: 0.7700 - recall: 0.7700 - f1: 0.7700 - accuracy: 0.7699 - val_loss: 0.4783 - val_precision: 0.7698 - val_recall: 0.7698 - val_f1: 0.7698 - val_accuracy: 0.7697\n",
      "\n",
      "Epoch 00013: val_f1 did not improve from 0.77082\n",
      "Epoch 14/30\n",
      " - 14s - loss: 0.4705 - precision: 0.7735 - recall: 0.7735 - f1: 0.7735 - accuracy: 0.7735 - val_loss: 0.4784 - val_precision: 0.7729 - val_recall: 0.7729 - val_f1: 0.7729 - val_accuracy: 0.7728\n",
      "\n",
      "Epoch 00014: val_f1 improved from 0.77082 to 0.77292, saving model to keras_fastetxt_centroids_model2\n",
      "Epoch 15/30\n",
      " - 15s - loss: 0.4688 - precision: 0.7728 - recall: 0.7728 - f1: 0.7728 - accuracy: 0.7727 - val_loss: 0.4800 - val_precision: 0.7678 - val_recall: 0.7678 - val_f1: 0.7678 - val_accuracy: 0.7676\n",
      "\n",
      "Epoch 00015: val_f1 did not improve from 0.77292\n",
      "Epoch 16/30\n",
      " - 14s - loss: 0.4663 - precision: 0.7734 - recall: 0.7734 - f1: 0.7734 - accuracy: 0.7734 - val_loss: 0.4787 - val_precision: 0.7689 - val_recall: 0.7689 - val_f1: 0.7689 - val_accuracy: 0.7687\n",
      "\n",
      "Epoch 00016: val_f1 did not improve from 0.77292\n",
      "Epoch 17/30\n",
      " - 14s - loss: 0.4635 - precision: 0.7762 - recall: 0.7762 - f1: 0.7762 - accuracy: 0.7761 - val_loss: 0.4722 - val_precision: 0.7735 - val_recall: 0.7735 - val_f1: 0.7735 - val_accuracy: 0.7734\n",
      "\n",
      "Epoch 00017: val_f1 improved from 0.77292 to 0.77352, saving model to keras_fastetxt_centroids_model2\n",
      "Epoch 18/30\n",
      " - 14s - loss: 0.4609 - precision: 0.7783 - recall: 0.7783 - f1: 0.7783 - accuracy: 0.7783 - val_loss: 0.4747 - val_precision: 0.7702 - val_recall: 0.7702 - val_f1: 0.7702 - val_accuracy: 0.7701\n",
      "\n",
      "Epoch 00018: val_f1 did not improve from 0.77352\n",
      "Epoch 19/30\n",
      " - 14s - loss: 0.4588 - precision: 0.7800 - recall: 0.7800 - f1: 0.7800 - accuracy: 0.7802 - val_loss: 0.4761 - val_precision: 0.7729 - val_recall: 0.7729 - val_f1: 0.7729 - val_accuracy: 0.7728\n",
      "\n",
      "Epoch 00019: val_f1 did not improve from 0.77352\n",
      "Epoch 20/30\n",
      " - 14s - loss: 0.4573 - precision: 0.7804 - recall: 0.7804 - f1: 0.7804 - accuracy: 0.7804 - val_loss: 0.4751 - val_precision: 0.7747 - val_recall: 0.7747 - val_f1: 0.7747 - val_accuracy: 0.7745\n",
      "\n",
      "Epoch 00020: val_f1 improved from 0.77352 to 0.77466, saving model to keras_fastetxt_centroids_model2\n",
      "Epoch 21/30\n",
      " - 14s - loss: 0.4556 - precision: 0.7822 - recall: 0.7822 - f1: 0.7822 - accuracy: 0.7822 - val_loss: 0.4725 - val_precision: 0.7718 - val_recall: 0.7718 - val_f1: 0.7718 - val_accuracy: 0.7717\n",
      "\n",
      "Epoch 00021: val_f1 did not improve from 0.77466\n",
      "Epoch 22/30\n",
      " - 14s - loss: 0.4533 - precision: 0.7837 - recall: 0.7837 - f1: 0.7837 - accuracy: 0.7838 - val_loss: 0.4733 - val_precision: 0.7711 - val_recall: 0.7711 - val_f1: 0.7711 - val_accuracy: 0.7710\n",
      "\n",
      "Epoch 00022: val_f1 did not improve from 0.77466\n"
     ]
    }
   ],
   "source": [
    "history = model.fit(X_train, y_train2,\n",
    "              batch_size=32,\n",
    "              epochs=30,\n",
    "              verbose = 2,\n",
    "              callbacks=my_callbacks,\n",
    "              validation_data=(X_dev, y_dev2),\n",
    "              shuffle=True)\n",
    "\n",
    "#model.save_weights(\"/content/gdrive/My Drive/keras_checkpoints/sentiment_keras_weights.h5\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "1lVvEKRAq5Pf"
   },
   "source": [
    "#### Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 730
    },
    "colab_type": "code",
    "id": "o3rBmXluxNDL",
    "outputId": "b077e904-57cf-490f-f85e-6db3ca28a927"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAt8AAALJCAYAAABshU9CAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdd3iU153//fdR7wUVQA0EiGJjOhiXuHtdE6c5xok3cZIN6XGSTbJOnt20Tfbx7u/3bHaTdeJ1i51mEjvNBcdx4riCbQTC9CpAEggEQl0aacp5/jijAqYI0Mw9Gn1e1zXXzNxl9B0pDp858z3nNtZaREREREQk8hK8LkBEREREZKxQ+BYRERERiRKFbxERERGRKFH4FhERERGJEoVvEREREZEoUfgWEREREYkShW8RkTHIGPOIMeZ7wzx2rzHmmpPsSzfGPGWMaTPGPD6yVYqIxB+FbxERORfvB8YDBdbaW40xE40xTxpjDhhjrDFmsrfliYjEFoVvERE5F5OAHdbaQPh5CPgT8D7vShIRiV0K3yIiMSrc7vFVY8wGY0yXMeYhY8x4Y8yzxpgOY8xfjDH5Q45/lzFmszGm1RjzojFm1pB9840x68Ln/RpIO+5n3WyMWR8+d5UxZs4w6vsO8E3gNmNMpzHm49baQ9baHwNrRu43ISISPxS+RURi2/uAa4HpwDuBZ4FvAEW4/w//AoAxZjrwGPDF8L6VwFPGmBRjTArwB+DnwDjgcYaMTBtj5gMPA58ECoD/BZ40xqSeqjBr7beAfwN+ba3NstY+NELvWUQkbil8i4jEth+FR5P3A68Ab1hra6y1PuD3wPzwcbcBz1hrn7fW+oH/C6QDFwNLgWTgv6y1fmvtExw7Mr0c+F9r7RvW2qC19lGgN3yeiIiMoCSvCxARkVM6NORxzwmeZ4UflwD7+ndYa0PGmHqgFAgC+621dsi5+4Y8ngR8xBjz+SHbUsKvKSIiI0jhW0QkPhwALuh/YowxQDmwH7BAqTHGDAngFcDu8ON64PvW2u9HsV4RkTFJbSciIvHhN8BNxpirjTHJwD/iWkdWAauBAPAFY0yyMea9wJIh5z4AfMoYc6FxMo0xNxljss+mEGNMGtDfL54afi4iIih8i4jEBWvtduAO4EfAEdzkzHdaa/ustX3Ae4E7gaO4/vDfDTm3GvgE8D9AC7ArfOzZ6gE6w4+3hZ+LiAhgjm0BFBERERGRSNHIt4iIiIhIlHgSvo0xDxtjmowxm06y3xhjfmiM2RW+uMSCaNcoIiIiIjLSvBr5fgS4/hT7bwCqwrflwE+iUJOIiIiISER5Er6ttS/jJv2czC3Az6zzOpBnjJkYnepERERERCIjVtf5LsWtO9uvIbytcehBxpjluJFxCgoKFk6ePDla9YmIiIjIGLV27VprrT2rQexYDd/DYq29H7gfYNGiRba6utrjikREREQk3hljznoJ1Vhd7WQ/7sps/crC20RERERERq1YDd9PAh8Or3qyFGiz1jae7iQRERERkVjmSduJMeYx4Aqg0BjTAHwLSAaw1t4HrARuxF1lrRv4qBd1ioiIiIiMJE/Ct7X29tPst8Bnz/Xn+P1+Ghoa8Pl85/pSMS8tLY2ysjKSk5O9LkVERERETmJUT7g8nYaGBrKzs5k8eTLGGK/LiRhrLc3NzTQ0NFBZWel1OSIiIiJyErHa8z0ifD4fBQUFcR28AYwxFBQUjIkRfhEREZHRLK7DNxD3wbvfWHmfIiIiIqNZ3IdvEREREZFYofAdYa2trfz4xz8+4/NuvPFGWltbI1CRiIiIiHhF4TvCTha+A4HAKc9buXIleXl5kSpLRERERDwQ16udxIK7776b3bt3M2/ePJKTk0lLSyM/P59t27axY8cO3v3ud1NfX4/P5+Ouu+5i+fLlAEyePJnq6mo6Ozu54YYbuPTSS1m1ahWlpaX88Y9/JD093eN3JiIiIiJnasyE7+88tZktB9pH9DXPK8nhW+88/5TH3HPPPWzatIn169fz4osvctNNN7Fp06aBJQEffvhhxo0bR09PD4sXL+Z973sfBQUFx7zGzp07eeyxx3jggQf4wAc+wG9/+1vuuOOOEX0vIiIiIrGqqzfAniNdx9xqj3Tx5Wunc/n0Iq/LOyNjJnzHiiVLlhyzFvcPf/hDfv/73wNQX1/Pzp073xa+KysrmTdvHgALFy5k7969UatXREREJBr8wRD1R7uPCdd7DrvHB9uPXU65NC+dysJMEkfham9jJnyfboQ6WjIzMwcev/jii/zlL39h9erVZGRkcMUVV5xwre7U1NSBx4mJifT09ESlVhEREZGRZK3lUHsvtUc6Xcg+PBi06452EwzZgWPzM5KpLMzkkmmFTCnKpLLQ3SYXZJKekujhuzg3YyZ8eyU7O5uOjo4T7mtrayM/P5+MjAy2bdvG66+/HuXqREREREZeW48/PILdyZ7DLlzXHu5ib3MX3X3BgePSkhOYXJDJrInZ3HjBBCoLs1zQLsgkPzPFw3cQOQrfEVZQUMAll1zC7NmzSU9PZ/z48QP7rr/+eu677z5mzZrFjBkzWLp0qYeVioiIiAxfKGTZ39rDjkMd7DjUyZ4jndSGR7Kbu/oGjkswUD4ug8rCTC6cMo4pRVlMCY9iT8hJIyFh9LWOnAtjrT39UaPAokWLbHV19THbtm7dyqxZszyqKPrG2vsVERGRyOtvFXEh2922H+pk56GOY0axi7JTqSzMZOpAi0gWlYWZVIzLICUpvla3NsZ0W2szT3/k22nkW0REREQAaO7sZfuhDnYe6gzfd7D9YAftvsHrkxRmpTJ9fBYfWFTO9PHZzJiQxbTibHLTkz2sfPRQ+BYREREZY9p6/C5Y9wftgx3sbOrgSOdgu0huejIzxmfzzrklzJiQTVVxNtPHZ1GQlXqKV5bTiWj4NsZcD/w3kAg8aK2957j9PwCuDD/NAIqttXnhff8B3IS7CufzwF02XnpkRERERKKgqzfAriY3ir3jYAc7mjrZcbDjmKX7MlMSqRqfzVUzi8Mj2dlMH59NcXYqZhQu5RfrIha+jTGJwL3AtUADsMYY86S1dkv/MdbaLw05/vPA/PDji4FLgDnh3a8ClwMvRqpeERERkVhgrcUftPgCQXz+IL3+EL2BID5/yD0PuPu3PQ8f0x2+IM2Opg7qjw4uT5ySlEBVcRYXTy2gKtwuMn18NiW56WNu0qOXIjnyvQTYZa2tBTDGrABuAbac5PjbgW+FH1sgDUgBDJAMHIpgrSIiIiIjot3nZ3dTJ7ua3FrWnb2Bk4TlEL3HhGl3TG8gSOgcvutPS06gYlwGc8vy+MDC8nDQzqZiXAaJCtmei2T4LgXqhzxvAC480YHGmElAJfACgLV2tTHmb0AjLnz/j7V26wnOWw4sB6ioqBjR4kVEREROxlrL4c5edjV1DgTtXYfd/aH23oHjkhMNWalJpCUnkpqU4O6TE0lLSiA3PZm07FTSkhNJS04gNcndu+fu+P5jh55/7ONj96UkJmgUO8bFyoTLZcAT1toggDFmGjALKAvvf94Y8w5r7StDT7LW3g/cD26pwSjWe9a+/e1vk5WVxVe+8hWvSxEREZHT6F/Leld/wG7qZGdTB7uaOo9ZASQrNYmpRZlcOq2IacVZA7fy/HSSEuNrmT05N5EM3/uB8iHPy8LbTmQZ8Nkhz98DvG6t7QQwxjwLXAS8coJzRURERM5JXyDE3uauY0L2rqZOao904vOHBo4rzEphalEW75xbckzInpCTpsmJMiyRDN9rgCpjTCUudC8DPnj8QcaYmUA+sHrI5jrgE8aY/xfXdnI58F8RrDWivv/97/Poo49SXFxMeXk5CxcuZPfu3Xz2s5/l8OHDZGRk8MADDzBx4kTmzJnDnj17SEhIoKuri5kzZ1JbW0tystbOFBEROVddvQF2Hz42YO863Mm+5m6CQxqty/LTmRaenDg0ZOdlxOclzyV6Iha+rbUBY8zngOdwSw0+bK3dbIz5LlBtrX0yfOgyYMVxywg+AVwFbMRNvvyTtfapcyro2bvh4MZzeom3mXAB3HDPKQ9Zu3YtK1asYP369QQCARYsWMDChQtZvnw59913H1VVVbzxxht85jOf4YUXXmDevHm89NJLXHnllTz99NNcd911Ct4iIiJnod3nZ1NDG281tLGhoZUNDW3sbx1c/SMpwTC5MJPpxdncOHviQMCeUpRJRkqsdOZKvIno/7KstSuBlcdt++Zxz799gvOCwCcjWVu0vPLKK7znPe8hIyMDgHe96134fD5WrVrFrbfeOnBcb6+bnHHbbbfx61//miuvvJIVK1bwmc98xpO6RURERhOfP8jmA+0DIfuthlZqD3cN7J9UkMGCSfl88MIKpha5kD2pIINk9WNLlI2dj3WnGaGOplAoRF5eHuvXr3/bvne961184xvf4OjRo6xdu5arrrrKgwpFRERiVyAYYvuhDjaER7Tfqm9jx6EOAuG2kfE5qcwpy+O980uZU5bHnLJctYtIzBg74dsjl112GXfeeSdf//rXCQQCPPXUU3zyk5+ksrKSxx9/nFtvvRVrLRs2bGDu3LlkZWWxePFi7rrrLm6++WYSExO9fgsiIiKeCYUse5u7BkazNzS0sWl/G70BNwkyNz2ZOWW5fHLmFOaU5TG3LI8JuWkeVy1ycgrfEbZgwQJuu+025s6dS3FxMYsXLwbgl7/8JZ/+9Kf53ve+h9/vZ9myZcydOxdwrSe33norL774ooeVi4iIRJe1lsY2nxvNHtKn3RFe0i89OZHZpTncsXQSc8pymVuWx6SCDK0yIqOKOXae4+i1aNEiW11dfcy2rVu3MmvWLI8qir6x9n5FRGR0O9rV50az69sGAveRTjcHKinBMHNiNnPDo9lzynOZVpSlNbMlJhhjuq21mWdzrka+RUREJCqa2n2srm1m1a5mVtc2U3e0GwBjYGpRFpdNL3RBuyyXWRNzSEtW66XEH4VvERERiYiWrj5er21m1e5mVu0+wu7w6iM5aUksnVLAhy6sYE5ZHrNLc8hO07K6MjbEffi21o6JXrB4aR8SEZHRq8PnZ83eo6za5QL31oPtWAsZKYksqRzHbYvLuXhqIbMm5pCYEP//NoucSFyH77S0NJqbmykoKIjrAG6tpbm5mbQ0ze4WEZHo6ekLsnZfC6t2H2HV7mY27m8jGLKkJCWwaFI+/3jtdC6aWsicslytpy0SFtfhu6ysjIaGBg4fPux1KRGXlpZGWVmZ12WIiEgc6wuEWF/fyupwG0lNXSt9wRBJCYa55Xl85oqpXDS1gAUV+erXjhZroeMgHN4Kh7dD01Zo2QNFM2HqVTD5UkjN9rpKGSKuVzsRERGRsxcMWTbtbxvo2a7e20KPP4gxMLskl4unFnDR1AIWTx5HZmpcj+d5z1poPwCHtw3emra5wN3bNnhcej7kT3b7Aj2QkARli2HKlTD1SihZAIn6W52rc1ntROFbREREAHdBmx1NHQM922/saR5YY3v6+CwunlrIRVMLWFpZQG6GJkhGhLXQvj8crLcNjmgf3g697YPHZRRA0SwomuFGuYtnuvvMIrd8TKAX6t+A3S/A7r9B41uAhdRcqHyHC+JTr4JxUzx7q6OZwjcK3yIiMrJqD3fywrYm/EFLbnoyOelJ5KQlk5Oe7J6nJZGTnjxqeplDIUtXX4B2X4AOn5/2nvC9z8/RLj/r9rXwem0zzV19AEwuyOCiqYVcPLWApVMKKMpOHbliDm6Et1bAuEqYcRPkTBy51x4trIW2+sFWkcPbw0F7B/R1DB6XWeRCddHMIUF7FmQWntnP62qGPS+5MF77ovvZAHmTXBCfciVUXgYZ40bsLcYzhW8UvkVE5Nztaupk5cZGVm5sZNvBjtOfgFvJw4VyF85dUB8M5y6oh/eHH/dvy05LImGYq370BULhsBygvcdPhy9Au89/XJB229p7+vf1H+unozfAqf7Jn5ibxkVTCwZGt0vz0odV17CFQrDreVj9P7DnZTCJYINuX+kimHUzzLwZCqtG9ud6LRQKh+yhrSLb4MgO6OscPC6zeHD0eugts2Dka7IWmndD7d9cGN/zigv8JgFK5g+2qJQtgaSUkf/5cUDhG4VvERE5OzsPdfBMOHDvONRJLp18tHgX70zfwOTO9diyxbQu+iJHc2bQ1uOnvcc/EHBP+NznPyYAn+qfWWMgK3XoiHoS2WnJ9AZCA6G5f6Ta5w+d8n0YA9mp7vycdBfsh34oyEnr3xe+D4f//g8K4zJTIrMyWF83vPUYvP4TaN4J2SVw4Sdh4Ueg4xBsewq2PQMHatzxhTNg5k0ujJcscG9sNAmF4NAm9wFjz8uwb9WxI9lZE9wIdnF/y0j43ssR56Af9q917Sm7X3CPbRCSM92Ezf6R8aIZo+/vESEK3yh8i4jI8Fhr2X6og5UbD7JyYyO7mjqYkdDAhwu2c01iDcVtb2FsyH3dX3ER1L7kJrTNuBEu+yqULhj2zwqFLJ19bvTZBfX+kenw8/DIdP+2/v2pyYlu5PyYsDwkPKeGR9eH7MtMGf4oelR0HIQ3H4Dqh6CnBSbOg4s+B+e/GxJP0C/e1gDbVrowvvc1F/6yS1wQn3mTC4EnOs9r1rpR7D0vu7aOva+69wtQMA0mvwNK5g22jaTne1vvcPja3Gh4/8j40Vq3PbtkMIhPuQKyirys0lMK3yh8i4jIyVlr2drY4VpKNjWy/3ALFyduYVnuFi4JrSPLd8AdOHEuVF0H0693X78nJEBPK7x5P6y+F3ytUPV3cNnXoHyxt28qVjVugNd/DBufgFDABeeLPus+yAx31LT7KOx4DrY9Dbv+6lbtSMt1f5eZN8G0ayDlrHLPubMWWva6sL33FXffecjtyy2Hystd73TlOyCnxJsaR1rLvnAQ/5v7gNH/4WL8BeGJm1e6v2/yCLcqxTCFbxS+RUTkWNZaNh9oZ+XGRp7ddJDuI/VcnVjD+7I3M9e/nqSgD5Iz3Cje9OtcqD7VxD9fO6x5AFb9D/QcdStFXP5PULE0em8qVoVCsPPP8Pq9LowmZ8L8O1x7ScHUc3vtvm4X/LY+DTuedcEvKc393WbdDNNviExf9FDtB9xIcH8rSVud25413o1sV17mbvmT478tIxSExvUuiNe+CHWvQ8gPiSmubz09D9LyTnyfnn+Cfbmx+Y3GaSh8o/AtIiIucG/a384zGxt5bmMDeS2buDppPTenbWCyf7c7KK/CjaBOvw4mXQrJZ3h14N5O10qx6kfQddiFrsv/ybVFjDV93fDWr8L93Lsgp9QF7gUfjkx7RTAAdatcj/i2Z9xERpMAFRcPtqfkTzr3n9N1ZHBUe88rrlcdXGCsfMfg6Hbh9PgP26fT2+n62ve9Cp2H3bdDPa3H3vu7T/0aKVknCOrHh/cTBfc8z9YsV/hG4VtEYoC1rm81PR9Ss7yuZsyw1vJWQxvPbmzkpY27mNL2Jlcnrufa5A3khFqxJhFTfqEL29OvH7lJY33dsPYReO2/XNtBxcVw+ddcL2y8B7L2RvctQPXDbiS6ZL7r5z7vluiNYlrr1q7e9oxrT2na4rZPuABmvtMF8fHnD+9v4WtzAbJ/ZPvQJrc9JQsmXRIO3Je5NouE0bG0ZEwJ9Lrf8UAob3l7QD/Z/emC+7vvg3m3R+d9DKHwjcK3iHgg0Ov+8a9/w331Wv8mdDW5kbjCGVC6EErnu/vi87Vk1wiy1lJT38rKDY1s2rCW87tWc01iDYsTtpNEkFBaPglV17rAPfWqyK4k4e+BdT+HV38AHQfc8myX/xNMuzr+QnjjW7D6x7Dpt0P6uT/nWm+8fq/NuwdHxOvfAKxrA5l5s6uz/EJICF/yvq/L/TfbH7Yb14MNuXaW8gvDbSSXu4mSo7AlIq4Eek8d0GfeBBNmR70shW8UvkUkCrqOuIBdHw7a+9dBsNfty5/s/tEuW+wmi+1f627dR9z+xFQ3Ile6MHxbAOOmjr5RNH+PW/ngaK3r/UxOh6RUF1oGbuHnyeHniakj8j5DIUtNfQvPrq+nadNfmdvzJlcn1jDZHAQgWDiLxJnXuwmTZYuj/3V0oBdqfuFCeFu9Wybv8n9yHwC8DqbnIhSCnc+5Cad7X3H93Av+3rWXxOrVETsOuf7wrU+7CYLBPsgodB+IWuuhYY3rU+6/9HrlZa53u2zxmbchyZik8I3Ct0jM6z7q/qFLy/G6kuEJhdzyYfVvDN6ad7l9CcluRKz8wvBtCWRPePtrWAutdXBgXTiMr4MD68Hf5fan5rrXGRrIY2F1hEAftO5zI4nNu+Bo+L65Ftobzu41E1NPEMxPHtp9NpmjfQk0+wxNPYaDXZZDHX5mBrZyWcJGskwPwYQUQpMvI3lmuH87r2Jkfw9nK9AHG1bAy//X/R4nzHHtKDNuGl0ftvq6hqzPvQtyyob0c+d5Xd3w+drdxX22PeMmCeZPGpwgWb5ULWJyVhS+UfgWiTmhIDRUu3/0dj7vvtYFN0Emr9xd0jivwi3NlVcxePPqH/W+bheS614Ph+033deaABkFgyG7fKkLzGe7pFYo6C4jvX/tYCg/tNl9hQ+QPdEF8ZL5g/eR+J2Egq4//ejucMgeErRb9g1eeRDc36xgWvg21d3GTXGrGwR8bsQ34AO/79jnAd8p9vdgAz76fD34errp83UR6PNh/T5M0EeK7SMVP6n4STaDtfSkjSdx5nWkzLrRhSevlpsbjqAfNj4OL/8f901B8flw+Vdh1i2xHcLbG93SitUPu/8GSha4pQKj2c8tEuMUvlH4FokJnU1uTd6df3YXZvC1uv7nsiVuXd6kFDcS3Fofvq8bHAXul5ozGMSPD+Z5FW4y40h8hd9+INyrHR7VPrhhMAAXzoCK/lHtpS5sRrJtwN8DBzcNtqocWDc4yg4u9PaPjpcscO0rw/lq3Fo3EbB515BwXRu+3zPYMgOulaA/WBdMcy0x/WH7HPulrbUcaPOx81AHu5o62Xmok51NHexs6qTDFxg4Ljc9menjs5hWnE1VcRZV47OoKs5mfFYiJtDrWgdG6u8fTcEAbP6dC+FHdriLrVz2VTj/PYM9yLHgwHq3Pvem37kPX/393OUXjr7fuUiEKXyj8C3iiZONbmcWQ9W1LnBPvfLkS45Z69pR2uoGw/jQYN5ad+xlmcGtPnDCYB4eTc8oeHtQCAXd6PLQiZH96/QmpbtQ2x+2yxZ7e5nnfj0t7nLb+9eFb2uh0/U2k5DkVnHoD+QT54V7sXe/PWj3dQ6+ZmKKG7EeN/XtQTt7wjkHrFDI0tDSMxCsdx7qZFeTC9xdfYOj14VZKUwrdsG6anzWwOPCrAhd3jxWhIKw5Q+uHaVpi/vdX/ZVmP3+6PSnB/3uqpMdje7DZ0dj+HGj+9/LgXXuv6/5/f3clZGvSWSUUvhG4VskavpHt3c97+6Hjm5XXQPTrnU9riPxtbq17vVPFMz7A7uv7dhzkjMGg3luqbsSXUP1YAjNnuhCdsVS10YyYc7o+Sq9/cDg6Pj+dS6c97Yfe4xJcB9CjhnBDj/OLRuRkdZQyLLvaDc7BkayXdjefbgTnz80cNz4nFSqirNduA6PYk8rzmJc5hhf9SUUckvjvfQfcGgj5FfCO/4R5i47u/8tWus+rA0N00MDdvsBF7q7DgPH/ZufmOI+eOWUupHuBR92Fz0RkVNS+EbhWyRiTjW6Pe0aF7inXOndaLGv7e2j5f3BvK3BTWAsXzoYtnPL4+cr9FDIjVg2vuUmjRVMc8F7BJc0bPf52dbYwbaD7WxtbGdrYwfbD3bQ4x8cyS7NSw+PXmeFR7JdyM5NHyUfarxiLWx/Fl76d/ffVV4FXPplmPehwb9hoG9IqD4uTA99HOh5++tnFEB2ibtqZ/aEIY/77yee+JsiETkthW8UvkVGVDRHtyUm9I9mb21sZ1tjO1vCgbuhZTDU5WUkM2tCDjMnZjNrQg4zJmQztTiLrFRvrjAXN6x1H2xf+nfYX+3CcUaBC9f9S1UOlZTmgnNOSThU9z8+bltSavTfi8gYofCNwrfIOYn10W0ZUe0+P9sPdgyMZG9tbD9mNDsxwTClMJOZE3OYFQ7asybmMD4nNb57sr1mrZuo/OYDgD0uVIdHqrMnjs5JpyJxRuEbhW+RMzZ0dHv3C65n1CS4CYdV12p0Ow6c8Wj2xBxmTcihanwWackxtAqHiEiMOZfwre8KRcaSln1utYUtf3ST9sCNbk+/QaPbo9xwR7PnV+TzwQsrNJotIuIRhW+ReNcfuDf/wS0lBu7CLVf+sxvh1uj2qNAXCNHY1kNDSw/7W3poaOmmoaWHhtYeGo52c6DNN3Bs/2j2siXlGs0WEYkxCt8i8ehkgfua77ir1Gn93pjTGwhyoNU3EKqPCdgtPRzq8DG0SzDBwMTcdErz01k6pYBp47M0mi0iMgoofIvECwXumObzB12obu05YcBu6ug95vjEBMPE3DTK8tO5tKqQsvx0SvPSKcvPoCw/nQm5aSQn6hsLEZHRRuFbZDRT4I4pgWCIV3Yeca0gQ0at97f0cKTz2HCdnGgoyXOB+ooZRZTlZ4TDdTpl4zIYn51KksK1iEjcUfgWGW0UuGNWgjEs/3k1/qAlJTGB0nwXpmfNKnahOj9jYFtxdhqJCWoNEREZaxS+RUYDBe5RISHB8LtPX0JxTipFWakkKFyLiMhxFL5FYtWJAvfEeQrcMe6CslyvSxARkRim8C0SSxS4RURE4prCt0hfFxyogYY17sIzvR2QkAyJyZCQFL5PhsQk9/yk+4ZsP2bfaV4HA/teVeAWEREZAxS+ZWyxFpp3u6Ddfzu0Gay7CiD5lZBZCEE/hALuFvRDyA/BQPjeP2R7YPDcc6XALSIiEvcUviW+9bS60eyG6vDIdjX0tLh9KdlQthDe8WUoWwyliyCz4Mx/RigUDurHB/P+58FT7AsH/OLzFLhFRETGAIVviR+hIBzeNjiiXb8GjmwP7zRQPAtmvdMF7bLFUDgdEkbgctsJCZCQAqSc+2uJiIhIXFP4ltGr87Abye4P2/vXQV+n25c+zgXsC26FskVQuhDScrytV0RERMY8hW8ZHQJ9cGjTsb3aLXvdvggEFWQAACAASURBVIQkGD8b5t4eHtVeBOOmgNEay7FiX3MXT29o5ILSXC6bXuR1OSIiIp5R+JbYFAzA7r/Cnpddv3bjegj43L7siS5kL/q4u584F1IyvK1X3qahpZtnNjTy9IZGNu5vA+BTl09V+BYRkTEtouHbGHM98N9AIvCgtfae4/b/ALgy/DQDKLbW5oX3VQAPAuWABW601u6NZL0SA1r2wrqfQc0vofMgJKZCyTxY/A9uRLtsMeSUalQ7RjW29QwE7vX1rQDMKcvlGzfO5MYLJlKWrw9JIiIytkUsfBtjEoF7gWuBBmCNMeZJa+2W/mOstV8acvzngflDXuJnwPettc8bY7KAUKRqFY8FemHbM7DuUah9EUwCTLsWFv6nu0/SRMZY1tTh49mNB3l6wwHW7HUryZw3MYevXT+Dmy8ooaJAgVtERKRfJEe+lwC7rLW1AMaYFcAtwJaTHH878K3wsecBSdba5wGstZ0RrFO8cniHC9xvPQbdzZBbDld8A+bfAbmlXlcnp9Dc2cuzmw7yzIZG3tjTTMjCjPHZfPna6dw8ZyJTirK8LlFERCQmRTJ8lwL1Q543ABee6EBjzCSgEnghvGk60GqM+V14+1+Au6099momxpjlwHKAioqKES1eIsTfA1v+CGsfhbpVbrLkjBthwUdg6pUjs/SfRERrdx/PbT7I0xsaWbW7mWDIMqUok89dVcXNcyYyfXy21yWKiIjEvFiZcLkMeGJIuE4C3oFrQ6kDfg3cCTw09CRr7f3A/QCLFi2y0SpWzsLBja6Xe8OvwdfmViO55jsw74OQVex1dXIS7T4/f958iKc3HODVnUcIhCyTCjL41OVTuOmCEmZNzMao/15ERGTYIhm+9+MmS/YrC287kWXAZ4c8bwDWD2lZ+QOwlOPCt8S43g7Y9Fs3yn1gnZs8ed673Cj35Es1aTJGdfYG+OvWQzz1ViMv7zhMXzBEaV46H7+0kpvnlDC7NEeBW0RE5CxFMnyvAaqMMZW40L0M+ODxBxljZgL5wOrjzs0zxhRZaw8DVwHVEaxVRoq17mI36x6BTb9zF70pmgXX3wNzboOMcV5XKCfQ3RfghW1NPP1WI3/b3kRvIMSEnDT+/qJJ3DRnIvPL8xS4RURERkDEwre1NmCM+RzwHG6pwYettZuNMd8Fqq21T4YPXQassNbaIecGjTFfAf5q3L/4a4EHIlWrjICeVtjwGzeB8tAmSM6A898LCz/ilgdUcIs5Pn+QF7c38fSGRv66tYkef5Ci7FSWLS7n5rklLKzIJyFBfzcREZGRZIZk3lFt0aJFtrpag+NRZS3UrXZtJVv+4C6CM3Guayu54FZdzj2K+gIhOnsDdPoCdPT66fQF3PPeAO2+QPi5P7w/QGu3nzdqm+nqCzIuM4UbZk/g5jklLKkcR6ICt4iIyCkZY7qttZlnc26sTLiU0aTriFsecN3P4MgOSMl2EycXfMRdEEfOSHdfgKb2Xjp7A3QMhObBoNwxEJ779/sHgnZ/uO4LnH4Z/MQEQ1ZqElmpSWSnJXHznBJunjuRi6YUkJSYEIV3KiIiIgrfMjyhEOx5ybWVbH0aQn4ovxBu+TGc/25IOasPf2NWMGR5bdcRHl/bwHObD54yPCclGLLTkshKSyIrNZns1CSKs9OYWuSCdFZaEtmp/Y+TB8J1dtrQ/cmkJSeob1tERMRjCt9yrGAAWvdB8y43qn1kZ/i23V0IJz0flnwCFnwYimd5Xe2os6+5iyfWNvDbtQ0caPORm57MssXlzCvPOyYoZ6UNjlCnJik0i4iIxAuF77HK1wZHwgG7eedg0D5aC8G+weMyCqBwursQzpQrYObNkJzmVdWjUldvgJUbG3l8bQNv7jlKgoF3VBXxjZtmcc2s8aQl68JCIiIiY4XCdzwLBaG17u2j2M07ofPQ4HEJSZBfCYVVMP06F7YLqtxzLQ14Vqy1rNnbwuPV9TyzsZHuviCVhZl89boZvG9BGRNy9QFGRERkLFL4jge9HccG6/6g3bwbgr2Dx6XlQdEMmHatC9aFVS5o50+GxGTPyo8njW09/HZtA0+sbWBvczeZKYm8c04Jty4qY+GkfLWPiIiIjHEK36PRjufc7cgON6rd0Ti4zyQMjmJPuzo8gj09PIpdoPW2I8DnD/L8lkP8prqeV3cdwVpYOmUcn7+qihsumEBGiv4zExEREUepYLTZthJWfBBSs12onnIlFE4bbBUZVwlJqV5XGfestWzc38bj1Q38cf1+2n0BSvPS+fxVVbx/QRkVBRlelygiIiIxSOF7NDm0GX73CbeW9p0rIUUBL9qOdPbyh5r9PF7dwPZDHaQmJXD97Al8YFE5F00p0BUhRURE5JQUvkeLriPwq2WQkgXLfqXgHUX+YIi/bWvi8bUN/G1bE4GQZV55Ht9/z2xunlNCbrr65UVERGR4FL5Hg0Av/PoO6GqCj66EnBKvKxoTth/s4PHqev6wfj9HOvsoyk7l45dW8v6FZVSNz/a6PBERERmFFL5jnbXw9JehbjW87yEoXeh1RXGtrcfPk28d4Inqet5qaCM50XD1zPHcuqiMy6cX6TLsIiIick4UvmPd6nth/S/gsq/BBe/3upq41dLVx/2v1PLoqr109wWZOSGbb958HrfMK6EgSxNYRUREZGQofMeyHX+G5/8FZr0Lrvi619XEpdbuPh54pZZHXttLtz/IzXNK+ORlUzi/JEdrcouIiMiIU/iOVU3b4ImPwfjZ8J77IEHtDiOprdvPg6/W8tPX9tLVF+DGCyZy19VVTFcvt4iIiESQwncs6mqGx26D5HS4/TFIyfS6orjR1uPnoVf38NNX99DRG+CmCybyhaurmDFBoVtEREQiT+E71gT64DcfhvZGuPMZyC3zuqK40O7z8/Cre3jo1T10+ALcMHsCd11TxcwJOV6XJiIiImOIwncssRae/SrsexXecz+UL/a6olGvw+fnp6/t5cFXamn3Bbju/PHcdfV0zitR6BYREZHoU/iOJW/eD2sfgUu/BHNv87qaUa3D5+eR1/by4Kt7aOvxc+1547nr6ipml+Z6XZqIiIiMYQrfsWLXX+FPd8OMm+Cqb3pdzajV2Rvg0VV7eeCVWlq7/Vwzq5gvXjNdoVtERERigsJ3LDi8Ax7/KBSfB++9XyubnIWu3gCPrt7LAy/X0tLt56qZxXzxmirmlOV5XZqIiIjIAIVvr3UfdSubJCa7lU1Ss7yuaFTp7gvws9X7uP/lWo529XHFjCK+eM105pUrdIuIiEjsUfj2UtAPj98JrfVw59OQV+F1RaNGd1+AX7y+j/99qZbmrj4un17EXddUsaAi3+vSRERERE5K4dtLf/o67HkJbvkxVCz1uppRoacvyC/f2Md9L+3mSGcf76gq5IvXTGfhJIVuERERiX0K315Z8yCseQAu/jzM/5DX1cQ8nz/IL9+o4ycv7uZIZy+XTivki9dUsWjyOK9LExERERk2hW8v1L4EK78GVdfBNd/xupqY5vMH+dUbdfzkpd0c7ujl4qkF/PhDC1hSqdAtIiIio4/Cd7Q173ZXsCycDu97EBISva4oJvn8QVa8WcePX9xNU0cvS6eM439un8+FUwq8Lk1ERETkrCl8R1NPK/zqNjAJbmWTNF1l8Xg+f5DH3qzjvpd2c6i9lyWV4/jvZfO5aKpCt4iIiIx+Ct/REgzAEx+Dlj3w4SdhXKXXFcWU/p7u+8LtJRdWjuO/blPoFhERkfii8B0tf/5n2P1XeOcPYfIlXlcTMwZXL6nlSGcvF00p4Ee3z2ep2ktEREQkDil8R8PaR+CNn8DSz8DCj3hdTUzo7gvwy9fr+N+X3ZKBF08t4N4PqqdbRERE4pvCd6TtfRWe+UeYejVc+69eV+O57r4APw9fkbK5q49LpxVy1zVVLNaSgSIiIjIGKHxH0tE98Ou/h3FT4NafQuLY/XV39brLwD/wirsMvLs4ThULJyl0i4iIyNgxdtNgpPna4bFlYENw+wpIy/W6Ik909gb42eq9PPByLS3dfi6bXsRdV1fpipQiIiIyJil8R0IoCL/9ODTvgjt+BwVTva4o6jp8/oGR7tZuP1fMKOILV1exoEKhW0RERMYuhe9I+Mu3YOef4ab/hCmXe11NVHX4/Dzy2l4efHUPbT1+rppZzBeurmJeeZ7XpYmIiIh4TuF7pNX8Elb9CBZ/AhZ/3OtqoqY9HLofCofuq8Ohe65Ct4iIiMgAhe+RVPc6PHUXTLkCrr/H62qioq3Hz09f28PDr+6h3RfgmlnjuevqKi4oG5s97iIiIiKnovA9Ulr2wYoPQV4F3PpI3K9s0tbt56HX9vDT1/bQ4Qvwd+eN5wtXVzG7VKFbRERE5GTiOyFGS28nPHY7hPzwwV9DevxOKmzt7uPhV/fw09f20tEb4LrzXeg+v0ShW0REROR0FL7PVSgEv1sOh7fBHU9AYZXXFUVEa3cfD76yh0dW7aWzN8ANsyfw+auqOK8kx+vSREREREYNhe9z9cJ3YfszcMN/wNSrvK4mIqr3HuWTP19Lc1cfN17gQvesiQrdIiIiImdK4ftc+H2w+2+w8KOwZLnX1UTE49X1fOP3GynNS+dnH1+i9hIRERGRc+BJ+DbGXA/8N5AIPGitvee4/RXAo0Be+Ji7rbUro17o6SSnwUefhcRkMMbrakZUMGS559mtPPDKHi6ZVsC9H1xAXkaK12WJiIiIjGpRD9/GmETgXuBaoAFYY4x50lq7Zchh/wz8xlr7E2PMecBKYHK0ax2WlAyvKxhxHT4/d61YzwvbmvjwRZP4l5vPIzkxweuyREREREY9L0a+lwC7rLW1AMaYFcAtwNDwbYH+puJc4EBUKxzD6pq7+fija6g90sW/vns2f790ktcliYiIiMSNiA5nGmOuN8ZsN8bsMsbcHd5cCtSH9/8A+BTwZWPMDmNMa/iYbwN3GGP2A6/1H3+C119ujKk2xlRH8n2MFa/XNnPLva/S1NHLzz+2RMFbREREZISdduTbGDMe+DegxFp7Q7gN5CJr7UOnOe+E7SVDj7HWfskYsw64ENgOzA/vuh14BKgAZgHXGmMSrLWh486/H7gfYNGiRfZ070VO7rE36/iXP2xiUkEGD31kMZMLM70uSURERCTuDGfk+xHgOaAk/HwH8MVhnDfQXmKt7QP620v2A+VDjisLb7sdeCy87ePAVmA88CvcpMvCYfxMOUOBYIjvPLWZr/9uIxdPK+R3n7lEwVtEREQkQoYTvguttb8BQgDW2gAQHMZ5A+0lYQ3hbWuAKmNMpTEmBVgW3lYJvBA+tg74T+ArwETcCP3hYfxMOQNtPX4+9mg1P31tLx+7pJKHP7KI3PRkr8sSERERiVvDmXDZZYwpwE2CxBizFGg72x9orQ0YYz6HG01PBB4GFuKC+k3Ak8BaXLvJM0A+8Bdr7dvaSowxy4HlABUVFWdb0pi050gXH390DXXN3dzz3gtYtkS/PxEREZFIG074/jIuEE81xrwGFAHvH8Z5J2svIbxm98C63caYGuCz1tpV4U2VuFH5XCADuNIYc4+19u4hr6ee77O0atcRPv3LdSQY+MU/XMjSKQVelyQiIiIyJpwyfIcnTV4evs0ADLDdWusfxmsPtJfgQvcy4IMn+BkzcaPbq/u3WWs/NGT/ncCi44O3nJ2fv76Pbz+5malFmTz44cVUFMTfOuUiIiIiseqU4dtaGzTG3G6t/QGw+Uxe+ETtJdbazcaY7wLV1tr+lU+WAStO1FYiI8cfDPHdp7bw89f3cdXMYv572Tyy09TfLSIiIhJN5nSZN7wWdzLwa6Crf7u1dl1kSzszixYtstXVWu77RNq6/XzmV2t5bVczyy+bwj9dP5PEBON1WSIiIiKjkjGm21p7VsvDDafne174/rtDtlngqrP5gRJdu5o6+cTPqtnf0sP/ef8cbl1UfvqTRERERCQiThu+rbVXRqMQGXkv7zjMZ3+1jpTEBH71iQtZNHmc1yWJiIiIjGmnXefbGJNrjPnP/su4G2P+P2NMbjSKk7NjreWnr+3hzp++SWleOn/83CUK3iIiIiIxYDgX2XkY6AA+EL61Az+NZFFy9voCIb7x+01856ktXD1rPL/99MWU5WtFExEREZFYMJye76nW2vcNef4dY8z6SBUkZ6+lq49P/WItb+w5ymeumMpX/m4GCZpYKSIiIhIzhhO+e4wxl1prXwUwxlwC9ES2LDlTOw918PFHqznY7uO/bpvHu+eXel2SiIiIiBxnOOH708CjQ/q8W4A7I1aRnLG/bWvi84/VkJacyIrlS1lQke91SSIiIiJyAsNZ7WQ9MNcYkxN+3h7xqmRYrLU89Ooe/m3lVmZNzOGBDy+iJC/d67JERERE5CSGs9rJvxlj8qy17dbadmNMvjHme9EoTk6uNxDka09s4HvPbOW68yfw+KcuUvAWERERiXHDWe3kBmtta/8Ta20LcGPkSpLTae7s5Y4H3+DxtQ184eoq7v3gAjJShtNBJCIiIiJeGk5iSzTGpFprewGMMelAamTLklP52KPVbGts50e3z+edc0u8LkdEREREhmk44fuXwF+NMf1re38UeDRyJcmptHT18VZ9K1+9boaCt4iIiMgoM5wJl/9ujHkLuCa86V+ttc9Ftiw5mfUNrgNIK5qIiIiIjD6nDd/GmEzgz9baPxljZgAzjDHJ1lp/5MuT49XUtZJgYE5Z7ukPFhEREZGYMpwJly8DacaYUuBPwN8Dj0SyKDm5mroWZkzIITNVEyxFRERERpvhhG9jre0G3gv8xFp7K3B+ZMuSEwmFLOvrW5lfked1KSIiIiJyFoYVvo0xFwEfAp4Jb0uMXElyMrVHOunwBZhfrvAtIiIiMhoNJ3zfBXwd+L21drMxZgrwt8iWJSeyrs5NtpyvyZYiIiIio9JwVjt5Gdf3jTFmgrW2FvhCpAuTt6upayUnLYkphZlelyIiIiIiZ2E4I99DrYxIFTIs6+tbmVueR0KC8boUERERETkLZxq+lfo80tUbYPvBdrWciIiIiIxiZxq+H4hIFXJaGxraCFm00omIiIjIKHZG4dta+2MAY0xWZMqRk6mpbwFgXpnCt4iIiMhodaYj3/22jGgVclo1da1MKcwkPzPF61JERERE5CyddLUTY8yXT7YL0Mh3FFlrqalr5bLphV6XIiIiIiLn4FQj3/8G5APZx92yTnOejLCGlh6OdPZqsqWIiIjIKHeqdb7XAX+w1q49focx5h8iV5Icr6Y+fHEdXdlSREREZFQ71Qj2fmCfMeauE+xbFKF65ATW17WSlpzAzAnZXpciIiIiIufgVOH7PCAF+JgxJt8YM67/BvijU56AW+lkTmkeSYnq9hEREREZzU7VdvK/wF+BKcBajr3Ajg1vlwjrDQTZvL+dj14y2etSREREROQcnXQo1Vr7Q2vtLOBha+0Ua23lkJuCd5RsOdBOXzCki+uIiIiIxIHT9jFYaz8djULkxGrqwpMttdKJiIiIyKinJuIYV1PfSkluGuNz0rwuRURERETOkcJ3jKupa9Got4iIiEicUPiOYU0dPhpaetTvLSIiIhInFL5j2Ppwv/c8XVxHREREJC4ofMew9fWtJCUYZpfmel2KiIiIiIwAhe8YVlPXynklOaQlJ3pdioiIiIiMAIXvGBUMWd5qaGW+Wk5ERERE4obCd4zacaiD7r6gVjoRERERiSMK3zFq8OI6GvkWERERiRcK3zGqpq6FcZkpVIzL8LoUERERERkhCt8xqqbe9XsbY7wuRURERERGiMJ3DGrr8bOrqVPre4uIiIjEGYXvGLShob/fW5MtRUREROKJwncMqqlrxRiYU66L64iIiIjEk4iGb2PM9caY7caYXcaYu0+w/wfGmPXh2w5jTGt4+zxjzGpjzGZjzAZjzG2RrDPW1NS1UFWcRU5asteliIiIiMgISorUCxtjEoF7gWuBBmCNMeZJa+2W/mOstV8acvzngfnhp93Ah621O40xJcBaY8xz1trWSNUbK6y11NS3ct15E7wuRURERERGWCRHvpcAu6y1tdbaPmAFcMspjr8deAzAWrvDWrsz/PgA0AQURbDWmLG3uZvWbr/W9xYRERGJQ5EM36VA/ZDnDeFtb2OMmQRUAi+cYN8SIAXYHYEaY05NXQugyZYiIiIi8ShWJlwuA56w1gaHbjTGTAR+DnzUWhs6/iRjzHJjTLUxpvrw4cNRKjWyaupayUpNYlpxlteliIiIiMgIi2T43g+UD3leFt52IssIt5z0M8bkAM8A/4+19vUTnWStvd9au8hau6ioKD66UmrqW5hTlktigi6uIyIiIhJvIhm+1wBVxphKY0wKLmA/efxBxpiZQD6wesi2FOD3wM+stU9EsMaY0tMXZGtjh/q9RUREROJUxMK3tTYAfA54DtgK/MZau9kY811jzLuGHLoMWGGttUO2fQC4DLhzyFKE8yJVa6zYdKCNYMgyv1z93iIiIiLxKGJLDQJYa1cCK4/b9s3jnn/7BOf9AvhFJGuLRf2TLedp5FtEREQkLsXKhEvBTbasGJdBYVaq16WIiIiISAQofMeQmrpW9XuLiIiIxDGF7xjR2NbDwXYf88sVvkVERETilcJ3jKipawV0cR0RERGReKbwHSNq6lpISUpg1sQcr0sRERERkQhR+I4RNXWtzC7JISVJfxIRERGReKWkFwP8wRAb97ep5UREREQkzil8x4BtjR30BkJa6UREREQkzil8x4CaendxHY18i4iIiMQ3he8YUFPXSnF2KiW5aV6XIiIiIiIRpPAdA2rqWphfkYcxxutSRERERCSCFL49drSrj73N3Wo5ERERERkDFL49tj7c7z1PV7YUERERiXsK3x6rqWslwcCcslyvSxERERGRCFP49lhNXSszJ+SQkZLkdSkiIiIiEmEK3x4KhSxv1bdqfW8RERGRMULh20O7D3fS0RvQZEsRERGRMULh20M1da0AGvkWERERGSMUvj1UU99CbnoylQWZXpciIiIiIlGg8O2hmrpW5pXnkZCgi+uIiIiIjAUK3x7p7A2w/VCH1vcWERERGUMUvj2yob4Va9XvLSIiIjKWKHx7pKbeTbbUyLeIiIjI2KHw7ZGaulamFGWSl5HidSkiIiIiEiUK3x6w1rK+voX55VrfW0RERGQsUfj2QENLD0c6+9TvLSIiIjLGKHx7YF1dC6DJliIiIiJjjcK3B2rqWklPTmTG+GyvSxERERGRKFL49kBNfSsXlOWSlKhfv4iIiMhYovQXZT5/kC0H2tRyIiIiIjIGKXxH2eYD7fiDViudiIiIiIxBCt9RVqPJliIiIiJjlsJ3lK2vb6U0L53xOWlelyIiIiIiUabwHWU1da3M06i3iIiIyJik8B1FTe0+9rf2ML9c4VtERERkLFL4jqKa+lYA5ldosqWIiIjIWKTwHUU1da0kJxrOL8nxuhQRERER8YDCdxTV1LVw3sQc0pITvS5FRERERDyg8B0lgWCIDQ1tajkRERERGcMUvqNk+6EOevxBre8tIiIiMoYpfEfJ+v7JlrqypYiIiMiYpfAdJTV1rRRkplA+Lt3rUkRERETEIwrfUVJT18L8ijyMMV6XIiIiIiIeUfiOgrZuP7sPd2mypYiIiMgYp/AdBesbXL/3PF3ZUkRERGRMU/iOgpq6FoyBOWW5XpciIiIiIh5S+I6CmrpWphdnk52W7HUpIiIiIuKhiIZvY8z1xpjtxphdxpi7T7D/B8aY9eHbDmNM65B9HzHG7AzfPhLJOiMpFLKsr2/V+t4iIiIiQlKkXtgYkwjcC1wLNABrjDFPWmu39B9jrf3SkOM/D8wPPx4HfAtYBFhgbfjclkjVGyl7mrto6/ErfIuIiIhIREe+lwC7rLW11to+YAVwyymOvx14LPz4OuB5a+3RcOB+Hrg+grVGzPq68MV1tNKJiIiIyJgXyfBdCtQPed4Q3vY2xphJQCXwwpmca4xZboypNsZUHz58eESKHmk19S1kpyYxrSjL61JERERExGOxMuFyGfCEtTZ4JidZa++31i6y1i4qKiqKUGnnpqaulbnleSQk6OI6IiIiImNdJMP3fqB8yPOy8LYTWcZgy8mZnhuzuvsCbDvYofW9RURERASIbPheA1QZYyqNMSm4gP3k8QcZY2YC+cDqIZufA/7OGJNvjMkH/i68bVTZ2NBGMGQ12VJEREREgAiudmKtDRhjPocLzYnAw9bazcaY7wLV1tr+IL4MWGGttUPOPWqM+VdcgAf4rrX2aKRqjZSael3ZUkREREQGRSx8A1hrVwIrj9v2zeOef/sk5z4MPByx4qKgpq6FSQUZFGSlel2KiIiIiMSAWJlwGXestdTUtTJfo94iIiIiEqbwHSGNbT6aOnq1vreIiIiIDFD4jpCagYvraORbRERERByF7wipqWshNSmBmRNyvC5FRERERGKEwneE1NS3Mrs0l5Qk/YpFRERExFEyjIC+QIiN+9s02VJEREREjqHwHQFbG9vpC4Q02VJEREREjuFJ+DbGXG+M2W6M2WWMufskx3zAGLPFGLPZGPOraNd4LmrqWgBNthQRERGRY0X0IjsnYoxJBO4FrgUagDXGmCettVuGHFMFfB24xFrbYowpjnad56KmvpXxOalMzE3zuhQRERERiSFejHwvAXZZa2v/f/buPDzq8t7///POTnYyISxJJCFBdggQQIIruKC1KEVBq1YOVrRqa23tObY9357uWq3nWH/aui91xV1R3JVqCQhBArJD2JKwhYSELGS/f3/MEAMChpCZz8zk9biuuTKfbeadMMYXN+/PfVtrG4EXgUuOOOd64EFr7X4Aa+1eH9d4UtyL6/TEGON0KSIiIiLiR5wI36lAcbvtEs++9k4FTjXGLDLGLDHGTD3aCxlj5hpjCowxBV6q9YSV1zSwo6JOLSciIiIi8g3+soIveAAAIABJREFUesNlGDAQOBu4EnjUGPONNGutfcRam2utzfVxfcdUWHxocR3dbCkiIiIih3MifJcC6e220zz72isB3rLWNllrtwIbcYdxv7diRyWhIYYRqQlOlyIiIiIifsaJ8L0MGGiMyTTGRABXAG8dcc4buEe9McYk425D2eLLIjtrRfF+BveJo0dEqNOliIiIiIif8Xn4ttY2A7cA7wPrgJestWuMMb83xkzznPY+UG6MWQt8CvzCWlvu61pPVEurZWVxlfq9RUREROSofD7VIIC1dgGw4Ih9v2n33AI/8zwCxua9NdQ0NDM6Xf3eIiIiIvJN/nrDZUDS4joiIiIicjwK311oxY5KEnqEk5kc43QpIiIiIuKHFL67UGFxJaNPSdTiOiIiIiJyVArfXaS6vomNe6vV7y0iIiIix6Tw3UVWlVRhrfq9RUREROTYFL67yKGbLUelK3yLiIiIyNEpfHeRFTsqyeoVQ0KPcKdLERERERE/pfDdBay1rCiuZPQp6vcWERERkWNT+O4COyrqqKhtVL+3iIiIiByXwncXWLGjEkAznYiIiIjIcSl8d4EVO/YTHRHKqb1jnS5FRERERPyYwncXKCyuZGRaAmGh+nGKiIiIyLEpLZ6k+qYW1uw8oJstRURERORbKXyfpDU7q2huteRofm8RERER+RYK3yfp65stFb5FRERE5PgUvk/Sih2VpCb2ICU+yulSRERERMTPKXyfpBU79mt+bxERERHpkDCnCwhk1loe+UEuoSHG6VJEREREJAAofJ8EYwzDUxOcLkNEREREAoTaTkREREREfEThW0RERETERxS+RURERER8ROFbRERERMRHFL5FRERERHxE4VtERERExEcUvkVEREREfEThW0RERETERxS+RURERER8ROFbRERERMRHFL5FRERERHwkzOkCukphYSG5ubmOvHdZWRm9evVy5L0lOOgzJCdLnyE5WfoMycnqZp+hiM5eGDThOycnh4KCAkfeOzc317H3luCgz5CcLH2G5GTpMyQnqzt9howxKzt7rdpORERERER8ROFbRERERMRHFL67wNy5c50uQQKcPkNysvQZkpOlz5CcLH2GOsZYa52uoUvk5uba7tJnJCIiIiLOMcYst9Z2aqYPjXyLiIiIiPiIwreIiIiIiI8ofJ+E9957j0GDBpGdnc1dd93ldDkSgDIyMhgxYgQ5OTmOzVMvgWfOnDmkpKQwfPjwtn0VFRWcd955DBw4kPPOO4/9+/c7WKH4s6N9fn7729+SmppKTk4OOTk5LFiwwMEKxd8VFxdzzjnnMHToUIYNG8bf/vY3QL+HOkrhu5NaWlq4+eabeffdd1m7di0vvPACa9eudbosCUCffvophYWF3WZuVDl5s2fP5r333jts31133cWUKVPYtGkTU6ZM0YCAHNPRPj8At912G4WFhRQWFnLRRRc5UJkEirCwMO69917Wrl3LkiVLePDBB1m7dq1+D3WQwncnLV26lOzsbAYMGEBERARXXHEFb775ptNliUg3cOaZZ5KUlHTYvjfffJNrr70WgGuvvZY33njDidIkABzt8yNyIvr27cuYMWMAiIuLY8iQIZSWlur3UAcpfHdSaWkp6enpbdtpaWmUlpY6WJEEImMM559/PmPHjuWRRx5xuhwJYHv27KFv374A9OnThz179jhckQSaBx54gJEjRzJnzhy1C0iHbdu2jRUrVjBhwgT9HuoghW8RB/373//myy+/5N133+XBBx/ks88+c7okCQLGGIwxTpchAeRHP/oRRUVFFBYW0rdvX37+8587XZIEgJqaGmbMmMF9991HfHz8Ycf0e+jYFL47KTU1leLi4rbtkpISUlNTHaxIAtGhz0xKSgrTp09n6dKlDlckgap3797s2rULgF27dpGSkuJwRRJIevfuTWhoKCEhIVx//fX6XSTfqqmpiRkzZnDVVVfxve99D9DvoY5S+O6kcePGsWnTJrZu3UpjYyMvvvgi06ZNc7osCSC1tbVUV1e3Pf/ggw8Om31A5ERMmzaNp59+GoCnn36aSy65xOGKJJAcCkwAr7/+un4XyXFZa7nuuusYMmQIP/vZz9r26/dQx4Q5XUCgCgsL44EHHuCCCy6gpaWFOXPmMGzYMKfLkgCyZ88epk+fDkBzczPf//73mTp1qsNVSSC48sorWbhwIfv27SMtLY3f/e533HHHHcycOZPHH3+c/v3789JLLzldpvipo31+Fi5cSGFhIcYYMjIyePjhh50uU/zYokWLeOaZZ9qmygX485//rN9DHaTl5UVEREREToCWlxcRERERCQAK3yIiIiIiPqLwLSIiIiLiIwrfIiIiIiI+ovAtIiIiIuIjCt8iInJcCxcu5OKLL3a6DBGRoKDwLSIiIiLiIwrfIiJB4tlnn2X8+PHk5ORwww030NLSQmxsLLfddhvDhg1jypQplJWVAVBYWMhpp53GyJEjmT59Ovv37wdg8+bNnHvuuYwaNYoxY8ZQVFQEQE1NDZdddhmDBw/mqquuIljWiBAR8TWFbxGRILBu3TrmzZvHokWLKCwsJDQ0lOeee47a2lpyc3NZs2YNZ511Fr/73e8A+MEPfsBf/vIXVq1axYgRI9r2X3XVVdx8882sXLmS/Px8+vbtC8CKFSu47777WLt2LVu2bGHRokWOfa8iIoFMy8uLiASBjz/+mOXLlzNu3DgADh48SEpKCiEhIcyaNQuAq6++mu9973tUVVVRWVnJWWedBcC1117L5ZdfTnV1NaWlpUyfPh2AqKiottcfP348aWlpAOTk5LBt2zZOP/10X36LIiJBQeFbRCQIWGu59tprufPOOw/b/4c//OGwbWNMp14/MjKy7XloaCjNzc2deh0Rke5ObSciIkFgypQpvPLKK+zduxeAiooKtm/fTmtrK6+88goAzz//PKeffjoJCQn07NmTzz//HIBnnnmGs846i7i4ONLS0njjjTcAaGhooK6uzplvSEQkSGnkW0QkCAwdOpQ//vGPnH/++bS2thIeHs6DDz5ITEwMS5cu5Y9//CMpKSnMmzcPgKeffpobb7yRuro6BgwYwJNPPgm4g/gNN9zAb37zG8LDw3n55Zed/LZERIKOCZY71nNzc21BQYHTZYiI+JXY2FhqamqcLkNEJKgYY5Zba3M7c63aTkREREREfEThW0QkiGnUW0TEvyh8i4iIiIj4iMK3iIiIiIiPKHyLiIiIiPiIwreIiIiIiI8ofIuIiIiI+IjCt4iIiIiIjyh8i4iIiIj4iMK3iIiIiIiPKHyLiIiIiPiIwreIiIiIiI8ofIuIiIiI+IjCt4hIEJs9ezb//d//3aFzMzIy+Oijj076dURE5NgUvkVEREREfEThW0RERETERxS+RUQclpGRwT333MPIkSOJiYnhuuuuY8+ePVx44YXExcVx7rnnsn///rbz33rrLYYNG0ZiYiJnn30269atazu2YsUKxowZQ1xcHLNmzaK+vv6w93r77bfJyckhMTGRvLw8Vq1a1amaH330UbKzs0lKSmLatGns3LkTAGstt912GykpKcTHxzNixAhWr14NwIIFCxg6dChxcXGkpqby17/+tVPvLSISyBS+RUT8wKuvvsqHH37Ixo0bmT9/PhdeeCF//vOfKSsro7W1lfvvvx+AjRs3cuWVV3LfffdRVlbGRRddxHe/+10aGxtpbGzk0ksv5ZprrqGiooLLL7+cV199te09VqxYwZw5c3j44YcpLy/nhhtuYNq0aTQ0NJxQrZ988gm//OUveemll9i1axf9+/fniiuuAOCDDz7gs88+Y+PGjVRVVfHSSy/hcrkAuO6663j44Yeprq5m9erVTJ48uYt+eiIigUPhW0TED/z4xz+md+/epKamcsYZZzBhwgRGjx5NVFQU06dPZ8WKFQDMmzeP73znO5x33nmEh4dz++23c/DgQfLz81myZAlNTU389Kc/JTw8nMsuu4xx48a1vccjjzzCDTfcwIQJEwgNDeXaa68lMjKSJUuWnFCtzz33HHPmzGHMmDFERkZy5513snjxYrZt20Z4eDjV1dWsX78eay1Dhgyhb9++AISHh7N27VoOHDhAz549GTNmTNf9AEVEAoTCt4iIH+jdu3fb8x49enxju6amBoCdO3fSv3//tmMhISGkp6dTWlrKzp07SU1NxRjTdrz9udu3b+fee+8lMTGx7VFcXNzWMtJRR9YQGxuLy+WitLSUyZMnc8stt3DzzTeTkpLC3LlzOXDgAOAe3V+wYAH9+/fnrLPOYvHixSf0viIiwUDhW0QkgPTr14/t27e3bVtrKS4uJjU1lb59+1JaWoq1tu34jh072p6np6fz61//msrKyrZHXV0dV1555UnVUFtbS3l5OampqQD85Cc/Yfny5axdu5aNGzdyzz33ADBu3DjefPNN9u7dy6WXXsrMmTM79TMQEQlkCt8iIgFk5syZvPPOO3z88cc0NTVx7733EhkZSV5eHhMnTiQsLIz777+fpqYmXnvtNZYuXdp27fXXX89DDz3EF198gbWW2tpa3nnnHaqrq0+ohiuvvJInn3ySwsJCGhoa+NWvfsWECRPIyMhg2bJlfPHFFzQ1NRETE0NUVBQhISE0Njby3HPPUVVVRXh4OPHx8YSE6H9BItL96DefiEgAGTRoEM8++yw//vGPSU5OZv78+cyfP5+IiAgiIiJ47bXXeOqpp0hKSmLevHl873vfa7s2NzeXRx99lFtuuYWePXuSnZ3NU089dcI1nHvuufzhD39gxowZ9O3bl6KiIl588UUADhw4wPXXX0/Pnj3p378/LpeLX/ziFwA888wzZGRkEB8fz0MPPcRzzz3XJT8TEZFAYtr/82Qgy83NtQUFBU6XISIiIiJBzhiz3Fqb25lrNfItIiIiIuIjCt8iIiIiIj6i8C0iIiIi4iMK3yIiIiIiPhLmdAFdZdu2beTmdqrvXURERETkRIzq7IVBE74zMjLQbCciIiIi4m3GmMbOXqu2ExERERERH1H4FhERERHxEYVvEREREREfCZqebxERERHxjaamJkpKSqivr3e6FK+KiooiLS2N8PDwLntNhW8REREROSElJSXExcWRkZGBMcbpcrzCWkt5eTklJSVkZmZ22euq7URERERETkh9fT0ulytogzeAMQaXy9Xlo/sK3yIiIiJywoI5eB/ije9R4VtERERExEcUvkVEREQkoFRWVvL3v//9hK+76KKLqKys9EJFHafwLSIiIiIB5Vjhu7m5+bjXLViwgMTERG+V1SGa7eQkWWs52NRCdIR+lCIiIiK+cMcdd1BUVEROTg7h4eFERUXRs2dP1q9fz8aNG7n00kspLi6mvr6eW2+9lblz5wKQkZFBQUEBNTU1XHjhhZx++unk5+eTmprKm2++SY8ePbxeuxLjSbDWMuXefzExy8Wfpo9wuhwRERERn/vd/DWs3XmgS19zaL94/ue7w455/K677mL16tUUFhaycOFCvvOd77B69eq2KQGfeOIJkpKSOHjwIOPGjWPGjBm4XK7DXmPTpk288MILPProo8ycOZNXX32Vq6++uku/j6NR+D4Jxhgyk2PILyp3uhQRERGRbmv8+PGHzcV9//338/rrrwNQXFzMpk2bvhG+MzMzycnJAWDs2LFs27bNJ7UqfJ+kvOxkPl6/l52VB+mX6P1/qhARERHxJ8cbofaVmJiYtucLFy7ko48+YvHixURHR3P22Wcfda7uyMjItuehoaEcPHjQJ7XqhsuTlJfl/luURr9FREREfCMuLo7q6uqjHquqqqJnz55ER0ezfv16lixZ4uPqjk8j3ydpUO84XDER5G/ex2Vj05wuR0RERCTouVwuJk2axPDhw+nRowe9e/duOzZ16lQeeughhgwZwqBBgzjttNMcrPSbFL5PUkiIYWKWi0VF+7DWdovVnkRERESc9vzzzx91f2RkJO++++5Rjx3q605OTmb16tVt+2+//fYur+9Y1HbSBfKyktlzoIEt+2qdLkVERERE/JjCdxeYlO3p+968z+FKRERERMSfKXx3gVOSoklN7MGizbrpUkRERESOzavh2xgz1RizwRiz2Rhzx1GOzzbGlBljCj2PH7Y7drcxZo0xZp0x5n7jx83UxhgmZbtYvKWc1lbrdDkiIiIi4qe8Fr6NMaHAg8CFwFDgSmPM0KOcOs9am+N5POa5Ng+YBIwEhgPjgLO8VWtXyMtKpupgE2t3de0KTyIiIiISPLw58j0e2Gyt3WKtbQReBC7p4LUWiAIigEggHNjjlSq7yKH5vhep71tEREREjsGb4TsVKG63XeLZd6QZxphVxphXjDHpANbaxcCnwC7P431r7Tov1nrSUuKjGJgSyyIttiMiIiLiU7/97W/561//6nQZHeL0DZfzgQxr7UjgQ+BpAGNMNjAESMMd2CcbY8448mJjzFxjTIExpqCsrMyHZR/dpOxklm2toLG51elSRERERMQPeTN8lwLp7bbTPPvaWGvLrbUNns3HgLGe59OBJdbaGmttDfAuMPHIN7DWPmKtzbXW5vbq1avLv4ETNTHLxcGmFgqLK50uRURERCSo/elPf+LUU0/l9NNPZ8OGDQAUFRUxdepUxo4dyxlnnMH69eupqqqif//+tLa6B0dra2tJT0+nqanJkbq9ucLlMmCgMSYTd+i+Avh++xOMMX2ttbs8m9OAQ60lO4DrjTF3Agb3zZb3ebHWLnHaABchxt33PT4zyelyRERERLzv3Ttg91dd+5p9RsCFdx3z8PLly3nxxRcpLCykubmZMWPGMHbsWObOnctDDz3EwIED+eKLL7jpppv45JNPyMnJ4V//+hfnnHMOb7/9NhdccAHh4eFdW3MHeS18W2ubjTG3AO8DocAT1to1xpjfAwXW2reAnxhjpgHNQAUw23P5K8Bk4CvcN1++Z62d761au0pCj3BGpCaQX7SP28471elyRERERILS559/zvTp04mOjgZg2rRp1NfXk5+fz+WXX952XkODu8Fi1qxZzJs3j3POOYcXX3yRm266yZG6wbsj31hrFwALjtj3m3bPfwn88ijXtQA3eLM2b5mYlcxjn2+hrrGZ6Aiv/nhFREREnHecEWpfam1tJTExkcLCwm8cmzZtGr/61a+oqKhg+fLlTJ482YEK3Zy+4TLoTMp20dxqWbq1wulSRERERILSmWeeyRtvvMHBgweprq5m/vz5REdHk5mZycsvvwyAtZaVK1cCEBsby7hx47j11lu5+OKLCQ0Ndax2he8ults/iYjQEPI15aCIiIiIV4wZM4ZZs2YxatQoLrzwQsaNGwfAc889x+OPP86oUaMYNmwYb775Zts1s2bN4tlnn2XWrFlOlQ2Asdb3y6EbY6YCf8PdC/6Ytfao/15hjJmBu/97nLW24HivmZubawsKjnuKz1zxyGKq65t55yffmB1RREREJOCtW7eOIUOGOF2GTxztezXG1FlrYzrzej4f+e7osvPGmDjgVuAL31Z48vKyklm76wD7axudLkVERERE/IgTbScdXXb+D8BfgHpfFtcVJmW7sBaWbFHriYiIiIh8zYnw/a3LzhtjxgDp1tp3jvdC7Ve47PoyO29kWiIxEaEsKtrndCkiIiIiXuFE67KveeN79LsbLo0xIcD/Aj//tnPbr3Dp/co6Ljw0hAkDXLrpUkRERIJSVFQU5eXlQR3ArbWUl5cTFRXVpa/rxETU37bsfBwwHFhojAHoA7xljJn2bTdd+pO8LBefrN/L7qp6+iR07R+aiIiIiJPS0tIoKSmhrKzM6VK8KioqirS0tC59TSfC93GXnbfWVgHJh7aNMQuB2wMpeIP7pktwLzU/Y2zX/qGJiIiIOCk8PJzMzEynywhIPm87sdY2A4eWnV8HvHRo2XnPUvNBYXCfOJJiItT3LSIiIiJtHFn//NuWnT9i/9m+qKmrhYQYJg5wsbjI3Q/laaERERERkW7M7264DCZ52S52VdWzdV+t06WIiIiIiB9Q+PaiSYf6vjXriYiIiIig8O1V/V3RpCb2IH+z+r5FREREROHbq4wxTMxysXhLOa2twTsPpoiIiIh0jMK3l03KdlFZ18TaXQecLkVEREREHKbw7WWH5vvO15SDIiIiIt2ewreX9Y6PIjsllkWbddOliIiISHen8O0DeVkulm2roLG51elSRERERMRBCt8+kJeVTF1jCytLKp0uRUREREQcpPDtAxMHuDAGFmnKQREREZFuTeHbBxKiwxneL4F8LbYjIiIi0q0pfPtIXraLFTv2U9fY7HQpIiIiIuIQhW8fmZSVTFOLZdm2/U6XIiIiIiIOUfj2kXEZSYSHGi01LyIiItKNKXz7SI+IUEaf0lN93yIiIiLdmMK3D03KSmb1zioq6xqdLkVEREREHKDw7UOTsl1YC0u2aPRbREREpDtS+PahUemJREeEaql5ERERkW5K4duHwkNDGJ+ZRH6RbroUERER6Y4Uvn1sUlYyRWW17K6qd7oUEREREfExhW8fy8t2AWj0W0RERKQb8mr4NsZMNcZsMMZsNsbccZTjs40xZcaYQs/jh+2OnWKM+cAYs84Ys9YYk+HNWn1lSJ94ekaHa8pBERERkW4ozFsvbIwJBR4EzgNKgGXGmLestWuPOHWetfaWo7zEP4E/WWs/NMbEAq3eqtWXQkIME7Nc5G/eh7UWY4zTJYmIiIiIj3hz5Hs8sNlau8Va2wi8CFzSkQuNMUOBMGvthwDW2hprbZ33SvWtvKxkdlbVs608aL4lEREREekAb4bvVKC43XaJZ9+RZhhjVhljXjHGpHv2nQpUGmNeM8asMMbc4xlJP4wxZq4xpsAYU1BWVtb134GXTMpOBmCRlpoXERER6VacvuFyPpBhrR0JfAg87dkfBpwB3A6MAwYAs4+82Fr7iLU211qb26tXL99U3AUyXNH0TYhisfq+RURERLoVb4bvUiC93XaaZ18ba225tbbBs/kYMNbzvAQo9LSsNANvAGO8WKtPGWPIy0omv2gfra3W6XJERERExEe8Gb6XAQONMZnGmAjgCuCt9icYY/q225wGrGt3baIx5tBw9mTgyBs1A9qkbBf765pYt/uA06WIiIiIiI94LXx7RqxvAd7HHapfstauMcb83hgzzXPaT4wxa4wxK4Gf4Gktsda24G45+dgY8xVggEe9VasT8rLcfd/5WmpeREREpNsw1gZH20Nubq4tKChwuowTMvnehfRPiubJ/xjvdCkiIiIi0kHGmDprbUxnrnX6hstubVJWMku3VtDUEhRTmIuIiIjIt1D4dtCkbBe1jS2sLK50uhQRERER8QGFbwedNsCFMbBIfd8iIiIi3YLCt4MSoyMY1i+e/CIttiMiIiLSHSh8O2xSVjIrdlRysLHF6VJERERExMsUvh2Wl51MY0sry7ZVOF2KiIiIiHiZwrfDxmX0JDzUkK+l5kVERESCnsK3w6Ijwhid3lN93yIiIiLdgMK3H8jLdvFVaRVVdU1OlyIiIiIiXqTw7QcmZSdjLSzeotYTERERkWCm8O0HRqUl0iM8lMVqPREREREJagrffiAiLITxmUks0k2XIiIiIkFN4dtPTMp2sXlvDXsO1DtdioiIiIh4icK3n8jLSgbQrCciIiIiQUzh208M7RtPYnQ4+ZvVeiIiIiISrBS+/URIiGHiABf5ReVYa50uR0RERES8QOHbj+RlJ1NaeZDt5XVOlyIiIiIiXqDw7UfyslwALFLft4iIiEhQUvj2IwOSY+gTH0W+phwUERERCUoK337EGENetovFReW0tqrvW0RERCTYKHz7mUlZyVTUNrJ+d7XTpYiIiIhIF1P49jN52e6+b833LSIiIhJ8FL79TN+EHgxIjlHft4iIiEgQUvj2Q3nZLr7YUk5TS6vTpYiIiIhIF1L49kN5WcnUNrawqqTS6VJEREREpAspfPuhiQNcGIOWmhcREREJMgrffqhnTARD+8ZrsR0RERGRIKPw7acmZSfz5fZKDja2OF2KiIiIiHQRhW8/NTHLRWNLKwXbK5wuRURERES6iMK3nxqfkURYiNGUgyIiIiJBROHbT8VEhjH6lETyN6vvW0RERCRYKHz7sbysZL4qraLqYJPTpYiIiIhIF/Bq+DbGTDXGbDDGbDbG3HGU47ONMWXGmELP44dHHI83xpQYYx7wZp3+Ki/LRauFJVvUeiIiIiISDLwWvo0xocCDwIXAUOBKY8zQo5w6z1qb43k8dsSxPwCfeatGfzf6lJ70CA9lsfq+RURERIKCN0e+xwObrbVbrLWNwIvAJR292BgzFugNfOCl+vxeRFgI4zKTWKS+bxEREZGg4M3wnQoUt9su8ew70gxjzCpjzCvGmHQAY0wIcC9w+/HewBgz1xhTYIwpKCsr66q6/UpelotNe2vYe6De6VJERERE5CQ5fcPlfCDDWjsS+BB42rP/JmCBtbbkeBdbax+x1uZaa3N79erl5VKdMSkrGYDF6vsWERERCXjeDN+lQHq77TTPvjbW2nJrbYNn8zFgrOf5ROAWY8w24K/AD4wxd3mxVr81tF88CT3C1XoiIiIiEgTCvPjay4CBxphM3KH7CuD77U8wxvS11u7ybE4D1gFYa69qd85sINda+43ZUrqD0BDDxAEuFm0ux1qLMcbpkkRERESkk7w28m2tbQZuAd7HHapfstauMcb83hgzzXPaT4wxa4wxK4GfALO9VU8gy8t2UVp5kB0VdU6XIiIiIiInwZsj31hrFwALjtj3m3bPfwn88lte4yngKS+UFzDyPH3f+UXl9HfFOFyNiIiIiHSW0zdcSgdk9Yqhd3yk+r5FREREApzCdwAwxjApK5nFReW0tlqnyxERERGRTlL4DhATs1yU1zayYU+106WIiIiISCcpfAeISdlf932LiIiISGBS+A4Q/RJ7kJkcQ776vkWmYPmvAAAgAElEQVREREQClsJ3AJmY5eKLrRU0t7Q6XYqIiIiIdILCdwCZlJVMTUMzK0uqnC5FRERERDpB4TuATMxyAbC4SK0nIiIiIoFI4TuAJMVEMLRvPIs266ZLERERkUCk8B1g8rJcLN+xn/qmFqdLEREREZETpPAdYCZlJ9PY3Mry7fudLkVERERETpDCd4AZn5lEWIjh7VU7sVarXYqIiIgEEoXvABMTGcbMcem8sLSY37+9VsvNi4iIiASQMKcLkBP3x0uGExkWwpOLtlFR28g9l40iIkx/jxIRERHxdwrfASgkxPCbi4fSKy6Su9/bQEVtIw9dPZaYSP1xioiIiPgzDZcGKGMMN52dzV9mjGDR5n18/9EllNc0OF2WiIiIiByHwneAmzXuFB6+Jpf1u6u5/KHFlOyvc7okERERETkGhe8gcN7Q3jxz3QT21TQw4x/5rN99wOmSREREROQoFL6DxPjMJF66cSIAMx9azLJtFQ5XJCIiIiJHUvgOIoP7xPPKjXkkx0Zy9WNf8OHaPU6XJCIiIiLtKHwHmfSkaF6+cSKD+8RxwzMFzFu2w+mSRERERMRD4TsIuWIjef7605iUncx/vfoVD366WathioiIiPgBhe8gFRMZxuPXjmPaqH7c8/4GrYYpIiIi4ge0KksQiwgL4b5ZObhiI3hy0TbKaxr56+VaDVNERETEKQrfQe7I1TD312k1TBERERGnaAi0Gzi0GubdM0ZqNUwRERERByl8dyMzx6UfthpmcYVWwxQRERHxJYXvbua8ob159odaDVNERETECQrfJ6t2HzTWOl3FCRmX4V4N0xi4/KHFLN2q1TBFREREfKFD4dsYc6sxJt64PW6M+dIYc763i/N7TfXw+Hnw2lxobXW6mhMyuE88r/4oj15xkVzz+Bd8sGa30yWJiIiIBL2OjnzPsdYeAM4HegLXAHd5rapAER4F42+A9W/Dx791upoTltYzmlduzGNwnzhufHa5VsMUERER8bKOhm/j+XoR8Iy1dk27fce+yJipxpgNxpjNxpg7jnJ8tjGmzBhT6Hn80LM/xxiz2Bizxhizyhgzq6PfkM9NuAFyr4NFf4Mv/+l0NScsKSZCq2GKiIiI+EhHw/dyY8wHuMP3+8aYOOC4fRbGmFDgQeBCYChwpTFm6FFOnWetzfE8HvPsqwN+YK0dBkwF7jPGJHawVt8yBi68G7Imw9u3wdbPna7ohB1aDfOSHPdqmL+br9UwRURERLyho+H7OuAOYJy1tg4IB/7jW64ZD2y21m6x1jYCLwKXdOTNrLUbrbWbPM93AnuBXh2s1fdCw+CyJyEpC+ZdDfs2O13RCYsIC+H/ZuYwZ1ImT+Vv49Z5hTQ2B1Yfu4iIiIi/62j4nghssNZWGmOuBv4bqPqWa1KB4nbbJZ59R5rhaS15xRiTfuRBY8x4IAIo6mCtzuiRCN+fByGh8PxMqAu8GURCQgz/7+Ih/OfUQcxfuZPrnl5GTUOz02WJiIiIBI2Ohu9/AHXGmFHAz3EH4a5ocJ4PZFhrRwIfAk+3P2iM6Qs8A/yHtfYbw7DGmLnGmAJjTEFZWVkXlHOSkjLhiuehqhhe+gE0Nzpd0QlrWw3zspHkF5VrNUwRERGRLtTR8N1s3XfhXQI8YK19EIj7lmtKgfYj2WmefW2steXW2kPJ7jFg7KFjxph44B3g19baJUd7A2vtI9baXGttbq9eftKVcsppcMmDsO1zeOc2CNCbF2fmpvPw1WPZsLuay7QapoiIiEiX6Gj4rjbG/BL3FIPvGGNCcPd9H88yYKAxJtMYEwFcAbzV/gTPyPYh04B1nv0RwOvAP621r3SwRv8xciac+Z+w4lnIv9/pajrtXM9qmOWe1TBX7NjvdEkiIiIiAa2j4XsW0IB7vu/duEex7zneBdbaZuAW4H3cofola+0aY8zvjTHTPKf9xDOd4ErgJ8Bsz/6ZwJnA7HbTEOacyDfmuLN/CcOmw4f/A+vedrqaThuXkcTLN+YRYgzT/57Pzc9/yZayGqfLEhEREQlIpqNzOhtjegPjPJtLrbV7vVZVJ+Tm5tqCggKnyzhc00F46juwdx38x7vQL7D+/tDegfomHv1sC499vpXGllZm5qZz65SB9EmIcro0EREREZ8yxtRZa2M6dW1HwrcxZibuke6FuBfXOQP4hT+1hPhl+Aao3gOPTYHWZrj+E4jv53RFJ6WsuoEHP93Mc19sJ8QYZk/K4EdnZZEYHeF0aSIiIiI+4YvwvRI479BotzGmF/CRtXZUZ97UG/w2fAPsXg1PXABJA2DOexDRqT8rv1JcUcf/fbiR1wtLiY0M48azsviPSRlER4Q5XZqIiIiIV51M+O5oz3fIEW0m5SdwrfQZDpc9AXtWw2tzoTXwF69JT4rmf2fl8O6tZzAhM4l73t/AmXcv5JnF27Q4j4iIiMgxdDRAv2eMed8YM9sYMxv3FIALvFdWEDr1Arjgz7D+bfj4t05X02UG94nnsWvH8cqNE8lMjub/vbmGc//3X7xZWKol6kVERESOcCI3XM4AJnk2P7fWvu61qjrBr9tODrEW3vk5FDwO0/4/GPMDpyvqUtZaFm4o4y/vrWf97moG94njv6YO5uxBvTDGOF2eiIiISJfwes93IAiI8A3Q0gTPXe5ehOeaNyDzDKcr6nKtrZb5q3Zy7wcb2VFRx/iMJP5z6iByM5KcLk1ERETkpHktfBtjqoGjnWAAa62N78ybekPAhG+Ag5Xw+PlQswd++DEkZztdkVc0Nrcyr6CY+z/eRFl1A1MGp3D7BYMY0tdvPjYiIiIiJ0wj3wRY+Aao2OqegjAqEX74EUQH76hwXWMzTy7axkP/KqKmoZlLc1K57dxTOcUV7XRpIiIiIidM4ZsADN8AO5bA09+F9Alw9WsQFtxzZVfWNfKPfxXx1KJttFrL98efwi2TB9IrLtLp0kREREQ6TOGbAA3fACvnwetzYfTVMO0B6AY3Ju6uquf+TzYxb1kxEaEhXHd6JnPPGkB8VLjTpYmIiIh8K1/M892ljDFTjTEbjDGbjTF3HOX4z4wxa40xq4wxHxtj+jtRp0+MmgVn/gJWPAv59ztdjU/0SYjiz9NH8OFtZzJlSAoPfLqZM+/+lEc+K6K+qcXp8kRERES8xucj38aYUGAjcB5QAiwDrrTWrm13zjnAF9baOmPMj4CzrbWzjve6ATvyDe5Fd16dA2vegFnPwpCLna7Ip1aXVnH3+xv4bGMZfeKj+Om5A7lsbBphoVrHSURERPxPoI18jwc2W2u3WGsbgReBS9qfYK391Fpb59lcAqT5uEbfCgmBS/8BqWPgtethZ6HTFfnU8NQE/jlnPM9fP4E+CVHc8dpXnP9/n7Hgq10ES1uUiIiICDgTvlOB4nbbJZ59x3Id8O7RDhhj5hpjCowxATrk3U54D7jiBYh2wQtXwIGdTlfkc3lZybx+Ux4PXzOWkBDDTc99yaUPLmLp1gqnSxMRERHpEn797/rGmKuBXOCeox231j5irc211ub6tjIviesNV74IDdXw/CxorHW6Ip8zxnDBsD68/9Mzufuykew50MDMhxcz958FbCmrcbo8ERERkZPiRPguBdLbbad59h3GGHMu8GtgmrW2wUe1Oa/PcLjsCdizGl6b6+4H74ZCQwwzc9P59Pazuf38U1m0eR/n/99n/M+bq6mobXS6PBEREZFOcSJ8LwMGGmMyjTERwBXAW+1PMMaMBh7GHbz3OlCjs069AC74M6x/Gz7+rdPVOKpHRCi3TB7Iwl+cw6xx6TyzZDtn3f0p/1iomVFEREQk8Dgyz7cx5iLgPiAUeMJa+ydjzO+BAmvtW8aYj4ARwC7PJTustdOO95oBPdvJ0VgL7/wMCp5wz/895hqnK/ILm/ZUc+e76/lk/V5SE3vwiwsGMW1UP0JCgn9+dBEREfEPWmSHIAzfAC1N8NzlsO1zuOYNyDzD6Yr8Rv7mffxpwTrW7DzAyLQEfnXREE4b4HK6LBEREekGFL4J0vANcLASHj8favbADz+G5GynK/Ibra2WNwpLuef9Deyqque8ob2548LBZPWKdbo0ERERCWIK3wRx+Aao2AqPTYGoRPjhRxCd5HRFfqW+qYXH/72Vfyws4mBTC1dNOIVbpwzEFRvpdGkiIiIShBS+CfLwDbB9MfxzGqRPgKtfg7AIpyvyO/tqGrjvo428sLSYHuGh3HROFnMmZRIVHup0aSIiIhJEAm2FS+mM/hPdN15u+xzeuc19Q6YcJjk2kj9eOoL3f3ompw1wcfd7G5j814W8vqKE1lb9vERERMR5Ct+BZNQsOPMXsOJZeOsWKC9yuiK/lJ0Sy2PX5vLC9aeRFBvBbfNWMu3Bf5NftM/p0kRERKSbU9tJoGlthQ9+DUsfhdZmGPwdmHgznDIRjKbbO1Jrq+XNlaXc894GdlbVc+6QFO64cDDZKXFOlyYiIiIBSj3fdKPwfUj1bncAL3gcDu6HfqNh4i0w9BIIDXe6Or9T39TCk4u28fdPN1PX1MKV49P56bmnkqybMkVEROQEKXzTDcP3IY11sPIFWPJ3KN8M8akw4QYYcy30SHS6Or9TXtPA/R9v4tkvdtAjPJQfne2+KbNHhG7KFBERkY5R+KYbh+9DWlth0wew2HNTZkQsjL7GHcSTMp2uzu8UldXwl3fX88HaPfRNiOL28wcxfXSqVsoUERGRb6XwjcL3YXathMV/h9WvgG2FwRe7W1LSx6sv/AhfbCnnTwvWsaqkimH94vn1RUPIy052uiwRERHxYwrfKHwf1YGdnr7wJ6C+ElJz3TdnDpkGoWFOV+c3Wlst81ft5O73NlBaeZDc/j2ZmZvORSP7Ehupn5OIiIgcTuEbhe/jaqyFwufdfeEVWyAhHSbcCGOugagEp6vzG/VNLTy7ZDvPL93BlrJaoiNCuWhEX2bmpjMuoydG/2ogIiIiKHwDCt8d0toCG9+HxQ/C9n9DRByM+YG7L7xnf6er8xvWWr7csZ+XC0qYv3IntY0tZLiiuTw3nRlj0uiTEOV0iSIiIuIgvw3fxpipwN+AUOAxa+1dRxyfDdwDlHp2PWCtfcxz7Frgvz37/2itffp476XwfYJ2rnD3ha95zd0XPmSapy98nNOV+ZW6xmYWfLWblwuK+WJrBSEGzjy1FzNz05kyJIXIMM2SIiIi0t34Zfg2xoQCG4HzgBJgGXCltXZtu3NmA7nW2luOuDYJKAByAQssB8Zaa/cf6/0UvjupqhSWPgwFT0FDFaSNd/eFD75YfeFH2LavlleWl/DqlyXsqqonMTqcS3NSmZmbztB+8U6XJyIiIj7ir+F7IvBba+0Fnu1fAlhr72x3zmyOHr6vBM621t7g2X4YWGitfeFY76fwfZIaaqDwOXdf+P5tkHgKTPgRjL4aohQs22tptfx78z5eKijmwzV7aGxpZVi/eGbmpnNJTj8SoyOcLlFERES8yF/D92XAVGvtDz3b1wAT2gdtT/i+EyjDPUp+m7W22BhzOxBlrf2j57z/Bxy01v71iPeYC8wFOOWUU8Zu377dK99Lt9LaAhsWuPvCdyyGyHhPX/iNkJjudHV+p7KukTcLd/JSQTFrdh4gIjSE84b1ZmZuOqdnJxOqecNFRESCTiCHbxdQY61tMMbcAMyy1k7uaPhuTyPfXlC63NMX/rp7O3uKuzd80EUQ43K2Nj+0ZmcVLxeU8EZhKZV1TfRNiGLGmDQuz02jv6tT/32KiIiIH/LX8P2tbSdHnB8KVFhrE9R24mcqi2HZo7D6dajaASYE+k9yB/HB34GEVKcr9CsNzS18vG4vLxUU89nGMlotTMhM4vLcdC4a0YfoCPXSi4iIBDJ/Dd9huFtJpuCezWQZ8H1r7Zp25/S11u7yPJ8O/Je19jTPDZfLgTGeU7/EfcNlxbHeT+HbB6yF3atg3Xz3o2y9e3/qWBjyXXcYd2U5W6Of2V1Vz6tflvByQTHbyuuIjQzj4pF9uTw3jTGnaO5wERGRQOSX4RvAGHMRcB/uqQafsNb+yRjze6DAWvuWMeZOYBrQDFQAP7LWrvdcOwf4leel/mStffJ476Xw7YCyjbDeE8R3rnDvSxnqCeLfhd7DtZy9h7WWZdv283JBMe98tYu6xhYG9IphZm463xudSkq85g4XEREJFH4bvn1J4dthlcWw/m13EN+x2D13eM+Mr0fEU3MhJMTpKv1CTUMzC1bt4uXlxSzbtp/QEMOEzCQmD05h8uAUBvSKdbpEEREROQ6FbxS+/UpNmXvGlHXzYctCaG2C2D7u/vAh34WM0yE03Okq/cKWshpe/bKED9fuYeOeGgAyXNGcMziFKYN7Mz4ziYgw/aVFRETEnyh8o/Dtt+qrYOMHsO4t2PwRNNVBVKJ7xpQh34WscyC8h9NV+oXiijo+3bCXT9bvJb+onMbmVmIiQjl9YDKTB6dwzqAUtaeIiIj4AYVvFL4DQmMdFH3ibk/ZsMAdzMNjYOB57iA+8Hwt6ONxsLGF/KJ9fLx+L5+u38uuqnoARqQmcI6nPWVkagIhmkdcRETE5xS+UfgOOC1NsO1zz8wpb0PtXgiNgAFnu4P4oIsgJtnpKv2CtZb1u6v5ZL17VHzFjv20WkiOjeDsQe4gfsbAZOKi1MojIiLiCwrfKHwHtNYWKFnmCeJvQWW7ucQzz4J+o6FfjsK4x/7aRv61sYyP1+/lXxv2cqC+mbAQw3jPTZvnDE5hQHKMpjEUERHxEoVvFL6DhrWw+yt3EF//Nuxd+/WxhFOg3yhPGB8NfXMgOsm5Wv1Ac0srX+6o5BNPe8qGPdUA9HdFt82eMj4ziciwUIcrFRERCR4K3yh8B636Kti10j2P+M5C99f9W78+ntj/6zDeL8cdyHskOlevw0r21/Hp+q9v2mzw3LQ5KTuZKUN006aIiEhXUPhG4btbObi/XSD3hPLK7V8fTxrgDuFtI+SjuuWNnIdu2jw0Kr7Tc9Pm8NR4Jg9K4bQBLoanJRCvXnEREZETovCNwne3V1fhDuK7Cr8O5FXFXx93ZR/ertJ3JETGOVevj1lr2bCnmo/XuYP4l56bNgEGJMcwIi2BEakJjEpPZFi/eKIjwpwtWERExI8pfKPwLUdRu+/rVpVDwfxAqeeggeRTv25X6Tca+oyAiE79dxRwKusaWVVSxVelVawsruSr0qq26QxDDGSnxDIiNZFR6e5QPqRvPFHh6hsXEREBhW9A4Vs6qHqPZ3S8XSiv2e0+ZkIgeZA7iKeOgX5joM9wCIt0tmYf2Vtdz+rSKlYWu0P5qpJK9tU0AhAWYji1d5wnjCcyMi2BU3vHafVNERHplhS+UfiWk3BgV7t2Fc+jtsx9LCQceg/7OoynjnEH9NDgb8uw1rL7QL0njFe2jZRX1jUBEBEawpC+cYxMS2REWgIj0xLI7hVLWKgCuYiIBDeFbxS+pQtZC1UlsPNLKP3S/XVnITQccB8Pj4Y+Iw8P5EkDoBvMq22tpbjiIKtKK/mqpIpVJVWsLq2iuqEZgB7hoQzrF98WxkemJZLpitFKnCIiElQUvlH4Fi9rbYWKonZhfIV7xpVmd580UQme/vExX4fy+H7dIpC3tlq2lte2hfFVJZWs2XmAg00tAMRGhjE8NZ6RaYmM7d+TCZlJJEZHOFy1iIhI5yl8o/AtDmhphrJ1Xwfy0i/diwK1ukeBie39zUAe43K2Zh9pbmmlqKyWVSXudpVVpVWs23mAxpZWjIEhfeI5bYCLiVkuxmckkRCt6Q5FRCRwKHyj8C1+oukg7F59eMvKvk2A57+zxFMOD+P9crrNlIeNza2sLKlkSVE5i7eUs3z7fhqa3WF8aN94Jg5wcdoAF+Myk0jooTAuIiL+S+EbhW/xY/UHPIsCtQvklTs8Bw0kD4SMM2DQhe6v4d1jBcqG5hZWFlexuKicJVvKWb5jP43NrYQYGNYvgdMGJLWFcS0EJCIi/kThG4VvCTC1+9x946VfQmkBbPs3NNVBeAxknQOnXgADL4C43k5X6jP1TS0UFleyZEs5i4vKWbGjksYWdxgfnprQNjKem9GTOIVxERFxkMI3Ct8S4JrqYdvnsOFd2Pg+HChx708dC6deCIOmQu/h3eIGzkPqm1r4csd+lmypYMmWcgo9YTw0xDA8td3IeEYSsZHBP/WjiIj4D4VvFL4liFgLe1bDhvdg47tQuty9Pz7NPSLezdpTDjnY2MKKHfvdI+NbyiksrqSpxRIaYhiRmsDELM/IeP+exCiMi4iIFyl8o/AtQax6D2z6ADa+B0WfeNpTomHAOe4R8W7WnnLIwcYWlm93h/ElnjDe3GoJCzGMTEvgtHZtKtERCuMiItJ1FL5R+JZu4lB7ysb33CPjR7annHoB9BnRrdpTDqlrbG4L44uLyllVUkVzqyU81DA6vScTs1xMyk4mJz2RiDCtwikiIp2n8I3Ct3RDak85rtqGZgq27ye/aB+Li8r5qrQKa92rcI7LTCIvy8WkrGSG9osnVCtwiojICVD4RuFbRO0px1dV18SSreXkb95HflE5m/bWAJDQI5zTBiSRl5VMXpaL7JRYTDf8lwMREek4hW8UvkUO01Tvnr5w47tHaU+Z6n70HgYhoc7W6aC9B+pZvKWc/M3lLCraR8n+gwD0iotsGxWfmOUiPSna4UpFRMTfKHyj8C1yTIe1p7znaU+xYEIhtjfE9YG4vkf/Gt8PevTsFj3kxRV1LPKMiucXlbOvpgGA9KQebUE8LyuZXnGRDlcqIiJOU/hG4Vukw2r2wuaPoLwIqnd5HrvdXw/u/+b5oRHHD+iHvkbGB01It9ayaW9NWxhfsqWc6vpmAE7tHdvWojJhgIuEHlrwR0Sku1H4RuFbpEs01UPN7q/D+FG/7oaGA9+8NjzaE8b7eb4eJaDHJENEHIQE1mwjLa2W1aVVnlHxfSzbVkF9k3v1Tfcc48lMynaR2z+JHhHdt5VHRKS7UPhG4VvEpxpqoGbPcQL6LjiwC5oPfvNaEwJRidAj0d3SEuX52qPnUfa1OxaV6DcztzQ0t7BiR6U7jG/e1zbHeHioYfQpPRmRmkBmcgwDesUwIDmW3vGRuolTRCSIKHyj8C3id6x1j5C3D+MHK9ytLQcrPV/3Q33l1/vqK8G2Hvs1w3ocJ6AnfnNfVCJEJ0FkgldH22sbmlm2rYL8Ivcc45v2VlPf9PX3ERMRSmavGDKTYxnQLpRn9oohVqtxiogEHIVvFL5FgkJrqzuwtw/khwX09vuqDt/XVHucFzbtQnpSu5H2nu5w3n67/SMqoVMzwrS2WnYdqGdLWQ1b99WypayWLftq2VJWQ2nlQdr/2k2Ji2SAJ5hn9YrxjJjHkt6zB2GhgdWeIyLSXSh8o/At0u01N34ztB/1UXH4dn3VcV7UuAP4NwJ6BwL8MdpM6pta2F5ex5ayGk8gr2XrPvfzyrqmtvPCQgynuKIZkBzrGSl3h/LM5BiSYyPUxiIi4qCTCd9e/fdOY8xU4G9AKPCYtfauY5w3A3gFGGetLTDGhAOPAWM8Nf7TWnunN2sVkQAXFgGxKe7HiWhpPmIU/YhwXtduu64cyjd/e2iPSYH08ZA2DtInQL8cCP//27vz4Ejv+s7j769aLaml7tZ9jTTj0Rx4ZjzYZmzPGJtkvUUg4FTZyYYlhsUBisQpgpNNNrAhe9RSUFsVQi3sUmEDJnFCCASIFzZTMQvhsgnHnL7mtD0eaWzd0ui+j/7tH79HrW6NNKM51N1qfV5VXf300093//rpRz2f+fX3+f0iAJSEQ9zcEOPmhtglDxsYn6G1f4xX+saDHnPfc/7jl/qYmV8sY4mVFKaF8W21ZdzWXKExyUVE1oE16/k2sxDwEvAWoB04CrzLOXd6yXYx4EmgCHg0CN/vBh5wzj1kZqXAaeA+51zbSq+nnm8RyajEvA/gE0vDej90vQDtR2DgvN+2IAyNt0LzftgcBPLy5lW/1HzC0TE4yfn+saCnfDy53DU8ldyuqSLCgZYqDmyr4kBLNTdVl6qHXERkDeRqz/d+4Jxz7jyAmX0NeBAfpFN9Avgk8JGUdQ4oM7NCIALMAMuMbSYikiUFIV9uUlq18jZjfdB+FF477K+P/w0c/gt/X2zTYhBv3u/DeeHyE/iEghKULdWl3Hdz+n0TM3Oc7xvn+IVBDrde5OmX+vjmsx0A1MeLOdBSzYFtVdy9rZptNWUK47lkeiw4GbnDn5A80gEjnX4koXDE/4ISrQ2u66CsdvE6pPHlRdartQzfTcBrKbfbgQOpG5jZPmCzc+5JM0sN30/gg3oXUAr8oXNuYOkLmNkjwCMAW7ZsubGtFxG5XtFa2HW/vwDMz0L3iSCQH/GX0//o7wsV+/KUhVKVzfv92OhXUFpUyN6mcvY2lfPee7binONc7xiHWgc4fP4iPz9/kYPPdwJQEy32QbyligPbqtlZF1UYXwvO+V9BFsL0wmU0ZXmkC6aXKV2KVEK0AWYnYLzPXy8nUpkeystqlwT1lOCeI0N0ioi3lmUn7wDe5pz7reD2w8AB59yjwe0C4IfA+5xzbWb2FPDhoOzkXuB3gfcBlcC/AG9f6EVfjspORGRdGu0OgnjQO975HMz7qe0p3+JD+MKlfu9V93g652jtH+dwEMYPnR+ge8SXqlSVFbF/62KZyq6GGAUFCuOXlZj3PdMrhupO35s9N7XkgbY44VR8U8qlaXFdrBGKltTtT4/BeK//FWW8189QO94XXKeu74OZ0eXbXBxP7zVfGs6jdRCth4oteTNLrchay8nRTszsjcDHnHO/HNz+E4CFEyfNrBx4BRgLHtIADAAPAO8HDjnnvhxs+zjwHefcN1Z6PYVvEckLc9OLNeMLveOjvueawgg07QtO5gwCeVnNVT29c45XByY4fH6AQ60XOXx+gI4hPxlSRWmYu7ZWcaDFl6nsbowTylYYd84H2KmRYPjJ4Ikqlo0AABqHSURBVHp+Fty8Hw8+MZ+ynPDLieB22nLiCuuXPp9bXE7M+bA70rVYEuLm09saKgoCdBPEG5eE6mBdtH7tS0VmJy8N5+N96QF9IcBPDV36+PrXwz2/B3v/jcpaRK4gV8N3If6EyzcDHfgTLt/tnDu1wvZPsdjz/cfALufc+82sLHjsQ865F1Z6PYVvEclbw+2LQbz9iA/niWBYwqptPojHGqCwxJcYFEaC6+ASjvh68hXWt485DreNcrhtgMOtA1y46EsdYiWFyTB+YFs1ezfFVz/2+OxUSmge9tdTw+lBOnm9wvr5mTXaoUuZr+G3ArBQynJBUNtfE4TqpsUe6mTQboLS6vXXYzw344P5QigfbIVjj0PfWYg3w90fhH2/CSXxbLdUJCflZPgGMLP7gf+JH2rwcefcfzezjwPHnHMHl2z7FIvhOwr8NbAHMOCvnXOfutxrKXyLyIYxO+nLUxZ6xzuO+2EQryesWoEP54XFzIdKmHRhRudDDM2EGJ4rZNqFmS0opqwsSkU8RnVFnOqiBKGZpUF6tcHZoDjmSyJK4sF1ecpy6vXC+pivjS8oWCYohy6zPuTD8bLrC9ZfcF4riQSc+x789LNw4Sd+v9/5PjjwQf8fDRFJytnwnUkK3yKy4SUSvlRj4TI7mbI8BXOTvqxlYf1scHtuMrg/ddvF55iZnmR8fIzpyXHmZyaw+WmKmWWaIubCMQoicUqilcQrqikqq1gSniuWD9RFMR+WJTd1HPch/MxB/x+VW9/pS1Lqdme7ZSI5QeEbhW8RkUzpH5vmSOsAx9oGOf7qIKc6hplL+H9LtteWcedNVdyxtZI7b6qkRcMbrm8DrXDof8MzX/b/SdvxFrj392HrL+gXA9nQFL5R+BYRyZbJmXmebx/i+IVBjrUNcPzCICNTc4AfUWXflkruDML43qZySsKhLLdYrtrEABz9Szj8BT+RVOPtPoTvfhBCazpZdm5bGFZyrMePXDTWC2PdMNrj1431+Nr6whI/J0Ckyg8TubBcGtyOVEFpcF0c169C64DCNwrfIiK5IpFwvNI3xrELgxxrG+SZVwdp7R8HoChUwOuby7njpsrkpSa6/ORCkoNmJ+H5v4ef/TkMvOKHJ7z7Q7DvYSi6phySm+bn/Mmoo92LIXq0xwfrsSXrlzu/IVwGsXo/yk1ZjS/lmhyEyQH/H5mpYfx8gsuwEEQqUsJ5amCvXBLYU+5fOkylrCmFbxS+RURyWf/YNMcvDCZ7x092jDAznwCgpaYsrXd8e21U443nukQCXvw2/Oyzfoz6kgq467fgwO/4ccNz1cyEH4d9aaBO7ake7fYnMC8XjkurfaCO1vsRhqJ1flKkWL2/jtb75eLY5duRmIfJofRAPjngby8sTwyk3B9crzTpEvje9dTe9JqdUH+LH0KybrdGrrnBFL5R+BYRWU+mZuc52TGc1js+MO57EMsj4bSe8duaK4gUqVQlZ7162Ifws0/6Mc9ve8ifnFmzM3ttSszDYBv0nISeU4uXwdZLty0oXF2gLquDwqKMv5U0S3vQlw3vg77Upe/F9FlUK26ChtcHgfwWP2lXZYtKXK6RwjcK3yIi69nCTJzHLgxyvG2QYxcGeKXPl6oUFhi3NJVzx5ZK9rdUctfWKqpVqpJ7+s/Bz/8cnvuqn6X15vvhnt+HLXev7cmZ4xeh91R6yO49408QBT+cZNV2qN8DdbdAxeaUoN3ge4nzMYA65+cI6DkFPScW983Fc35SKYBwKdTt8WF8IZjX7fFlL3JZCt8ofIuI5JvB8RmeeXUwGcifbx9ies6Hhh11Ufa3+AmA7tpaxaaKSJZbK0ljfXDkMTj6Rd8T23yXD+G7fsWPrX6t5qah/6X0kN1zypeNLCitXuzVXQiStbtUD51qZsJPppTchyeh+0T6rKflW1J6yINgXrXt+j6/G2l+FqZHfe18aXVWSmoUvlH4FhHJdzNzCU50DHG4dYCjwVCHo9N+VJXmykgyjO9vqWZrdamGOMy2mXF49iu+N3zogg9vb3wUbn+3n111Jc7BSMelIfviy5DwnzehIqi9OT1k1+/1JSP63K+ec74WvvtkSqnOSeh/Gdy836YwAnW7gn2+dzGYl1at/nUSCZgZ88F5eiQI0MHEXGm3R1Nmv1267ejirxoAv/YY3PYbN3Z/rILCNwrfIiIbzXzCcaZrhCOtAxxpHeBo2wAXg7rxmmhxEMT95eb6mE7izJb5OT9Zz88+C53P+p7K/Y/AXb/ta6h7zywpGTkVjAYSKN+8GPQWQnb1dgiFs/eeNorZKeh/0X8uyWB+MjghNRBv8p9N7S7ALROgR9ID9EqjvCQtnf02WC6OpdwuX7y95Y1Q1bKGO2GFVip8K3yLiGx0zjle6RsPwvhFjrQO0Dk8BUC8pJC7ti6G8b1N5YRDeVjnm8ucgws/9TNnvvxdKAhDYnbx/qJoSsAOSkfqdqv+ONc450eFST2ZtfukD+kFhUuC8sJy+TIhOr58wC6KrosafIVvFL5FRORS7YMTyZ7xI60DnA/GG4+EQ9xxU2UykL9hS4Um/8mk3jPw7N/5IQrrb/EnQ5ZvWRehS1bg3IYq+VH4RuFbRESurHd0imNtgxxpHeBw6wBnu0dwDsIh47bmCva3VHFXSxV33lRJrERlDSKyPIVvFL5FROTqDU/McuzCAEfafM/4ifZh5hKOAoM9m+LctbWKnXUxNlWU0FQRYVNFhLLiDTyduogACt+AwreIiFy/iZk5nn11KFmm8syrg8nhDRdUlIbZVB6hqTISBPISNlX45aaKCDXRYp3cKZLnrid867/vIiIigdKiQu7dUcO9O2oAmJtP0Ds6TefQJB3BpXNoks6hKV4bmODQKxeTwx0uCIeMxvJIsqe8aSGcV/rbm8ojmrFTZANT+BYREVlBYajAB+aKCHeusM3I1GwQyCfpGJykY2gqeftnr/TTMzJFYsmPzFVlRWmlLEuva6JFGqdcJE8pfIuIiFyHeEmYeEOYXQ3Lz7I3O5+gZ2SKzqEpOoYmgmsfzs/3jfMvL/czMTOf9pjySJibG2Lsbohxc0OcXY0xbq6Pqd5cJA/or1hERGQNhUMFNFeW0lxZClw6G6BzjpHJuWRZS/vgBC/3jnG2a4QnjrcznhLMt1SVsqsh5i+NcW5uiLG1uoyQasxF1g2FbxERkSwyM8pLw5SXhtmzKb33PJFwdAxNcrZ7lLNdI5zt8dffP9OTLGUpCRews24xkC+E8+pocRbejYhciUY7ERERWWemZuc51zvGma4RXuwe9eG8e5T+senkNjXRYnYH5SoLoXxHXVSTCYncABrtREREZAMpCYfY21TO3qbytPX9Y9O82D2aFsq/fOhCcrjEUIHRUlOWXk/eEKO5MqITPEUyROFbREQkT9REi6nZUZwcKhFgPuFouzjO2a5RXuwe4Uz3KCfah3nyha7kNtHiQvY0xoNA76+310ZVSy6yBlR2IiIisgGNTc/xUs8oZ7tGOds9wqnOEU53jjA560/wjIRD7G6MJXvY924qZ2d9lHCoIMstF8k+zXCJwreIiMj1mk84zveNcaJjmJMdI5zsHOZ05whjwURCRYUF7G6IcUtTOa8PAvnrGqIUF6qOXDYWhW8UvkVERNZCIihbOdk5wsmO4eRlZMoH8nDIeF19jL2bFktWdjfGdWKn5DWFbxS+RUREMsU5x2sDk5zsHA56yf1lcGIW8Cd27qiNptWQ72mMa5IgyRsK3yh8i4iIZJNzjs7hqbTe8RMdI8nhD81gW02ZL1dpKueWTeXs2RSnPBLOcstFrp7CNwrfIiIiuahnZCGQj3CiY5hTncN0DU8l72+qiLC7Mcbuxjh7GuPsboyzpaqUAo20IjlM43yLiIhITqqPl1AfL+HNu+uT6/rHpjnZMcyZLj8m+emuEX54tjc5a2dZUciPRR6E8d3BJEEqW5F8oJ5vERERybqp2Xle6vFh/EzXKKe7RjjTNcJocGKnGWytLvO95A1BKN8UZ1N5iSYIkoxTz7eIiIisayXhELc2V3Brc0VynXOOjqFJTneOJHvJT3WO8O0T3cltyiNhdjXE2LMpnixd2VEX1WgrkrPWtOfbzN4G/C8gBPylc+5PV9ju14EngLucc8eCdbcCXwDiQCK4b2q5x4N6vkVERDaKsek5Xuwe4XTXaBDMR3ixezQ5QVCowNheW5ZWtrK7MUZdrCTLLZd8kZMnXJpZCHgJeAvQDhwF3uWcO71kuxjwJFAEPOqcO2ZmhcAzwMPOuefNrBoYcs7Nr/R6Ct8iIiIb13zCceHieLKHfOHSmXJyZ020iO21UVpqythaU0ZLcNlSVaqecrkquVp2sh8455w7D2BmXwMeBE4v2e4TwCeBj6SseyvwgnPueQDn3MU1bKeIiIisc6ECY1ttlG21UX7l1sbk+qGJmaB+fJSzXSOc7x/ne6d7uDg+k9zGDDaVR4JQXkpLTZSWmlK2VpexuaqUcKggG29J8tRahu8m4LWU2+3AgdQNzGwfsNk596SZpYbv1wHOzL4L1AJfc8792Rq2VURERPJQRWkR92yv4Z7tNWnrhydnaesfp+3iOK39/tLWP87B5zqTs3eCD/WbKyNpPeVbq/31pooIIQ2JKFcpaydcmlkB8GngfcvcXQi8CbgLmAB+YGbHnXM/WPIcjwCPAGzZsmVN2ysiIiL5ozwS5rbNFdy2uSJtvXOOgfGZIJRP0No/Rlv/BK394xxpHWBiZrECtihUwJbq0ktCeUtNGfXxYo3CIstay/DdAWxOud0crFsQA/YCTwUHZwNw0MwewPeS/9g51w9gZt8G9gFp4ds59xjwGPia77V5GyIiIrJRmBnV0WKqo8XccVNV2n3OOXpHp5O95Mke84vjPP1SHzNzieS2kXAo6C0vpbE8QkO8hLp4MfXxEhqCsc8jRaoz34jWMnwfBXaaWQs+dD8EvHvhTufcMJD8DcjMngI+HJxw+QrwH82sFJgB/hXwmTVsq4iIiMhlmVly0qC7t1Wn3TefcHQNTwa95GO09k/QdnGcs12j/OhsX3IkllSxksJkEK+LFyeX64OQXh8voTZWrJrzPLNm4ds5N2dmjwLfxQ81+Lhz7pSZfRw45pw7eJnHDprZp/EB3gHfds49uVZtFREREbkeoQKjubKU5spS3rQzvb7cOcfo9By9I1N0D0/TMzJFz+gUPcNT9IxM0zM6xaFXxugdnWYukf5DvhlUlxWnBPLFYJ66XFVaRIHqz9cFzXApIiIikgMSCcfAxAzdw1P0jvpgvtxy/9jMJY8Nh4y6WFDaEithU0WE5soITZURmoLl8khYdeg3SK4ONSgiIiIiq1RQYNREi6mJFgPlK243M5egbyzoQR+eCnrSp/3y6BQv947y9EuXlrqUFYVorixNBvLUYN5UGaE2qpNEM0HhW0RERGQdKSos8OG5IrLiNgujtnQMTdIxOEnH0CTtg/7SMTTJsbaBtCEVlz5v85KA3lTpTxotVP35dVP4FhEREckzqaO23Npcsew2o1OzyXC+EMo7BidpH5rk+2d66R+bTts+VGA0xEtoqozQHARyH9JLaQ6WFc6vTOFbREREZAOKlYTZ1RBmV0N82funZufTes4Xe9AnOHT+It0jU6SeHxoOGS01Zeyoi7KjLsaOuig766K01JRREtawigsUvkVERETkEiXhENtro2yvjS57/+x8gu7hKTqGJnltYILz/eO83DPGma5RvnOyOxnMCww2V5Wysy7K9rooO4NgvqMuSrR440XRjfeORUREROS6hUMFbK4qZXNV6SXjnk/NztN20Yfxc72Ll6df6mN2frG7vLG8JBnEU0N5VVlRpt9Oxih8i4iIiMgNVRIOsashfklJy9x8glcHJni5Nz2Uf/3oa0zMLI7OUl1WFPSSpwfz+vj6H5FF4VtEREREMqIwVMC22ijbaqP88i2L6xMJR+fwZFogf7l3jH96oYvhydnkdrHiQrYnA3mUX9pTv2JZTK5S+BYRERGRrCpImSH0vpvrkuudc/SNTaeF8oXylSeOt9NYEVH4FhERERG5EcyCmTtjJdyzvSbtvuGJWcKF668EReFbRERERNad8tJwtptwTTQSuoiIiIhIhih8i4iIiIhkiMK3iIiIiEiGKHyLiIiIiGSIwreIiIiISIYofIuIiIiIZIjCt4iIiIhIhih8i4iIiIhkSFbCt5m9zcxeNLNzZvbRZe4vNrOvB/cfNrOtmW+liIiIiMiNlfHwbWYh4HPA24E9wLvMbM+SzT4ADDrndgCfAT6Z2VaKiIiIiNx42ej53g+cc86dd87NAF8DHlyyzYPAl4LlJ4A3m5llsI0iIiIiIjdcYRZeswl4LeV2O3BgpW2cc3NmNgxUA/2pG5nZI8AjKbcn1qLBq1AIzGXpteXy1stns17amS/ycX/n43vKtI2+Dzf6+1+J9svqbaR9FbnWB2YjfN8wzrnHgMcAzOyYc+7ObLQjm68tl7dePpv10s58kY/7Ox/fU6Zt9H240d//SrRfVm8j7SszO3atj81G2UkHsDnldnOwbtltzKwQKAcuZqR1IiIiIiJrJBvh+yiw08xazKwIeAg4uGSbg8B7g+V3AD90zrkMtlFERERE5IbLeNlJUMP9KPBdIAQ87pw7ZWYfB4455w4CfwV82czOAQP4gH4lj61Zo3P7teXy1stns17amS/ycX/n43vKtI2+Dzf6+1+J9svqbaR9dc3v1dShLCIiIiKSGZrhUkREREQkQxS+RUREREQyZN2F71yamv5KbZHMWcVx8R/M7LSZvWBmPzCzm7LRzqXMrM3MTpjZc9czbJEsb7V/o2b262bmzGxdDJFlZo+bWa+ZnUxZV2Vm3zOzl4Prymy2MZet5rgws3cG3xmnzOyrmW7jWlrh+PmYmXUE30XPmdn92WxjNqzi35EtZvYjM3s2+Ldkw+2jBWa2OdgXC38j/z5Yn1ffQ8v9rSy538zss8Ex84KZ7VvVEzvn1s0Ff4LmK8A2oAh4HtizZJvfBT4fLD8EfD1bbdElp46Lfw2UBssfXKvj4hra3gbUZLsd+XhZ7d8oEAN+DBwC7sx2u1f53n4R2AecTFn3Z8BHg+WPAp/Mdjtz8bLK74udwLNAZXC7LtvtzsDx8zHgw9luW44fF48BHwyW9wBt2W53FvdXI7AvWI4BLwX7JK++h5b7W1ly//3A/wMMuBs4vJrnXW8937k0Nf1q2iKZccXPwjn3I+fcwgyoh/Djy0t+W+3f6CeATwJTmWzc9XDO/Rg/ElSq1O++LwG/mtFGrR+rOS5+G/icc24QwDnXm+E2rqkVjp+NbjXHhQPiwXI50JnB9uUU51yXc+6ZYHkUOIOfnTyvvodW8bfyIPC3zjsEVJhZ45Wed72F7+Wmpm9aaRvn3BywMDV9NtoimXG1n8UH8P9TzQUO+GczO25mj2S7MXnmisdF8BPhZufck5ls2Bqpd851BcvdQH02G5PDVvN98TrgdWb2UzM7ZGZvy1jrsuvR4Kfzx9d7ucA1WM1x8THgPWbWDnwb+L3MNC23BeW9bwAOs/G+h64pC6638C1yXczsPcCdwKey3ZbAm5xz+4C3Ax8ys1/MdoM2CjMrAD4N/FG223KjOf97qMaRvXaF+NKT+4B3AV80s4qstmjt/QWwHbgd6AL+R3abk5PeBfyNc64ZX27w5eB7ZMMysyjwf4A/cM6NpN6n76GVrbeDJpempl9NWyQzVvVZmNkvAf8ZeMA5N52htl2Wc64juO4FvoX/6VNujCsdFzFgL/CUmbXh6/UOrpeTLpfRs/BzZ3CdV6USN9Bqvi/agYPOuVnnXCu+nnVnhtqXFc65HufcvHMuAXyRjfddtJrj4gPANwCccz8HSoCajLQuB5lZGB+8v+Kc+2aweqN9D11TFlxv4TuXpqZfTVskM674WZjZG4Av4IN3TnwZmFmZmcUWloG3AsueUS3X5LLHhXNu2DlX45zb6pzbij8X4AHn3HoddSb1u++9wD9msS25bDXf3f8X3+uNmdXgy1DOZ7KRmbakTvXX2HjfRas5Ll4F3gxgZrvx4bsvo63MEcG5dH8FnHHOfTrlro32PXQQ+M1g1JO7geGUspsVZXx6+evh1m5q+hvWlrV4Lbm8VR4XnwKiwD8E59++6px7IGuN9uqBbwXtKQS+6pz7TnablD9WeVysS2b29/hwWBPUn/434E+Bb5jZB4ALwDuz18Lctcrj4rvAW83sNDAPfMQ5txa/oGbFCsfPfWZ2O75MoA34naw1MAtWeVz8Eb4E6Q/x++l9a9S5tx7cCzwMnDCz54J1/4k8+x5a4W8lDOCc+zy+9v9+4BwwAbx/Vc+7cY8bEREREZHMWm9lJyIiIiIi65bCt4iIiIhIhih8i4iIiIhkiMK3iIiIiEiGKHyLiIiIiGSIwreIiFyWmd1nZv+U7XaIiOQDhW8RERERkQxR+BYRyRNm9h4zO2Jmz5nZF8wsZGZjZvYZMztlZj8ws9pg29vN7JCZvWBm3zKzymD9DjP7vpk9b2bPmNn24OmjZvaEmZ01s68EM9yJiMhVUvgWEckDwXTXvwHc65y7HT8z478DyvAz9N0CPI2foQ3gb4E/ds7dCpxIWf8V4HPOuduAe4CFqZLfAPwBsAfYhp/hTkRErtK6ml5eRERW9GbgDuBo0CkdAXqBBPD1YJu/A75pZuVAhXPu6WD9l4B/MLMY0OSc+xaAc24KIHi+I8659uD2c8BW4Cdr/7ZERPKLwreISH4w4EvOuT9JW2n2X5ds567x+adTlufRvx8iItdEZSciIvnhB8A7zKwOwMyqzOwm/Pf8O4Jt3g38xDk3DAya2S8E6x8GnnbOjQLtZvarwXMUm1lpRt+FiEieU8+FiEgecM6dNrP/AvyzmRUAs8CHgHFgf3BfL74uHOC9wOeDcH0eeH+w/mHgC2b28eA5/m0G34aISN4z5671F0gREcl1ZjbmnItmux0iIuKp7EREREREJEPU8y0iIiIikiHq+RYRERERyRCFbxERERGRDFH4FhERERHJEIVvEREREZEMUfgWEREREcmQ/w/Z2dP4XT+LGAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x864 with 3 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# summarize history for f1\n",
    "fig, ax = plt.subplots(figsize=(12,12))\n",
    "ax = fig.add_subplot(2,1,1)\n",
    "\n",
    "ax.plot(history.history['f1'])\n",
    "ax.plot(history.history['val_f1'])\n",
    "ax.set_title('model f1')\n",
    "ax.set_ylabel('f1-score')\n",
    "ax.set_xlabel('epoch')\n",
    "ax.legend(['train', 'dev'], loc='upper left')\n",
    "\n",
    "ax = fig.add_subplot(2,1,2)\n",
    "# summarize history for loss\n",
    "ax.plot(history.history['loss'])\n",
    "ax.plot(history.history['val_loss'])\n",
    "ax.set_title('model loss')\n",
    "ax.set_ylabel('loss')\n",
    "ax.set_xlabel('epoch')\n",
    "ax.legend(['train', 'dev'], loc='upper right')\n",
    "fig.savefig(\"/content/gdrive/My Drive/keras_checkpoints/History_keras.pdf\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 319
    },
    "colab_type": "code",
    "id": "4j4bXU2K9sVa",
    "outputId": "c718df85-6ee4-45cd-cb9d-364e0eb64618"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9976/9976 [==============================] - 1s 103us/step\n",
      "\n",
      "Test Binary_cross_entropy: 0.4631\n",
      "\n",
      "Test precision: 0.7815\n",
      "\n",
      "Test recall: 0.7815\n",
      "\n",
      "Test f1: 0.7815\n",
      "\n",
      "Test accuracy: 0.7815\n",
      "+-------+-------------+----------+-------+\n",
      "|       |   precision |   recall |    f1 |\n",
      "|-------+-------------+----------+-------|\n",
      "| 0     |       0.78  |    0.792 | 0.786 |\n",
      "| 1     |       0.783 |    0.77  | 0.777 |\n",
      "| macro |       0.782 |    0.781 | 0.781 |\n",
      "+-------+-------------+----------+-------+\n"
     ]
    }
   ],
   "source": [
    "score = model.evaluate(\n",
    "    X_test,\n",
    "    y_test2,          \n",
    "    batch_size=32,\n",
    "    verbose=1)\n",
    "\n",
    "print('\\nTest Binary_cross_entropy: %.4f' %  (score[0]))\n",
    "print('\\nTest precision: %.4f' %  (score[1]))\n",
    "print('\\nTest recall: %.4f' %  (score[2]))\n",
    "print('\\nTest f1: %.4f' % (score[3]))\n",
    "print('\\nTest accuracy: %.4f'% (score[4]))\n",
    "\n",
    "#128 64 32 -> best 0.7696\n",
    "y_calc = model.predict_proba(X_test)\n",
    "prec_dict, rec_dict, f1_dict = get_scores(y_test, y_calc, 0.5)\n",
    "scores_df = pd.DataFrame.from_dict({'precision': prec_dict, 'recall': rec_dict, 'f1': f1_dict})\n",
    "print(tabulate(scores_df.round(3), headers='keys', tablefmt='psql'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 454
    },
    "colab_type": "code",
    "id": "IdVCfI2D4lR5",
    "outputId": "6773c7d3-6657-4040-88b6-4c1be6baf0d4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Keras MLP classifier- Train scores:\n",
      "+-------+-------------+----------+-------+\n",
      "|       |   precision |   recall |    f1 |\n",
      "|-------+-------------+----------+-------|\n",
      "| 0     |       0.787 |    0.818 | 0.803 |\n",
      "| 1     |       0.814 |    0.782 | 0.797 |\n",
      "| macro |       0.8   |    0.8   | 0.8   |\n",
      "+-------+-------------+----------+-------+\n",
      "---------------------------------------------------------\n",
      "Keras MLP classifier- Dev scores:\n",
      "+-------+-------------+----------+-------+\n",
      "|       |   precision |   recall |    f1 |\n",
      "|-------+-------------+----------+-------|\n",
      "| 0     |       0.766 |    0.783 | 0.774 |\n",
      "| 1     |       0.776 |    0.759 | 0.768 |\n",
      "| macro |       0.771 |    0.771 | 0.771 |\n",
      "+-------+-------------+----------+-------+\n",
      "---------------------------------------------------------\n",
      "Keras MLP classifier- Test scores:\n",
      "+-------+-------------+----------+-------+\n",
      "|       |   precision |   recall |    f1 |\n",
      "|-------+-------------+----------+-------|\n",
      "| 0     |       0.78  |    0.792 | 0.786 |\n",
      "| 1     |       0.783 |    0.77  | 0.777 |\n",
      "| macro |       0.782 |    0.781 | 0.781 |\n",
      "+-------+-------------+----------+-------+\n"
     ]
    }
   ],
   "source": [
    "print_classifier_scores(model, 'Keras MLP classifier', X_train, y_train\n",
    "  , X_dev, y_dev\n",
    "  , X_test, y_test,0.5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "vTXN29bhnAfM"
   },
   "source": [
    "#### Precision - Recall Curve\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 981
    },
    "colab_type": "code",
    "id": "_cTzdA55Fkf5",
    "outputId": "8a67ce13-11d9-479d-bfeb-d6c120f4b737"
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsoAAAHiCAYAAADxgeqGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdeXxU5fn///eVFYEgYXFBNncBUZAgSq3iQlUK2FariHVv/eqn2vqrXahtraXVj+3HtrTWurQutRXB1o0q1mIr4hpZRBEQQYQQECSsYQlZ5v79cc6Ek8mZZJIMOZPwej4ePMicc+ace5bANddc93Wbc04AAAAA6sqKegAAAABAJiJQBgAAAEIQKAMAAAAhCJQBAACAEATKAAAAQAgCZQAAACAEgTLQjpjZ581sWQrH3Wpmf26NMbUGM5ttZl/3f77KzF6PekzNYWYHm9kcMys3s19HPZ6mMLO+ZrbDzLKjHgsApAuBMtoNM1tlZpVm1iNh+7tm5sysfzQjqx3HbDOr8IOJMjN72swOTec1nHOvOeeOTeG4O51zX0/nteP812G3/zjXm9mjZtZ5X1yrOcwsz8xuN7PlZrbTH+/DUb8/fNdJKpPUxTl3S0tP5j/WX5tZqf96rDKzKS0fZu3rfE78tnOuxDnX2TlXk47zN3EszsyOamB/nQ9PZtbFzN4ws6fMLK91RpkaM+vsv1Yvhuyr9zj99/LfAre7mNkUMyvxz/Oxf7tH4vkaGUc3M3vG/x1ZbWYTGzg238zuN7MNZrbZzP5pZocF9gf/7duR+GHezG4ys0/MbLuZzTOz05oyVmBfIlBGe/OJpEvjN8xssKSO++JCzcyc3eic6yzpGEldJf025Lw5LR1bBhjnP84hkoZK+mHE4wn6h6TxkiZKOlDSiZLmSzq7qSfaB69VP0lLXDNWgkoylh9KKpJ0sqQCSaMkLWjJANs6MyuU9B9JqyVd4pyrbMJ9W+N380JJeySNNrNDmnJHP+j/j6RBks6T1EXSqZI2yXsPNMW9kiolHSzpMkn3mdmgJMd+27/OCZJ6Sdoi6Z6EY270P0h1Dn6YN7MRku6SdJG838eHJD3DNxPIFATKaG/+KumKwO0rJT0WPMDMvuhnmbeb2Rozuz1h/2lm9qaZbfX3X+Vvf9TM7jOzmWa2U9KZZjbAz5ZsNbPFZjY+lUE65zZLekrS8f65V5nZD8zsfUk7zSzHzE4JjOM9MxsVGGM3M3vEzNaZ2RYze9bfPsrMSgPH/cDM1pr3Vf4yMzvb356YhRrvj3+r/3gGBPatMrPvmtn7ZrbNzKabWYcUH+d6SS/JC5jj52vO4yo0s+fNbKO//Xkz653KGIL8DOhoSRc45+Y656qdc9ucc/c65x4KPN5zAvepfa7MrL+f1bvWzEok/dfMXjSzGxOu856ZfcX/+Tgzm+Vn2paZ2cVJxvaovPfr9/2s2zl+pm6K/3ys83/O948fZV6m+Admtl7SIyGnHS7pGefcOudZ5Zx7LHDNXuZlVTf6Gb1vJTzuJ83sMf/9s9jMivx9f5XUV9I//bF+P/Dc5PjHzDazX/iv9Q7zsozdzexx/3dvrgWy+A09T/7v3r1m9oI/lmIzO9LfN8c/7D3/Opc08Pr3lPSKpA8kfc05V+1vH2tmC/335JtmdkLgPmG/m5PMy9SWm9kSM/ty4PijzOxV/3elzMymJxtPEldKul/S+5K+1sT7XiHvdfmyc26Jcy7mnPvMOfdz59zMVE9iZp3kBew/cc7tcM69LmmGpMuT3OVwSS855zY45yokTZcXrKeiv6TFzrn5/gfExyT1kHRQquMF9innHH/40y7+SFol6RxJyyQNkJQtqVRels5J6u8fN0rSYHkfFE+QtEHSl/x9/SSVy8tK50rqLmmIv+9RSdskfc6/b4GkFZJulZQn6Sz/vscmGd9sSV/3f+4h6b+S/hoY+0JJfSQdIOkweVmgMf61Rvu3e/rHvyDvP6NCf5xnBB5bqf/zsZLWSOrl3+4v6Uj/59sl/c3/+RhJO/1r5Er6vv+48gJje0depqibpKWSrm/sdfB/7i1pkaTf+beb+7i6y/uPu6P/vP9d0rNJnturJL2eZGx3SXo1lfdR4Hbwueov7730mKRO/mt1haQ3AscPlLRVUr5/zBpJV0vKkZddL5M0MMm1H5X0i8DtyZLelhc09JT0pqSfB17rakm/9K91QMj5fiypRNL/yHvPW2BflrxM+m3y3r9HSFop6dzA467wX6tsSf8r6e0Gnqf4c5MTeE1WSDpSXqZwiaSP5P2O5vjP4SP+sQ0+T/7zEs+K5kh6XNK0wLWdpKMaeE2v8q+/WNJ9Cc/DUEmfSRrhP84r/ceWH/a76W/7qrzfhyxJl8j7/TnU3/eEpB/5+zpIOi1wreclTWpgnP0kxeS9h26R9H7C/nqPU3Xfn9Mk/aWR9/fz8t6fYX+eDzwnuxLu911J/0xyziJJb/jPSUdJUyVNSfj93Oi/pm9IGhXY10Xe+zD+/N8k6d3ga8Qf/kT5h4wy2qN4Vnm0vKBubXCnc262c26R87It78v7j+0Mf/dESS87555wzlU55zY55xYG7v6cc+4N51xMXpa0s6S7nHOVzrn/yvtP6FIl93sz2yrpPUmfSvpOcJ9zbo1zbre8TNJM59xMf5yzJM2TNMa8uubz5QWrW/xxvhpyrRp5AdRAM8t1Xjbx45DjLpH0gnNulnOuStLd8gLAkQljW+e8TPg/FcgQJ/GsmZXLC34+k/RTf3uzHpf/OjzlnNvlnCuXdIf2vmZN0V3e895Stzvndvqv1TOShphZP3/fZZKeds7tkTRW0irn3CPOy16/K++bhK+meJ3LJE12XlZwo6SfqW5WLybpp865Pf5YEv2vvED6MnnP81ozu9LfN1zeB5TJ/vt3paQ/SZoQuP/r/mtVI+/36sQUxx33iHPuY+fcNkkvSvrYOfey8zK5f5cXkEmpPU/POOfe8e/7uBp/DybqI+9D4aPOuWBpy3WSHnDOFTvnapxzf5FX+nBK4Jjg76acc3/3fx9izrnpkpZrb2lDlbyAt5dzrsJ52Vj59xvrnLurgTFeLi84XiIv6B1kZkMbOD5Ro+9vfwxdk/wZ6x/WWdL2hLtuk/chNcxyeb/ra/37DZD3IS/uB/I+iB0m6UF530Qc6e8rl/davy7vef+ppOsSXiMgMgTKaI/+Ki/gvUoJZReSVxNnZq/4Xzdvk3S9vAyv5P1nGhZMxq0J/NxL0ho/aI5bLe8/g2S+5f+HdJhz7jI/+Ak7dz9JX/W/Ct7qB9enSTrUH+Nm59yWBq4j59wKSTfLyzh9ZmbTzKxXyKG9/HHH7xfzxxJ8HOsDP++S9x+p/LKD+ASdywLHfMk5F6+JPU57n99mPS4z62hmD5g3qWi7pDmSulrT6xg3+ddqqdrXyg/cX9DeAPNSeYGc5D3eEQmP9zJJqdae1nlt/J+Dr+FG533VHcoP/O51zn1OXk38HZIeNq+0pp+kXglju1VeTWpc4uvewZpWp7sh8PPukNvxSZ6pPE+h78EmeE9eVvTFhOCzn6RbEq7dR3Wf5+DvpszsikCpxlZ5JVTx9/j3JZmkd/xylWuaMMYr5L93nHNrJb0qL8MdVyPvm5agXHnBuZS+9/cOeZneoC7ygtow98r7UN5d3rcDT8v7YCRJ8j+ElPsf6P4iL6s8xt99rbxvEgbJ+2bja5KeT/JvFdDqCJTR7jjnVsub1DdG3j/YiabKq7fr45w7UF49oPn71sj7qjjp6QM/r5PUx8yCv0d9lZDBbsrQAz+vkVeWEcz2dPKzUWskdTOzro2e0LmpzrnTtLf85Jchh63z90uSzMzkBQqNPg7n3Plu7wSdx0P2vyrva/O7W/i4bpFXSjLCOddF0unx4TY2xgQvSzrZGq5v3qm6E0DDgtrEbNcTki41s1Plfd3+ir99jbxSj+Dj7eycuyHF8dZ5beS9v9Y1MI6knHO7nXP3yptoNdAf2ycJYytwzo1p+ExNv3YKWvo8pcQ59zt55TezzOz4wLXvSLh2R+fcE8G7xn/wvzn4k6QbJXV3znWVV/Ns/jXWO+e+4ZzrJen/SfqjNdCRI3DekZKOlvRD87rFrJdXjjAx8OGkRF6JS9Dh2vth6mVJ5/o1xsmuE/xwm/gnHtx+JCnHzI4O3PVEeaUrYYbIy9Rv9r9JuUfe71myThtOe393h8gr+fjIz9D/S15WfGSS+wKtikAZ7dW1ks5yzu0M2VcgL3NZYWYny8s+xz0u6Rwzu9iftNPdzJJ9xVssL7P1fTPLNW9S2jh5X5m21N8kjTOzc80s28w6mDd5q7dz7lN52Zo/mjfJLdfMTk88gZkda2ZnmTf5q0JeBi+WeJykJyV90czONrNceUHpHnn1sOkwRd4M/hNb8LgK/PFvNbNu2lvK0STOuZclzZI3q36Y/xoXmNn1gczfQkkT/OsXyZuN35iZ8gLayZKmB75leF7SMWZ2uX++XDMbboHJko14QtKPzaynH3TcJu85TImZ3ew/vwf4j/VKec/lu/LqzsvNm6h2gP96HG9mw1M8/QZ5X6enQ0ufp5TH4pz7laTfSXrZzI6VF/Reb943TWZmncyb8JuszKCTvEBvoySZ2dXyJ+X6t78a+CC2xT827Pcu0ZXy3psD5QWPQ/zzHiCvJEny6vd/bGa9zSzLvEmn4+R1cpG8b9PWSHrKvMmRWf6/Ybea2Rj/8Qc/3Cb+Od8/Zqe8JMNk//n4nKQL/POHmSvpCjM70P835H8krXPOlZlZV//3vYP/HrxM3gfdfwXu+0UzO8J//kfLK5H5IIXnDNjnCJTRLvl1kfOS7P4fef8BlMsLPJ4M3K9EXib6Fkmb5QVNoXWZzmsrNU7ef2Jlkv4o6Qrn3IdpGP8aef8x3SrvP+Q1kr6nvb+zl8v7uvVDeTXAN4ecJl9e9qxM3tfWBymkTZtzbpm8rzvv8Y8dJ6+9W8ptsxp5LBvllcDc1oLHNUVewFAmb3Jb/D/Z5rhIXmA7XV7d5QfyJiO97O//ibxvFbbIqwmemsJj3CMvsDgneLxflvEFeWUZ6+S9DvHJd6n4hbza4vflTYpc4G9L1S5Jv/avWybpm5IudM6tdF7d8Vh5Adkn/v4/y5t4l4r/lRe0bTWz7zZhTPWk4Xm6XdJf/LGEdhVJuN7P5T3W/8h7nb8h6Q/+zyvklW0lu+8Sec/pW/IC9MHySgnihksqNrMd8r65+rbz6r/j2dxbE89pXheZiyXd42ek438+kRecxssvJsv7APu6P9ZfSbrMOfeBP7Y98t6DH8oLurfL+0DUQ94H+6b4H3m/c5/J+8B2g3NusT/ez/uPL+678j6ML5f3ez1GUrwTSK6892x8Mt9N8kqzPvL3PyYvuTDbH+/vJf2/dPw7CqSDUS8PAAAA1EdGGQAAAAhBoAwAAACEIFAGAAAAQhAoAwAAACEIlAEAAIAQTVlhKa169Ojh+vfvH9XlAQAAsJ+YP39+mXOuZ1PvF1mg3L9/f82bl6zNLQAAAJAeZra68aPqo/QCAAAACEGgDAAAAIQgUAYAAABCRFajDAAAEJWqqiqVlpaqoqIi6qEgjTp06KDevXsrNzc3LecjUAYAAPud0tJSFRQUqH///jKzqIeDNHDOadOmTSotLdXhhx+elnNSegEAAPY7FRUV6t69O0FyO2Jm6t69e1q/JSBQBgAA+yWC5PYn3a8pgTIAAEAE7rjjDg0aNEgnnHCChgwZouLi4madZ+HChZo5c2bt7RkzZuiuu+5K1zBDzZ49W2+++WbovkcffVQ33nijJCkWi+nKK6/UNddcI+fcPh3TvkCNMgAAQCt766239Pzzz2vBggXKz89XWVmZKisrm3WuhQsXat68eRozZowkafz48Ro/fnw6h1vP7Nmz1blzZ40cOTLpMc45XX/99aqqqtIjjzySUra3urpaOTmZE56SUQYAAEjB/NVbdO8rKzR/9ZYWn+vTTz9Vjx49lJ+fL0nq0aOHevXq5V1n/nydccYZGjZsmM4991x9+umnkqRRo0bpBz/4gU4++WQdc8wxeu2111RZWanbbrtN06dP15AhQzR9+vQ6Gd2rrrpKN9xwg0455RQdccQRmj17tq655hoNGDBAV111Ve14/v3vf+vUU0/VSSedpK9+9avasWOHJG8l5Z/+9Kc66aSTNHjwYH344YdatWqV7r//fv32t7/VkCFD9Nprr4U+xm9961vatGmTHnvsMWVlZWnnzp265pprdPLJJ2vo0KF67rnnJHkZ6PHjx+uss87S2WefrR07dujss8+uvWb8uJ07d+qLX/yiTjzxRB1//PGaPn16i1+HxmROyA4AABCBn/1zsZas297gMeUVVfpwfbliTsoy6bhDClTQIXkLsoG9uuin4wYl3f+FL3xBkydP1jHHHKNzzjlHl1xyic444wxVVVXppptu0nPPPaeePXtq+vTp+tGPfqSHH35YkpdxfeeddzRz5kz97Gc/08svv6zJkydr3rx5+sMf/iDJCzyDtmzZorfeekszZszQ+PHj9cYbb+jPf/6zhg8froULF6p37976xS9+oZdfflmdOnXSL3/5S/3mN7/RbbfdJskL4hcsWKA//vGPuvvuu/XnP/9Z119/vTp37qzvfve7oY9v6tSpGjBggGbPnl2bIb7jjjt01lln6eGHH9bWrVt18skn65xzzpEkLViwQO+//766deum6upqPfPMM+rSpYvKysp0yimnaPz48frXv/6lXr166YUXXpAkbdu2rcHXLB0IlAEAABqxvaJaMb/ENua82w0Fyo3p3Lmz5s+fr9dee02vvPKKLrnkEt11110qKirSBx98oNGjR0uSampqdOihh9be7ytf+YokadiwYVq1alVK1xo3bpzMTIMHD9bBBx+swYMHS5IGDRqkVatWqbS0VEuWLNHnPvc5SVJlZaVOPfXU0Gs+/fTTKV3zpJNO0ocffqh33nmn9rz//ve/NWPGDN19992SvM4jJSUlkqTRo0erW7dukrySjVtvvVVz5sxRVlaW1q5dqw0bNmjw4MG65ZZb9IMf/EBjx47V5z//+ZTG0hIEygAAYL/WUOY3bv7qLbrsz2+rqjqm3Jws/W7CUA3rV9ii62ZnZ2vUqFEaNWqUBg8erL/85S8aNmyYBg0apLfeeiv0PvFSjezsbFVXV6d0nfh9srKyan+O366urlZ2drZGjx6tJ554Im3XPO644zR58mRdfPHFeumllzRo0CA55/TUU0/p2GOPrXNscXGxOnXqVHv78ccf18aNGzV//nzl5uaqf//+qqio0DHHHKMFCxZo5syZ+vGPf6yzzz67Nuu9r1CjDAAA0Ihh/Qr1+NdP0Xe+cKwe//opLQ6Sly1bpuXLl9feXrhwofr166djjz1WGzdurA2Uq6qqtHjx4gbPVVBQoPLy8maP5ZRTTtEbb7yhFStWSPJqgT/66KMWX3PkyJG67777NHbsWJWUlOjcc8/VPffcU9v94t133w2937Zt23TQQQcpNzdXr7zyilavXi1JWrdunTp27Kivfe1r+t73vqcFCxY09aE2WaMZZTN7WNJYSZ85544P2W+SfidpjKRdkq5yzu37kQMAALSiYf0KWxwgx+3YsUM33XSTtm7dqpycHB111FF68MEHlZeXp3/84x/61re+pW3btqm6ulo333yzBg1KnvU+88wzddddd2nIkCH64Q9/2OSx9OzZU48++qguvfRS7dmzR5L0i1/8Qsccc0zS+4wbN04XXXSRnnvuOd1zzz1JyyDGjRunsrIynXfeeZo1a5YmT56sE044QbFYTIcffrief/75eve57LLLNG7cOA0ePFhFRUU67rjjJEmLFi3S9773PWVlZSk3N1f33Xdfkx9rU1ljPe3M7HRJOyQ9liRQHiPpJnmB8ghJv3POjWjswkVFRW7evHnNGjQAAEBLLF26VAMGDIh6GNgHwl5bM5vvnCtq6rkazSg75+aYWf8GDrlAXhDtJL1tZl3N7FDn3KeNnXv+6i2aPrdEzjmNO/EwmUnvl27TKUd0T9snNgAAAKA50jGZ7zBJawK3S/1tDQbKuyprdMkDb6nan0L69/lrJUkmKT83Ky31PwAAAEBztepkPjO7zszmmdm8zzZvqw2Sg5ykquqY3l65qTWHBgAAANSRjkB5raQ+gdu9/W31OOcedM4VOeeKDup2oLKz9i5lmJNtit/MzcnSKUd0T8PQAAAAgOZJR+nFDEk3mtk0eZP5tqVSn9wxL1tfGd5XfyterUuKeuvi4X1VvHKTfvXSMv3kiwNDyy7e+WSTZi/bqN6FHbVlV6U652drx54aapoBAACQdqm0h3tC0ihJPcysVNJPJeVKknPufkkz5XW8WCGvPdzVqV68V2EHSdLPLjheHXKzdXiPTvrVS8u0dXdVnePe+rhMv3t5ud7+ZHPoefKys3RRUW8d3+tALVq7VTUxp0uG9yV4BgAAQLOl0vXi0kb2O0nfbM7F453pzC+56NYpT8cdUqC3V27SN888Sn+ft0YPzlmp5Z/taPA8lTUxTS0uqbPtqQWlOvu4g9WjIF8XntS7TtA8f/UWvb1yU215x9MLSrWxfI96FOTr+F4HasuuSrLUAABgnzIzfec739Gvf/1rSdLdd9+tHTt26Pbbb0/p/hs2bNC1116rNWvWqKqqSv3799fMmTM1e/Zs3X333fV6FM+YMUNLlizRpEmTdPvtt6tz58767ne/q6uuukpjx47VRRddlO6H2OZFuoR1vIdzlu2tVT7liO6aWlyir/zxDS0o2VrvPiZvwl/872RqYtK/l2yQJE1/p0Tf+PwRKjggV1t3VerhN1apJuaUZSbnXOh58rJNFxX1qRM4S6oNsAmiAQBAS+Tn5+vpp5/WD3/4Q/Xo0aPJ97/ttts0evRoffvb35Ykvf/++w0eP378eI0fP75ZY91fRRoox5teWGBbz855qqyJ1QuSTd4kv4uG9a4NXgs75mnxum36+7w1qo45hTTRkCTVOOn+OStDrp881K6scXWy1FnmZcCdpNws01eH99aFJ/VJS8Acz3AXdsyrk80OZr4TrzN/9RY9vaBUTqqXMQcAAJkvJydH1113nX7729/qjjvuqLNv1apVuuaaa1RWVqaePXvqkUceUd++fesc8+mnn+oLX/hC7e0TTjih3jXmzp2r6667Tv/4xz/02muvad68efrDH/6wbx5QOxRxRtn72wIZ5YrqWJ1jggFysoDwKyf1rg00F6/bpo3le/TfDz8LbT8XlG0my5Kqaxo+TlKdILwq5jS1eI2mzy3VN047XOV7qpMGrMEgePG6bXKSju91oNZt3a0aF9NHG3bo1WUbVRPbm9nOzTZ97sjuev3jTYrFnLKzTF8ZepiOO7SLNpbvkcnpgTmf1D6+J+eu0U/HDdT2iup6wTYAAEjBI1+sv23Ql6STvyFV7pIe/2r9/UMmSkMvk3Zukp68ou6+q19I6bLf/OY3dcIJJ+j73/9+ne033XSTrrzySl155ZV6+OGH9a1vfUvPPvtsvftecskl+sMf/qBzzjlHV199tXr16lW7/80339RNN92k5557Tn379tVrr72W0piwV8QZ5Xjpxd5to449SH96baWqqmPKzm44QI4LW3t9/uoteuDVj/WfpRsUc3XLNUxSdpZp8gXH69hDCurVKCdmqU1SbnaWJKeqmr0BbU3M1clUPzl3jS4e3qf2HBu2V2j2RxtVUxNe3pFMVY3T7I/K9j5PNU7T55UmPb465vST5xbX2ZaTZbry1H4q7JSnU4/sQdAMAEAG6tKli6644gr9/ve/1wEHHFC7/a233tLTTz8tSbr88svrBdKSdO6552rlypX617/+pRdffFFDhw7VBx98IMlbxvm6667Tv//97zrBM5om2oyy/3cwozysX6Ee//opLa4FHtavUA9eUVSvrCEs49pYljpYo/z0glJNe6dEYUno6pirN6kwVY3VXCcea7Y3y50lKZZwTHXM6aE3VkmSsm25Lj25jzaU71GWmT5/dA9t211F1hkAgLiGMsB5HRve36l7yhnkMDfffLNOOukkXX11yo3DanXr1k0TJ07UxIkTNXbsWM2ZM0fdu3fXoYceqoqKCr377rsEyi0QaaAs52RWf3NYhri5mnuuZPcb1q9Qg3odqNue+6BOuURDwoLg+LYsSTmB2ut4Nrsm5pSdnaVRx/TU7GWf1Wa3syTl5WbptrGD6pRyTH5+sSqrY6F12jXO6W+BAP6lxesleVn1b5x2uAoOyK3zQSJ43vjP1EEDALBvdOvWTRdffLEeeughXXPNNZKkkSNHatq0abr88sv1+OOP6/Of/3y9+/33v//VKaecoo4dO6q8vFwff/yx+vbtq507d6pr16566KGHNHr0aHXq1EmjRo1q5UfVPkQ+mS8kTs54E0f01bF+G7tgXfQryz5TTcKkwpxs08VFfeoEncHJiGH1xPFsduKkvobqj+PjKd9dpT+//kltEJ8lKSvLQuu1E0tHGhIvKyFgBgAg/W655ZY6k+zuueceXX311fq///u/2sl8iebPn68bb7xROTk5isVi+vrXv67hw4dr9uzZkqSDDz5Yzz//vM4//3w9/PDDrfVQ2hVzDXR+2JeKiorcJXf8TQ+8ulIr7hwTyRjSLWziXhSBZVi5ye3/9DLOLZWbbfreucdqddkuOasf9DOZEADQFixdulQDBgyIehjYB8JeWzOb75wrauq5os8ot8WUchLpLBlJ9zjikxadpC75OXWyzqn2ppa8iYZ3zvyw0TFkm+nCYYepS4dc5edmaVXZLnXKz2bFRAAA0GZE3h7O2lOknMESg+fRgw4JneQYVqO8sXyPZvtlJTLz/m5EjXN6MqRTx9/nlWrM4EP0uaN6plyKAgAAEIXIV+YjTI5GU7PfwXKOxDKOpmSknaQXFq3XC4vWJz0mO8t01nEH6cxjD2IyIQAAiEzk7eFIKLcNwcA6WMaRmA1OnEzYlLZ3cTUxp1lLNmiWvwS5xGRCAED6Oef4ZrudSffcu8gzylm8QducxrLRYWUdwRKOsOXGGwuo4z2q/z53jS4cdphO6F1Yr781mWcAQKo6dOigTZs2qWvC8FoAACAASURBVHv37gTL7YRzTps2bVKHDh3Sds7oJ/NFOQDsEw0F0smW9I5no//kZ6OTqYo5TZtbqmlzw1cqDK6OuGVXpTrnZ2vZ+h1SoEMHNdAAgN69e6u0tFQbN26MeihIow4dOqh3795pO1/kk/nIKO9fUslGB8s6gpnoqhSWAm9sdcT4cuSnHd1d3TvlaWjfbqGTFyWpR0G+LjzJ+2WL97WO/8zEQwBo23Jzc3X44YdHPQxkuIgzyqSUUVeyQHr+6i16ekGp/j5vTZ1VCnNyshRzTtVha4qHcJIqa2L674deBuHv89c2ePy0d0ok590vO8sUc3XLRrLNdNaAuhMPyVwDANA+RLuEtcgoIzXxADq+amEwoyt5NcqJNdDNmUiYKBgUh65u6OpPPIzLyTJ93V8inKAZAIC2J/KMMnEymiJZxjm+LWxVwsTSiunz1qScgW6J6sAS4TlZpqF9u6rrAXk68qBOWre1Qrsra2TmlXgEyz7it2cv+0wbtlfokuF9NXFE39rzxh8jwTcAAPsWNcpoV1LpD/2Vk3rXq4MOq1GWVJuhNjOZ/JIPM4094VD98/1PU1p8RfKC5rmrtkiSZi1t2mN6r3SR/vrWKvUq7KA9VTG9vXKzamJOWfSbBgBgn4o+oxzlALBfaspiK8HsraQ6mdzLT+0fGnAnLhGeDkvXl2vp+vI628L6TU97p0RFfQvVtVNebWaaemkAAJqHBUeABiQG1Yk/Jws+g72kW7PcI+akd1Zvqbc920zD+nXVUQcX1AnqyUADAJBc5KUXNPlGe5QYRIeVewTb0IXVKD/7bqneWRUS9GZ5vzspVn1I8iYdvrNqS73zTSsu0dkDD9aZxx5UZ4VF2t8BABB5oEzpBfYPTSn3iJs4om9tW7yw3s7BwHv2ss/08tINTQqeJSkmJe3aIUm52aaRR3ZXfk42pRwAgP1O5BllJvMByTUUYAe3Jwuqu+TnNLraYUOqapxe/ais3vYsk4b26arunfNrrxUPooNLlsf3UeIBAGiLop/MR5wMpEWyoDq+2mFi67nEDHRT+k7HnDS/ZGvKY5v2TokmntxXB3XJV4/OHaiRBgC0CZFP5iOjDOxbYQF0MAMdXE2wfHdVbceOrCwvdK6JtXwMMSf9LWRp8WnvlOjsAQfr+jOOJGAGAGScyDPKAKLRUAY62BIvmI1urJSjqashxpxXI/2fJRt09sCDdWSPTlr86XYNOrSLyvdU1+lpLYk6aQBAq4p2CWsnZWVFOgIACRpqiSfVLeWIS1ajvHVXpeav3qKY89tBKjyQrp1U6N9+bXn9uuhEWSadcNiBysvJ0p7qWL0VDAEAaKnIM8pG3wugTWlqB4+wZcWb26UjKOakhaXbam/HVzAc2q+Q2mcAQFpEXqNMiTLQvjVUI/3Aqx+3OGAOiq9gOLW4RAMOKSBoBgC0CO3hAERiWL9CPXhFUZ1JhV3ycxqtUU615V08aJ4+t0RnHXtwvQ/ltK0DADQmA0ovAOzPmrMYS7BOes3mXVq6vjzpsTUxadbS8AVVniguUVG/QhV2yiNwBgDUQ+kFgDYnMbiOZ6UXrN7SYNCcyEmau3rvst7Tikt0Yp8DVdgpTzn+TON4Fjue6S44IJeOGwCwn4h+CWsiZQAtFAycpxaXaPrcEuXnZKlrxzz9d9lnqq5JrQg6JundNduS7o9348jOMl1xSl9VVjtt3MEKhADQXmVAjXKUIwDQ3kwcUbdNXOLS3pK0dVel5vlt65qjJub0yJur622fWlyi4f0L9eWhvVl9EADagQyoUSZSBrDvJKuBDgbQLQ2cg+au2qK5q/aWczxRXKJDD8xX78KO6toxTxLZZwBoKyLPKFN5ASAKyeqck3XaWPzpdnXvlKfn3/9UNTGX8gqETtK6bXu0blvd8z7hZ5+DwTOrDgJAZok4oyxqlAFkhFS7b1x+av/aBVTiKxA21nkjjJP0TiDzHJRl0tC+XdW9U7627qrUnuqYDu/RSZt2Vur84w9lBUIAaCXRLmEt2sMBaFuSBdTxSYQHdemgM489SLOXfaZZSzaknHkOijlp/uqtdba9569C+NryMv31rVXq061jnf3xzPdbKzcpPydLRx1cQHkHALSQOZemJbGaqKioyJ14433aUF6h52/6fCRjAIB9Kb764MqNO9StU566dsxLaz10Y0zSBUN6qVN+jpZvKNfmnZU6vGdnXX/GkQTQAPYrZjbfOVfU1PsxmQ8A9pH46oOJwuqhm7LqYKqcpGcXrquzbcXGnXp5yQYN71+oow4u0PG9DqwtIQmOhWw0AEQ9mU+0hwOw/2moHjq46qCk2hrlAw/I1WsrypSOLwHj9dHJaqSlvasWHn1IgTrnZav4k82UdADY70Q+mY+2FwCwV0NBdLLOHFLdGuWtu6q0evOuFo0jvmphcOVCyQuwpxaXaMAhBerTrSPZZwDtWuQr85FRBoDUpNqZQ/ImF774wacadGgXle+p1vIN5Vq7rULrtuxu1gTDREvXl9d2+niiuESDDztQHXKzqIMG0K5E30c5ygEAQDuVuEJhXDArHe/dHKxRbs5kQyfp/bV7l/4O1kF37ZhH1hlAmxVxjbJTFqUXANBqUslKJwbTzSnpSOwT/URxiY4/rIt6FuQrJytL0t7660uGhwf1ABC1aGuUY5QoA0CmaSiYjveLzs/xgt1Us89O0qK120P3vVe6SA+8+rGOPaSA7DOAjBJ5Rpn2cADQdiSWdASzz1t3VTa7Dnr15l212eqpxSXq162junbMJdsMIFLR1ygTJwNAmxWWfU4MnpuzwIoXOHvZ5vtfXaHjDukiiR7PAFpX5IGyX6oGAGgnEoPnZG3tPtpQrlWbGq95Ltm8WyWbd9fefqK4RL26dtBhXQ+QJG3eWalunfLo8Qwg7SIvvcgyImUAaM9SrXkur6iubTnXECdp7dYKrd1asXfjxp11ejwXdMhhoiCAFot8wRFKLwBg/5VY8xwPnDdsr9D67fUXVklFMNiOTxQ8uEt+beaZlnUAUpUBC44QKQMAPMHAOZht7toxT1t3VWruqi0tmiiojTtrtz9RXKLjDilQVU2MRVIAhIo8owwAQJiwRVPmr96iB179WCs37lC3TnmSvBrlqhrX5GW7nfZmn1ds3KlZSzboML/2mawzACnqjLJERhkAkLJh/Qr14BVFofuCGejPyvekNFEwUWLt8xPFJTq8R0eZmbpTtgHsdyIvvSBOBgCkQ7J65/jiKJt3Vio3O0sfri9PuXzDSVpZ5gXcHyeUbZzUt6sKO+Vp++4qJg4C7VTk7eGIkwEA+0JY6YYUvkjK2i27Q86QnJM0v2RrnW3vlS7Sb2YtU8/O+aqqidGyDmgHMqA9HKEyAKD1JFskJV773NSsc1DZjkqV7aj0bvgt64J9nwmcgbYl2sl8MdrDAQCil1j7nJh1bk7ZRlyw73M8cKbbBtA2RD6Zz4iUAQAZJtkiKWGrDK7ZvCulhVLiwrptDO9fqEnnDyBgBjJM9JP5ohwAAABN0FAAHSzdqKqJNSkDPXfVFl1435vqUZCn/OwsyjSADBH5ZD5qlAEAbV2ytnXxAHrxp9tTmjBYVu7VN8fLNKYWl5BtBiIU8YIjtIcDALRfwQA6WLbRlHKNeLa5X7eOys026pqBVhR5jTIZZQDA/iCxbKOp2eb4yoPxuuZ44JybnaW8nCz6OAP7QOQZZYqUAQD7o8Rsczxo3lNdU1uC0ZDEJbvfK12k37/8kTp3yCHrDKRJpIGyiJMBAAhtT/fAqx/r3ZIt2rij8aA5bn35Hql8T23WeQBt6IAWofQCAIAMEwyc40txV1bHVLZjT5MC58Q2dId17aCBvQ4kaAZSFHnpBXEyAADJJS7FHQycq2pi2rGnWuu372ngDHvFFz6ZtWSDenTO09C+hQTNQANoDwcAQBuSGDhLe4Pnrbuq6tUuJ1O2o1KzlmyoDZqP6NGJ3s1AgugzylEOAACAdiAYPDenDV3ZjkqV7aikdzOQIKVA2czOk/Q7SdmS/uycuythf19Jf5HU1T9mknNuZmPndY4lrAEASKewNnRPLyjV8g3l+qRsZ0o1zvHezQMOKVDZjj2qiTldXNRHk8YM2JdDBzJOo4GymWVLulfSaEmlkuaa2Qzn3JLAYT+W9KRz7j4zGyhppqT+jZ3bUaMMAMA+lRg4Ty0u0R9fWa7SrRWN3jeYjb5/zkr99e3VGnlUD+qasd9IJaN8sqQVzrmVkmRm0yRdICkYKDtJXfyfD5S0LpWLe10vUh4rAABooXiZRnN6N++srKlT18xkQLR3qQTKh0laE7hdKmlEwjG3S/q3md0kqZOkc1K5uFejTKQMAEBrC+vd/MsXl+qdVVtSun9wMuCAQwo0tF8hEwHR7qRrMt+lkh51zv3azE6V9FczO945FwseZGbXSbpOkvr27auDnSi9AAAgAwzrV6gnrx9ZZzLg1l2VWvLpdu3YU9PgfZeuL9fS9eWaWlyiHp3zlJ+TpS4dcllaG21eKoHyWkl9Ard7+9uCrpV0niQ5594ysw6Sekj6LHiQc+5BSQ9KUlFRkXNiMh8AAJkksaZZ8uqaH37jE5WVV2jr7uoG71/mTxZcK68G+r3SRfrNrGWUaaBNSiVQnivpaDM7XF6APEHSxIRjSiSdLelRMxsgqYOkjY2dmMl8AABkvmD7uaZMBoxLLNPo062jJKlHQT7lGshojQbKzrlqM7tR0kvyWr897JxbbGaTJc1zzs2QdIukP5nZ/ydvjt5VzjnX+LmZzAcAQFsSNhlw7ZbdKd8/XqYR90RxiY7s2UnXnHYEJRrIOJZCPLtPFBUVuZrxd2rsCb308y8dH8kYAABAywV7Na/dViE5p5ysrJRXCYzrlJ+tQ7t0IGhG2pnZfOdcUeNH1hXtEtYiowwAQFsXVtcsNb2Txs49NVqxcadufWaRfv/yRzr4wA5MBkSkol3COuaYzAcAQDsV7KTxwKsfa+XGHerWKU+St/pfQ99pry/fo/Xle/ReKUEzohN5Rpk4GQCA9i2xZ7O0t1zjjRVlWrWp4RKNYNB8x8wllGeg1UQbKDux4AgAAPuhYLlGvP3chm27Vd5Iz+ZgeQZt57CvRRwo0x4OAID9XWL7uelzS7Rhe4XWb9/T4P2CbecO69pB3zzzaLLMSKvISy+YzAcAAOKaGzSv3VrBJECkXbST+RyT+QAAQLjEoDmV1QETJwGeNfBgFjVBs0Vfo0ycDAAAGhEMmlNtO7e+fI+mFpdoanGJrj/9CE0aM6A1hop2JPpAmcl8AACgCRLbzs1dtUlbdiXPMkvS/XNW6ol5Jeqcl6OBvQ5kAiBSEnGNsqNGGQAANEuw7Vy8nnnN5l3avKsq9Phtu6q1bVe11m6t0KwlG9StY64uLupDphlJRVyjTOkFAABouWBpxl0zl+qvb6/WzsqGW81t3lWl++es1MNvfKLO+TkEzagnK8qLO+eURaQMAADSaNKYAVo8+Txdf/oR6tKh8ZxgZY2rDZrP+NUrmr86tSW30f5Fn1GOcgAAAKDdmjRmgCaNGVBby/xuyRZt3FHZ4H1Wb96lC+97UwcekKOenfNZAXA/F2mgLInaCwAAsE8Fa5njS2f/Z+mGBnszb9tdrW27q3XrM4v0838u1gF52ZRm7IciD5SZzAcAAFpLfOnsO748WFOLS/THV5ardGtFg/fZXR3T7uqY7p+zUn8tXq2uB+TSOWM/EXmgTHs4AAAQhfgEwFT7MkvSzj012rmnprZzxvD+hZp0/gAC5nYqskDZ+X+TUQYAAFFK7Mu8+NPt2r6rUuV7Gu6aIUlzV23Rhfe9qcO6dtA3zzyaeuZ2JrqMsh8pU6IMAAAyQbCWWdq7bPaGbbsbDZrXbq3Qrc8s0m3PLdLYE3ppyoSh+3q4aAXRl14QKQMAgAyUuGx2KqsAVsekZxeu0wvvr9O1p7FsdlsXaR9liYwyAADIfPFs87u3nas7vzxYvbt2aPD4qpi3bPaxP5qpu2YubaVRIt0yoEaZSBkAALQdwUmAjWWZ99Q43T9npR59a5V6dz2AvsxtTISlF16oTJgMAADaomBN89TiEv3vzCVJa5krqmJasXGnbn1mke54YYlGHtWD9nJtQAbUKEc9AgAAgJaJZ5mnFpfoVy99qK27qpIeu7OyRrOWbNCsJRvUo3OehvYtJGjOUJHXKFN6AQAA2ouJI/pq4W1f0J1fHqyuHXMbPb5sR6VmLdmgC+97U+dPmaP5qxvv5YzWE1mg7FzjxwAAALRFwYD5qIM664DcxkOupevLdeF9b+qr979JwJwhyCgDAADsIxNH9NXL3zlDS39+vq4//Qh16dB41Wt8EZMrHipuhRGiIZEHysTJAABgfzBpzAC9f/u5euqGkfrCwIPVs3Neg8fPWV6m4378or7x2DwyzBGhPRwAAEArCnbLmL96i255cqFWbdoVemxFdax24t/1p7OASWsjowwAABCRYf0KNft7Z6a0iMn9c1bqxJ+9pKnFJa00OmRAoEykDAAA9m8TR/TV65PO1lM3jNShXfKTHrdtd7VufWaRbp72biuObv8VXaDs114QJgMAAHiG9SvUW7eeozu/PFgHFSSvYX524ToV/WIW2eV9LLr2cP7fJJQBAADqmjiir9750egG+zGX7ajUrc8sov/yPhR56QWT+QAAAMLF+zEP6X1g0mPi/ZdpJ5d+EQbKXk6ZMBkAAKBhz954mr40pFeDx8xZXqaBP3lRtz6ziAxzmpBRBgAAaAOmTBiqp24YqYGHFiQ9ZldVTFOLS3ThfW/qrplLW3F07VP0S1gTJwMAAKRkWL9Czfz26Sm3k/vSH15vpZG1T2SUAQAA2phgO7mGlsVeWLpNR9/6AtnlZoo8UCZMBgAAaJ5h/Qr1/u3n6ktDeik3Ozyqqop52WUC5qaLPFDOinwEAAAAbduUCUO1/I4xDU74iwfMg277F/2XUxR9H2VyygAAAGkRn/DX0Op+OytrdOsziwiYUxB5PpcSZQAAgPQJru5XkJ+d9Lh4wEz/5eQyIFAmUgYAAEi3iSP6atHPztP1px+hvCT1y5LXf/nEn71EdjlEdIGyX3tBmAwAALDvTBozQB/dMabBgHnb7mrd+swiAuYEkdco0x4OAABg34sHzKcf3SPpMfGAmf7LnuiXsCZOBgAAaDWPXTtCT90wUv27d0x6zMLSbTruxy/u99nlyGuUswiUAQAAWtWwfoWa/b0zdf3pRyQ9pqI6tt9P9ou89IIqZQAAgGhMGjNAT90wUif3L1ROkqhwzvKy/bYUg4wyAADAfmxYv0I9ef1Irbjzizq6Z6fQYxaWbtMZv3pF81dvaeXRRSv6rhcUKQMAAGSEWbeMSrq63+rNu3ThfW/uV3XLkWeUCZMBAAAyx5QJQ3Xnlwcn3b8/1S1HHihnRT4CAAAABE0c0bfBYHl/qVuOfDKfkVMGAADIOBNH9G2wjdzC0m26edq7rTyq1hV5PpcSZQAAgMwUbyOXbJGSZxeua9eZ5QxYcIRIGQAAIJM9du2IpD2XF5Zua7fBcuQZZdrDAQAAZL5JYwYk7YjRXtvHRVejHG8PR40yAABAmzBlwtBG28e1p7plMsoAAABIWWPt455duK7dtI+LPFAmoQwAANC2pNI+buBPXtRdM5e24qjSj/ZwAAAAaLLG2sftqorp/jkr23R2OfKMMqUXAAAAbVO8fVyyumXJyy631cxy5IEy7eEAAADatikThiZtHydJ989Z2SaD5cgDZTLKAAAAbd+kMQP01A0jdXL/wtD9989Z2eb6LUffHo5AGQAAoF0Y1q9QT14/ssF+y6fd9Z9WHlXzRZ5RpvQCAACgfZkyYaiG9D4wdF/p1god+cMX9NX738z4BUqiD5SjHgAAAADS7tkbT0uaWa5x0txVWzJ+gZIIA2Wv9iKLjDIAAEC7NGXCUD11w0h16ZCT9JhnF67L2Nrl6PsoEycDAAC0W8P6Fer9289NWoohebXLmdgVIwNKL4iUAQAA2rtnbzxNd355sArys0P3P/jayoyrWY4uUKbrBQAAwH5l4oi+WvSz80Jrl2NOGVezHH1GmUAZAABgvzJlwlD16xa+9PWzC9dp9K9nt+6AkkgpUDaz88xsmZmtMLNJSY652MyWmNliM5ua8gCIlAEAAPY7v7lkSNJ9yzfuzIjMcqOBspllS7pX0vmSBkq61MwGJhxztKQfSvqcc26QpJsbOy+T+QAAAPZfw/oV6qkbRmrgoQWh+597b10rj6i+VDLKJ0ta4Zxb6ZyrlDRN0gUJx3xD0r3OuS2S5Jz7LOUBECkDAADsl4b1K9TMb5+up24YWa+9g3OKfBW/VALlwyStCdwu9bcFHSPpGDN7w8zeNrPzUh0AYTIAAMD+bVi/Qv2/04+ot710a0WkPZbTNZkvR9LRkkZJulTSn8ysa+JBZnadmc0zs3nbt2+Pb0vTEAAAANBWTRozQL27dqi3fWHptsiC5VQC5bWS+gRu9/a3BZVKmuGcq3LOfSLpI3mBcx3OuQedc0XOuaKCgi6SqFEGAACA5/VJZ4f2WY4qWE4lUJ4r6WgzO9zM8iRNkDQj4Zhn5WWTZWY95JVirExlAMTJAAAAiHv0mhGh2xeWbtMVDxW36lgaDZSdc9WSbpT0kqSlkp50zi02s8lmNt4/7CVJm8xsiaRXJH3PObepkTN7AyClDAAAAN+wfoW688uDQ/fNWV7WqsFySjXKzrmZzrljnHNHOufu8Lfd5pyb4f/snHPfcc4NdM4Nds5Na/Sc/t/EyQAAAAiaOKKvrg+Z3Cd5wfJdM5e2yjgiX5mPjDIAAAASTRozIHSpa0l68LWUKnxbLPJAGQAAAAgzZcLQ0GA55tQqK/dFFyj7tRdZWWSUAQAAEG7KhKE6umenettbY+W+yALl2hrlqAYAAACANmHWLaPqbXNOGv3r2fv0upGXXlCjDAAAgMaElWAs37hzn07sizxQJk4GAABAY6ZMGBq6ct+fXv9kn10z+kA56gEAAACgTXh90tn1YseamNtnvZWjr1EmpQwAAIAUXRBSgjFneZnmr96S9mtFn1EmTgYAAECKpkwYGrr9xsfnp/1a0beHI1IGAABAE4RN7Pt0+560T+yLsPTCi5QJkwEAANAUyXorP5TmiX2Rl16QUQYAAEBTzbpllHITItmqmEtrb+XIA2VSygAAAGiOa087ot62dPZWjjxQZgVrAAAANMekMQPUrWNuve3pKsGIPFCmPRwAAACa609XDq+3rSpNvZWjm8znd70gTAYAAEBzDetXGNoFIx29lSPPKDOZDwAAAC0xZcJQdcipH9b+8sWW1SpHHigTJwMAAKClbhs3qN62eW01o7x3CeuoRgAAAID2YuKIvspOiCtjTrp52rvNPmf0GWWqlAEAAJAG406sX6v8/PufNvt8kQfKtIcDAABAOkyZMLReCrY65pTVsWuP5pwvukA53vWC2gsAAACkSVH/wnrbsjt1PbQ554qwRtmLlMkoAwAAIF0mnT+g3jbLys5pzrkiL70gowwAAIB0GdavUNn1I9xmBZyRB8oAAABAOqUrDRtpoEzZBQAAANKtJpa4pXklDJH2UabsAgAAAOlWr/SimSEnGWUAAAC0K2NPqN9PuTkibQ/HYiMAAABItykThqblPJFmlKm8AAAAwL6QuJx1c0RcoxzV1QEAANCe1biWnyPiGmUiZQAAAKRfm84oS+nrcQcAAAAEjTux5RP6Iq5RJlQGAABA+qVjQl+ENcqOGmUAAADsM3ktrL+IuD0cAAAAsG+cP/jQFt0/2sl8rDgCAACAfeR3E4bqS0N6ycVi1c25P5P5AAAA0G5NmTBUVZ+tfK859420jzLt4QAAAJCpWJkPAAAACEF7OAAAACAENcoAAABAiOhqlB2lFwAAAMhc0baHI1IGAABAhqL0AgAAAAjBZD4AAAAgRIR9lB01ygAAAMhY1CgDAAAAIaILlOl6AQAAgAxGRhkAAAAIEWGNMl0vAAAAkLkizSgTKQMAACBTUXoBAAAAhGDBEQAAACBEdDXKjowyAAAAMlfEK/NFeXUAAAAgOZawBgAAAEJEGCg7apQBAACQsSLto5wVbXM6AAAAIKmIu16QUwYAAEBmii5QdkzmAwAAQOZiMh8AAAAQItIaZcJkAAAAZKqIl7CO8uoAAABAcpReAAAAACHIKAMAAAAhoqtRdrSHAwAAQOaKuPQiyqsDAAAAyREoAwAAACEiDJQdpRcAAADIWCkFymZ2npktM7MVZjapgeMuNDNnZkWNndNJyoo0nw0AAAAk12ioambZku6VdL6kgZIuNbOBIccVSPq2pOJUL05GGQAAAJkqlZzuyZJWOOdWOucqJU2TdEHIcT+X9EtJFalenBplAAAAZKpUAuXDJK0J3C71t9Uys5Mk9XHOvZDqhZ1jwREAAABkrhZXCZtZlqTfSLolhWOvM7N5ZjavurqaBUcAAACQsVIJlNdK6hO43dvfFlcg6XhJs81slaRTJM0Im9DnnHvQOVfknCvKycmhQhkAAAAZK5VAea6ko83scDPLkzRB0oz4TufcNudcD+dcf+dcf0lvSxrvnJvX2IkpvQAAAECmajRQds5VS7pR0kuSlkp60jm32Mwmm9n45l7YSZReAAAAIGPlpHKQc26mpJkJ225Lcuyo1C9PpAwAAIDMFN2SH46MMgAAADJXpGvjUaIMAACATBVZoOzklEWkDAAAgAxFRhkAAAAIEXGgTKQMAACAzBRtoBzlxQEAAIAGRFijTEYZAAAAmYv2cAAAAEAISi8AAACAEJEGyrSHAwAAQKaKtEaZlDIAAAAyFRllAAAAIAQ1ygAAAECIiLteECoDAAAgM0VYo+xYwhoAAAAZK+IlrKO8OgAAAJBcxIEykTIAAAAyE5P5AAAAgBCR9lFmMh8AAAAyFTXKAAAAQAjawwEAAAAhol3CGgAAAMhQ8/S7/gAADHVJREFULGENAAAAhKBGGQAAAAhBezgAAAAgRLSlF1mEygAAAMhM0U3mc46MMgAAADIWS1gDAAAAIZjMBwAAAISIuD1clFcHAAAAkot0wRGjShkAAAAZiowyAAAAECLSQJkiZQAAAGQqFhwBAAAAQkRcekGoDAAAgMxEezgAAAAgBJP5AAAAgBCszAcAAACEoPQCAAAACBFx1wsiZQAAAGQmapQBAACAEJReAAAAACEovQAAAABCkFEGAAAAQtAeDgAAAAjBZD4AAAAgRMQ1ygAAAEBmijajTEoZAAAAGYqMMgAAABCCyXwAAABACNrDAQAAACFYcAQAAAAIQXs4AAAAIASlFwAAAEAIJvMBAAAAIWgPBwAAAISIuEaZUBkAAACZiRplAAAAIAQZZQAAACAEGWUAAAAgRKSBMgAAAJCpKL0AAAAAQlB6AQAAAIQgowwAAACEYMERAAAAIARLWAMAAAAhqFEGAAAAQlCjDAAAAISgRhkAAAAIQekFAAAAECKlQNnMzjOzZWa2wswmhez/jpktMbP3zew/ZtYvpYsTKQMAACBDNRoom1m2pHslnS9poKRLzWxgwmHvSipyzp0g6R+SfpXS1YmTAQAAkKFSySifLGmFc26lc65S0jRJFwQPcM694pzb5d98W1LvlC5ORhkAAAAZKpVA+TBJawK3S/1tyVwr6cVULk6YDAAAgEyV1sl8ZvY1SUWS/i/J/uvMbJ6ZzZPIKAMAACBzpRIor5XUJ3C7t7+tDjM7R9KPJI13zu0JO5Fz7kHnXJFzrsi7T9MHDAAAALSGVALluZKONrPDzSxP0gRJM4IHmNlQSQ/IC5I/S/XiBMoAAADIVI0Gys65akk3SnpJ0lJJTzrnFpvZZDMb7x/2f5I6S/q7mS00sxlJTleHUaUMAACADJWTykHOuZmSZiZsuy3w8znNuTgZZQAAAGSqSFfmYzIfAAAAMhVLWAMAAAAhIs4oR3l1AAAAILlIA2WWHAEAAECmIqMMAAAAhIi4RplIGQAAAJmJjDIAAAAQItqMMjXKAAAAyFC0hwMAAABCECgDAAAAISi9AAAAAEJEO5kv4i7OAAAAQDJklAEAAIAQtIcDAAAAQjCZDwAAAAjBynwAAABAiIhrlAEAAIDMREYZAAAACMFkvv+/vbuNlbQ86wD+v2RhSXWpDesLFijYLo2UNqKUUpuIhsayNbJGagVL+iKlfsFaXwPRVIIfautLDRFrJV37Ri0UTHNMwU1ssSS1INtgsDTZukWFlTYVirQRkQKXH2Y+7O65OWe24cxMl98v2fDMuZ+ZuUiuzPmfe655BgAABlweDgAABlz1AgAABgRlAAAYWPCMsqQMAMBysqMMAAADdpQBAGDAF44AAMDAQoOypAwAwLIyegEAAANGLwAAYMCOMgAADLg8HAAADCw4KEvKAAAsJzvKAAAwYEYZAAAGXPUCAAAGjF4AAMCA0QsAABhY7FdYAwDAkrKjDAAAA2aUAQBgwI4yAAAMuDwcAAAMGL0AAICBBQdlSRkAgOXk8nAAADAgKAMAwMDCgrKhCwAAlpkdZQAAGBCUAQBgYHFB2ewFAABLbIEzypIyAADLy+gFAAAMCMoAADAgKAMAwIDrKAMAwIAdZQAAGHB5OAAAGLCjDAAAA2aUAQBgwI4yAAAMCMoAADAgKAMAwMACZ5RNKQMAsLxcHg4AAAaMXgAAwICgDAAAA66jDAAAA3aUAQBgQFAGAICBxY1emL0AAGCJzRSUq+rcqtpTVXur6rLB+uaqum66fntVnfR0FwoAAPO0blCuqiOSXJ1ke5JTk1xYVacedNrFSR7q7hckeXeSdz7dhQIAwDzNsqN8ZpK93X1Pdz+W5KNJdhx0zo4kH5ge35DknKq1hyueeLLzuf946FDrBQCAudg0wznPTXLffrf3JXnZU53T3Y9X1cNJjk3ywFM96Il9f57YuT3fOO6YbNl85OSHL/rZ5MxLksceSa79+dV3+uFfTE5/XfI/DybXv371+kt/KTnt/OThfcnf/PLq9R+7NHnh9uSBf03+9m2r13/8N5Pn/2Ty5buSv7t89fo5b09OfFly7+3JJ69cvX7uO5LjXpJ86Zbk1j9avf4zf5ps3ZbsuTn5xz9bvf5z702efXzy+RuTO3auXn/tB5PvPDa589rknz+yev11H0uOelbyT9ckd3989fqbPjH572euSr6468C1I49OLrpxcvzpdyX3fPrA9Wc9J/mFD0+O//6K5L47Dlw/5geS86+ZHN98WfKVfzlw/djnJ+ddNTleeWvy4JcOXP/+Fyfb/2ByfOMlydfvP3D9hJcmr7xicnzdRckjB/2R9YNnJ2f/9uT4w+cn33z0wPVTXpW84q2T47/66ayi9/Reovf03oHrek/v6b3Ds/cOwVw/zFdVb6mq3VW1O0me7OTr//v4PEsAAICZVHevfULVy5Nc0d2vmt6+PEm6+x37nbNres5nq2pTkq8k+Z5e48E3H7etT37zVbn2zWflR5/3nKfhfwUAAFarqs919xmHer9ZdpTvSLKtqk6uqqOSXJBk5aBzVpK8YXr8miSfWiskJ8n3HXO0kAwAwNJad0Z5OnN8aZJdSY5IsrO7766qK5Ps7u6VJO9L8qGq2pvka5mE6TV975bNQjIAAEtrlg/zpbtvSnLTQT97+37HjyYZTGMDAMC3J19hDQAAA4IyAAAMCMoAADAgKAMAwICgDAAAA4IyAAAMCMoAADAgKAMAwICgDAAAA4IyAAAMCMoAADAgKAMAwICgDAAAA4IyAAAMCMoAADBQ3b2YJ676RpI9C3lyltnWJA8sugiWjr5gRF8woi8YeWF3bznUO23aiEpmtKe7z1jg87OEqmq3vuBg+oIRfcGIvmCkqnZ/K/czegEAAAOCMgAADCwyKP/lAp+b5aUvGNEXjOgLRvQFI99SXyzsw3wAALDMjF4AAMDAhgflqjq3qvZU1d6qumywvrmqrpuu315VJ210TSzeDH3x61X1haq6q6o+WVXPW0SdzNd6fbHfeedXVVeVT7Y/A8zSF1X12ulrxt1V9ZF518j8zfB75MSquqWq7pz+Lnn1IupkfqpqZ1V9tao+/xTrVVVXTXvmrqr6kfUec0ODclUdkeTqJNuTnJrkwqo69aDTLk7yUHe/IMm7k7xzI2ti8WbsizuTnNHdL0lyQ5J3zbdK5m3GvkhVbUnyq0lun2+FLMIsfVFV25JcnuQV3f2iJG+be6HM1YyvF7+b5PruPj3JBUn+fL5VsgDvT3LuGuvbk2yb/ntLkves94AbvaN8ZpK93X1Pdz+W5KNJdhx0zo4kH5ge35DknKqqDa6LxVq3L7r7lu5+ZHrztiTHz7lG5m+W14sk+f1M/qB+dJ7FsTCz9MUlSa7u7oeSpLu/Oucamb9Z+qKTHDM9fnaS++dYHwvQ3bcm+doap+xI8sGeuC3Jd1fVcWs95kYH5ecmuW+/2/umPxue092PJ3k4ybEbXBeLNUtf7O/iJDdvaEUsg3X7Yvo22Qnd/Yl5FsZCzfJ6cUqSU6rqM1V1W1WttaPE4WGWvrgiyUVVtS/JTUl+ZT6lscQONX8s9Jv5YF1VdVGSM5KcvehaWKyq+o4kf5LkjQsuheWzKZO3Un8ik3efbq2qF3f3fy+0KhbtwiTv7+4/rqqXJ/lQVZ3W3U8uujC+fWz0jvJ/Jjlhv9vHT382PKeqNmXy9siDG1wXizVLX6SqXpnkd5Kc193/N6faWJz1+mJLktOS/ENV/XuSs5Ks+EDfYW+W14t9SVa6+5vd/W9JvphJcObwNUtfXJzk+iTp7s8mOTrJ1rlUx7KaKX/sb6OD8h1JtlXVyVV1VCbD9CsHnbOS5A3T49ck+VS7uPPhbt2+qKrTk7w3k5Bs3vCZYc2+6O6Hu3trd5/U3SdlMrt+XnfvXky5zMksv0c+nslucqpqayajGPfMs0jmbpa+uDfJOUlSVT+USVD+r7lWybJZSfL66dUvzkrycHd/ea07bOjoRXc/XlWXJtmV5IgkO7v77qq6Msnu7l5J8r5M3g7Zm8kA9gUbWROLN2Nf/GGS70ryselnO+/t7vMWVjQbbsa+4Blmxr7YleSnquoLSZ5I8lvd7Z3Jw9iMffEbSa6pql/L5IN9b7QRd3irqr/O5I/mrdPZ9N9LcmSSdPdfZDKr/uoke5M8kuRN6z6mngEAgNV8Mx8AAAwIygAAMCAoAwDAgKAMAAADgjIAAAwIygAAMCAoAwDAgKAMAAAD/w9ghdcdZpmlkAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsoAAAHiCAYAAADxgeqGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdeXhU9dn/8c+dBcISICyySgBFZBXKJmjFBVTUotbWBauIVqpWrZc+9qF2s1YrPtpHq1JtfaRWBdGfKO57RbQqSwRlCZRFEjbZDBBAtuT7++OcCZPJSTJJJpmZ5P26Li8zc86c8z0zE/KZe+7zPeacEwAAAIDSUuI9AAAAACAREZQBAACAAARlAAAAIABBGQAAAAhAUAYAAAACEJQBAACAAARlIImY2ffNbGUU691hZv9XF2OqC2Y2x8x+6v98lZl9Eu8xVYeZtTezuWZWaGZ/jvd4gphZVzPbY2ap1Xjs42b229oYV6KK9ncSQHIiKCNpmNk6MztoZm0j7l9kZs7MusVnZCXjmGNm+/2Qsd3MXjKzjrHch3PuY+dcryjW+5Nz7qex3HeI/zp85x/nN2b2lJk1r419VYeZNTKzO81slZnt9cc7Ld7vD98kSdsltXDO3VbTjdXGhwbnXL5zrrlzrqiq+3bOXeec+2NV95no76mKRPs7GStm1tx/nt4KWObM7NiI++40s2fDbrcws4fMLN/fzhr/dtvI7VUyjtZm9rL/O5ZnZuMrWLex/yFqi5l9a2avmVnnsOU3mtlCMztgZk9FPPZyf5yh//b5xzm4KuMFqougjGTztaTLQjfMrL+kprWxo+pU1CTd6JxrLuk4Sa0kPRiw3bSaji0B/MA/zoGSBkn6VZzHE+5FSeMkjZfUUtIJknIknVHVDdXCa5UtabmrxpWe6sn7piK1/p6qJ8/hRZIOSBpjZh2q8kAzayTpA0l9JZ0tqYWkEZJ2SBpWxXFMlXRQUntJl0t6zMz6lrPuL/z9DJDUSVKBpEfClm+SdLekaZEPdM5N9z+4NfffHzdIWivpiyqOF6gWgjKSzTOSrgy7PUHS0+ErmNm5fpV5t5mtN7M7I5afbGafmtlOf/lV/v1PmdljZvamme2VdJqZ9fYrxTvNbJmZjYtmkM65byXNktTP3/Y6M/tvM/tK0l4zSzOzE8PG8aWZnRo2xtZm9g8z22RmBWY227//VDPbELbef5vZRv+r/JVmdoZ/f2QVaZw//p3+8fQOW7bOzP7LzL4ys11m9ryZZUR5nN9IekdeuAltrzrHlWVmr5vZNv/+182sSzRjCGdmoyWNkXS+c26Bc+6wc26Xc26qc+7JsOMdHfaYkufKzLr51aprzCxf0r/M7C0zuzFiP1+a2Q/9n483s/f8StlKM7u4nLE9Je/9+ku/Mjbar7Q95D8fm/yfG/vrn2pmG/zX+BtJ/6jiczHSzBb4r+kCMxsZtqy7HWkBed/MpgY8B2n+7avMbK2/7tfmVfh6S3pc0gj/WHaGjtHM7g7bz/lmttj/XVxjZmdXNu5qvKeiOZaS19O//2ozy/Xfa++YWbZ/v5nZg2a21R/zEjML/Q6fY2bL/f1sNLP/Cn+dwsZT7r8Z/vMz1cze8Lczz8yOif5VleS9hx6X9JWkn1TxsVdK6irpQufccudcsXNuq3Puj865N6PdiJk1kxfYf+uc2+Oc+0TSq5KuKOch3SW945zb4pzbL+l5eWFdkuSce8k5N1teYK/MBElPV+fDJlAdBGUkm88ltfD/GKVKulTSsxHr7JX3B6GVpHMlXW9mF0iS/wfxLXnVjHby/hgvDnvseEn3SMqUNE/Sa5LelXSUpJskTTezSr9mNe9rzIskLQq7+zJ/PK3kVWHekFdFaS3pvyTNMrN2/rrPyKuU9/X3HVSZ7iXpRklDnXOZks6StC5gveMkPSfpFv+Y35T0mnnVpZCL5VWYusur+lxV2TH62+4iaayk1f7tztU8rhR5QTBb3h/y7yQ9Gs0YIoyWNN85t74ajw03SlJvec/pcyr9LUYff5xv+IHhPUkz5B3PpZL+6q9TinPuKknTJf2PXx17X9KvJZ0o7314gryq3m/CHtZB3vOYLa9tIypm1lre6/CwpDaS/tcfbxt/lRmS5vvL7lQ5Acc/vocljfXfYyMlLXbO5Uq6TtJn/rG0CnjsMHkfYm+X954/RQHvz4DHVfU9Fc2xlLyeZna+pDsk/VDe78PH8l5jSTrTH+dx8r6NuFhHwtuTkn7mPw/95IfuiLGnq/J/My6V9AdJWf4x3hP2+NfNbHIFz022pFPlvY+mq3TRIBqjJb3tnNtTwT5e90N+0H+v+6sdJ+mwc+4/YQ/9UmHhN8KTkk4ys05m1lReBbpM60hl/OM/RRHFEaA2EZSRjEJV5TGSciVtDF/onJvjnFviV0u+kvdHcJS/eLyk951zzznnDjnndjjnwoPyK865fzvniuWFl+aSpjjnDjrn/iXpdYWFpgAP+9W1LyVtlnRr+DLn3Hrn3HfyKkFvOufe9Mf5nqSFks4xr695rKTrnHMF/jg/CthXkaTGkvqYWbpzbp1zbk3AepdIesM5955z7pCkByQ1kRd6wse2ya+Ev6awal45ZptZoaT1krZK+r1/f7WOy38dZjnn9jnnCuWFh1Fl9lq5NvKe95q60zm313+tXpY0MFR1lPdH/iXn3AFJ50la55z7h1+9XiTvm4QfR7mfyyXd5Vf1tskLUOFBr1jS751zB/yxROtcSaucc8/443pO0gpJPzCzrpKGSvqd/74OVQPLUyypn5k1cc5tds4ti3IM10ia5r/vip1zG51zKypYvzrvqWiPJfz1vE7Svc65XOfcYUl/0pHX95C8D8nHSzJ/ndD76ZC837UW/vs36Kv/E1X5vxkvO+fm+/uerrDfNefcec65KRU8R1dI+so5t1zSTEl9zWxQBetHqvT3wx9Dq3L+O89frbmk3REP3SXvuQuySt7rutF/XG9Jd1Vh3CFXSvrYOfd1NR4LVAtBGcnoGXmB9yoFVBbMbLiZfWje1/i75P1hDJ2ocrSkoDAZEl6J7CRpvR+aQ/IkdVb5bvb/oHR2zl3uh5+gbWdL+nF4tUbSyZI6+mP81jlXUMF+5JxbLa9KfKekrWY208w6BazayR936HHF/ljCj+ObsJ/3yftDKPPaDkIn0Vwets4FfmXtVHmhIvT8Vuu4zKypmf3NvJOCdkuaK6mVVb1PfIe/r5oqea384P6GvEqg5IWe6f7P2ZKGRxzv5fIqwdEo9dr4P4e/htv8r6qrKnK7oW139pd965zbF7YssALvnNsr74PWdZI2+y0Dx0c5hsp+1yJV5z0V7bFE/u79JWxb30oySZ39YPuovP7brWb2dzNr4T/uIknnSMozs4/MbETAfqL5NyPwdy1KV8p/7znnNkr6SF4rQkiRpPSIx6TLC/lS7H4/9sjrbw7XQlJhOetPlfehvo2kZpJeUjUqyvKO/5/VeBxQbQRlJB3nXJ68k/rOkfcPbqQZ8qpKRzvnWsrr5zN/2XpJFfUEhve9bZJ0tJmF/550VUQFuypDD/t5vaRnIqo1zfxq0npJrc2szNfZZTbo3Azn3Mny/vg7SfcFrLbJXy7J68OUF2IqPQ7n3Fh35ESa6QHLP5L0lLwqdU2O6zZJvSQNd861kPf1qnTkdYvW+5KGWcX9zXtV+gTQoFAb2f/4nKTL/HCUIelD//71kj6KON7mzrnroxxvqddG3vtrUwXjiFbkdkPb3iivotja/wo85OjyNuSce8c5N0ZewFoh6Ykox1bZ71p5+6vKeyraY4n83ftZxPaaOOc+9ff/sHNusKQ+8loMbvfvX+CcO19eS8VsSS8E7CfW/2aUMK/HvKekX5k3M8g3koZLGm9HTlLMl9Qt4qHddeRD0/vy2k+aVbCf8A/Hkf+Fwu1/JKWZWc+wh54gqbxvGwZKeso5963/Tcwj8n5Po55pw8xOkvdB5MVoHwPEAkEZyeoaSaf7Fa9ImfKqTPv9PsnwaYumSxptZhebd0JdGzMrr81gnryKzy/NLN28E4h+IO8rz5p6Vt7X4GeZWaqZZZh3UlAX/6vet+T1umb5+z4lcgNm1svMTjfv5K/98vp6iyPXk/cH/VwzO8PvobxN3lnzn8bgOCTpIXln4J9Qg+PK9Me/07z+2t8H7qkSzuv7fU/Sy2Y22H+NM83sOjO72l9tsaRL/f0PkfSjKDb9przgeZek58Mqhq9LOs7MrvC3l25mQy3sZMlKPCfpN2bWzg8Nv1PZnvvKmP88l/znj/c4MxvvPweXyAt+r/sfNBdKutO8qfRGyHtfB224vXkn5DWT957ZoyPvsS2SuljpXvdwT0qa6L/vUsyscxWq0dG+p6I+ljCPywubff1jbGlmP/Z/HmreN1Lp8j5Q7ZdU7G/7cjNr6bz2pd0K/l2rzX8zJsh7b/eRFzwHyuuVbiKvpUnyTpL7jZl18Z/z0f7+Q+HyGXkfFGaZdxJqiv9v4B1mdo5U5sNx5H9j/XX2yitS3GVmzfwQe76//SALJF3pP9fp8mau2OSc2y55s5H479tUSaHXOHKGkgmSZvnf8AB1hqCMpOScW+OcW1jO4hvk/QNeKC94vBD2uHx5lejb5H3lulheJSRoHwfl/ZEZK2/u279KurKSPstox79e3h+WOyRtk/fH63Yd+Z28Qt7XpSvk9WveErCZxpKm+GP7Rl6lq8yUWs65lfL6PB/x1/2BvKm4Dtb0OPztb5PXAvO7GhzXQ/L+4G+Xd8Lm2zUY0o/kBcXn5fVNLpU0RF41TZJ+K6/SWSCvJ3hGFMd4QF4wGB2+vv9H+0x5bRmb5L0O98l7baJxt7yg95WkJfKmvLq7wkeUNVLeh4zw/3bJ65++Td7X7b+UdF4omMhrDwlNC3a3vOfqQMC2U+T12W+S9/sySlKoWv4veRXEb8xse+QDnXPzJU2Ud8LmLnltApFV7kBVfE9Feyyhbb8s7zWaaV6bz1IdCZot5FXMC+RVYXdIut9fdoWkdf5jrvP3G7ntGv2b4Vdz7wi4P0PeiYWPOOe+Cfvva3nhNNR+cZe8D8Cf+MfwP5Iud84t9cd3QN57eIW80L1b3omQbeWF/Kq4Qd7v7FZ5H/iud37/unkXYQk/YfC/5H3oWCXvNTxH0oVhy38j7307Wd6/Vd8p7KTWsOOn7QJ1zhwzrABAg2Zmz0ta4ZyrViU/kdSnYwEQf1SUAaCB8VsMjvG/ej9bXsV2drzHVR316VgAJJ76cJUiAEDVdJDXStJG0gZ5X5svqvghCas+HQuABEPrBQAAABCA1gsAAAAgQKVB2cymmXfd+6XlLDcze9jMVpvZV2b2vdgPEwAAAKhb0fQoPyXvSkXlXVt9rLxJ0HvKm/z8Mf//FWrbtq3r1q1bVIMEAAAAqisnJ2e7c65dVR9XaVB2zs01s24VrHK+pKed1+z8uZm1MrOO/sUFytWtWzctXFjeNLgAAABAbJhZXuVrlRWLHuXO8iaAD9mg0te1L2Fmk8xsoZkt3LZtWwx2DQAAANSOOj2Zzzn3d+fcEOfckHbtqlz9BgAAAOpMLILyRklHh93u4t8HAAAAJK1YXHDkVUk3mtlMeSfx7aqsPxkAACBWDh06pA0bNmj//v3xHgriLCMjQ126dFF6enpMtldpUDaz5ySdKqmtmW2Q9HtJ6ZLknHtc0puSzpG0WtI+SRNjMjIAAIAobNiwQZmZmerWrZvMLN7DQZw457Rjxw5t2LBB3bt3j8k2o5n14rJKljtJP4/JaAAAAKpo//79hGTIzNSmTRvFcsIIrswHAACSHiEZUuzfBwRlAACAGmrevHmNt7Fw4ULdfPPN5S5ft26dZsyYEfX6kU499VT16tVLJ5xwgoYOHarFixfXaLyx9Oqrr2rKlCnxHkYZBGUAAIAEMGTIED388MPlLo8MypWtH2T69On68ssvdcMNN+j222+v9ljDFRUV1Xgb48aN0+TJk2MwmtgiKAMAgAYnJ69AUz9crZy8glrbx+LFi3XiiSdqwIABuvDCC1VQ4O1rwYIFGjBggAYOHKjbb79d/fr1kyTNmTNH5513niTpo48+0sCBAzVw4EANGjRIhYWFmjx5sj7++GMNHDhQDz74YKn19+zZo4kTJ6p///4aMGCAZs2aVeHYRowYoY0bvdl89+7dq6uvvlrDhg3ToEGD9Morr0iS9u3bp4svvlh9+vTRhRdeqOHDh5dcVbl58+a67bbbdMIJJ+izzz7Ts88+q2HDhmngwIH62c9+pqKiIhUVFemqq65Sv3791L9/fz344IOSpIcfflh9+vTRgAEDdOmll0qSnnrqKd14442SvA8Ep59+ugYMGKAzzjhD+fn5kqSrrrpKN998s0aOHKkePXroxRdfjM0LVYFYTA8HAACQEP7w2jIt37S7wnUK9x/Sim8KVeykFJOO75CpzIzypxPr06mFfv+DvlUey5VXXqlHHnlEo0aN0u9+9zv94Q9/0EMPPaSJEyfqiSee0IgRI8qtoj7wwAOaOnWqTjrpJO3Zs0cZGRmaMmWKHnjgAb3++uuSvGAd8sc//lEtW7bUkiVLJKkklJfn7bff1gUXXCBJuueee3T66adr2rRp2rlzp4YNG6bRo0frscceU1ZWlpYvX66lS5dq4MCBJY/fu3evhg8frj//+c/Kzc3Vfffdp3//+99KT0/XDTfcoOnTp6tv377auHGjli5dKknauXOnJGnKlCn6+uuv1bhx45L7wt10002aMGGCJkyYoGnTpunmm2/W7NmzJUmbN2/WJ598ohUrVmjcuHH60Y9+FM1LUW1UlAEAQIOye/9hFTvv52Ln3Y61Xbt2aefOnRo1apQkacKECZo7d6527typwsJCjRgxQpI0fvz4wMefdNJJuvXWW/Xwww9r586dSkuruLb5/vvv6+c/PzIJWVZWVuB6l19+ubp376577rmnZP13331XU6ZM0cCBA3Xqqadq//79ys/P1yeffFJS8e3Xr58GDBhQsp3U1FRddNFFkqQPPvhAOTk5Gjp0qAYOHKgPPvhAa9euVY8ePbR27VrddNNNevvtt9WiRQtJ0oABA3T55Zfr2WefDTyuzz77rOR5ueKKK/TJJ5+ULLvggguUkpKiPn36aMuWLRU+J7FARRkAANQb0VR+c/IKdPn/fa5Dh4uVnpaiv1w6SIOzg4NlvEyePFnnnnuu3nzzTZ100kl65513YrLd6dOna/Dgwbr99tt100036aWXXpJzTrNmzVKvXr2i3k5GRoZSU1MlefMXT5gwQffee2+Z9b788ku98847evzxx/XCCy9o2rRpeuONNzR37ly99tpruueee0qq4NFo3Lhxyc/eDMW1i4oyAABoUAZnZ2n6T0/UrWf20vSfnlgrIblly5bKysrSxx9/LEl65plnNGrUKLVq1UqZmZmaN2+eJGnmzJmBj1+zZo369++v//7v/9bQoUO1YsUKZWZmqrCwMHD9MWPGaOrUqSW3K2q9MDP98Y9/1Oeff64VK1borLPO0iOPPFISPBctWiTJq2q/8MILkqTly5eXG2jPOOMMvfjii9q6dask6dtvv1VeXp62b9+u4uJiXXTRRbr77rv1xRdfqLi4WOvXr9dpp52m++67T7t27dKePXtKbW/kyJElz8v06dP1/e9/v9xjqW1UlAEAQIMzODsrpgF537596tKlS8ntW2+9Vf/85z913XXXad++ferRo4f+8Y9/SJKefPJJXXvttUpJSdGoUaPUsmXLMtt76KGH9OGHHyolJUV9+/bV2LFjlZKSotTUVJ1wwgm66qqrNGjQoJL1f/Ob3+jnP/+5+vXrp9TUVP3+97/XD3/4w3LH26RJE9122226//779eijj+qWW27RgAEDVFxcrO7du+v111/XDTfcoAkTJqhPnz46/vjj1bdv38Cx9unTR3fffbfOPPNMFRcXKz09XVOnTlWTJk00ceJEFRcXS5LuvfdeFRUV6Sc/+Yl27dol55xuvvlmtWrVqtT2HnnkEU2cOFH333+/2rVrV/K8xYPVRdk6yJAhQ1zozEkAAIDqys3NVe/eveM9jKjt2bOnZN7lKVOmaPPmzfrLX/4S51GVVVRUpEOHDikjI0Nr1qzR6NGjtXLlSjVq1CjeQ6tQ0PvBzHKcc0Oquq24VZT3HSzS1A9X68QebRKuLwgAAKC2vPHGG7r33nt1+PBhZWdn66mnnor3kALt27dPp512mg4dOiTnnP76178mfEiOtbhVlBt37Ok6TnhIjVJNz00aUSdhOSevQC99sUFO0kXf60JABwCgHki2ijJqV72oKIccLHKa9cWGKofWnLwCfb52R5mKdE5egV7K2aCthQe067uD+nbvQaWnpmjz7u+0c9+R6V+em5ev8wd20tJNu/XdwcNqkZGu3QcOq0laiq4+uYfGD+8as2MEAABA8ol7UJak7YUHqrT+jHn5+s3sJd5E4ZL6dm6hy4ZlS5J+PXuJoimSO0mzF28qub1R+0t+vuPlJZr/9Q71bJ9JawgAAEADlRBB+YPcLZryZq4ym6QHBtNQy8S2wgP6evterdp6ZBqRYklLNu7Wkpejn4MvGqEQbZKGdstSq6aN1DazMS0bAAAADURCBOUiJz0+d60klelZzskr0MWPf6qi+LRSy0mav+7IXIQz5uXr+A7NdXRWM7VrQXAGAACorxLugiMHi5z+9tEaSV4oveLJz2MWkpump6hDZuMy97dt3ki9O2SqXfPozuRc8c0evZe7RTPm5etHj32qix/3/hv95zm6+PFPdcfLS5STV/5E3zl5BZr64eoK1wEAAMnDzHTbbbeV3H7ggQd05513Rv34LVu26LzzztMJJ5ygPn366JxzzpEkzZkzR+edd16Z9V999VVNmTJFknTnnXfqgQcekCRdddVVevHFF2twJAiXEBXlSO8u36JbZi4q1UNcFe2aN1L3ts104HCxRvRoU6alo6LZL6a8mVtS3Y5GZMVZ2/Zq/roCPTcvX706ZOpwUbFaN2ukVk0baee+g9q48ztt3On1Q6eYdEbv9rpu1DFUpQEASGKNGzfWSy+9pF/96ldq27ZtlR//u9/9TmPGjNEvfvELSdJXX31V4frjxo3TuHHjqjVWRC8hg7KkSkNy2+aNdOuYXurVIVO3vbBY63bskySlp5oev2JIhcGzoqvxTD6nt7q2aabnF+SrcVpKScAtFYaj4CSt+Ma/zOS2vYHrFDvpveVb9EHuFg3pmiWZSsJ94YHDTGMHAECSSEtL06RJk/Tggw/qnnvuKbVs3bp1uvrqq7V9+/aSK8117Vp6dq3NmzfrzDPPLLk9YMCAMvtYsGCBJk2apBdffFEff/yxFi5cqEcffbR2DgiSEjgoB0lPNWW3blpm+rY5t59W7nRx1TF+eNcy08Pl5BXobx+t0dpte3SoyCnv23012ke4YifND2vD+HLDrpKfZ87P190X9Ge6OgAAovWPc8ve1/cCadi10sF90vQfl10+cLw06HJp7w7phStLL5v4RlS7/fnPf64BAwbol7/8Zan7b7rpJk2YMEETJkzQtGnTdPPNN2v27NllHnvJJZfo0Ucf1ejRozVx4kR16tSpZPmnn36qm266Sa+88oq6du2qjz/+OKoxoWaSJiibSTMruDBJrK/ZHrT9v195ZJ7qGfPyy1SdN+7ar40F38V0v8XOm67umc/W6ejWTZl5AwCABNWiRQtdeeWVevjhh9WkSZOS+z/77DO99NJLkqQrrriiTJCWpLPOOktr167V22+/rbfeekuDBg3S0qVLJXkX0Jg0aZLefffdUuEZtS9pgvI9F/RPqHAYVHWWSk9lt3Ofd8GT8B7lyNsL8wpUHMXJirnfFCrXb+V4fn6+BmdnlWzr2PaZUYXnoN7s8PESwgEA9UJFFeBGTSte3qxN1BXkILfccou+973vaeLEiVV+bOvWrTV+/HiNHz9e5513nubOnas2bdqoY8eO2r9/vxYtWkRQrmNxD8pN0lJ0sKhYxc7r6w0yrFtW0rQeVLWyHRmsDxwuVve2zfTql5vKDdBFLuwEQv/kwRnz8tW7Q6aObt1Ukkq2Nbx7a23fc1Drvt2rRXk7S57j5+fn67j2mSXhO+TFhev13KQRkqSXvtigVVsKdeBwsS4ZGvzBAAAAHNG6dWtdfPHFevLJJ3X11VdLkkaOHKmZM2fqiiuu0PTp0/X973+/zOP+9a9/6cQTT1TTpk1VWFioNWvWqGvXrtq7d69atWqlJ598UmPGjFGzZs106qmn1vFRNVxxD8rHts/UneP66vO1O7R4/U69t3xLmXV6ts+Mw8jqRnnB+ooR3XTfW7lVOokwvOocEt7vHK7Iqcy6kjc93+0vfqm87XtLTcv35Qav/SMzI43gDABABW677bZSJ9k98sgjmjhxou6///6Sk/ki5eTk6MYbb1RaWpqKi4v105/+VEOHDtWcOXMkSe3bt9frr7+usWPHatq0aXV1KA2euWiu91wLGnfs6TpOeEgdW2bo0fHfK2kDuOyJz3XwcHHJemmppucr6E2u70K90AcPF2vlN4UqrvwhdSa8gn2oqEgpZkpNOTI1d9vMxurXqaUK9h1U4XeH9NnaHSU93aFlyzbtou0DAFAjubm56t27d7yHgQQR9H4wsxzn3JByHlKuuAdlScpIT9H0n55Iz2wlQjN7ZDVtpDkrt1Z5Bo7UFMkVK6HCdrhUk/7IDB8AgCoiKCNcLINy3FsvJOnQ4WJ9vnZHSRsCwThY+HMTHiYjZ+CQjvQoR87JLKkkbC/btEtOUr9OLXXna8vKVPJP73WU1n+7L7BFozYUOenXLy/Rc/Pz1LFFE8m8qnSLxmlatnm3+nZsUebiMQAAALUlrkE5LUVyTkpPS9GJPdrEcyhJrbwZOMoTFDJ7dcgst5IfXuWv7eDsJC3ZuFtLNu4us+zjVdslefNpn9brqJL7+eYBAADUhjhXlE2XDDuakJMAKqrkRy4LD84VCVWD/++Tr3W42MlMOr59Zsl80KEe5VVbCrVgXUG5s55EOlTk9G7ESZ/PzcvX949rq1E922nttr36Zvd3Jf3S9EMDQP3nnJOZxXsYiLNYtxTHNSgXO6fOrZoQWHAG/wQAACAASURBVJJMVdtjxvTtUOlVE2fMy9dvX1mqomgmlQ7gJM39z3bN/c/2qNafOS9fQ7pllZxYSHAGgOSVkZGhHTt2qE2bNoTlBsw5px07digjIyNm24zryXzZ1zys5649kYACScGV6lBV+rO1O7R0464K59uuiRSTzurbQXsOHKYXGgCSzKFDh7Rhwwbt378/3kNBnGVkZKhLly5KT08vdX9Sznrx1Csf6LJhzHCA6ITP+hFqo6jK1Q2rKjXFNLhrq6ivfAgAABJTUgblz+bN1/e6Ej5QM+GX5g7vRQ6J7IeuTrA2Sf07t1SHFhnejYjthl8WHAAAJJakDMqfz5uvQQRl1LHIy4bn5BWUugphdaWlmE4/3puNg75nAAASB0EZqKZQS0fhd4e0bPNutWnWSK99tbnaJxaGmKTTjm+rM47voOWbd5eZJSSrabrSU1OU9+0++qIBAKhFBGUghkJV5+q2alRXWorpvAEd9fX2vRVe7jt0aXCCNQAAlUvqK/MBiSZ8Crzy5o2ODLD/WrlVh2vYw3G42Gn24k1Rr5+aYrr25O6lrr5IcAYAIDbiWlGeN3+BBh7dKi77B2ItXlXocCkmTRzZTfsPF5c5oZEQDQBoqJKy9YKgjPqqspk4pCNzRIf6ol//arMO12K6TjXp2u/3KKk+h8/YQSsHAKA+IygDSS78pMLP1u6osEc5dGnwomIX0wuwpJo3d3SrZlyxEABQfxCUgQYm/AIsc1Zu1Qe5W2J+5cLUFOmkY9qqcVqK2rXIYN5oAEBSSsqgPH/+Ap1AUAZiIujKhZJK5oqOeYgOa+UIVboJzwCARERQBlCuyBAd2aPconGanvBbOWoixaQh2Vk6tn0mfc8AgIRBUAZQI0HT4MViyru0FNNPT+7OBVUAAHGTlPMom8Vz7wDChc8dHRIenkMnFVa1H/pwsdPjc9dK8uZ9Hty1lY5tn0mbBgAg4cW1orxgwQIN6EJFGUg2ka0c1Zk7OtWkk45tq3aZjdWjbTOtL/hO3+49WGaWD4l5oAEANZOUrRcEZaD+iKw+x3oKu9QU6fReR5WafYNLegMAopGUrRcA6o+g1o0xfTuUzA1d05MFi4ql93K3VrhOqB+amTgAALEQ14rywgUL1b9Ly7jsH0DdijxZMBYnCkYjNUU64/j2khR48RaCNADUf0nZekFQBhqu8i7zHdSjLNVesPZaOtqrXYsjLRxZTRvRygEA9QitFwCSSlCrRkUqCtY16Yf2Wjq2BC5LMWlA55bq07klfdAA0AARlAEkhcqCdagfOvLKhDWpRBc7afGGXVq8YVfJfeF90KHQToAGgPqJeZQB1AvlBemguaArCtKmiueHDp8XOlyKSaf0bKsz+3YsueIh/c8AkNzi2qOcs3Ch+nWmRxlA/ES2dBTsOxiTWTokLzxfNuxotWueoaNaZFB5BoA4oUcZAKqhvEr0mL4dajwvdLGTps9bX+q+tBTTKce1VVpKCrNuAECCo6IMAFGKvCKhk9SicVqNqs8pJk36fo+SnmeCMwDEXlJOD0dQBlAfRLZvzFm5VR/kblGx8/qdK+t7Dpci6Yw+7XVar6No1QCAGCEoA0ACCa8+h+ZmXrZpl55fuL5Ks3CkpZjG9u+g49tnavXWvSo8cFBHtWjCbBsAUAUEZQBIAqHq86othcrJKyipOldXqplO732Urht1DIEZAMqRlEH5i5yF6tuJoAygYQqvOs9ZuVXv++0a1ZEi6fLh2TpYVKTUVFO/Tq2oOAOAj6AMAEku6OqD/2/heh0udtUP0CadfGxbndm3g3I37y6ZxYOTBgE0JARlAKiHImfaCJ+uriazbZik049vpzN6d+ACKQDqPYIyADQwOXkF+ttHa0rNsFETaSmm048/ioozgHonKYPyopwc9enUIi77B4D6ojbmd05NkX48+GgN6NKqVCWbAA0gGRGUAQClBPU8r9pSqIX+bBvVkWretkYc06bkIimhbdO+ASBREZQBAFEJCtDbCg/oXyu3VmmO5yApJl19UjdlNWtcqsJNgAYQTwRlAECNhAL0tsIDmrNya41m24jEpboBxFNSBuXFX+Sod0eCMgAkmqDZNnbuO6j56wpisn0u1Q2gLhGUAQC1bsa8fL21dLP6dmxRqkd5zsqtpWbfMFVtFo5Uf8YNgjOA2kBQBgDEVXgVumDfwZIrDlZn+rrQpbkJzgBiISmD8peLcnR8B4IyANRnsbhUdyg4XzfqGAIzgCojKAMAkkL4rBtVne+ZkwIBVAdBGQCQlGoUnCUN6ZalVk0bqW1mY/Xr1JJWDQBlVDcop0W58bMl/UVSqqT/c85NiVjeVdI/JbXy15nsnHuzqoMBADQ8g7OzSoXaMX07RB2ci6XAmThSzTQ4u5WObZ9J1RlAtVVaUTazVEn/kTRG0gZJCyRd5pxbHrbO3yUtcs49ZmZ9JL3pnOtW0XYbd+zpvlr0hXp1yKzhIQAA6rOcvAL97aM11TopUPKuJnit367BpbiBhqk2K8rDJK12zq31dzRT0vmSloet4ySFeihaStpU1YEAABBkcHaW/n7lkGqfFFjkpMfnri1138z5+bp8eLaKi52c0esMIFg0FeUfSTrbOfdT//YVkoY7524MW6ejpHclZUlqJmm0cy6nou1SUQYA1ET4lQQlqW1m4yr3OIeEep1p1QDqp1rtUY7CZZKecs792cxGSHrGzPo554ojBjlJ0iRJatTh2BjtGgDQEEX2NoeEepxXbSlUTl5BVO0aoV7n+esK9Pz8/JJWDWbWABq2aCrKIyTd6Zw7y7/9K0lyzt0bts4yeVXn9f7ttZJOdM5tLW+7jTv2dEsWf6Hj2lNRBgDUjshLcVclPIeEX2572aZdhGcgCdXa9HBmlibvZL4zJG2UdzLfeOfcsrB13pL0vHPuKTPrLekDSZ1dBRsnKAMA4iEWF0BJMWlINq0aQLKo1XmUzewcSQ/Jm/ptmnPuHjO7S9JC59yr/kwXT0hqLu9D+i+dc+9WtE2CMgAgEYR6natTbZa80HxG7/ZcbhtIYEl5wZGli79QT4IyACBBxORy2ymmwV2ZwxlIJARlAABiLPyqgf06taxyeGY2DSAxEJQBAKgD4a0aC/1WjWikmDSJC58AcUFQBgCgjoVXnKszh3Oov/m6UccQmIFalJRBedmXX+jYowjKAID6oSbVZgIzUHsIygAAJJDqzKYRPu1cv04tS+Zt7tepJTNqADVAUAYAIEEFXfikKhXnkPRU00nHtlGnVk1LgjT9zkDlCMoAACSRnLwC/e2jNdWagi5IWorp9OOPKrlNeAaOSNKgvEjHHtU8LvsHACARxDowh0s16Vp/pg0uvY2GjKAMAEASC/U0h1opInuUl23apecXrtfhopr93R7aLUuTx/YmMKNBISgDAFDPRV4AJdSjLEn/Wrk16hBt8gIzF0JBQ0FQBgCgAQuvSO/cdzDqmTZSU6RRx7VTWkpKyX30N6O+ScqgvPyrRTqmHUEZAIBYC59pY/aiDZq/rqBKj09NkU7t1U6pRoBG8iMoAwCAcs2Yl69p//5aa7buqbTKXBEujoJkRFAGAACVCm/RkKrW2xwuRdIQ+pyRJJIyKOd+tUg9CMoAAMRNZHAOqUqApsqMREdQBgAAMRMZoHfuO1jp1QRTU6TTe7WXmXebnmYkCoIyAACoVdW5OApT0SEREJQBAECdCFWbV20prLTKHI6+ZsRLUgblFUsWq3vbZnHZPwAAqLnwFo2q9jXffUF/jR/etZZHCBCUAQBAnEVe9CSaavOx7Zrp6pN7EJhRqwjKAAAgoVSlRaN3h0wNys6iJQO1gqAMAAASVuhEwPeWb6nwgiehPuZWTRsxawZiJimD8soli9WNoAwAQIMRbWAO4QRAxAJBGQAAJI2cvALd91au5q8riPoxJukYeppRDQRlAACQdKo71dzQblmaPLY3FWZEJSmD8n+WLlZ2G4IyAACo+qwZKSYNyaYtA5UjKAMAgHolvNq8YF1BhT3Naamm5yeNICwjEEEZAADUW9H0NB97VHPdd9EAwjLKICgDAIB6LzRrxvu5WwLbMkzS6D7tdd2oYwjMKJGUQXnV0i/VtU3TuOwfAAAkr1Bgfnf5lsDlKfL6l1s1Yz5mEJQBAEADNGNevn49e4kqizMmb6YMLmTSMFU3KKfVxmAAAADqQmg+5d/MXlLhDBlOKtXf/Ny8fB3TrplG926vzCbpOrFHG4IzyiAoAwCApDZ+eFf16pBZpfmYnaTV2/Zq9ba1kryp5s7oTW8zSotr68XqZV/q6Na0XgAAgNip6nzMkbiYSf2TlD3KBGUAAFDbIoNzZXMyhwzjZMB6g6AMAAAQhdCMGWu37dHa7XujqjabpGPaNdPVJ/co6YtG8iAoAwAAVFGo2vxFXoFyvymM6jG9O2RqUHYWVeYkQlAGAACogRnz8jXt319rzdY9UbVmmKROrTLUp1NLTgJMcARlAACAGKhuT/OxtGYkLIIyAABALcjJK9B9b+WWmoe5Ip1bZejnp/UkMCeQpAzKa5Z/qS5ZBGUAAJD4QpXmVVsKo6oy08ucOAjKAAAAdSQ0c8ayzbu1seC7Stdnbub4IigDAADEQVVaMzpz8l9cEJQBAADiqCqB2eRVmY9tn0lrRh1IyqC8NvcrdW7VJC77BwAAqA2htox3l2+Jan0uZlL7CMoAAAAJpKp9zJKU3bqpWjVN1yVDuxKaY4igDAAAkKDCQ/Omgu+impc5u3VTndSzLa0ZMUBQBgAASAJVnWbOJP3slB6afE7vuhhevZSUQfnr3K/UiaAMAAAaqKrOmMGFTKqHoAwAAJCkQlXmf6/ernU79lW47gUDO+mhSwfV0cjqB4IyAABAPTBjXr6eX5CvnfsOKe/b4NDctnkj3TqmF9XlKCVlUF634it1bElQBgAACDLlzVz9be7acvuYaceIDkEZAACgHsrJK9BvZy/R8s2F5a7Tu0OmBmVnMUNGOQjKAAAA9dgtMxdp9uJNFa7DxUuCVTcop9TGYAAAABBbD106SH+6sL+6tMoodx0nafW2vbrj5SW68sl5dTe4eiquFeW8FUvUoWX5LzYAAADKinZaOU768yRl6wVBGQAAoPpCV/x7b/mWCi9c0tADM0EZAACggQrNw/xB7hZ9s/tAues11FkyCMoAAACI6qS/od2yNHls7wYzQ0ZSBuX8lUvUvgVBGQAAIJaimVJOajiBmaAMAACAUmbMy9dfP1ylDTv3V7jesfV8SjmCMgAAAAJFO0tGfe1hJigDAACgQjPm5evXs5eosvjXu0Om7r6wf71pyUjKC45YPHcOAADQwIwf3lUvXjdSw7pVHIBzvynURY99qltmLqqjkSWmuFaU169coqOoKAMAANS50JRyX+QVKPeb8k/6qw9zMCdl6wVBGQAAIP6imSUju3VT/e8lA5OyHSMpWy/ovQAAAIi/wdlZevMXp+i6U3qUu07et/t00WOfasqbuXU4sviKb1AGAABAwph8Tm/Nun6k+nTMLHedx+eu1ZVPzqvDUcUPQRkAAAAlQtXlWdePVLc2TQPXmbtqu0b9z4fKyat4urlkR1AGAABAGYOzszTn9tN0wcBOgcsbQitGnKeHo0kZAAAgkT106SDNun6kWjVJD1xen1sxqCgDAACgQoOzs/TkVUPLLXHW11YMgjIAAAAqNTg7Sy9eX/7FSkKtGDPm5dfxyGoPQRkAAABRGZydpReuG6k/Xdi/3HXueHlJvaksRxWUzexsM1tpZqvNbHI561xsZsvNbJmZzYhuu1UZKgAAABLB+OFdK5wV49K/f6Zrn16Y9IG50ivzmVmqpP9IGiNpg6QFki5zzi0PW6enpBckne6cKzCzo5xzWyvabuOOPd3GVUvVtnnjmh4DAAAA4uTKJ+dp7qrt5S6/YGAnPXTpoDocUVm1eWW+YZJWO+fWOucOSpop6fyIda6VNNU5VyBJlYVkAAAA1A9PXzNcA7u0LHf57MWbdOI97ydldTmaoNxZ0vqw2xv8+8IdJ+k4M/u3mX1uZmdHs3M6LwAAAJLf7BtP1ik925a7/JvCA0k553KsTuZLk9RT0qmSLpP0hJm1ilzJzCaZ2UIzWxij/QIAACABPH3NcP3pwv7q0iqj3HUen7tWQ+5+L2lmxogmKG+UdHTY7S7+feE2SHrVOXfIOfe1vJ7mnpEbcs793Tk3pDo9IgAAAEhs44d31SeTz9Cs60eqY4vg89C27zmoO15ekhRhOZqgvEBSTzPrbmaNJF0q6dWIdWbLqybLzNrKa8VYG8NxAgAAIEkMzs7SZ3eMrrAd47evLE34sFxpUHbOHZZ0o6R3JOVKesE5t8zM7jKzcf5q70jaYWbLJX0o6Xbn3I7Ktm3MDwcAAFBvPX3NcF13So/AZUXFTne8vCShL39d6fRwtaVxx55u8+plat2sUVz2DwAAgLqRk1eg215YrHU79gUuz27dVP97yUANzg6+6l9N1eb0cAAAAEC1Dc7O0pzbTyu3FSN0+etEm0KOoAwAAIA68fQ1w3XBwE7lLp/4j/l1OJrKxTUo06EMAADQsDx06aBy+5Z37z+sk6d8UMcjKh8VZQAAANSpyef01qzrR6p5o9Qyyzbs3K8LHv0kDqMqi6AMAACAOjc4O0v/vGZ44LLFG3YlxGwYBGUAAADExeDsLM26fmRgO+7cVdvjXlmOb48yTcoAAAAN2uDsLP2snJ7lxRt2acyf59TtgMJQUQYAAEBcTT6nd7lTx63atlffu+vduEwdR1AGAABA3FV0Fb9v9x3SRY99qilv5tbpmOI8PRy9FwAAAPBMPqd3uWFZkh6fu1Yz5uXX2XioKAMAACBhVBaW//DasjobC0EZAAAACSU0z3KbZulllh04XFxnJ/gRlAEAAJBwBmdnKee3ZyqzcdmLkqzatrdOpo6Lb1CmRRkAAAAVeOrq8i9KUtuVZSrKAAAASFiDs7PK7VletW2vbpm5qNb2TVAGAABAQqtonuVXFm+qtf0SlAEAAJDwnr5muC4Y2KnM/U6qtX5lLmENAACApPDQpYPUumnZmTAWb9hVK/MrU1EGAABA0nhiwtDA+3/98pKY74ugDAAAgKQxODtLA7u0LHO/kzT07vdiuq84X8IaAAAAqJrZN56sJmllY+y2PQdj2oJBRRkAAABJJ/fusYH3//7VpTHbB0EZAAAASSlofuVDRU5T3syNyfYJygAAAEhKk8/pHXj/43PXxmT7cZ4eji5lAAAAVF95FyI55ldv1HjbVJQBAACQtJ6+ZnjgiX1FThrz5zk12jZBGQAAAEmtvBP7Vm3bW6PtEpQBAACQ9GZdPzLw/pOnfFDtbTKPMgAAAJLe4OwsNU4tmy437Nxf7W1SUQYAAEC9MGPSiMD709t161ed7RGUAQAAUC8Mzs4KvN9S0xpXZ3sEZQAAANQbQRchqa44z6Mcz70DAACgvinvIiTVQUUZAAAA9UrzRqkx2Q5BGQAAAPXK0rvOjsl24jw9HL0XAAAASExUlAEAAIAABGUAAADUO+umnFvjbRCUAQAAUC+tm3KuurTKkFz1Hm/OVfORNdS4Y0+3K3+FMtJjc1YiAAAAEMTMcpxzQ6r6OCrKAAAAQACCMgAAABCAoAwAAAAEICgDAAAAAQjKAAAAQACCMgAAABAgvpew5grWAAAASFBUlAEAAIAABGUAAAAgAEEZAAAACBDfHmXRpAwAAIDEREUZAAAACEBQBgAAAAIQlAEAAIAAzKMMAAAABKCiDAAAAAQgKAMAAAABCMoAAABAgDjPowwAAAAkJirKAAAAQACCMgAAABAgztPD0XwBAACAxERFGQAAAAhAUAYAAAACEJQBAACAAEwPBwAAAASgogwAAAAEICgDAAAAAQjKAAAAQIA4z6Mcz70DAAAA5aOiDAAAAAQgKAMAAAABuIQ1AAAAECCqoGxmZ5vZSjNbbWaTK1jvIjNzZjYkdkMEAAAA6l6lQdnMUiVNlTRWUh9Jl5lZn4D1MiX9QtK8WA8SAAAAqGvRVJSHSVrtnFvrnDsoaaak8wPW+6Ok+yTtj+H4AAAAgLiIJih3lrQ+7PYG/74SZvY9SUc7596I4dgAAACAuKnxyXxmliLpfyXdFsW6k8xsoZktrOl+AQAAgNoUTVDeKOnosNtd/PtCMiX1kzTHzNZJOlHSq0En9Dnn/u6cG+Kc42Q/AAAAJLRogvICST3NrLuZNZJ0qaRXQwudc7ucc22dc92cc90kfS5pnHOOqjEAAACSVqVB2Tl3WNKNkt6RlCvpBefcMjO7y8zG1fYAAQAAgHgw51xcdty4Y093YPOquOwbAAAADYeZ5VSn9ZdLWAMAAAABCMoAAABAAIIyAAAAEICgDAAAAAQgKAMAAAABCMoAAABAAIIyAAAAEICgDAAAAAQgKAMAAAAB4haULV47BgAAAKJARRkAAAAIQFAGAAAAAhCUAQAAgAAEZQAAACAAQRkAAAAIQFAGAAAAAhCUAQAAgAAEZQAAACAAQRkAAAAIQFAGAAAAAhCUAQAAgAAEZQAAACAAQRkAAAAIQFAGAAAAAhCUAQAAgAAEZQAAACAAQRkAAAAIQFAGAAAAAsQvKFvc9gwAAABUiooyAAAAEICgDAAAAASIW1A2ei8AAACQwKgoAwAAAAEIygAAAEAAgjIAAAAQgKAMAAAABCAoAwAAAAEIygAAAEAAgjIAAAAQgKAMAAAABCAoAwAAAAEIygAAAEAAgjIAAAAQgKAMAAAABCAoAwAAAAEIygAAAEAAgjIAAAAQgKAMAAAABCAoAwAAAAEIygAAAEAAgjIAAAAQgKAMAAAABCAoAwAAAAEIygAAAECAuAVli9eOAQAAgChQUQYAAAACEJQBAACAAARlAAAAIABBGQAAAAhAUAYAAAACEJQBAACAAPELyswPBwAAgARGRRkAAAAIQFAGAAAAAhCUAQAAgAAEZQAAACAAQRkAAAAIQFAGAAAAAhCUAQAAgAAEZQAAACAAQRkAAAAIQFAGAAAAAsQtKHMFawAAACQyKsoAAABAgKiCspmdbWYrzWy1mU0OWH6rmS03s6/M7AMzy479UAEAAIC6U2lQNrNUSVMljZXUR9JlZtYnYrVFkoY45wZIelHS/8R6oAAAAEBdiqaiPEzSaufcWufcQUkzJZ0fvoJz7kPn3D7/5ueSusR2mAAAAEDdiiYod5a0Puz2Bv++8lwj6a2aDAoAAACIt7RYbszMfiJpiKRR5SyfJGmSJGV0OCaWuwYAAABiKpqK8kZJR4fd7uLfV4qZjZb0a0njnHMHgjbknPu7c26Ic25ISgoTbgAAACBxRZNWF0jqaWbdzayRpEslvRq+gpkNkvQ3eSF5a+yHCQAAANStSoOyc+6wpBslvSMpV9ILzrllZnaXmY3zV7tfUnNJ/8/MFpvZq+VsDgAAAEgK5pyLy46bdT7O7d34n7jsGwAAAA2HmeU454ZU9XE0CgMAAAABCMoAAABAAIIyAAAAEICgDAAAAAQgKAMAAAABCMoAAABAAIIyAAAAEICgDAAAAAQgKAMAAAABCMoAAABAAIIyAAAAECBuQdlk8do1AAAAUCkqygAAAEAAgjIAAAAQgKAMAAAABIhfUKZFGQAAAAmMijIAAAAQgKAMAAAABCAoAwAAAAEIygAAAEAAgjIAAAAQgKAMAAAABCAoAwAAAAEIygAAAEAAgjIAAAAQgKAMAAAABCAoAwAAAAEIygAAAEAAgjIAAAAQgKAMAAAABCAoAwAAAAEIygAAAEAAgjIAAAAQgKAMAAAABIhbULZ47RgAAACIAhVlAAAAIABBGQAAAAhAUAYAAAACEJQBAACAAARlAAAAIABBGQAAAAhAUAYAAAACEJQBAACAAARlAAAAIABBGQAAAAhAUAYAAAACEJQBAACAAARlAAAAIABBGQAAAAgQv6BscdszAAAAUCkqygAAAEAAgjIAAAAQIG5Bmc4LAAAAJDIqygAAAEAAgjIAAAAQgKAMAAAABCAoAwAAAAEIygAAAEAAgjIAAAAQgKAMAAAABCAoAwAAAAEIygAAAEAAgjIAAAAQgKAMAAAABCAoAwAAAAEIygAAAEAAgjIAAAAQgKAMAAAABCAoAwAAAAEIygAAAEAAgjIAAAAQgKAMAAAABCAoAwAAAAGiCspmdraZrTSz1WY2OWB5YzN73l8+z8y6xXqgAAAAQF2qNCibWaqkqZLGSuoj6TIz6xOx2jWSCpxzx0p6UNJ9sR4oAAAAUJeiqSgPk7TaObfWOXdQ0kxJ50esc76kf/o/vyjpDDOzijZaVOyUk1dQ1fECAAAAdSItinU6S1ofdnuDpOHlreOcO2xmuyS1kbS9vI12dZtUNG2sCju2UGbjdO/OvhdIw66VDu6Tpv+47IMGjpcGXS7t3SG9cGXZ5UOvlvpdJO3aIL30s7LLR94o9RorbV8lvXZL2eWn/Jd0zGnS5q+kt39VdvkZv5O6Dpfy50kf3FV2+dn3Sh0HSGs+lOY+UHb5Dx6S2vaUVr4lffpo2eU//JvUsou0dJa0YFrZ5Rc/LTVrIy2aLi2eUXb55f9PatRUmv+EtGx22eUT3/D+/++Hpf+8U3pZeob0k1nezx/9j7T2o9LLm2ZJlzzr/fz+ndL6BaWXt+gkXfSE9/Nbk6VvlpRe3uYYadzD3s+v3iztWFN6eYf+0tgp3s+zrpV2byq9/Oih0ug7vZ+f/4m0L+JDVo9R0qhfej8/e5F0aH/p5cedJZ10s/fzP85VGbz3eO9JvPd475VeznuP9x7vvfr53quCOj2Zz8wmmdlCM1soScVO2v3d4bocAgAAABAVc85VvILZCEl3OufO8m////buLtSyuozj+PfXTI4XjRWdCklzBjpGZoE1xEQXBUaoFzMXSYwgaYx1ZfRGkBQVdhEVFQj2ioMlvZkXsaGJuUhDEEc8IUgjTBwsbHrBShsCqomeVgAABTdJREFUsbKeLta6OBz/nb1G2mttz3w/cGDtvf9rrefAj7WfvdZ/7X0TQFV9fsOYY/2Y+5PsBP4EvLy22Piu81dr7w238L0b9vPmi176f/hXJEmSpGdL8suq2nem6w05o/wgsJpkb5JzgEPAbNOYGXBdv3w1cPdWTTLAK8871yZZkiRJS2vuHOV+zvGNwDFgB3Ckqk4kuRlYq6oZcBtwR5J14Am6ZnpLr9i9yyZZkiRJS2vIzXxU1VHg6KbnPr1h+WmgMRtbkiRJen7yl/kkSZKkBhtlSZIkqcFGWZIkSWqwUZYkSZIabJQlSZKkBhtlSZIkqcFGWZIkSWqwUZYkSZIabJQlSZKkBhtlSZIkqcFGWZIkSWqwUZYkSZIabJQlSZKkBhtlSZIkqcFGWZIkSWpIVU2z4+TvwMlJdq5ltgL8ZeoitHTMhVrMhVrMhVpeW1W7z3SlnYuoZKCTVbVvwv1rCSVZMxfazFyoxVyoxVyoJcnac1nPqReSJElSg42yJEmS1DBlo/ytCfet5WUu1GIu1GIu1GIu1PKccjHZzXySJEnSMnPqhSRJktSw8EY5yRVJTiZZT/KJxuu7kvyof/2BJHsWXZOmNyAXH03ySJKHk/w8yUVT1KlxzcvFhnHvTlJJvLP9LDAkF0ne0x8zTiT5/tg1anwD3kdeneSeJA/17yVXTVGnxpPkSJLHk/zqf7yeJLf0mXk4yZvmbXOhjXKSHcCtwJXAJcA1SS7ZNOww8GRVvQb4KvCFRdak6Q3MxUPAvqp6I3AX8MVxq9TYBuaCJLuBDwEPjFuhpjAkF0lWgZuAt1XV64EPj16oRjXwePEp4M6qugw4BHxt3Co1gduBK7Z4/Upgtf/7APD1eRtc9BnltwDrVfVoVf0T+CFwcNOYg8B3+uW7gMuTZMF1aVpzc1FV91TVU/3D48AFI9eo8Q05XgB8ju4D9dNjFqfJDMnF+4Fbq+pJgKp6fOQaNb4huSjgvH75xcAfRqxPE6iqe4EnthhyEPhudY4DL0ly/lbbXHSj/Crgdxsen+qfa46pqmeA08DLFlyXpjUkFxsdBn620Iq0DObmor9MdmFV/XTMwjSpIceLi4GLk9yX5HiSrc4oaXsYkovPAtcmOQUcBT44TmlaYmfaf0z6y3zSXEmuBfYBb5+6Fk0ryQuArwDXT1yKls9Oukup76C7+nRvkjdU1d8mrUpTuwa4vaq+nOStwB1JLq2q/0xdmJ4/Fn1G+ffAhRseX9A/1xyTZCfd5ZG/LrguTWtILkjyTuCTwIGq+sdItWk683KxG7gU+EWS3wL7gZk39G17Q44Xp4BZVf2rqn4D/Jqucdb2NSQXh4E7AarqfuBcYGWU6rSsBvUfGy26UX4QWE2yN8k5dJPpZ5vGzIDr+uWrgbvLL3fe7ubmIsllwDfpmmTnG54dtsxFVZ2uqpWq2lNVe+jmrh+oqrVpytVIhryP/ITubDJJVuimYjw6ZpEa3ZBcPAZcDpDkdXSN8p9HrVLLZga8t//2i/3A6ar641YrLHTqRVU9k+RG4BiwAzhSVSeS3AysVdUMuI3ucsg63QTsQ4usSdMbmIsvAS8Cftzf2/lYVR2YrGgt3MBc6CwzMBfHgHcleQT4N/DxqvLK5DY2MBcfA76d5CN0N/Zd74m47S3JD+g+NK/0c9M/A7wQoKq+QTdX/SpgHXgKeN/cbZoZSZIk6dn8ZT5JkiSpwUZZkiRJarBRliRJkhpslCVJkqQGG2VJkiSpwUZZkiRJarBRliRJkhpslCVJkqSG/wKWPIvrYrfqewAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_precision_recall_curve(estimator, estimator_name, X, y_true):\n",
    "  # predict using given estimator\n",
    "  y_pred = estimator.predict_proba(X)\n",
    "  precisions = []\n",
    "  recalls = []\n",
    "  \n",
    "  # thresholds list\n",
    "  thres = np.linspace(0,1,1000)\n",
    "\n",
    "  # keep scores for each threshold\n",
    "  for t in thres:\n",
    "    prec_dict, rec_dict, _ = get_scores(y_true, y_pred, t)\n",
    "\n",
    "    precisions.append(prec_dict['macro'])\n",
    "    recalls.append(rec_dict['macro'])\n",
    "\n",
    "  # calculate area-under-curve\n",
    "  area = auc(recalls, precisions)\n",
    "\n",
    "  # plot\n",
    "  fig, ax = plt.subplots(figsize=(12,8))\n",
    "  ax.set_title('Macro Precision-Recall Curve for %s: AUC=%0.3f' % (estimator_name,area))\n",
    "  ax.plot(recalls, precisions, marker='.', label=estimator_name)\n",
    "  ax.plot(thres,np.zeros(len(thres)), linestyle='--', label='No Skill')\n",
    "  ax.set_xlim(0,1)\n",
    "  ax.legend()\n",
    "  fig.savefig(\"/content/gdrive/My Drive/keras_checkpoints/\"+str(estimator_name)+\"_PR_CURVE.pdf\")\n",
    "\n",
    "\n",
    "pipeline_list = [\n",
    "                [model,'Sentiment Keras']\n",
    "               ,[sgd,'Logistic Regression']\n",
    "               ]\n",
    "\n",
    "for pipe in pipeline_list:\n",
    "  plot_precision_recall_curve(pipe[0], pipe[1], X_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "TvaW8CjXHagM"
   },
   "source": [
    "#### Bootstrap Statistical Significance Tests"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 67
    },
    "colab_type": "code",
    "id": "zCSTTXlaZj-x",
    "outputId": "3ed4c9e6-1830-4bd2-e991-5158780be85f"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Base macro-f1: 0.3304\n",
      "SGD macro-f1: 0.7492\n",
      "Keras macro-f1: 0.7814\n"
     ]
    }
   ],
   "source": [
    "# Initialize classifiers ##########################################\n",
    "# Base\n",
    "y_pred = dummy.predict_proba(X_test)\n",
    "_, _, bs_f1 = get_scores(y_test, y_pred, 0.5)\n",
    "print('Base macro-f1: {:.4f}'.format( bs_f1['macro']))\n",
    "\n",
    "# SGD\n",
    "y_pred = sgd.predict_proba(X_test)\n",
    "_, _, sgd_f1 = get_scores(y_test, y_pred, 0.5)\n",
    "print('SGD macro-f1: {:.4f}'.format(sgd_f1['macro']))\n",
    "\n",
    "# Keras\n",
    "y_pred = model.predict_proba(X_test)\n",
    "_, _, ks_f1 = get_scores(y_test, y_pred, 0.5)\n",
    "print('Keras macro-f1: {:.4f}'.format( ks_f1['macro']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 67
    },
    "colab_type": "code",
    "id": "tNlIgHPqHiNW",
    "outputId": "df1c314d-1a5b-43db-b299-6d0bfa3f4da5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Keras vs Baseline, p_value:  0.0000\n",
      "Keras vs Logistic Regression, p_value: 0.0020\n",
      "Elapsed time 148 seconds\n"
     ]
    }
   ],
   "source": [
    "# Bootstrapping significance test ###################################\n",
    "\n",
    "def bootstrap_sign_test(best_estimator, comp_estimator, X, y, b, N):\n",
    "  \"\"\"\n",
    "  Input:\n",
    "    best_estimator: best scoring pipeline in terms of macro-f1\n",
    "    comp_estimator: pipeline to be compared with the best\n",
    "    X: array with X dataset\n",
    "    y: \n",
    "    b: number of sampling datasets\n",
    "    N: size of sampling datasets\n",
    "\n",
    "  Output:\n",
    "    p_value: p-value for the bootstrapping significance test\n",
    "  \"\"\"\n",
    "  y1 = best_estimator.predict_proba(X)\n",
    "  _, _, e1_f1 = get_scores(y, y1, 0.5)\n",
    "\n",
    "  y2 = comp_estimator.predict_proba(X)\n",
    "  _, _, e2_f1 = get_scores(y, y2, 0.5)\n",
    "  \n",
    "  # calculate true difference\n",
    "  delta = e1_f1['macro'] - e2_f1['macro']\n",
    "\n",
    "  diffs = []\n",
    "\n",
    "  # calculate a diff for each simulated sample dataset\n",
    "  for i in range(b):\n",
    "    # generate N random indexes with replacement\n",
    "    random_idx = np.random.choice(X.shape[0], N, replace=True) \n",
    "    bootstrap = X[random_idx, :]\n",
    "    y_best = best_estimator.predict_proba( bootstrap )\n",
    "    y_comp = comp_estimator.predict_proba( bootstrap )\n",
    "\n",
    "    _, _, best_f1 = get_scores(y[random_idx], y_best, 0.5)\n",
    "    _, _, comp_f1 = get_scores(y[random_idx], y_comp, 0.5)\n",
    "    diffs.append(best_f1['macro'] - comp_f1['macro'])\n",
    "  \n",
    "  # calculate p-value\n",
    "  s = 0\n",
    "  for i in range(b):\n",
    "    if diffs[i] > 2 * delta:\n",
    "      s += 1\n",
    "  p_value = s/b\n",
    "  \n",
    "  return p_value\n",
    "\n",
    "# print p-values\n",
    "start_time = time.time()\n",
    "\n",
    "print('Keras vs Baseline, p_value:  {:.4f}' .format( bootstrap_sign_test(model, dummy, X_test, y_test, b = 1000, N = 1000) ))\n",
    "print('Keras vs Logistic Regression, p_value: {:.4f}' .format( bootstrap_sign_test(model, sgd, X_test, y_test, b = 1000, N = 1000) ))\n",
    "\n",
    "elapsed_time = time.time() - start_time\n",
    "\n",
    "print('Elapsed time {:.0f} seconds'.format(elapsed_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "4ok-0yQWD-7n"
   },
   "source": [
    "#### Learning Curves"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 768
    },
    "colab_type": "code",
    "id": "37qeZKNGw_TH",
    "outputId": "45cfd5d4-8505-4267-aef1-28989b24d42a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Process ended for trainsize: 10.0 % :\n",
      "Process ended for trainsize: 16.42857142857143 % :\n",
      "Process ended for trainsize: 22.857142857142858 % :\n",
      "Process ended for trainsize: 29.285714285714292 % :\n",
      "Process ended for trainsize: 35.71428571428572 % :\n",
      "Process ended for trainsize: 42.142857142857146 % :\n",
      "Process ended for trainsize: 48.57142857142858 % :\n",
      "Process ended for trainsize: 55.00000000000001 % :\n",
      "Process ended for trainsize: 61.42857142857143 % :\n",
      "Process ended for trainsize: 67.85714285714286 % :\n",
      "Process ended for trainsize: 74.28571428571429 % :\n",
      "Process ended for trainsize: 80.71428571428572 % :\n",
      "Process ended for trainsize: 87.14285714285715 % :\n",
      "Process ended for trainsize: 93.57142857142858 % :\n",
      "Process ended for trainsize: 100.0 % :\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAtcAAAHwCAYAAABtz0NOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nOzdeXzdVZ3/8den6b5Q6AoUuiClFFCKVHahAgoMKsryY+kgImMdZHdBx23qUlccEQWxMmxaBKSKiKigEEQQpMgylJ3aQstOS2kJXXN+f5wbcpPepGn7vU3SvJ6Px/fx3e89N/mmfefkLJFSQpIkSdKG69beBZAkSZI2FYZrSZIkqSCGa0mSJKkghmtJkiSpIIZrSZIkqSCGa0mSJKkghmtJ2ggi4t0R8Xh7l6OjiYhfRsSH2rscDSKiV0Q8FhFD27sskjonw7WkTV5EzI2Ig9uzDCmlO1JK46r1+hFxSET8NSKWRMTLEXF7RHywWu9XhIh4B7Ar8NvS/kcj4m9l5zeLiDsjYmZE9NwYZUopLQcuBT6/Md5P0qbHcC1JBYiImnZ876OBXwFXAtsAw4GvAB9Yj9eKiNhY/zd8ApiRKsxmFhFbAH8B5gHHppRWtPVFI6L7BpbrKuCkiOi1ga8jqQsyXEvqsiKiW0R8PiKejohXI+LaiBhUdv5XEfFCRCwu1QrvXHbu8oj4SUTcFBFvAO8p1ZB/JiIeKt1zTUT0Ll0/KSLml93f4rWl8+dGxPMR8VxE/EdEpIjYvsJnCOB/gK+nlC5JKS1OKdWnlG5PKX28dM3UiPhF2T2jS6/XvbRfGxHTIuJOoA74bETMavY+50TEDaXtXhFxXkQ8ExEvRsTFEdGndG5IRNwYEa9FxMKIuKOVsH4YcHuFzzQUuA14GPj3lNKq0vH3R8QDpde+q1TzXf71/FxEPAS8ERHdy763SyLikYj4cNn125dq9xdHxCsRcU3DuZTSfGARsFcL5ZakFhmuJXVlZwAfAg4AtiYHqgvLzv8BGAsMA/4JzGh2/wnANGAA0NCc4f8BhwJjgHcAH23l/SteGxGHAp8CDga2Bya18hrjgG2B61q5pi1OBKaQP8vFwLiIGFt2/gRyjS7At4EdgAml8o0g15QDfBqYDwwl16B/AahUM92P/Lmbt0MfBNQCfwc+llKqL12/G7m5xieAwcBPgRua1S4fDxwObF4K5E8D7wYGAl8FfhERW5Wu/TpwM7AFubb/R83K8Si5yYokrRPDtaSu7D+BL6aU5pfa2k4Fjm6o0U0pXZpSWlJ2bteIGFh2/29TSneWaoqXlY5dkFJ6LqW0EPgdOYC2pKVr/x9wWUppdkqprvTeLRlcWj/f1g/dgstL77cqpbSY3A76eIBSyN6RHGaDHMLPSSktTCktAb4JHFd6nZXAVsColNLKUlvzNcI1sHlpvaTZ8W3Jwf3yZvdNAX6aUronpbQ6pXQFsJymtcsXpJSeTSm9CZBS+lXp61ufUroGeBLYo6yco4CtU0rLUkp/o6klZWWUpDYzXEvqykYBvyk1M3iNXFu5GhgeETUR8e1Ss4LXgbmle4aU3f9shdd8oWy7Dujfyvu3dO3WzV670vs0eLW03qqVa9qi+XtcRSlck2utry8F/aFAX+C+sq/bH0vHAb4HPAXcHBFzIqKljoGvldYDmh1/EPgM8IdSbXWDUcCnG96z9L7bkr9WFT9DRHykrBnJa8AuNH7/zgUC+EdEzI6IjzUrx4CyMkpSmxmuJXVlzwKHpZQ2L1t6p5QWkAPlEeSmGQOB0aV7ouz+SjWyRXie3FShwbatXPs4+XMc1co1b5ADcYMtK1zT/LPcAgyNiAnkkN3QJOQV4E1g57Kv2cCUUn+AUk3/p1NK2wEfBD4VEQet8WYpvUFutrFDhXM/JDc9uSUidikdfhaY1ux71Tel9MtKnyEiRgE/A04HBqeUNie34Y7Se7yQUvp4SmlrclOTi5q1aR9PDvqStE4M15K6ih4R0bts6U5uWzytFMSIiKERcUTp+gHkZgevkoPpNzdiWa8FTo6I8RHRF/hySxeWmk58CvhyRJwcefi6bhGxX0RML132ALB/RIwsNWv5r7UVIKW0kjwCyffI7aBvKR2vJ4fWH0TEMICIGBERh5S231/qLBjAYvJfAupbeJubyO3dK73/d4EfAn+OiHGl9/zPiNgzsn4RcXhENK/5btCPHLZfLpXrZHLNNaX9YyKi4ReYRaVrG9p3jyh95rtb+xpJUiWGa0ldxU3kGteGZSo5vN1AbsKwhBym9ixdfyV5GLgFwCNsxKCVUvoDcAF5xIynyt57eQvXXwccC3wMeA54EfgGpfGjU0q3ANcADwH3ATe2sShXkWvuf9UwYkfJ5xrKVWoy82dyx0rIHUD/DCwld0q8KKV0WwuvPx2YXArilT7X14FLyEPyLQI+Dvy4tP0UrXQWTSk9Any/VIYXgbcDd5Zd8i7gnohYSn4GzkopzSmdOwG4otTWXpLWSVTuZyJJ6igiYjy5SUOvZiG304uIq4BrU0rXt3dZIA8zSG4Osn9K6aX2Lo+kzsdwLUkdUGlM5pvITVKuAOpTSh1mmnBJUmU2C5GkjukTwEvkTn+rgVPbtziSpLaw5lqSJEkqiDXXkiRJUkEM15IkSVJBulfrhSPiUuD9wEsppV0qnA/yMFj/Rp6Z7KMppX+Wzp0EfKl06TdK09y2asiQIWn06NEFlV4b4o033qBfv37tXQx1QD4baonPhlris6HWtNfzcd99972SUhpa6VzVwjVwOXk80itbOH8YeTzUseRxZX8C7BkRg4D/BiaSB/W/LyJuSCktau3NRo8ezaxZswoqujZEbW0tkyZNau9iqAPy2VBLfDbUEp8Ntaa9no+ImNfSuao1C0kp/RVY2MolRwBXpuxuYPOI2Ao4BLglpbSwFKhvAQ6tVjklSZKkorRnm+sRwLNl+/NLx1o6LkmSJHVo1WwWUnURMQWYAjB8+HBqa2vbt0ACYOnSpX4vVJHPhlris6GW+GyoNR3x+WjPcL0A2LZsf5vSsQXApGbHayu9QEppOjAdYOLEick2WR2D7ePUEp8NtcRnQy3x2VBrOuLz0Z7NQm4APhLZXsDilNLzwJ+A90XEFhGxBfC+0jFJkiSpQ6vmUHy/JNdAD4mI+eQRQHoApJQuBm4iD8P3FHkovpNL5xZGxNeBe0sv9bWUUmsdIyVJkqQOoWrhOqV0/FrOJ+C0Fs5dClxajXJJkiRJ1eIMjZIkSVJBDNeSJElSQQzXkiRJUkEM15IkSVJBDNeSJElSQQzXkiRJUkEM15IkSVJBDNeSJElSQQzXG2DGDBg9Grp1y+sZM9q7RJIkSWpPVZuhcVM3YwZMmQJ1dXl/3ry8DzB5cvuVS5IkSe3Hmuv19MUvNgbrBnV18OlPr3lckiRJXYM11+vpmWcqH3/xRRg4EHbfHfbdF/bbL6+HDdu45ZMkSdLGZ831eho5svLxoUPhM5+BHj3gwgvhyCNh+HDYYQc4+WT43/+Fxx6DlDZueSVJklR91lyvp2nTmra5BujbF37wg8Y218uXw333wZ13wt/+Br/7HVx+eT43ZAjss09jzfbuu0OvXhv9Y0iSJKlAhuv11BCgv/jF3ERk5MgcuMs7M/bqlQP0PvvAZz+ba6sff7wxbN95J9xwQ+O173pXY9jeZx8YNGjjfy5JkiStP8P1Bpg8ed1GBomAHXfMyymn5GMvvgh33dUYts87D7797Xxup50aw/Z++8GYMfk1JEmS1DEZrtvZ8OHw4Q/nBXIzk3vvbQzb11wD06fnc1tu2TRs77prbtstSZKkjsFw3cH07QsHHJAXgPp6mD27MWz/7W9w3XWN1+61V2PY3msv2Gyz9iu7JElSV+doIR1ct27w9rfDqafCL34Bc+fCs8/C1VfnpiWLFuW23occAltsAbvtBqefns8/+2zj6zibpCRJUvVZc90JbbMNHHtsXgCWLIG7726s2b788jwMIOSOliNGwKxZsHJlPuZskpIkSdVhuN4EDBgA731vXgBWrYIHH2wM2zNn5uYl5erq4D//E+bPzzXZo0blZfjwXLstSZKkdWe43gR1757Hzd59dzjzzJbD8tKl8PnPNz3Wq1eu7W4I26NGNQ3fI0bk15ckSdKajEldwMiRuSlIc6NGwf/9Xz43b15uz12+feONeajAcjU1uVlKS+G7pZkrJUmSugLDdRfQ0myS06blJiW77JKXSt58M3eMLA/eDeG7thYWLFizycngwXszdmzl8D1qFPTvv/Yyz5jR+gQ9kiRJHZHhugtoy2ySLenTB3bYIS+VrFyZA3Z5+L777oUsX74Vs2bBr3/d2JGyweDBLQfv0aPhppua/jJgB0xJktRZGK67iHWdTbKtevTIgXj06MZjtbWPM2nSVkCu1X7hhco13489Bn/6U9MadcizUKbU9FhdHZx7bh4hxTbfkiSpozKmqKq6dYOtt87LPvuseT4leOWVpsH705+u/FrPPZebs4wZA2PHrrlsu21uEy5JktReDNdqVxEwdGheJk7Mxy64oHIHzMGD4eMfhyefzMtttzWt9e7ZE7bbrnLw3mYbhxiUJEnVZ7hWh9NSB8wf/rBp05aUcm12Q9huWJ56Cm65BZYta7y2d29429saw/b22zdub721wVuSJBXDcK0Op60dMCPyuNsjRsCkSU3P1dfnjpbNg/fjj+cOkytWNF7bp0/TsF2+bLllfp/WOLKJJElqYLhWh7ShHTC7dcttsLfdFg48sOm51avz8IJPPdU0eM+eDb/7XdPRTfr1azl4DxsGV13lyCaSJKmR4VpdTk1N4wgnBx/c9NyqVbkGunmN94MPwvXX5/MNBgzITU+aDzVYV5drsg3XkiR1PYZrqUz37rlT5HbbwSGHND23cmWumS4P3T/+ceXXmTcvd8zcbz94xzscPlCSpK7C//KlNurRIzcR2X57OOywfOx3v6s8sklNDZx1Vt7u3x/23jsH7X33hT33bNsslZIkqfNxjARpA0yblkcyKde3L1xxRQ7dV10FH/kIvPgiTJ2am6Fsvjm8611wzjkwc2aeZEeSJG0arLmWNsDaRjYZORKOPz5vv/Ya3H03/O1vebn4Yjj//Hxu++0ba7b32w/GjVv7KCWSJKnjMVxLG6itI5tsvjkcemheIA8H+M9/5qB9551w441w+eX53JAhjUF7v/3gne/Mk+RIkqSOzXAttZOePWGvvfLymc/kSXGeeKKxZvtvf4Pf/jZf27t3bqvdELj33juHdUmS1LEYrqUOIiI3Bxk3Dk45JR974YVcq33nnTlsf+c78M1v5mvf/vbGmu399stjekuSpPZluJY6sC23hKOOygvAG2/APfc01mxfeSVcdFE+N3Jk03bbO++cRy2RJEkbj+Fa6kT69cszTjbMOrlqFTz0UGPN9m235RFKAAYOhH32aazZfte78lTvTtcuSVL1GK6lTqx799zZ8Z3vhDPOyO22585t2m77D3/I1/boAaNG5fMNM006XbskScUyXEubkAgYMyYvJ56Yjy1cCHfdlYP2+ec3ncId8nTtU6bk0D1+POy4Yx4a0NFJJElad4ZraRM3aBC8//15+e53K19TVwdf+lLjfvfu8La35aA9fnxj6N5xR9hss41TbkmSOiPDtdSFjBxZebr2UaPg4Yfh8cfh0Ufz8thjef373zet7R4xYs3QPX587nzpxDeSpK7OcC11IdOm5SYgdXWNx/r2zcf794fdd89LuZUrYc6cNUP3FVfAkiWN1w0c2DRsN6zHjMk14ZIkdQX+lyd1IWubrr2SHj0ax9/+0Icaj6cEzz23Zuj+4x8bZ5qE3HZ77NgctPv0Gc1zz+XtceNysG+NI5tIkjobw7XUxbR1uva1ichNREaMgIMPbnrutdcaw3bD+oEHYM6cUfz8543XjRq1ZvOS8ePz9O8zZjStZXdkE0lSZ2C4llS4zTdvnNq93M03/5Wttz6gSeh+9FG4/XZ4883G6wYPhqVLYfnypvfX1eWabMO1JKmjMlxL2mh69kzssgvsskvT4/X1uelHeej+2c8qv8Yzz8CyZdC7d/XLK0nSujJcS2p33brB6NF5OeywfOzmmyuPbJISDBuWhxY86qh8/drabkuStLF0a+8CSFIl06atGZr79oVzz4Vjj4VbboGjj87ts48+Gn75S3j99fYpqyRJDQzXkjqkyZNh+vTc6TEir6dPh+98JzcZef55+Mtf4OST4c474YQTco32Bz+YhwlctKi9P4EkqSsyXEvqsCZPztOy19fndXlHxu7d4cAD4cILYcECuOMOOPXUPCrJRz+ag/ahh8Ill8DLL7fTB5AkdTmGa0mdXrdusN9+8IMf5Hba99wDn/oUPPkkfPzjefbIAw+Eiy7KNd6SJFWL4VrSJiUC9tgjNx956im4/374whdyqD7ttDwu9377wfnn55FHJEkqkuFa0iYrAiZMgK9/PQ/vN3s2TJ2ap20/55zcjnvPPeG734Wnn27v0kqSNgWGa0ldxk47wVe+Ag8+CE88Ad/6Vm7P/bnPwfbbw267wTe+kYO4JEnrw3AtqUsaOxY+/3m4917417/g+9+HPn3gy1/OIXznnXMQf+ihPLa2JEltYbiW1OWNHp07QN51F8yfDz/6EQwdmsfa3nVX2GGHHMRnzTJoS5JaZ7iWpDIjRsDpp0Ntbe4E+dOfwpgxcN558K535e2GIF5f396llSR1NIZrSWrBsGEwZUqeiv3FF+HSS2GXXfLY2vvuC9tuC2eckYP46tUwY0auBW+Yzn3GjHb+AJKkja57exdAkjqDwYPzbJAnnwyLF8ONN8LMmXmSmh//GAYMgDffhFWr8vXz5uVgDk0nv5EkbdqsuZakdTRwYA7Mv/51nv3x2mtzzXVDsG5QV5fbakuSug7DtSRtgP794Zhjcq11JfPn50ltvvY1uO8+22lL0qbOcC1JBRg5svLxgQNzG+ypU2HixNxh8j/+A37zmzyZjSRp02K4lqQCTJsGffs2Pda3b+78ePfd8MILcMUVsP/+8KtfwZFH5nbc730v/PCHeap2SVLnZ7iWpAJMngzTp+cp1SPyevr0xs6Mw4bBRz4C11wDr7wCt90GZ50FCxbA2WfnSW3GjcvD/N16K6xY0b6fR5K0fgzXklSQyZNh7tzcrnru3JZHCenRAyZNgu99Dx55BJ5+Gi64II+hfeGFcNBBMGQIHH00XH55HgZQktQ5GK4lqZ1tt10eL/uPf4RXX4Xrr4fjjoO//z0P/bfllrlT5Fe/aqdISeroDNeS1IH07w9HHJGblMyfD//8J3z961BTk8N1Q6fIU07JQwHaKVKSOhbDtSR1UBGw227wpS/lWuwXX2zsFDlzJhx1VGOnyPPPhyefbO8SS5IM15LUSQwd2tgp8uWX87TrDZ0izzkHdtihsVPkX/5ip0hJag+Ga0nqhHr0gAMOaOwUOWcO/OhHuVPkRRfBwQc3doq87LI1O0XOmAGjR+cxuEePzvuSpA3Xvb0LIEnacGPGwOmn5+WNN3LN9e9/n5eZM/M1EyfC+9+fA/W3v52nZweYNw+mTMnbLY1wIklqG8O1JG1i+vWDD34wLynBgw82Bu2vfjUfa66uDj7/eTj2WOju/wyStN6q+k9oRBwK/BCoAS5JKX272flRwKXAUGAh8O8ppfmlc6uB/ytd+kxK6YPVLKskbYoiYMKEvHzxi7mt9rBhla+dPx969oRBg3L77mHDGtfl2+XrQYPySCaSpKxq4ToiaoALgfcC84F7I+KGlNIjZZedB1yZUroiIg4EvgWcWDr3ZkppQrXKJ0ld0dChefbIefPWPLfFFnDmmTmAv/RSXj/ySO44uXBh5Rrvbt1y2+5KwbtSKN9883zPupoxI/9y8MwzMHJknm7eJiySOqJq1lzvATyVUpoDEBFXA0cA5eF6J+BTpe3bgOurWB5JEjmYTpnS2OYaoG/f3CGypcC6alWe4KY8eL/0UtPtl1+GBx7I60WLKr9O9+45jDcP3kuXjuTJJ9cM5ZttBldd1bS8thGX1JFFqlQVUcQLRxwNHJpS+o/S/onAniml08uuuQq4J6X0w4g4EpgJDEkpvRoRq4AHgFXAt1NKawTviJgCTAEYPnz47ldffXVVPovWzdKlS+nfv397F0MdkM9Gx/HnPw/jkku246WXejFs2HL+4z/mcPDBLxX2+qtWBYsX92DRoh689lpPXnutfN24vWhRTxYv7sEbb1Su6+nRo57Vq4P6+ljj3JAhy7j22ruJNU9pE+K/G2pNez0f73nPe+5LKU2sdK69w/XWwI+BMcBfgaOAXVJKr0XEiJTSgojYDrgVOCil9HRL7zdx4sQ0a9asqnwWrZva2lomTZrU3sVQB+SzoZbcfPPt7LTTAWvUhL/0EnznOy3f169fHtt7xx1h/Pi83nFHGDsWevXaeOVX9fjvhlrTXs9HRLQYrqvZLGQBsG3Z/jalY29JKT0HHAkQEf2Bo1JKr5XOLSit50RELbAb0GK4liR1Xj17JrbZBrbZZs1zV19duY34oEF5Up1HH4U778zNRxp06wbbbdcYtsuD96BB1fscklTNcH0vMDYixpBD9XHACeUXRMQQYGFKqR74L/LIIUTEFkBdSml56Zp9ge9WsaySpA6qpTbiF1zQtM31G2/AE0/AY481Lo8+CrfcAsuXN143dGjTsN0QvkeOXL/OlpJUrmrhOqW0KiJOB/5EHorv0pTS7Ij4GjArpXQDMAn4VkQkcrOQ00q3jwd+GhH15Fkkv91slBFJUhfREKDXNlpIv36w2255Kbd6da75fvTRpsF75szcSbNBnz55CvnmwXuHHfI5SWqLqo5znVK6Cbip2bGvlG1fB1xX4b67gLdXs2ySpM5j8uT1HxmkpiY3EdluOzj88KbnXnmlaS33Y4/BP/4B11zTOPRgRJ4ivlITk6FD13w/hw2Uujbn4ZIkdVlDhsB+++Wl3JtvwpNPrhm8a2vzuQaDBzet5X7hBbjwQli2LJ932ECp6zFcS5LUTJ8+8I535KVcfX2ukW7ervt3v4P//d/Kr1VXl2uyDddS12C4liSpjbp1y01ERo+GQw9tem7hwlwTXmmE23nz4Gtfg6OPhp122hglldRe7BctSVIBBg3Kbawr6dULpk6FnXfO4forX4GHHqocxCV1boZrSZIKMm1aHiawXN++ucnIggW5PfaWW+brdt01j0TyX/8F991n0JbWxYwZ+S9IBx54AKNH5/2OwnAtSVJBJk+G6dNh1Kg8ysioUXl/8mTYaiv45Cfh1lvh+efhpz+FMWPge9+DiRPzaCaf/Szcc49BW2rNjBm5o/C8eZBSvNVxuKMEbMO1JEkFmjwZ5s7NnR/nzq3ckXHYsBwGbr4ZXnwx12yPHw8//CHstVcO5eeck2eerK/f2J9A6jjq6+Hll3Mzqj/9CS6/HE4/vemkUtDYcbgjsEOjJEntaPBg+NjH8rJoUR555Lrr4KKL4Pzzc433UUflzpD77ZfH7ZY6u1Wr8i+Wzz/f+vLCC/natnjmmeqWua0M15IkdRBbbAEf+UheXn8dfv/7HLQvuQR+/ONc433kkTloH3AAdPd/cRWkqMmPli1be2B+/vlcG12p+dPQofkXyq22yh2AG7bLl0mT4Nln17y3pQ7FG5s/lpIkdUCbbQbHH5+XpUvhD3/IQfvKK+Hii3ON94c+lIP2gQdCz57tXWJ1Vg1tmBuaWjSf/CglWLKkbaH5tdfWfP2amtyRd6utcgDec8/KoXn4cOjRY+3l/da3mpYXcsfhadM2/GtRBMO1JEkdXP/+cMwxeamry21Pr7sOrr02t9fefHM44ogctN/73jz0n9RWX/hC5TbMp5wC//3fOTQ3Pw/Qu3djMN5pJzjooMqheciQPEZ8URpq1HNNe2LkyFjvmvZqMFxLktSJ9O0LH/5wXpYtg1tugZkz4be/hSuuyDXeH/hADtqHHJJnm5QarFyZZxW9/3544IG8bqmt8vLlsMcelQPzVlvBwIF5VJz2MHlyXmprb2fSpEntU4gWGK4lSeqkevfOQfoDH4AVK/Iwf9ddB7/5Tf5Tf79+8P7356B92GF5X13H0qXw4INNg/TDD+dnBfIvau94R/7LyNKla94/ahRcddXGLfOmwKH4JEnaBPTsmadkv+SSPMLCLbfAv/97DtzHHJM7ih19NFx9dW4/26BhMo6Gqd07yljBWjcvvgh//GNuj3zssXmCos02yyPMnHFG/svG4MFw9tk5MD/6aO40+/e/5zb8lSY/6ihtmDsba64lSdrE9OgBBx+clwsvhDvuyDXaM2fmpVev3GRk661zU5I338z3Ne/Ipo6nvh7mzGlaG33//fkXqgZjxsBuu8GJJ+b1brvl73VLTTiatmHesNFCZLiWJGmTVlOThy6bNAkuuADuuisH7OuugxtuWPP6hsk4DFbtb8UKmD27aZB+8MHGvzx07547Eh5yCEyYkEP0rrvmDq7rqqENszac4VqSpC6iW7fcTGC//eD738/hrNJYw/Pm5fP7758Dm+NpV9/rr+cAXV4b/cgjuQMi5HbRu+6ax0BvqI3eaafc7l4diz8ukiR1Qd265T//z5u35rnu3eEzn8nbAwbAvvvmoH3AATBxomNqt1WliVlOOCEPbde8WcecOY33DR+ew/Nhh+X1hAmw/fbFDmen6jFcS5LURU2bVnkyjunT88Q0f/0r3H57Xn/hC/l8nz6w9945aB9wQJ4QxNrTNVWamOUjH4FTT23aoXT77eGd78xjSjcE6a22ap8yqxiGa0mSuqi1dWQ79ti8QJ6u+o47GgP31Km5SUnPnjlgN9Rs77NP1xvyb+VKePrp3D76kUfy+te/bmzS0aC+Pi8XXJBD9K675hE9tGkxXEuS1IW1tSPb0KFw5JF5AVi0CO68Mwft22+Hb387B/Pu3WH33RtrtvfdN082silYsQKeeqoxQDesn3iiaZAeM2bNYN2gri4PjadNl+FakiStsy22yBPUvP/9eX/JkjwSSUPY/sEP4Lvfze2EJ0zIQXv//eHd787jLXdkK1bkwPzII02D9BNPwKpV+ZoI2G673KnwAx/I6512gqDPx4QAACAASURBVB13zDX3o0dXbs8+cuRG/ShqB4ZrSZK0wQYMyEPCHXJI3q+rg7vvbmyz/ZOf5MAN8Pa3NzYj2X//3IGvPSxfngNzQ3huCNJPPgmrV+drunXLIXrnneGII/J6p51g3Lg1J14p11J7didm2fQZriVJUuH69s2dIg88MO8vXw7/+Edjm+3LL88T3EAOqg3NSPbfH7bZptiyLFsGjz/etBb6kUdyE4/yEL399jk4H3VUXu+8c57psE+fdX9PJ2bpugzXkiSp6nr1yk1C3v3uHDhXroR//rOxGcnVV+dRSiDXFDcE7V69epNSboZRaWi78rD65ps5RJe3h37kkdzZsL4+X1NTA2PH5uB8zDGNNdE77FD8qCdOzNI1Ga4lSdJG16NHHmVkzz3h3HNzDfKDDzY2I/ntb+GyywD24nOfg223hVmzcntoyO2ZP/YxuOaaHLxnz85jRTdMitO9ew7Ru+4Kxx/fWBM9dmwO+lK1GK4lSVK7q6nJ4z2/851wzjm5pnn2bLjkkid4/vkdmDmzsfa5wYoV8Lvf5dD8znfCiSc2diwcO9bJbtQ+DNeSJKnD6dYtd3z88IefY9KkHVqcnTACHn5445ZNao0TaUqSpA6vpSHsHNpOHY3hWpIkdXjTpq059J1D26kjMlxLkqQOb/LkPJrIqFG5KcioUXnf0TjU0djmWpIkdQoObafOwJprSZIkqSCGa0mSJKkghmtJkiSpIIZrSZIkqSCGa0mSJKkghmtJkiSpIIZrSZIkqSCGa0mSJKkghmtJkiSpIIZrSZIkqSCGa0mSJKkghmtJkiSpIIZrSZIkqSCGa0mSJKkghmtJkiSpIIZrSZIkqSCGa0mSJKkghmtJkiSpIIZrSZIkqSCGa0mSJKkghmtJkiSpIIZrSZIkqSCGa0mSJKkghmtJkiSpIIZrSZIkqSCGa0mSJKkghmtJkiSpIIZrSZIkqSCGa0mSJKkghmtJkiSpIIZrSZIkqSCGa0mSJKkghmtJkiSpIIZrSZIkqSCGa0mSJKkghmtJkiSpIIZrSZIkqSCGa0mSJKkghmtJkiSpIIZrSZIkqSCGa0mSJKkghmtJkiSpIIZrSZIkqSBVDdcRcWhEPB4RT0XE5yucHxURf4mIhyKiNiK2KTt3UkQ8WVpOqmY5JUmSpCJULVxHRA1wIXAYsBNwfETs1Oyy84ArU0rvAL4GfKt07yDgv4E9gT2A/46ILapVVkmSJKkI1ay53gN4KqU0J6W0ArgaOKLZNTsBt5a2bys7fwhwS0ppYUppEXALcGgVyypJkiRtsGqG6xHAs2X780vHyj0IHFna/jAwICIGt/FeSZIkqUPp3s7v/xngxxHxUeCvwAJgdVtvjogpwBSA4cOHU1tbW4Uial0tXbrU74Uq8tlQS3w21BKfDbWmIz4f1QzXC4Bty/a3KR17S0rpOUo11xHRHzgqpfRaRCwAJjW7t7b5G6SUpgPTASZOnJgmTZrU/BK1g9raWvxeqBKfDbXEZ0Mt8dlQazri81HNZiH3AmMjYkxE9ASOA24ovyAihkREQxn+C7i0tP0n4H0RsUWpI+P7SsckSZKkDqtq4TqltAo4nRyKHwWuTSnNjoivRcQHS5dNAh6PiCeA4cC00r0Lga+TA/q9wNdKxyRJkqQOq6ptrlNKNwE3NTv2lbLt64DrWrj3UhprsiVJkqQOzxkaJUmSpIIYriVJkqSCGK4lSZKkgrQpXEdEn4gYV+3CSJIkSZ3ZWsN1RHwAeAD4Y2l/QkTc0PpdkiRJUtfTlprrqcAewGsAKaUHgDFVLJMkSZLUKbUlXK9MKS1udixVozCSJElSZ9aWca5nR8QJQE1EjAXOBO6qbrEkSZKkzqctNddnADsDy4GrgMXA2dUslCRJktQZtVpzHRE1wO9TSu8BvrhxiiRJkiR1Tq3WXKeUVgP1ETFwI5VHkiRJ6rTa0uZ6KfB/EXEL8EbDwZTSmVUrlSRJktQJtSVc/7q0SJIkSWrFWsN1SumKiOgJ7FA69HhKaWV1iyVJkiR1PmsN1xExCbgCmAsEsG1EnJRS+mt1iyZJkiR1Lm1pFvJ94H0ppccBImIH4JfA7tUsmCRJktTZtGWc6x4NwRogpfQE0KN6RZIkSZI6p7bUXM+KiEuAX5T2JwOzqlckSZIkqXNqS7g+FTiNPO05wB3ARVUrkSRJktRJtSVcdwd+mFL6H3hr1sZeVS2VJEmS1Am1pc31X4A+Zft9gD9XpziSJElS59WWcN07pbS0Yae03bd6RZIkSZI6p7aE6zci4p0NOxGxO/Bm9YokSZIkdU5taXN9NvCriHiOPInMlsCxVS2VJEmS1Am1ZfrzeyNiR2Bc6ZDTn0uSJEkVrLVZSEQcQ253/TDwIeCa8mYikiRJkrK2tLn+ckppSUTsBxwE/C/wk+oWS5IkSep82hKuV5fWhwM/Syn9HuhZvSJJkiRJnVNbwvWCiPgpuRPjTRHRq433SZIkSV1KW0Ly/wP+BBySUnoNGAR8tqqlkiRJkjqhtowWUgf8GiAipqSUpgPPV7tgkiRJUmezrs07/rMqpZAkSZI2AesarqMqpZAkSZI2Aesarj9QlVJIkiRJm4B1CtcppfkAEXFydYojSZIkdV7rO6TeVwsthSRJkrQJaHG0kIh4qKVTwPDqFEeSJEnqvFobim84cAiwqNnxAO6qWokkSZKkTqq1cH0j0D+l9EDzExFRW7USSZIkSZ1Ui+E6pXRKK+dOqE5xJEmSpM6rxQ6NEXFk2fYWG6c4kiRJUufV2mghXyrb/ku1CyJJkiR1dq2F62hhW5IkSVIFrXVo7BMRu5EDeO/S9lshO6X0z2oXTpIkSepMWgvXzwP/U9p+oWwbIAEHVqtQkiRJUmfU2mgh79mYBZEkSZI6u/Wd/lySJElSM4ZrSZIkqSCGa0mSJKkgrXVofEtEfBDYv7R7e0rpd9UrkiRJktQ5rbXmOiK+BZwFPFJazoyIb1a7YJIkSVJn05aa68OBCSmleoCIuAK4H/hCNQsmSZIkdTZtbXO9edn2wGoURJIkSers2lJz/U3g/oi4jTxD4/7A56taKkmSJKkTajVcR0Q3oB7YC3hX6fDnUkovVLtgkiRJUmfTarhOKdVHxLkppWuBGzZSmSRJkqROqS1trv8cEZ+JiG0jYlDDUvWSSZIkSZ1MW9pcH1tan1Z2LAHbFV8cSZIkqfNaa7hOKY3ZGAWRJEmSOru2TCJzWkRsXra/RUR8srrFkiRJkjqftrS5/nhK6bWGnZTSIuDj1SuSJEmS1Dm1JVzXREQ07EREDdCzekWSJEmSOqe2dGj8I3BNRPy0tP+J0jFJkiRJZdoSrj9HDtSnlvZvAS6pWokkSZKkTqoto4XUAz8pLZIkSZJasNZwHRFjgW8BOwG9G46nlBznWpIkSSrTlg6Nl5FrrVcB7wGuBH5RzUJJkiRJnVFbwnWflNJfgEgpzUspTQUOr26xJEmSpM6nLR0al0dEN+DJiDgdWAD0r26xJEmSpM6nLTXXZwF9gTOB3YETgZOqWShJkiSpM2rLaCH3ljaXAidXtziSJElS59ViuI6IG1q7MaX0weKLI0mSJHVerdVc7w08C/wSuAeIVq6VJEmSurzWwvWWwHuB44ETgN8Dv0wpzd4YBZMkSZI6mxY7NKaUVqeU/phSOgnYC3gKqC2NGCJJkiSpmVZHC4mIXhFxJHnSmNOAC4DftPXFI+LQiHg8Ip6KiM9XOD8yIm6LiPsj4qGI+LfS8dER8WZEPFBaLl63jyVJkiRtfK11aLwS2AW4CfhqSunhdXnhiKgBLiQ3LZkP3BsRN6SUHim77EvAtSmln0TETqX3Gl0693RKacK6vKckSZLUnlqruf53YCx5nOu7IuL10rIkIl5vw2vvATyVUpqTUloBXA0c0eyaBGxW2h4IPLduxZckSZI6jhZrrlNKbZlgpjUjyKONNJgP7NnsmqnAzRFxBtAPOLjs3JiIuB94HfhSSumODSyPJEmSVFVtmf68mo4HLk8pfT8i9gZ+HhG7AM8DI1NKr0bE7sD1EbFzSqlJjXlETAGmAAwfPpza2tqNXHxVsnTpUr8XqshnQy3x2VBLfDbUmo74fFQzXC8Ati3b36Z0rNwpwKEAKaW/R0RvYEhK6SVgeen4fRHxNLADMKv85pTSdGA6wMSJE9OkSZOq8DG0rmpra/F7oUp8NtQSnw21xGdDremIz8eGNv1ozb3A2IgYExE9geOA5rM+PgMcBBAR44HewMsRMbTUIZKI2I7c9ntOFcsqSZIkbbCq1VynlFaVxsT+E1ADXJpSmh0RXwNmpZRuAD4N/CwiziF3bvxoSilFxP7A1yJiJVAP/GdKaWG1yipJkiQVoaptrlNKN5GH1ys/9pWy7UeAfSvcNxOYWc2ySZIkSUWrZrMQSZIkqUsxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQUxXEuSJEkFMVxLkiRJBTFcS5IkSQWpariOiEMj4vGIeCoiPl/h/MiIuC0i7o+IhyLi38rO/Vfpvscj4pBqllOSJEkqQvdqvXBE1AAXAu8F5gP3RsQNKaVHyi77EnBtSuknEbETcBMwurR9HLAzsDXw54jYIaW0ulrllSRJkjZUNWuu9wCeSinNSSmtAK4Gjmh2TQI2K20PBJ4rbR8BXJ1SWp5S+hfwVOn1JEmSpA6rajXXwAjg2bL9+cCeza6ZCtwcEWcA/YCDy+69u9m9I5q/QURMAaYADB8+nNra2iLKrQ20dOlSvxeqyGdDLfHZUEt8NtSajvh8VDNct8XxwOUppe9HxN7AzyNil7benFKaDkwHmDhxYpo0aVJ1Sql1Ultbi98LVeKzoZb4bKglPhtqTUd8PqoZrhcA25btb1M6Vu4U4FCAlNLfI6I3MKSN90qSJEkdSjXbXN8LjI2IMRHRk9xB8YZm1zwDHAQQEeOB3sDLpeuOi4heETEGGAv8o4pllSRJkjZY1WquU0qrIuJ04E9ADXBpSml2RHwNmJVSugH4NPCziDiH3LnxoymlBMyOiGuBR4BVwGmOFCJJkqSOrqptrlNKN5GH1ys/9pWy7UeAfVu4dxowrZrlkyRJkorkDI2SJElSQQzXkiRJUkEM15IkSVJBDNeSJElSQQzXkiRJUkEM15IkSVJBDNeSJElSQQzXkiRJUkEM15IkSVJBDNeSJElSQQzXkiRJUkEM15IkSVJBDNeSJElSQQzXkiRJUkEM15IkSVJBDNeSJElSQQzXkiRJUkEM15IkSVJBDNeSJElSQQzXkiRJUkEM15IkSVJBDNeSJElSQQzXkiRJUkEM15IkSVJBDNeSJElSQQzXkiRJUkEM15IkSVJBDNeSJElSQQzXkiRJUkEM15IkSVJBDNeSJElSQQzXkiRJUkEM15IkSVJBDNeSJElSQQzXkiRJUkEM15IkSVJBDNeSJElSQQzXkiRJUkEM15IkSVJBDNeSJElSQQzXkiRJUkEM15IkSVJBDNeSJElSQQzXkiRJUkEM15IkSVJBDNeSJElSQQzXkiRJUkEM15IkSVJBDNeSJElSQQzXkiRJUkG6t3cBJEmSqmHlypXMnz+fZcuWtXdRVCUDBw7k0Ucfrdrr9+7dm2222YYePXq0+R7DtSRJ2iTNnz+fAQMGMHr0aCKivYujKliyZAkDBgyoymunlHj11VeZP38+Y8aMafN9NguRJKkr+9cMuH40XNUtr/81o71LVJhly5YxePBgg7XWS0QwePDgdf7LhzXXkiR1Vf+aAf+YAqvr8n7dvLwPMGZy+5WrQAZrbYj1eX6suZYkqStavQwe+GxjsH7reB08+MX2KdMm5tVXX2XChAlMmDCBLbfckhEjRry1v2LFilbvnTVrFmeeeeZa32OfffYpqrgqiDXXkiRtqlYuhaVPw5KnYOlTpXVpv24+kCrfVzcPbjsMNtsxLwPH53WvobAJ1wTPmAFf/CI88wyMHAnTpsHkDajAHzx4MA888AAAU6dOpX///nzmM5956/yqVavo3r1yFJs4cSITJ05c63vcdddd61/AKmrts23quuanliRpU7FiESxpHqBL62UvNr2211AYsD0MmwQD3gaP/whWvLrma9b0zfe+dDusfrPxeM8tGgP3ZuMbt/uPgW6dO1LMmAFTpkBdqSJ/3ry8DxsWsJv76Ec/Su/evbn//vvZd999Oe644zjrrLNYtmwZffr04bLLLmPcuHHU1tZy3nnnceONNzJ16lSeeeYZ5syZwzPPPMPZZ5/9Vq12//79Wbp0KbW1tUydOpUhQ4bw8MMPs/vuu/OLX/yCiOCmm27iU5/6FP369WPfffdlzpw53HjjjU3KNXv2bE4++WRWrFhBfX09M2fOZOzYsVx55ZWcd955RATveMc7+PnPf87cuXP52Mc+xiuvvMLQoUO57LLLGDly5Bqf7bTTTuO0007j5Zdfpm/fvvzsZz9jxx13LO6L2UF17p8ESZI2dSnB8pdzWC6veW4I0CsWNr2+z9Y5QG99eF4P2B76vy0vPQc2vbb/9k3bXEMO1ntMz22uUz3UPQuLH4PXy5bnboI5lzXe060nDBhbFrwblnHQozojOayrs8+GUiVyRXffDcuXNz1WVwennAI/+1nleyZMgPPPX/eyzJ8/n7vuuouamhpef/117rjjDrp3786f//xnvvCFLzBz5sw17nnssce47bbbWLJkCePGjePUU09dY3i4+++/n9mzZ7P11luz7777cueddzJx4kQ+8YlP8Ne//pUxY8Zw/PHHVyzTxRdfzFlnncXkyZNZsWIFq1evZvbs2XzjG9/grrvuYsiQISxcmJ+1M844g5NOOomTTjqJSy+9lDPPPJPrr79+jc920EEHcfHFFzN27FjuuecePvnJT3Lrrbeu+xeskuWvQt0C+tevgEU9oe8I6DW4mNfeQIZrqav514zcnrLuGeg7Enadtsl0XNI68DmonvX52qZ6ePP5ys03ljwFq5Y0Xhvd8usO2B5GHlMKzw0hejvo3rftZW0oV0vljW7Qb1Retj6k6b0rFsHrjzcN3a/9H8y/HtLqxuv6blMhdI+HPlt1qCYmzYP12o5viGOOOYaamhoAFi9ezEknncSTTz5JRLBy5cqK9xx++OH06tWLXr16MWzYMF588UW22WabJtfssccebx2bMGECc+fOpX///my33XZvDSV3/PHHM3369DVef++992batGnMnz+fI488krFjx3LrrbdyzDHHMGTIEAAGDRoEwN///nd+/etfA3DiiSdy7rnnrvHZli5dyl133cUxxxzz1rnlRX0xl78Kb8yDVE8A1K/I+9AhArbhWupKusDIAGqDis/Bx/Of/7c7GbrVtG/5OrPWfsZGHZdrgRvCc5MgPadp84vonpta9N8ehu6Xa50baqH7jYaaXsWVeczk9fv577kFDNkrL+VWr8i/GDQE7sWP5vWcK5r+ktB9QFmb7rJmJv3fBjU9K7/nBvxSuLYa5tGjc1OQ5kaNgtraNr1Fm/Xr1++t7S9/+cu85z3v4Te/+Q1z585l0qRJFe/p1avxe15TU8OqVavW65qWnHDUIey540B+/6fb+LdDD+anF65HlTyNn62+vp7NN9/8rTbnQP4lsn41UJ+3U31pO1U41tLxlP9ak+qbvnGqh7oFhutOz5ofdTQpwcrXYfkrpeXlxu1lL8OTF1YeGeAfU+Cl2vzn4O59S+t+ZdutHSutqxHI/BnbMPUrc23Okqeb1oY+9wdIzf7TXf1mDtj/+DjU9IHu/fPSo3/jdkvHevSHmn6Vjzdsd+u5YbWU7fkspJRrxuqXw+rlpfWysu2y9T/PqfwzdvdH4Z6T8/ekQU3vXNPcf3vY6pCyGui35c/YWdsw1/TMHSAHjm96PKVcO//6Y/D6o43h+6XbYO7PG6+LmhywSx0pt6wDXumda8XvO7tqlQPTpjVtcw3Qt28+Xk2LFy9mxIgRAFx++eWFv/64ceOYM2cOc+fOZfTo0VxzzTWNJ1MCEix/hTmP3M12I7fizCnH8cz8F3jon3/jve89jA+fcD6f+uRHGDx4cxa+upBBWwxknz134+orLuTE4z7EjF9cy7v3ficseRJWLs4dZRfPZrOUGLPtcH512fc45oiDSPWreWj2k+y6yw7r90GiG9At/zvSPFg3qG99BJaNpZP+5HYA1gCqQTX/01+9vGlIXlYhNDfZfqXpf97luvVs+R+e1XW5DeWqN/J2S6/Rmm69mgbu7v3WCOHjXlsMs37dttD+Yi08/NXG2jx/xipb9Wau9Vz6VFmILq3fmNf0z/M1fXKAax6sy739q7BqaV5WLm3cXrUUlr3Q9Fh5TevaRPdWw/oOry2B+35b+ZpX/wGP/zAHWMjPwj2nwGsPwtB9K4fct9YthOCK62WVjxfxH3ZaBeM/19h8Y8D2uW10dKERcSOg79Z52fLApudWLlmzicnrj8Hzf2DH+pVw83cqv+bqOrj3k/lnoKZ3fsbL16vG5AoHupW+1pHX5UGNgIi3Oi0WOVpIQ7tg6lfAmy9C7zV/wTz33HM56aST+MY3vsHhhx/e8mulBPWrSsEywaq6/HUDWL4QVizOz+ubz+ef+5VLYNkr9Fn1HBd9/0sc+r4D6de3D+/abWfoUw8L72/y78O119/Mz6+9iR49urPlsMF84ZyTGbTFQL549okccPC/UVNTw25v34HLfzyVH33zLE4+/at87wcXMXTIFlz242+U/b/RLf9/QzdmXHoBp57zZb7xP5exctUqjjv6CHbd48Cyr39L35fmx6LpL+eLHqr8c9mthb94bGSRUgvD8HQyEydOTLNmzdp4b3j96PwPfHM1fWDUsdB9M+hRvgxotr9Z4zUt/fmrSBux1qe2trbFP2ttcpr/kgVNOwOVS/W5jeKyZmG4ee1y+fFVS1t444Beg6DXkLJlaOXt3qX97v3ht2MqP7d9R8GH5jbu16/MwWlVXWPgXlXXuF7PY8vfWESvmlWlAL+egaVbT9j6MOg1DHoPz0uf4Y3bvYdDj4Edqi3nBluxuKwTW7P1mwuaXttj86ad2MrXvbfMX5eW/v1q/hysTf1qWP3GmiF81RuVw/ka1zUeW/7GQnrVrMjHWqqVWh/deuZf/Gp6VVj3buH4Blx3x5H5l5Dm1vVrq6x+FffcejV77jgQ/vrBdb790R3+wPgxQ9Z+YYshL2g9mLcSEBt+IS3PWRH53+WaPo1NHVJDM4nVjftvNYMoP7aOeS1qSks3lr7xJv0HbEYiOO1TX2Xs9ttxzhkff+s8dQtafp3Nd2n8vE0+ezsqa3P9lob+AVVoFvLoo48yfnzTv8RExH0ppYpjJVpzvb7qnql8fPWb8MJf8m/KK1+nxTFEy3Xr2XLwbimYv3VN2bnu/SrXgljLXh2rl/3/9u49uqryzOP490kIieESqEAQUC4dVJQkhHLRsUGKDDhqsYzjUC91gfXaKRbssLTVVVlOpyrqoFZHoF0WcWktOspYYaqukYwg4iRgQFAKctESqwItGMBwSZ75Y++Ek+ScJIRzA36ftc46Z7/73fs85+Q9J89597vfDRV3RD8EXHYL/Ok/GybK0caI1WnXoWFi3PmsMDHuHj2Bbt+1bYeMi/4t+o+BokbHPTOygltW56N/jma8E/nDq/bwkQS+Zl+YjEck5v97GVE/P7UHg16qne8EP0ai1clo3zDZzsmHnB5RyvKDHymp7j2snw1ic/Qe6AM7G9bP6RkMHeg5tmkCnf21lp+vte2gJRmZkNE5Lu2kvm24B5+tyAR8SRHRv0sNLi5vmuRm5gSPj3UoSlsUPxSf91YCGe34ql0f6DM6+IES60fhhI/CoTrVwfdK3X1lLXT+euxxvHhEIuuN6tWV1YAfCrZrsp+jTHjdw++tCJZxJMm1DCBMijOyIsozOZLIZza8J2Lb+rKGPb2/+s1snn76aQ4ePEhxcTE33/aTYMxLneodsXuCM3OO7jUmQ10Cvb8Srz2IZWi2kBND7hkt9/y4Bz04dYn24aojjyNv0cq/qgwOiR0Ol2tac117a5Rsh493LG96+LZmfzA2MLc3tO8SJGtZXYLtU51opEpNddhzvCO8/yJi+Yum5ZEn5jR2eG8w/iy7G+SdGztJrru1OyU5r7GlmQGSKaMdZHSKPU1Xc5+xS9YGj2trwh7/z2PcvoCvPoW/vhc8jjYcwtoFf59oiXfjhDy7e/Njy5s7QlR3sk2THugwiW7Qngw6nBEkzKf/w5FxuB3D2SCyOrbqLY4pndpBY2bB56HdKUD3oCxmWzgDvjY0qeG1KJ3f2+Ndcz8KM9pBDWJ0vQAAFCNJREFURsemn43PPkzsVIANEu5GJ999uSH2dl2LOJIsJ/4H4PTp05k+fXrsCrm9o/cE5/ZOeGxtln0qZJ/K3qoqOnVKj+ke6yi5bqvW9PyYBR/0rI5Ar2N7vpqDEUl4o2S8LgFvXF53izUu8sAO+J9vNSyzjOCQelaYcLdvdN+4vMFyl/j9wo3HMJZ4JcsZWUFSldMjuO84IEy6usOHDzedYxaCBPDS94/+dSdDW2cGSLbWfMYyMoMhIafkt7y/+mE5YeL9VXh/4IuGy3s+DO7rxvc2EB7SbZyEn5IfJMhbFzQcF7xyctBGaquD3vbIfWZkQYf+QW9z95KGvc/xng0imuOlHUD8etqT5Xh6b48n6fjDxSzsVY7yozvWeS4Z7YPPfzqJ6Amm9mAQYxr1BB9vlFy3VbI/5JntIfPUtjX0WOMrc06DC56Fg7vD21/hUHgfufzlh0eWWzqBKTOH8z0XXs2PnZA3KI8oy8oLkvtYw1j8MPS8KEiMGyfJdY8jy1tKlusS5o4Dwp7LHg3L6+6zOsfuWcjte3z90z+exPszZhn1PR3kndN83bpZVyJ7wKP1jO9aGdwf3hdjP4dhzzrofRn0vjRiLuKvQ+7pmvKutdIxqZLUOJ5+uBxvvcHZbcwxpAkl18fiePmQx+r1KX4Q8r8Ve7toag4EiXaDJHw3HDqSkO/6+AN65Z0S1Kn+PDgDvC5Rb/ZEJQsS2cN7G85wAEemsoq6WXhYPzvsTa5LlhskyRHr43mym/7pJ1aqPmNmwZXs2udB51ZMG3V4HyzsRNTxl34YRr0U9xBPOsfL961IHfUGn7SUXJ8M4pkAZma3eBh+455SepWMbrrCPehNPhildzyybONjsZ9/xNwjSXJ9z3KKZ4bQP31p16H5ccEicnJSb/BJScn1ySIdEkCzIydbdmgm4dj+X7FPZPubmxIXn8ixON7GBYtIUnz22WdMmzaNsrIyunTpQn5+Po888ghnntnGi6kkyPz58ykvL+fxxx9nzpw55Obmct111zWos23bNi677DLWrVsXcz/btm1jxYoVXH311QCUl5ezYMECHnusmY6zE8xJOi2EpLWifwuSkkhKUiTd9b8mmN88ty9gwX20+c5FJH1tfTY4T+m5jOB+67PHtDt3Z+LEiYwePZrNmzezatUq7rvvPj7//PMG9Y7mMuXJcMsttzRJrFtr27ZtPPfcc/XLw4YNS8vEOpHvuZJrST9KUuR41f+aYCrOq2uDe7VZkeNH3cn0+z8G/MjJ9MeQYC9dupSsrCxuueWW+rKioiJKSkooLS2lpKSECRMmcM4551BdXc2UKVMoKCiguLiYpUuXArB+/XpGjBjBkCFDKCwsZNOmTezbt49LL72UoqIiBg8e3PCS5kBtbS39+vVj9+7d9WUDBw7k888/5/e//z0jR46kuLiYsWPHNkn0AWbOnMlDDz0EwKpVqygqKqKoqIgnnniivs62bdsoKSlh6NChDB06lBUrVgBw5513smzZMoYMGcLs2bMpLS3lsssuA+Avf/kL3/nOdygsLOS8885j7dq19c93/fXXM3r0aAYMGBA1Ga+pqWHy5MkMHjyYgoICZs+eDcDmzZsZO3YsRUVFDB06lM2bN+PuzJgxo75u3fvT+D2vqalhxowZDB8+nMLCQubOnXuUf+HoNCxE0lM6DGMREZETx6pp8NeK2Ot3rmw6BWfNfnj3+7D5V9G36ToEvvFIzF2uW7eOb3zjGzHXr169mnXr1tG/f38efvhhzIz333+fDRs2MG7cODZu3MicOXP40Y9+xDXXXMPBgwepqalhyZIl9OrVi8WLFwOwZ8+eBvvNyMjg8ssv5+WXX2bKlCm8++679O3bl/z8fL75zW+ycuVKzIxf//rXzJo1i4cffjhmjFOmTOHxxx9n1KhRzJgxo768R48evPHGG+Tk5LBp0yauuuoqysvLuf/++3nooYd49dVXgSChrXPPPfdQXFzMokWLePPNN7nuuuuoqAj+Jhs2bGDp0qVUVVVx1llnceutt5KVdWTKwoqKCiorK+uHpNT9cLjhhhu46667mDhxItXV1dTW1vLSSy9RUVHBmjVr2LlzJ8OHD2fUqFFN3vN58+aRl5dHWVkZBw4c4IILLmDcuHH0798/5vvRGuq5FhEREYk6t30z5XEwYsSI+kRu+fLlXHvttQCcffbZ9O3bl40bN3L++efzi1/8ggceeICPP/6YU045hYKCAt544w3uuOMOli1bRl5eXpN9T5o0qb7H9vnnn2fSpEkAbN++nfHjx1NQUMCDDz7I+vXrY8a3e/dudu/eXZ+Yfu9736tfd+jQIW688UYKCgq48sor+eCDD1p8vcuXL6/fx5gxY9i1axdffvklAJdeeinZ2dl069aNHj16NOlRHzBgAFu2bGHq1Kn84Q9/oHPnzlRVVfHnP/+ZiRMnApCTk0Nubi7Lly/nqquuIjMzk/z8fC688ELKysqavOevv/46CxYsYMiQIYwcOZJdu3axadOmFl9HS9RzLSIiIie+ZnqYgdjXhMjtC2NL2/SU5557Li+++GLM9R06dGhxH1dffTUjR45k8eLFXHLJJcydO5cxY8awevVqlixZwt13381FF13E+PHjufnmmwG49957+fa3v81HH33Ejh07WLRoEXfffTcAU6dO5fbbb2fChAmUlpYyc+bMNr222bNnk5+fz5o1a6itrSUn59guIpedfeTCWZmZmU3GRHft2pU1a9bw2muvMWfOHBYuXMijjz561M8T+Z67O7/85S8ZP3582wOPIqE912Z2sZn90cw+MrM7o6yfbWYV4W2jme2OWFcTse6VRMYpIiIiJ7kEnEw/ZswYDhw4wLx58+rL1q5dy7Jly5rULSkp4dlng/HdGzdu5JNPPuGss85iy5YtDBgwgNtuu43LL7+ctWvX8umnn5Kbm8u1117LjBkzWL16NSNHjqSiooKKigomTJiAmTFx4kRuv/12Bg0axKmnBlMC7tmzh969gwvZPP30083G36VLF7p06cLy5csB6uOr289pp51GRkYGzzzzDDU1wfUpOnXqRFVV9Iu4Rb7G0tJSunXrRufOnVv1Xu7cuZPa2lquuOIKfv7zn7N69Wo6depEr169WLRoEQAHDhxg//79lJSU8Lvf/Y6amhp27NjBW2+9xYgRI5rsc/z48Tz55JMcOnSo/n3fty/GRcGOQsJ6rs0sE3gC+DtgO1BmZq+4e/1xA3efHlF/KlAcsYuv3H1IouITERERqZeAi4KZGS+//DLTpk3jgQceICcnh379+vHII49QWVnZoO4PfvADbr31VgoKCmjXrh3z588nOzubhQsX8swzz5CVlUXPnj356U9/SllZGTNmzCAjI4OsrCyefPLJqM8/adIkhg8fzvz58+vLZs6cyZVXXknXrl0ZM2YMW7dubfY1/OY3v+H666/HzBg3blyDeK+44goWLFjAxRdfXN8jXFhYSGZmJkVFRUyePJni4iOpXd2Ji4WFheTm5raY3EeqrKxkypQp1NYGF6O77777AJg3bx4//vGP+dnPfkZWVhYvvPACEydO5J133qGoqAgzY9asWfTs2ZMNGzY02OcNN9zAtm3bGDp0KO5O9+7d6xP1Y2HuUa4oFgdmdj4w093Hh8s/AXD3+2LUXwHc4+5vhMt73b1ja59v2LBhXl5efuyByzErLS1l9OjRqQ5D0pDahsSitiGxHEvb+PDDDxk0aFB8A5K0UlVVRadOnRL6HNHakZmtcvdh0eonclhIb+BPEcvbw7ImzKwv0B94M6I4x8zKzWylmX0ncWGKiIiIiMRHupzQ+F3gRXeviSjr6+6VZjYAeNPM3nf3zZEbmdlNwE0A+fn5DaZ7kdTZu3ev/hYSldqGxKK2IbEcS9vIy8uLOf5XTgw1NTUJ/xtXV1cfVRtMZHJdCZwesdwnLIvmu8A/Rxa4e2V4v8XMSgnGY29uVGceMA+CYSE6pJgedHhXYlHbkFjUNiSWYx0WkughA5JayRgWkpOT02DseEsSOSykDBhoZv3NrD1BAt1k1g8zOxvoCrwTUdbVzLLDx92AC4CWJ1AUERERiZCoc8vk5NCW9pOw5NrdDwM/BF4DPgQWuvt6M7vXzCZEVP0u8Lw3jH4QUG5ma4ClwP2Rs4yIiIiItCQnJ4ddu3YpwZY2cXd27dp11HN4J3TMtbsvAZY0KvtZo+WZUbZbARQkMjYRERE5sfXp04ft27ezY8eOVIciCVJdXX3MF7BpTk5ODn369DmqbdLlhEYRERGRuMrKyqq/1LWcmEpLS49qPHQyJPQKjSIiIiIiJxMl1yIiIiIicaLkWkREREQkThJ2+fNkM7MdwMepjkMA6AbsTHUQkpbUNiQWtQ2JRW1DmpOq9tHX3btHW3HCJNeSPsys3N2HpToOST9qGxKL2obEorYhzUnH9qFhISIiIiIicaLkWkREREQkTpRcSyLMS3UAkrbUNiQWtQ2JRW1DmpN27UNjrkVERERE4kQ91yIiIiIicaLkWtrEzC42sz+a2UdmdmeU9beb2QdmttbM/sfM+qYiTkmNltpHRL0rzMzNLK3O9JbEaU3bMLN/Cr8/1pvZc8mOUVKjFf9XzjCzpWb2Xvi/5ZJUxCnJZ2ZPmdkXZrYuxnozs8fCtrPWzIYmO8ZISq7lqJlZJvAE8PfAOcBVZnZOo2rvAcPcvRB4EZiV3CglVVrZPjCzTsCPgHeTG6GkSmvahpkNBH4CXODu5wLTkh6oJF0rvzfuBha6ezHwXeA/khulpNB84OJm1v89MDC83QQ8mYSYYlJyLW0xAvjI3be4+0HgeeDyyAruvtTd94eLK4E+SY5RUqfF9hH6V+ABoDqZwUlKtaZt3Ag84e5/BXD3L5Ico6RGa9qGA53Dx3nAp0mMT1LI3d8C/tJMlcuBBR5YCXQxs9OSE11TSq6lLXoDf4pY3h6WxfJ94L8TGpGkkxbbR3jI7nR3X5zMwCTlWvPdcSZwppm9bWYrzay53io5cbSmbcwErjWz7cASYGpyQpPjwNHmJQnVLlVPLCcHM7sWGAZcmOpYJD2YWQbw78DkFIci6akdwaHd0QRHvN4yswJ3353SqCQdXAXMd/eHzex84BkzG+zutakOTCSSeq6lLSqB0yOW+4RlDZjZWOAuYIK7H0hSbJJ6LbWPTsBgoNTMtgHnAa/opMaTQmu+O7YDr7j7IXffCmwkSLblxNaatvF9YCGAu78D5ADdkhKdpLtW5SXJouRa2qIMGGhm/c2sPcGJJa9EVjCzYmAuQWKtMZMnl2bbh7vvcfdu7t7P3fsRjMmf4O7lqQlXkqjF7w5gEUGvNWbWjWCYyJZkBikp0Zq28QlwEYCZDSJIrnckNUpJV68A14WzhpwH7HH3P6cqGA0LkaPm7ofN7IfAa0Am8JS7rzeze4Fyd38FeBDoCLxgZgCfuPuElAUtSdPK9iEnoVa2jdeAcWb2AVADzHD3XamLWpKhlW3jx8CvzGw6wcmNk11XwjspmNlvCX50dwvH3N8DZAG4+xyCMfiXAB8B+4EpqYk0oCs0ioiIiIjEiYaFiIiIiIjEiZJrEREREZE4UXItIiIiIhInSq5FREREROJEybWIiIiISJwouRYRiRMzO9XMKsLbZ2ZWGbHcvoVth5nZY614jhXxizj1zGyymT2e6jhEROJF81yLiMRJOB/zEAAzmwnsdfeH6tabWTt3Pxxj23KgxQvpuPvfxidaERFJBPVci4gkkJnNN7M5ZvYuMMvMRpjZO2b2npmtMLOzwnqjzezV8PFMM3vKzErNbIuZ3Raxv70R9UvN7EUz22Bmz1p4xSYzuyQsW2Vmj9Xtt1FcmWb2oJmVmdlaM7s5LJ9uZk+FjwvMbJ2Z5TYT92QzW2Rmb5jZNjP7oZndHtZbaWZfC+uVmtmjYS/+OjMbESWm7mb2n2FMZWZ2QVh+YcQRgPfMrFNc/0giInGknmsRkcTrA/ytu9eYWWegJLwi3VjgF8AVUbY5G/gW0An4o5k96e6HGtUpBs4FPgXeBi4ws3JgLjDK3beGVzaL5vsElwgebmbZwNtm9jrwKFBqZhOBu4Cb3X2/mW1oJu7BYSw5BFdIu8Pdi81sNnAd8EhYL9fdh5jZKOCpcLtIjwKz3X25mZ1BcLW+QcC/AP/s7m+bWUegOsZrEhFJOSXXIiKJ94K714SP84CnzWwgwSWcs2Jss9jdDwAHzOwLIB/Y3qjO/7n7dgAzqwD6AXuBLe6+NazzW+CmKPsfBxSa2T9GxDUwTMgnA2uBue7+diviXuruVUCVme0Bfh+Wvw8URtT7LYC7v2Vmnc2sS6OYxgLnhB3wAJ3DZPpt4N/N7FngpbrXLCKSjpRci4gk3r6Ix/9KkIxONLN+QGmMbQ5EPK4h+vd1a+rEYsBUd38tyrqBBEl6r4iy5uKOjKM2Yrm2UUze6HkaL2cA57l7457p+81sMXAJQQ/7eHffEO1FiYikmsZci4gkVx5QGT6enID9/xEYECbAAJNi1HsNuNXMsgDM7Ewz62BmecBjwCjg1EY928ca96Twub5JMCRlT6P1rwNT6xbMrO7k0K+7+/vu/gBQRjBkRkQkLSm5FhFJrlnAfWb2Hgk4eujuXwE/AP5gZquAKqBxEgvwa+ADYLWZrSMYp90OmA084e4bCcZl329mPeIUd3W4/Zxw343dBgwLT7D8ALglLJ8WngS5FjgE/Hcbn19EJOHMvfFROREROZ6ZWUd33xvOHvIEsMndZ6c4plLgX8IpB0VETljquRYROfHcGJ7guJ5gOMfcFMcjInLSUM+1iIiIiEicqOdaRERERCROlFyLiIiIiMSJkmsRERERkThRci0iIiIiEidKrkVERERE4kTJtYiIiIhInPw/X9afBIPTE9AAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 864x576 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light",
      "tags": []
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def plot_learning_curve(estimator, title, X, y, X_dev, y_dev, ylim=None, train_sizes=np.linspace(.1, 1.0, 5), is_keras=False):\n",
    "   \n",
    "  fig, ax = plt.subplots(figsize=(12,8))\n",
    "  ax.set_title(title)\n",
    "  if ylim is not None:\n",
    "    ax.set_ylim(*ylim)\n",
    "  ax.set_xlabel(\"Training examples\")\n",
    "  ax.set_ylabel(\"Macro F1-score\")\n",
    "\n",
    "  f1_train = []\n",
    "  f1_dev = []\n",
    "\n",
    "  # for each training subset, fit and predict\n",
    "  for t_sz in train_sizes:\n",
    "    X_subset = X[:int(t_sz * total_length)]\n",
    "    y_subset = y[:int(t_sz * total_length)]\n",
    "\n",
    "    # make y 2-dim if estimator is keras\n",
    "    if is_keras==True:\n",
    "      # fit MLP estimator\n",
    "      y_subset2 = np.vstack(( 1-y_subset, y_subset)).T\n",
    "      y_dev2 = np.vstack(( 1-y_dev, y_dev)).T\n",
    "      \n",
    "      estimator.fit(X_subset, y_subset2,\n",
    "              batch_size=32,\n",
    "              epochs=30,\n",
    "              verbose = 0,\n",
    "              #callbacks=my_callbacks, we dont need callbacks now\n",
    "              #validation_data=(X_dev, y_dev2),\n",
    "              shuffle=True)\n",
    "    else:\n",
    "        # fit sklearn estimator\n",
    "        estimator.fit(X_subset, y_subset2)\n",
    "      \n",
    "    y_pred = estimator.predict_proba(X_subset)\n",
    "    y_pred_dev = estimator.predict_proba(X_dev)\n",
    "\n",
    "    _, _, f1_dict = get_scores(y_subset, y_pred, 0.5)\n",
    "    _, _, f1_dict_dev = get_scores(y_dev, y_pred_dev, 0.5)\n",
    "    f1_train.append(f1_dict['macro'])\n",
    "    f1_dev.append(f1_dict_dev['macro'])\n",
    "    print(\"Process ended for trainsize: {} % :\".format(t_sz*100))\n",
    "\n",
    "  ax.grid()\n",
    "  ax.plot(train_sizes, f1_train, 'o-', color=\"b\",\n",
    "            label=\"Training score\")\n",
    "  ax.plot(train_sizes, f1_dev, 'o-', color=\"orange\",\n",
    "            label=\"Cross-validation score\")\n",
    "\n",
    "  ax.legend(loc=\"lower right\")\n",
    "  fig.savefig(\"/content/gdrive/My Drive/keras_checkpoints/\"+title+'.pdf')\n",
    "\n",
    "plot_learning_curve(model, \"Learning Curves (Keras)\"\n",
    "                    , X_train, y_train\n",
    "                    , X_dev, y_dev\n",
    "                    , train_sizes = np.linspace(.1, 1.0, 15)\n",
    "                    ,is_keras = True\n",
    "                    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "WwIncvGN-vMh"
   },
   "source": [
    "### Appendix A - Randomized GridSearch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 0,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "ZTYBJba2uP-_",
    "outputId": "3cfaa91f-3fc9-435a-e232-d8bb25be303d"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fitting 3 folds for each of 10 candidates, totalling 30 fits\n",
      "[CV] layer_3_sz=64, layer_2_sz=128, layer_1_sz=256, drp_2=0.5, drp_1=0.5 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Using backend SequentialBackend with 1 concurrent workers.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "106426/106426 [==============================] - 21s 198us/step - loss: 0.5898 - precision: 0.6917 - recall: 0.6474 - f1: nan - accuracy: 0.6828\n",
      "Epoch 2/30\n",
      "106426/106426 [==============================] - 19s 175us/step - loss: 0.5282 - precision: 0.7458 - recall: 0.7192 - f1: 0.7254 - accuracy: 0.7383\n",
      "Epoch 3/30\n",
      "106426/106426 [==============================] - 18s 173us/step - loss: 0.5151 - precision: 0.7538 - recall: 0.7325 - f1: 0.7361 - accuracy: 0.7482\n",
      "Epoch 4/30\n",
      "106426/106426 [==============================] - 19s 174us/step - loss: 0.5066 - precision: 0.7601 - recall: 0.7374 - f1: 0.7417 - accuracy: 0.7533\n",
      "Epoch 5/30\n",
      "106426/106426 [==============================] - 19s 176us/step - loss: 0.5004 - precision: 0.7642 - recall: 0.7413 - f1: 0.7458 - accuracy: 0.7577\n",
      "Epoch 6/30\n",
      "106426/106426 [==============================] - 19s 174us/step - loss: 0.4961 - precision: 0.7658 - recall: 0.7446 - f1: 0.7485 - accuracy: 0.7596\n",
      "Epoch 7/30\n",
      "106426/106426 [==============================] - 18s 174us/step - loss: 0.4920 - precision: 0.7706 - recall: 0.7480 - f1: 0.7524 - accuracy: 0.7634\n",
      "Epoch 8/30\n",
      "106426/106426 [==============================] - 19s 180us/step - loss: 0.4879 - precision: 0.7708 - recall: 0.7496 - f1: 0.7536 - accuracy: 0.7648\n",
      "Epoch 9/30\n",
      "106426/106426 [==============================] - 19s 174us/step - loss: 0.4846 - precision: 0.7740 - recall: 0.7524 - f1: 0.7567 - accuracy: 0.7682\n",
      "Epoch 10/30\n",
      "106426/106426 [==============================] - 19s 182us/step - loss: 0.4819 - precision: 0.7771 - recall: 0.7540 - f1: 0.7591 - accuracy: 0.7703\n",
      "Epoch 11/30\n",
      "106426/106426 [==============================] - 19s 176us/step - loss: 0.4790 - precision: 0.7789 - recall: 0.7546 - f1: 0.7600 - accuracy: 0.7714\n",
      "Epoch 12/30\n",
      "106426/106426 [==============================] - 19s 175us/step - loss: 0.4766 - precision: 0.7819 - recall: 0.7580 - f1: 0.7635 - accuracy: 0.7746\n",
      "Epoch 13/30\n",
      "106426/106426 [==============================] - 18s 172us/step - loss: 0.4743 - precision: 0.7813 - recall: 0.7570 - f1: 0.7626 - accuracy: 0.7740\n",
      "Epoch 14/30\n",
      "106426/106426 [==============================] - 18s 171us/step - loss: 0.4710 - precision: 0.7827 - recall: 0.7587 - f1: 0.7644 - accuracy: 0.7753\n",
      "Epoch 15/30\n",
      "106426/106426 [==============================] - 19s 178us/step - loss: 0.4698 - precision: 0.7840 - recall: 0.7603 - f1: 0.7654 - accuracy: 0.7761\n",
      "Epoch 16/30\n",
      "106426/106426 [==============================] - 19s 179us/step - loss: 0.4661 - precision: 0.7855 - recall: 0.7632 - f1: 0.7682 - accuracy: 0.7788\n",
      "Epoch 17/30\n",
      "106426/106426 [==============================] - 19s 179us/step - loss: 0.4640 - precision: 0.7874 - recall: 0.7644 - f1: 0.7697 - accuracy: 0.7802\n",
      "Epoch 18/30\n",
      "106426/106426 [==============================] - 19s 179us/step - loss: 0.4620 - precision: 0.7888 - recall: 0.7665 - f1: 0.7713 - accuracy: 0.7818\n",
      "Epoch 19/30\n",
      "106426/106426 [==============================] - 18s 173us/step - loss: 0.4592 - precision: 0.7923 - recall: 0.7662 - f1: 0.7727 - accuracy: 0.7836\n",
      "Epoch 20/30\n",
      "106426/106426 [==============================] - 19s 179us/step - loss: 0.4584 - precision: 0.7920 - recall: 0.7686 - f1: 0.7741 - accuracy: 0.7839\n",
      "Epoch 21/30\n",
      "106426/106426 [==============================] - 19s 175us/step - loss: 0.4566 - precision: 0.7924 - recall: 0.7689 - f1: 0.7745 - accuracy: 0.7848\n",
      "Epoch 22/30\n",
      "106426/106426 [==============================] - 19s 178us/step - loss: 0.4536 - precision: 0.7958 - recall: 0.7688 - f1: 0.7759 - accuracy: 0.7867\n",
      "Epoch 23/30\n",
      "106426/106426 [==============================] - 19s 174us/step - loss: 0.4517 - precision: 0.7943 - recall: 0.7725 - f1: 0.7771 - accuracy: 0.7871\n",
      "Epoch 24/30\n",
      "106426/106426 [==============================] - 19s 178us/step - loss: 0.4513 - precision: 0.7946 - recall: 0.7752 - f1: 0.7787 - accuracy: 0.7882\n",
      "Epoch 25/30\n",
      "106426/106426 [==============================] - 19s 179us/step - loss: 0.4492 - precision: 0.7965 - recall: 0.7715 - f1: 0.7780 - accuracy: 0.7884\n",
      "Epoch 26/30\n",
      "106426/106426 [==============================] - 18s 173us/step - loss: 0.4468 - precision: 0.7978 - recall: 0.7755 - f1: 0.7805 - accuracy: 0.7904\n",
      "Epoch 27/30\n",
      "106426/106426 [==============================] - 18s 173us/step - loss: 0.4458 - precision: 0.7981 - recall: 0.7746 - f1: 0.7802 - accuracy: 0.7907\n",
      "Epoch 28/30\n",
      "106426/106426 [==============================] - 19s 176us/step - loss: 0.4444 - precision: 0.7982 - recall: 0.7765 - f1: 0.7815 - accuracy: 0.7913\n",
      "Epoch 29/30\n",
      "106426/106426 [==============================] - 18s 174us/step - loss: 0.4424 - precision: 0.8013 - recall: 0.7776 - f1: 0.7833 - accuracy: 0.7935\n",
      "Epoch 30/30\n",
      "106426/106426 [==============================] - 18s 173us/step - loss: 0.4418 - precision: 0.8024 - recall: 0.7773 - f1: 0.7837 - accuracy: 0.7938\n",
      "53214/53214 [==============================] - 2s 36us/step\n",
      "[CV]  layer_3_sz=64, layer_2_sz=128, layer_1_sz=256, drp_2=0.5, drp_1=0.5, score=0.773, total= 9.6min\n",
      "[CV] layer_3_sz=64, layer_2_sz=128, layer_1_sz=256, drp_2=0.5, drp_1=0.5 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   1 out of   1 | elapsed:  9.6min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 19s 175us/step - loss: 0.5891 - precision: 0.6933 - recall: 0.6649 - f1: 0.6687 - accuracy: 0.6836\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 18s 171us/step - loss: 0.5312 - precision: 0.7464 - recall: 0.7162 - f1: 0.7238 - accuracy: 0.7363\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 18s 172us/step - loss: 0.5177 - precision: 0.7533 - recall: 0.7281 - f1: 0.7333 - accuracy: 0.7449\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 18s 172us/step - loss: 0.5097 - precision: 0.7579 - recall: 0.7372 - f1: 0.7404 - accuracy: 0.7503\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 19s 180us/step - loss: 0.5050 - precision: 0.7633 - recall: 0.7384 - f1: 0.7439 - accuracy: 0.7547\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 18s 172us/step - loss: 0.4990 - precision: 0.7651 - recall: 0.7438 - f1: 0.7477 - accuracy: 0.7580\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 18s 170us/step - loss: 0.4954 - precision: 0.7661 - recall: 0.7458 - f1: 0.7493 - accuracy: 0.7596\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 18s 167us/step - loss: 0.4912 - precision: 0.7699 - recall: 0.7486 - f1: 0.7529 - accuracy: 0.7628\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 18s 170us/step - loss: 0.4884 - precision: 0.7719 - recall: 0.7499 - f1: 0.7544 - accuracy: 0.7644\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 19s 177us/step - loss: 0.4859 - precision: 0.7732 - recall: 0.7505 - f1: 0.7551 - accuracy: 0.7650\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 18s 172us/step - loss: 0.4819 - precision: 0.7753 - recall: 0.7524 - f1: 0.7575 - accuracy: 0.7675\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 18s 168us/step - loss: 0.4793 - precision: 0.7775 - recall: 0.7553 - f1: 0.7598 - accuracy: 0.7699\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 18s 172us/step - loss: 0.4766 - precision: 0.7807 - recall: 0.7586 - f1: 0.7629 - accuracy: 0.7722\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 18s 168us/step - loss: 0.4751 - precision: 0.7811 - recall: 0.7556 - f1: 0.7618 - accuracy: 0.7722\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 17s 164us/step - loss: 0.4726 - precision: 0.7816 - recall: 0.7605 - f1: 0.7647 - accuracy: 0.7738\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 19s 175us/step - loss: 0.4697 - precision: 0.7830 - recall: 0.7602 - f1: 0.7652 - accuracy: 0.7751\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 18s 171us/step - loss: 0.4678 - precision: 0.7875 - recall: 0.7634 - f1: 0.7689 - accuracy: 0.7786\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 18s 167us/step - loss: 0.4649 - precision: 0.7864 - recall: 0.7662 - f1: 0.7701 - accuracy: 0.7789\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 18s 171us/step - loss: 0.4647 - precision: 0.7869 - recall: 0.7631 - f1: 0.7686 - accuracy: 0.7780\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 18s 166us/step - loss: 0.4624 - precision: 0.7886 - recall: 0.7653 - f1: 0.7709 - accuracy: 0.7801\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 18s 169us/step - loss: 0.4605 - precision: 0.7898 - recall: 0.7679 - f1: 0.7726 - accuracy: 0.7817\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 18s 170us/step - loss: 0.4576 - precision: 0.7926 - recall: 0.7673 - f1: 0.7735 - accuracy: 0.7839\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 18s 169us/step - loss: 0.4568 - precision: 0.7920 - recall: 0.7691 - f1: 0.7747 - accuracy: 0.7845\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 18s 171us/step - loss: 0.4550 - precision: 0.7910 - recall: 0.7692 - f1: 0.7739 - accuracy: 0.7834\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 18s 170us/step - loss: 0.4524 - precision: 0.7960 - recall: 0.7714 - f1: 0.7774 - accuracy: 0.7866\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 18s 167us/step - loss: 0.4507 - precision: 0.7948 - recall: 0.7750 - f1: 0.7789 - accuracy: 0.7878\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 19s 174us/step - loss: 0.4498 - precision: 0.7955 - recall: 0.7739 - f1: 0.7788 - accuracy: 0.7876\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 18s 172us/step - loss: 0.4484 - precision: 0.7968 - recall: 0.7742 - f1: 0.7795 - accuracy: 0.7885\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 19s 174us/step - loss: 0.4467 - precision: 0.7978 - recall: 0.7754 - f1: 0.7805 - accuracy: 0.7895\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 18s 173us/step - loss: 0.4461 - precision: 0.7994 - recall: 0.7740 - f1: 0.7804 - accuracy: 0.7902\n",
      "53213/53213 [==============================] - 2s 36us/step\n",
      "[CV]  layer_3_sz=64, layer_2_sz=128, layer_1_sz=256, drp_2=0.5, drp_1=0.5, score=0.778, total= 9.2min\n",
      "[CV] layer_3_sz=64, layer_2_sz=128, layer_1_sz=256, drp_2=0.5, drp_1=0.5 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   2 out of   2 | elapsed: 18.7min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 18s 171us/step - loss: 0.5954 - precision: 0.6897 - recall: 0.6361 - f1: nan - accuracy: 0.6783\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.5318 - precision: 0.7445 - recall: 0.7168 - f1: 0.7236 - accuracy: 0.7360\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 18s 170us/step - loss: 0.5176 - precision: 0.7544 - recall: 0.7291 - f1: 0.7349 - accuracy: 0.7464\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 18s 172us/step - loss: 0.5095 - precision: 0.7581 - recall: 0.7360 - f1: 0.7400 - accuracy: 0.7506\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 18s 167us/step - loss: 0.5027 - precision: 0.7625 - recall: 0.7404 - f1: 0.7445 - accuracy: 0.7552\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 18s 166us/step - loss: 0.4976 - precision: 0.7647 - recall: 0.7453 - f1: 0.7484 - accuracy: 0.7583\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 18s 170us/step - loss: 0.4940 - precision: 0.7666 - recall: 0.7472 - f1: 0.7502 - accuracy: 0.7606\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 18s 169us/step - loss: 0.4897 - precision: 0.7700 - recall: 0.7464 - f1: 0.7514 - accuracy: 0.7624\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 18s 169us/step - loss: 0.4866 - precision: 0.7727 - recall: 0.7522 - f1: 0.7559 - accuracy: 0.7658\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 18s 171us/step - loss: 0.4839 - precision: 0.7741 - recall: 0.7518 - f1: 0.7562 - accuracy: 0.7663\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 18s 167us/step - loss: 0.4812 - precision: 0.7761 - recall: 0.7554 - f1: 0.7591 - accuracy: 0.7691\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 18s 166us/step - loss: 0.4790 - precision: 0.7779 - recall: 0.7570 - f1: 0.7609 - accuracy: 0.7707\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 18s 165us/step - loss: 0.4762 - precision: 0.7778 - recall: 0.7608 - f1: 0.7630 - accuracy: 0.7724\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 18s 171us/step - loss: 0.4727 - precision: 0.7803 - recall: 0.7616 - f1: 0.7648 - accuracy: 0.7737\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 18s 167us/step - loss: 0.4711 - precision: 0.7802 - recall: 0.7632 - f1: 0.7654 - accuracy: 0.7745\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4675 - precision: 0.7822 - recall: 0.7656 - f1: 0.7679 - accuracy: 0.7768\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 18s 165us/step - loss: 0.4653 - precision: 0.7850 - recall: 0.7648 - f1: 0.7687 - accuracy: 0.7782\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4646 - precision: 0.7848 - recall: 0.7663 - f1: 0.7692 - accuracy: 0.7786\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 17s 164us/step - loss: 0.4617 - precision: 0.7878 - recall: 0.7670 - f1: 0.7712 - accuracy: 0.7809\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.4600 - precision: 0.7882 - recall: 0.7679 - f1: 0.7718 - accuracy: 0.7817\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 18s 167us/step - loss: 0.4590 - precision: 0.7894 - recall: 0.7689 - f1: 0.7731 - accuracy: 0.7822\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.4564 - precision: 0.7907 - recall: 0.7721 - f1: 0.7754 - accuracy: 0.7843\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4539 - precision: 0.7926 - recall: 0.7759 - f1: 0.7783 - accuracy: 0.7865\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.4514 - precision: 0.7917 - recall: 0.7747 - f1: 0.7772 - accuracy: 0.7858\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4496 - precision: 0.7929 - recall: 0.7789 - f1: 0.7801 - accuracy: 0.7883\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 17s 164us/step - loss: 0.4483 - precision: 0.7944 - recall: 0.7772 - f1: 0.7800 - accuracy: 0.7889\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 18s 168us/step - loss: 0.4479 - precision: 0.7951 - recall: 0.7794 - f1: 0.7813 - accuracy: 0.7896\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 18s 165us/step - loss: 0.4468 - precision: 0.7965 - recall: 0.7811 - f1: 0.7826 - accuracy: 0.7906\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4445 - precision: 0.7978 - recall: 0.7820 - f1: 0.7839 - accuracy: 0.7919\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4417 - precision: 0.7994 - recall: 0.7805 - f1: 0.7843 - accuracy: 0.7934\n",
      "53213/53213 [==============================] - 2s 36us/step\n",
      "[CV]  layer_3_sz=64, layer_2_sz=128, layer_1_sz=256, drp_2=0.5, drp_1=0.5, score=0.773, total= 8.9min\n",
      "[CV] layer_3_sz=128, layer_2_sz=256, layer_1_sz=256, drp_2=0.3, drp_1=0.5 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   3 out of   3 | elapsed: 27.6min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "106426/106426 [==============================] - 18s 170us/step - loss: 0.5721 - precision: 0.7066 - recall: 0.6597 - f1: nan - accuracy: 0.6986\n",
      "Epoch 2/30\n",
      "106426/106426 [==============================] - 18s 168us/step - loss: 0.5211 - precision: 0.7492 - recall: 0.7216 - f1: 0.7277 - accuracy: 0.7410\n",
      "Epoch 3/30\n",
      "106426/106426 [==============================] - 18s 167us/step - loss: 0.5096 - precision: 0.7577 - recall: 0.7307 - f1: 0.7371 - accuracy: 0.7502\n",
      "Epoch 4/30\n",
      "106426/106426 [==============================] - 18s 172us/step - loss: 0.5012 - precision: 0.7616 - recall: 0.7372 - f1: 0.7424 - accuracy: 0.7542\n",
      "Epoch 5/30\n",
      "106426/106426 [==============================] - 18s 166us/step - loss: 0.4960 - precision: 0.7681 - recall: 0.7433 - f1: 0.7487 - accuracy: 0.7600\n",
      "Epoch 6/30\n",
      "106426/106426 [==============================] - 18s 172us/step - loss: 0.4906 - precision: 0.7696 - recall: 0.7450 - f1: 0.7505 - accuracy: 0.7622\n",
      "Epoch 7/30\n",
      "106426/106426 [==============================] - 18s 166us/step - loss: 0.4865 - precision: 0.7716 - recall: 0.7482 - f1: 0.7533 - accuracy: 0.7645\n",
      "Epoch 8/30\n",
      "106426/106426 [==============================] - 18s 167us/step - loss: 0.4828 - precision: 0.7749 - recall: 0.7504 - f1: 0.7560 - accuracy: 0.7667\n",
      "Epoch 9/30\n",
      "106426/106426 [==============================] - 18s 170us/step - loss: 0.4799 - precision: 0.7766 - recall: 0.7531 - f1: 0.7583 - accuracy: 0.7693\n",
      "Epoch 10/30\n",
      "106426/106426 [==============================] - 18s 168us/step - loss: 0.4763 - precision: 0.7804 - recall: 0.7544 - f1: 0.7609 - accuracy: 0.7717\n",
      "Epoch 11/30\n",
      "106426/106426 [==============================] - 18s 169us/step - loss: 0.4738 - precision: 0.7805 - recall: 0.7550 - f1: 0.7611 - accuracy: 0.7725\n",
      "Epoch 12/30\n",
      "106426/106426 [==============================] - 18s 170us/step - loss: 0.4704 - precision: 0.7840 - recall: 0.7593 - f1: 0.7651 - accuracy: 0.7756\n",
      "Epoch 13/30\n",
      "106426/106426 [==============================] - 18s 167us/step - loss: 0.4679 - precision: 0.7836 - recall: 0.7589 - f1: 0.7649 - accuracy: 0.7758\n",
      "Epoch 14/30\n",
      "106426/106426 [==============================] - 18s 172us/step - loss: 0.4662 - precision: 0.7863 - recall: 0.7601 - f1: 0.7667 - accuracy: 0.7779\n",
      "Epoch 15/30\n",
      "106426/106426 [==============================] - 18s 168us/step - loss: 0.4642 - precision: 0.7862 - recall: 0.7619 - f1: 0.7674 - accuracy: 0.7783\n",
      "Epoch 16/30\n",
      "106426/106426 [==============================] - 18s 168us/step - loss: 0.4599 - precision: 0.7905 - recall: 0.7645 - f1: 0.7713 - accuracy: 0.7817\n",
      "Epoch 17/30\n",
      "106426/106426 [==============================] - 18s 172us/step - loss: 0.4597 - precision: 0.7904 - recall: 0.7638 - f1: 0.7706 - accuracy: 0.7815\n",
      "Epoch 18/30\n",
      "106426/106426 [==============================] - 18s 167us/step - loss: 0.4573 - precision: 0.7918 - recall: 0.7662 - f1: 0.7724 - accuracy: 0.7829\n",
      "Epoch 19/30\n",
      "106426/106426 [==============================] - 18s 173us/step - loss: 0.4551 - precision: 0.7915 - recall: 0.7671 - f1: 0.7729 - accuracy: 0.7839\n",
      "Epoch 20/30\n",
      "106426/106426 [==============================] - 18s 165us/step - loss: 0.4542 - precision: 0.7938 - recall: 0.7674 - f1: 0.7742 - accuracy: 0.7849\n",
      "Epoch 21/30\n",
      "106426/106426 [==============================] - 18s 171us/step - loss: 0.4519 - precision: 0.7950 - recall: 0.7682 - f1: 0.7751 - accuracy: 0.7859\n",
      "Epoch 22/30\n",
      "106426/106426 [==============================] - 18s 171us/step - loss: 0.4490 - precision: 0.7971 - recall: 0.7693 - f1: 0.7768 - accuracy: 0.7878\n",
      "Epoch 23/30\n",
      "106426/106426 [==============================] - 18s 172us/step - loss: 0.4491 - precision: 0.7969 - recall: 0.7696 - f1: 0.7768 - accuracy: 0.7879\n",
      "Epoch 24/30\n",
      "106426/106426 [==============================] - 18s 167us/step - loss: 0.4462 - precision: 0.7983 - recall: 0.7727 - f1: 0.7790 - accuracy: 0.7889\n",
      "Epoch 25/30\n",
      "106426/106426 [==============================] - 18s 171us/step - loss: 0.4447 - precision: 0.7981 - recall: 0.7743 - f1: 0.7802 - accuracy: 0.7906\n",
      "Epoch 26/30\n",
      "106426/106426 [==============================] - 18s 166us/step - loss: 0.4421 - precision: 0.8002 - recall: 0.7743 - f1: 0.7810 - accuracy: 0.7916\n",
      "Epoch 27/30\n",
      "106426/106426 [==============================] - 18s 168us/step - loss: 0.4402 - precision: 0.8021 - recall: 0.7748 - f1: 0.7823 - accuracy: 0.7933\n",
      "Epoch 28/30\n",
      "106426/106426 [==============================] - 18s 171us/step - loss: 0.4392 - precision: 0.8041 - recall: 0.7754 - f1: 0.7835 - accuracy: 0.7943\n",
      "Epoch 29/30\n",
      "106426/106426 [==============================] - 18s 169us/step - loss: 0.4377 - precision: 0.8014 - recall: 0.7783 - f1: 0.7839 - accuracy: 0.7940\n",
      "Epoch 30/30\n",
      "106426/106426 [==============================] - 18s 166us/step - loss: 0.4372 - precision: 0.8053 - recall: 0.7775 - f1: 0.7855 - accuracy: 0.7955\n",
      "53214/53214 [==============================] - 2s 38us/step\n",
      "[CV]  layer_3_sz=128, layer_2_sz=256, layer_1_sz=256, drp_2=0.3, drp_1=0.5, score=0.773, total= 9.0min\n",
      "[CV] layer_3_sz=128, layer_2_sz=256, layer_1_sz=256, drp_2=0.3, drp_1=0.5 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   4 out of   4 | elapsed: 36.6min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 18s 173us/step - loss: 0.5754 - precision: 0.7054 - recall: 0.6778 - f1: 0.6808 - accuracy: 0.6959\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 18s 166us/step - loss: 0.5242 - precision: 0.7478 - recall: 0.7244 - f1: 0.7288 - accuracy: 0.7403\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 18s 169us/step - loss: 0.5135 - precision: 0.7547 - recall: 0.7331 - f1: 0.7370 - accuracy: 0.7479\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.5053 - precision: 0.7593 - recall: 0.7381 - f1: 0.7419 - accuracy: 0.7524\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 18s 166us/step - loss: 0.4990 - precision: 0.7656 - recall: 0.7412 - f1: 0.7465 - accuracy: 0.7570\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 18s 167us/step - loss: 0.4940 - precision: 0.7697 - recall: 0.7441 - f1: 0.7501 - accuracy: 0.7607\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4900 - precision: 0.7695 - recall: 0.7459 - f1: 0.7512 - accuracy: 0.7620\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4857 - precision: 0.7731 - recall: 0.7495 - f1: 0.7548 - accuracy: 0.7654\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4833 - precision: 0.7748 - recall: 0.7520 - f1: 0.7567 - accuracy: 0.7667\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.4801 - precision: 0.7770 - recall: 0.7521 - f1: 0.7579 - accuracy: 0.7684\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4767 - precision: 0.7780 - recall: 0.7558 - f1: 0.7605 - accuracy: 0.7700\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4739 - precision: 0.7798 - recall: 0.7568 - f1: 0.7619 - accuracy: 0.7720\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4714 - precision: 0.7816 - recall: 0.7585 - f1: 0.7636 - accuracy: 0.7735\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 18s 167us/step - loss: 0.4692 - precision: 0.7844 - recall: 0.7596 - f1: 0.7656 - accuracy: 0.7755\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4674 - precision: 0.7841 - recall: 0.7604 - f1: 0.7658 - accuracy: 0.7757\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 18s 166us/step - loss: 0.4655 - precision: 0.7856 - recall: 0.7631 - f1: 0.7680 - accuracy: 0.7776\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4613 - precision: 0.7870 - recall: 0.7638 - f1: 0.7690 - accuracy: 0.7788\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4601 - precision: 0.7903 - recall: 0.7650 - f1: 0.7713 - accuracy: 0.7805\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 18s 171us/step - loss: 0.4580 - precision: 0.7907 - recall: 0.7690 - f1: 0.7736 - accuracy: 0.7828\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4564 - precision: 0.7905 - recall: 0.7690 - f1: 0.7737 - accuracy: 0.7831\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4531 - precision: 0.7932 - recall: 0.7693 - f1: 0.7751 - accuracy: 0.7845\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4523 - precision: 0.7944 - recall: 0.7693 - f1: 0.7755 - accuracy: 0.7847\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 18s 165us/step - loss: 0.4498 - precision: 0.7955 - recall: 0.7703 - f1: 0.7767 - accuracy: 0.7861\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4481 - precision: 0.7949 - recall: 0.7720 - f1: 0.7774 - accuracy: 0.7868\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.4475 - precision: 0.7979 - recall: 0.7730 - f1: 0.7794 - accuracy: 0.7889\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4461 - precision: 0.7970 - recall: 0.7745 - f1: 0.7798 - accuracy: 0.7891\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4442 - precision: 0.7984 - recall: 0.7744 - f1: 0.7803 - accuracy: 0.7895\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4416 - precision: 0.8010 - recall: 0.7771 - f1: 0.7830 - accuracy: 0.7923\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4397 - precision: 0.8003 - recall: 0.7799 - f1: 0.7841 - accuracy: 0.7926\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 18s 168us/step - loss: 0.4383 - precision: 0.8002 - recall: 0.7780 - f1: 0.7831 - accuracy: 0.7923\n",
      "53213/53213 [==============================] - 2s 36us/step\n",
      "[CV]  layer_3_sz=128, layer_2_sz=256, layer_1_sz=256, drp_2=0.3, drp_1=0.5, score=0.778, total= 8.7min\n",
      "[CV] layer_3_sz=128, layer_2_sz=256, layer_1_sz=256, drp_2=0.3, drp_1=0.5 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   5 out of   5 | elapsed: 45.4min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.5764 - precision: 0.7019 - recall: 0.6778 - f1: 0.6806 - accuracy: 0.6947\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 18s 165us/step - loss: 0.5237 - precision: 0.7466 - recall: 0.7244 - f1: 0.7282 - accuracy: 0.7395\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 17s 164us/step - loss: 0.5127 - precision: 0.7543 - recall: 0.7347 - f1: 0.7373 - accuracy: 0.7479\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.5052 - precision: 0.7587 - recall: 0.7406 - f1: 0.7427 - accuracy: 0.7527\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 18s 167us/step - loss: 0.4988 - precision: 0.7621 - recall: 0.7433 - f1: 0.7461 - accuracy: 0.7561\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 18s 169us/step - loss: 0.4938 - precision: 0.7661 - recall: 0.7488 - f1: 0.7505 - accuracy: 0.7605\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 18s 168us/step - loss: 0.4902 - precision: 0.7674 - recall: 0.7507 - f1: 0.7523 - accuracy: 0.7614\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4852 - precision: 0.7715 - recall: 0.7516 - f1: 0.7549 - accuracy: 0.7650\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4824 - precision: 0.7742 - recall: 0.7554 - f1: 0.7582 - accuracy: 0.7677\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.4778 - precision: 0.7760 - recall: 0.7573 - f1: 0.7603 - accuracy: 0.7697\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 18s 169us/step - loss: 0.4750 - precision: 0.7768 - recall: 0.7609 - f1: 0.7622 - accuracy: 0.7713\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4734 - precision: 0.7782 - recall: 0.7616 - f1: 0.7634 - accuracy: 0.7728\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4706 - precision: 0.7812 - recall: 0.7611 - f1: 0.7647 - accuracy: 0.7749\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4669 - precision: 0.7831 - recall: 0.7643 - f1: 0.7672 - accuracy: 0.7769\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4646 - precision: 0.7847 - recall: 0.7665 - f1: 0.7693 - accuracy: 0.7781\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 17s 164us/step - loss: 0.4629 - precision: 0.7852 - recall: 0.7647 - f1: 0.7687 - accuracy: 0.7788\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4594 - precision: 0.7882 - recall: 0.7679 - f1: 0.7717 - accuracy: 0.7808\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.4579 - precision: 0.7873 - recall: 0.7703 - f1: 0.7725 - accuracy: 0.7816\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4558 - precision: 0.7889 - recall: 0.7718 - f1: 0.7743 - accuracy: 0.7834\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.4548 - precision: 0.7897 - recall: 0.7730 - f1: 0.7753 - accuracy: 0.7840\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4515 - precision: 0.7922 - recall: 0.7725 - f1: 0.7763 - accuracy: 0.7856\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 18s 165us/step - loss: 0.4493 - precision: 0.7940 - recall: 0.7760 - f1: 0.7789 - accuracy: 0.7876\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4485 - precision: 0.7939 - recall: 0.7775 - f1: 0.7794 - accuracy: 0.7880\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 18s 169us/step - loss: 0.4459 - precision: 0.7958 - recall: 0.7775 - f1: 0.7806 - accuracy: 0.7893\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 18s 165us/step - loss: 0.4445 - precision: 0.7966 - recall: 0.7771 - f1: 0.7809 - accuracy: 0.7896\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.4446 - precision: 0.7947 - recall: 0.7787 - f1: 0.7806 - accuracy: 0.7891\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4418 - precision: 0.7968 - recall: 0.7802 - f1: 0.7824 - accuracy: 0.7912\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 18s 165us/step - loss: 0.4401 - precision: 0.7968 - recall: 0.7800 - f1: 0.7828 - accuracy: 0.7913\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4388 - precision: 0.7998 - recall: 0.7807 - f1: 0.7844 - accuracy: 0.7926\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4376 - precision: 0.8008 - recall: 0.7846 - f1: 0.7867 - accuracy: 0.7949\n",
      "53213/53213 [==============================] - 2s 34us/step\n",
      "[CV]  layer_3_sz=128, layer_2_sz=256, layer_1_sz=256, drp_2=0.3, drp_1=0.5, score=0.776, total= 8.7min\n",
      "[CV] layer_3_sz=32, layer_2_sz=128, layer_1_sz=128, drp_2=0.0, drp_1=0.5 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   6 out of   6 | elapsed: 54.1min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "106426/106426 [==============================] - 17s 159us/step - loss: 0.5857 - precision: 0.6940 - recall: 0.6728 - f1: 0.6743 - accuracy: 0.6882\n",
      "Epoch 2/30\n",
      "106426/106426 [==============================] - 17s 155us/step - loss: 0.5279 - precision: 0.7453 - recall: 0.7181 - f1: 0.7241 - accuracy: 0.7373\n",
      "Epoch 3/30\n",
      "106426/106426 [==============================] - 17s 159us/step - loss: 0.5158 - precision: 0.7536 - recall: 0.7266 - f1: 0.7331 - accuracy: 0.7457\n",
      "Epoch 4/30\n",
      "106426/106426 [==============================] - 17s 159us/step - loss: 0.5084 - precision: 0.7578 - recall: 0.7326 - f1: 0.7383 - accuracy: 0.7505\n",
      "Epoch 5/30\n",
      "106426/106426 [==============================] - 16s 154us/step - loss: 0.5025 - precision: 0.7629 - recall: 0.7352 - f1: 0.7422 - accuracy: 0.7548\n",
      "Epoch 6/30\n",
      "106426/106426 [==============================] - 17s 158us/step - loss: 0.4990 - precision: 0.7652 - recall: 0.7400 - f1: 0.7459 - accuracy: 0.7581\n",
      "Epoch 7/30\n",
      "106426/106426 [==============================] - 17s 158us/step - loss: 0.4952 - precision: 0.7672 - recall: 0.7390 - f1: 0.7458 - accuracy: 0.7586\n",
      "Epoch 8/30\n",
      "106426/106426 [==============================] - 17s 157us/step - loss: 0.4915 - precision: 0.7703 - recall: 0.7443 - f1: 0.7505 - accuracy: 0.7626\n",
      "Epoch 9/30\n",
      "106426/106426 [==============================] - 17s 159us/step - loss: 0.4886 - precision: 0.7716 - recall: 0.7464 - f1: 0.7522 - accuracy: 0.7642\n",
      "Epoch 10/30\n",
      "106426/106426 [==============================] - 18s 165us/step - loss: 0.4862 - precision: 0.7744 - recall: 0.7452 - f1: 0.7529 - accuracy: 0.7652\n",
      "Epoch 11/30\n",
      "106426/106426 [==============================] - 17s 161us/step - loss: 0.4842 - precision: 0.7735 - recall: 0.7482 - f1: 0.7543 - accuracy: 0.7660\n",
      "Epoch 12/30\n",
      "106426/106426 [==============================] - 17s 161us/step - loss: 0.4811 - precision: 0.7771 - recall: 0.7506 - f1: 0.7573 - accuracy: 0.7686\n",
      "Epoch 13/30\n",
      "106426/106426 [==============================] - 17s 160us/step - loss: 0.4789 - precision: 0.7780 - recall: 0.7518 - f1: 0.7585 - accuracy: 0.7697\n",
      "Epoch 14/30\n",
      "106426/106426 [==============================] - 17s 161us/step - loss: 0.4767 - precision: 0.7799 - recall: 0.7545 - f1: 0.7606 - accuracy: 0.7715\n",
      "Epoch 15/30\n",
      "106426/106426 [==============================] - 17s 160us/step - loss: 0.4758 - precision: 0.7810 - recall: 0.7544 - f1: 0.7612 - accuracy: 0.7723\n",
      "Epoch 16/30\n",
      "106426/106426 [==============================] - 17s 162us/step - loss: 0.4732 - precision: 0.7817 - recall: 0.7571 - f1: 0.7627 - accuracy: 0.7737\n",
      "Epoch 17/30\n",
      "106426/106426 [==============================] - 17s 158us/step - loss: 0.4722 - precision: 0.7822 - recall: 0.7552 - f1: 0.7623 - accuracy: 0.7739\n",
      "Epoch 18/30\n",
      "106426/106426 [==============================] - 17s 162us/step - loss: 0.4704 - precision: 0.7825 - recall: 0.7567 - f1: 0.7632 - accuracy: 0.7744\n",
      "Epoch 19/30\n",
      "106426/106426 [==============================] - 17s 160us/step - loss: 0.4691 - precision: 0.7835 - recall: 0.7601 - f1: 0.7653 - accuracy: 0.7760\n",
      "Epoch 20/30\n",
      "106426/106426 [==============================] - 17s 157us/step - loss: 0.4677 - precision: 0.7851 - recall: 0.7601 - f1: 0.7664 - accuracy: 0.7770\n",
      "Epoch 21/30\n",
      "106426/106426 [==============================] - 16s 154us/step - loss: 0.4659 - precision: 0.7871 - recall: 0.7619 - f1: 0.7682 - accuracy: 0.7786\n",
      "Epoch 22/30\n",
      "106426/106426 [==============================] - 17s 160us/step - loss: 0.4648 - precision: 0.7861 - recall: 0.7596 - f1: 0.7665 - accuracy: 0.7777\n",
      "Epoch 23/30\n",
      "106426/106426 [==============================] - 17s 159us/step - loss: 0.4630 - precision: 0.7873 - recall: 0.7643 - f1: 0.7693 - accuracy: 0.7798\n",
      "Epoch 24/30\n",
      "106426/106426 [==============================] - 17s 159us/step - loss: 0.4620 - precision: 0.7876 - recall: 0.7622 - f1: 0.7687 - accuracy: 0.7795\n",
      "Epoch 25/30\n",
      "106426/106426 [==============================] - 17s 162us/step - loss: 0.4602 - precision: 0.7897 - recall: 0.7623 - f1: 0.7696 - accuracy: 0.7815\n",
      "Epoch 26/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.4594 - precision: 0.7912 - recall: 0.7632 - f1: 0.7711 - accuracy: 0.7826\n",
      "Epoch 27/30\n",
      "106426/106426 [==============================] - 17s 159us/step - loss: 0.4583 - precision: 0.7916 - recall: 0.7649 - f1: 0.7720 - accuracy: 0.7828\n",
      "Epoch 28/30\n",
      "106426/106426 [==============================] - 16s 155us/step - loss: 0.4569 - precision: 0.7915 - recall: 0.7663 - f1: 0.7727 - accuracy: 0.7835\n",
      "Epoch 29/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.4561 - precision: 0.7930 - recall: 0.7678 - f1: 0.7740 - accuracy: 0.7845\n",
      "Epoch 30/30\n",
      "106426/106426 [==============================] - 17s 160us/step - loss: 0.4555 - precision: 0.7918 - recall: 0.7662 - f1: 0.7730 - accuracy: 0.7842\n",
      "53214/53214 [==============================] - 2s 34us/step\n",
      "[CV]  layer_3_sz=32, layer_2_sz=128, layer_1_sz=128, drp_2=0.0, drp_1=0.5, score=0.771, total= 8.5min\n",
      "[CV] layer_3_sz=32, layer_2_sz=128, layer_1_sz=128, drp_2=0.0, drp_1=0.5 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   7 out of   7 | elapsed: 62.6min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 18s 167us/step - loss: 0.5939 - precision: 0.6879 - recall: 0.6382 - f1: nan - accuracy: 0.6806\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.5332 - precision: 0.7408 - recall: 0.7183 - f1: 0.7221 - accuracy: 0.7337\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.5213 - precision: 0.7483 - recall: 0.7256 - f1: 0.7297 - accuracy: 0.7407\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 17s 164us/step - loss: 0.5125 - precision: 0.7556 - recall: 0.7310 - f1: 0.7365 - accuracy: 0.7470\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 18s 166us/step - loss: 0.5068 - precision: 0.7581 - recall: 0.7334 - f1: 0.7388 - accuracy: 0.7502\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 18s 169us/step - loss: 0.5024 - precision: 0.7625 - recall: 0.7379 - f1: 0.7435 - accuracy: 0.7541\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4987 - precision: 0.7657 - recall: 0.7403 - f1: 0.7457 - accuracy: 0.7567\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4961 - precision: 0.7677 - recall: 0.7415 - f1: 0.7479 - accuracy: 0.7591\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 18s 165us/step - loss: 0.4924 - precision: 0.7696 - recall: 0.7449 - f1: 0.7506 - accuracy: 0.7610\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4903 - precision: 0.7699 - recall: 0.7441 - f1: 0.7503 - accuracy: 0.7612\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4869 - precision: 0.7740 - recall: 0.7462 - f1: 0.7534 - accuracy: 0.7644\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4858 - precision: 0.7732 - recall: 0.7445 - f1: 0.7522 - accuracy: 0.7639\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4829 - precision: 0.7761 - recall: 0.7476 - f1: 0.7548 - accuracy: 0.7664\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4809 - precision: 0.7773 - recall: 0.7522 - f1: 0.7580 - accuracy: 0.7680\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4791 - precision: 0.7787 - recall: 0.7533 - f1: 0.7593 - accuracy: 0.7699\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4777 - precision: 0.7780 - recall: 0.7540 - f1: 0.7593 - accuracy: 0.7697\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4757 - precision: 0.7796 - recall: 0.7547 - f1: 0.7607 - accuracy: 0.7712\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4734 - precision: 0.7804 - recall: 0.7564 - f1: 0.7622 - accuracy: 0.7723\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.4719 - precision: 0.7817 - recall: 0.7590 - f1: 0.7640 - accuracy: 0.7738\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4709 - precision: 0.7839 - recall: 0.7588 - f1: 0.7650 - accuracy: 0.7750\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4689 - precision: 0.7839 - recall: 0.7607 - f1: 0.7658 - accuracy: 0.7753\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 18s 166us/step - loss: 0.4687 - precision: 0.7838 - recall: 0.7600 - f1: 0.7655 - accuracy: 0.7753\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4663 - precision: 0.7871 - recall: 0.7628 - f1: 0.7686 - accuracy: 0.7783\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4655 - precision: 0.7855 - recall: 0.7626 - f1: 0.7674 - accuracy: 0.7773\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4637 - precision: 0.7862 - recall: 0.7635 - f1: 0.7686 - accuracy: 0.7781\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4630 - precision: 0.7899 - recall: 0.7636 - f1: 0.7701 - accuracy: 0.7798\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4629 - precision: 0.7877 - recall: 0.7637 - f1: 0.7693 - accuracy: 0.7787\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4605 - precision: 0.7878 - recall: 0.7681 - f1: 0.7718 - accuracy: 0.7808\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4599 - precision: 0.7888 - recall: 0.7673 - f1: 0.7717 - accuracy: 0.7809\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4579 - precision: 0.7892 - recall: 0.7691 - f1: 0.7729 - accuracy: 0.7822\n",
      "53213/53213 [==============================] - 2s 32us/step\n",
      "[CV]  layer_3_sz=32, layer_2_sz=128, layer_1_sz=128, drp_2=0.0, drp_1=0.5, score=0.777, total= 8.6min\n",
      "[CV] layer_3_sz=32, layer_2_sz=128, layer_1_sz=128, drp_2=0.0, drp_1=0.5 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   8 out of   8 | elapsed: 71.2min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.5882 - precision: 0.6962 - recall: 0.6505 - f1: nan - accuracy: 0.6856\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.5305 - precision: 0.7416 - recall: 0.7218 - f1: 0.7243 - accuracy: 0.7350\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.5191 - precision: 0.7502 - recall: 0.7293 - f1: 0.7328 - accuracy: 0.7435\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.5117 - precision: 0.7556 - recall: 0.7333 - f1: 0.7375 - accuracy: 0.7486\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.5055 - precision: 0.7594 - recall: 0.7369 - f1: 0.7418 - accuracy: 0.7526\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.5015 - precision: 0.7626 - recall: 0.7413 - f1: 0.7452 - accuracy: 0.7555\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4980 - precision: 0.7654 - recall: 0.7434 - f1: 0.7476 - accuracy: 0.7584\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4951 - precision: 0.7685 - recall: 0.7437 - f1: 0.7493 - accuracy: 0.7600\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4919 - precision: 0.7701 - recall: 0.7457 - f1: 0.7512 - accuracy: 0.7617\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4897 - precision: 0.7726 - recall: 0.7487 - f1: 0.7539 - accuracy: 0.7644\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4874 - precision: 0.7724 - recall: 0.7502 - f1: 0.7548 - accuracy: 0.7651\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 16s 155us/step - loss: 0.4845 - precision: 0.7719 - recall: 0.7504 - f1: 0.7547 - accuracy: 0.7652\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4821 - precision: 0.7768 - recall: 0.7543 - f1: 0.7587 - accuracy: 0.7690\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 16s 155us/step - loss: 0.4803 - precision: 0.7767 - recall: 0.7556 - f1: 0.7595 - accuracy: 0.7694\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4787 - precision: 0.7761 - recall: 0.7599 - f1: 0.7618 - accuracy: 0.7707\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4765 - precision: 0.7778 - recall: 0.7577 - f1: 0.7615 - accuracy: 0.7711\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4746 - precision: 0.7790 - recall: 0.7604 - f1: 0.7634 - accuracy: 0.7727\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4739 - precision: 0.7801 - recall: 0.7617 - f1: 0.7645 - accuracy: 0.7737\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4713 - precision: 0.7801 - recall: 0.7621 - f1: 0.7648 - accuracy: 0.7744\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4709 - precision: 0.7812 - recall: 0.7622 - f1: 0.7655 - accuracy: 0.7747\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4685 - precision: 0.7831 - recall: 0.7623 - f1: 0.7665 - accuracy: 0.7764\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4675 - precision: 0.7829 - recall: 0.7653 - f1: 0.7678 - accuracy: 0.7769\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4659 - precision: 0.7841 - recall: 0.7650 - f1: 0.7686 - accuracy: 0.7782\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.4659 - precision: 0.7854 - recall: 0.7657 - f1: 0.7695 - accuracy: 0.7785\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.4629 - precision: 0.7860 - recall: 0.7702 - f1: 0.7719 - accuracy: 0.7805\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4628 - precision: 0.7854 - recall: 0.7672 - f1: 0.7702 - accuracy: 0.7795\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4607 - precision: 0.7881 - recall: 0.7690 - f1: 0.7725 - accuracy: 0.7811\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 16s 148us/step - loss: 0.4603 - precision: 0.7875 - recall: 0.7707 - f1: 0.7728 - accuracy: 0.7815\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4583 - precision: 0.7890 - recall: 0.7715 - f1: 0.7742 - accuracy: 0.7834\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 15s 145us/step - loss: 0.4569 - precision: 0.7890 - recall: 0.7714 - f1: 0.7742 - accuracy: 0.7831\n",
      "53213/53213 [==============================] - 2s 34us/step\n",
      "[CV]  layer_3_sz=32, layer_2_sz=128, layer_1_sz=128, drp_2=0.0, drp_1=0.5, score=0.775, total= 8.2min\n",
      "[CV] layer_3_sz=128, layer_2_sz=128, layer_1_sz=64, drp_2=0.5, drp_1=0.3 \n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done   9 out of   9 | elapsed: 79.4min remaining:    0.0s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "106426/106426 [==============================] - 17s 159us/step - loss: 0.5946 - precision: 0.6906 - recall: 0.6336 - f1: nan - accuracy: 0.6797\n",
      "Epoch 2/30\n",
      "106426/106426 [==============================] - 17s 158us/step - loss: 0.5305 - precision: 0.7441 - recall: 0.7164 - f1: 0.7231 - accuracy: 0.7369\n",
      "Epoch 3/30\n",
      "106426/106426 [==============================] - 17s 157us/step - loss: 0.5180 - precision: 0.7518 - recall: 0.7260 - f1: 0.7320 - accuracy: 0.7444\n",
      "Epoch 4/30\n",
      "106426/106426 [==============================] - 17s 159us/step - loss: 0.5104 - precision: 0.7573 - recall: 0.7344 - f1: 0.7387 - accuracy: 0.7503\n",
      "Epoch 5/30\n",
      "106426/106426 [==============================] - 17s 163us/step - loss: 0.5041 - precision: 0.7624 - recall: 0.7389 - f1: 0.7438 - accuracy: 0.7556\n",
      "Epoch 6/30\n",
      "106426/106426 [==============================] - 18s 166us/step - loss: 0.5013 - precision: 0.7645 - recall: 0.7404 - f1: 0.7456 - accuracy: 0.7573\n",
      "Epoch 7/30\n",
      "106426/106426 [==============================] - 17s 160us/step - loss: 0.4972 - precision: 0.7658 - recall: 0.7447 - f1: 0.7484 - accuracy: 0.7595\n",
      "Epoch 8/30\n",
      "106426/106426 [==============================] - 17s 157us/step - loss: 0.4950 - precision: 0.7681 - recall: 0.7444 - f1: 0.7495 - accuracy: 0.7609\n",
      "Epoch 9/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.4915 - precision: 0.7723 - recall: 0.7457 - f1: 0.7522 - accuracy: 0.7637\n",
      "Epoch 10/30\n",
      "106426/106426 [==============================] - 17s 159us/step - loss: 0.4893 - precision: 0.7729 - recall: 0.7462 - f1: 0.7531 - accuracy: 0.7647\n",
      "Epoch 11/30\n",
      "106426/106426 [==============================] - 17s 158us/step - loss: 0.4859 - precision: 0.7752 - recall: 0.7506 - f1: 0.7562 - accuracy: 0.7679\n",
      "Epoch 12/30\n",
      "106426/106426 [==============================] - 17s 159us/step - loss: 0.4841 - precision: 0.7750 - recall: 0.7537 - f1: 0.7574 - accuracy: 0.7681\n",
      "Epoch 13/30\n",
      "106426/106426 [==============================] - 17s 164us/step - loss: 0.4823 - precision: 0.7756 - recall: 0.7542 - f1: 0.7581 - accuracy: 0.7688\n",
      "Epoch 14/30\n",
      "106426/106426 [==============================] - 17s 160us/step - loss: 0.4796 - precision: 0.7784 - recall: 0.7562 - f1: 0.7610 - accuracy: 0.7715\n",
      "Epoch 15/30\n",
      "106426/106426 [==============================] - 17s 162us/step - loss: 0.4786 - precision: 0.7802 - recall: 0.7556 - f1: 0.7614 - accuracy: 0.7722\n",
      "Epoch 16/30\n",
      "106426/106426 [==============================] - 17s 162us/step - loss: 0.4757 - precision: 0.7808 - recall: 0.7575 - f1: 0.7629 - accuracy: 0.7738\n",
      "Epoch 17/30\n",
      "106426/106426 [==============================] - 18s 169us/step - loss: 0.4733 - precision: 0.7811 - recall: 0.7588 - f1: 0.7638 - accuracy: 0.7744\n",
      "Epoch 18/30\n",
      "106426/106426 [==============================] - 17s 163us/step - loss: 0.4730 - precision: 0.7830 - recall: 0.7598 - f1: 0.7649 - accuracy: 0.7759\n",
      "Epoch 19/30\n",
      "106426/106426 [==============================] - 18s 166us/step - loss: 0.4718 - precision: 0.7837 - recall: 0.7581 - f1: 0.7645 - accuracy: 0.7758\n",
      "Epoch 20/30\n",
      "106426/106426 [==============================] - 18s 165us/step - loss: 0.4691 - precision: 0.7856 - recall: 0.7627 - f1: 0.7679 - accuracy: 0.7785\n",
      "Epoch 21/30\n",
      "106426/106426 [==============================] - 17s 161us/step - loss: 0.4686 - precision: 0.7834 - recall: 0.7623 - f1: 0.7665 - accuracy: 0.7769\n",
      "Epoch 22/30\n",
      "106426/106426 [==============================] - 17s 161us/step - loss: 0.4669 - precision: 0.7860 - recall: 0.7642 - f1: 0.7688 - accuracy: 0.7791\n",
      "Epoch 23/30\n",
      "106426/106426 [==============================] - 18s 165us/step - loss: 0.4663 - precision: 0.7874 - recall: 0.7643 - f1: 0.7697 - accuracy: 0.7800\n",
      "Epoch 24/30\n",
      "106426/106426 [==============================] - 18s 168us/step - loss: 0.4651 - precision: 0.7870 - recall: 0.7632 - f1: 0.7687 - accuracy: 0.7796\n",
      "Epoch 25/30\n",
      "106426/106426 [==============================] - 17s 160us/step - loss: 0.4634 - precision: 0.7894 - recall: 0.7642 - f1: 0.7706 - accuracy: 0.7812\n",
      "Epoch 26/30\n",
      "106426/106426 [==============================] - 18s 165us/step - loss: 0.4615 - precision: 0.7888 - recall: 0.7680 - f1: 0.7722 - accuracy: 0.7825\n",
      "Epoch 27/30\n",
      "106426/106426 [==============================] - 17s 164us/step - loss: 0.4616 - precision: 0.7895 - recall: 0.7676 - f1: 0.7724 - accuracy: 0.7826\n",
      "Epoch 28/30\n",
      "106426/106426 [==============================] - 17s 162us/step - loss: 0.4597 - precision: 0.7894 - recall: 0.7682 - f1: 0.7728 - accuracy: 0.7828\n",
      "Epoch 29/30\n",
      "106426/106426 [==============================] - 17s 157us/step - loss: 0.4582 - precision: 0.7912 - recall: 0.7702 - f1: 0.7747 - accuracy: 0.7848\n",
      "Epoch 30/30\n",
      "106426/106426 [==============================] - 17s 158us/step - loss: 0.4581 - precision: 0.7913 - recall: 0.7677 - f1: 0.7733 - accuracy: 0.7837\n",
      "53214/53214 [==============================] - 2s 34us/step\n",
      "[CV]  layer_3_sz=128, layer_2_sz=128, layer_1_sz=64, drp_2=0.5, drp_1=0.3, score=0.768, total= 8.6min\n",
      "[CV] layer_3_sz=128, layer_2_sz=128, layer_1_sz=64, drp_2=0.5, drp_1=0.3 \n",
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.6031 - precision: 0.6682 - recall: 0.6096 - f1: nan - accuracy: 0.6676\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.5363 - precision: 0.7417 - recall: 0.7150 - f1: 0.7212 - accuracy: 0.7332\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.5229 - precision: 0.7506 - recall: 0.7252 - f1: 0.7311 - accuracy: 0.7426\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.5142 - precision: 0.7551 - recall: 0.7302 - f1: 0.7359 - accuracy: 0.7469\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.5085 - precision: 0.7605 - recall: 0.7369 - f1: 0.7416 - accuracy: 0.7523\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.5045 - precision: 0.7606 - recall: 0.7383 - f1: 0.7427 - accuracy: 0.7531\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.4996 - precision: 0.7670 - recall: 0.7398 - f1: 0.7465 - accuracy: 0.7574\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4963 - precision: 0.7672 - recall: 0.7420 - f1: 0.7481 - accuracy: 0.7586\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4934 - precision: 0.7687 - recall: 0.7458 - f1: 0.7506 - accuracy: 0.7611\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4908 - precision: 0.7715 - recall: 0.7465 - f1: 0.7523 - accuracy: 0.7625\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4887 - precision: 0.7738 - recall: 0.7462 - f1: 0.7534 - accuracy: 0.7643\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4869 - precision: 0.7744 - recall: 0.7500 - f1: 0.7556 - accuracy: 0.7660\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4849 - precision: 0.7751 - recall: 0.7498 - f1: 0.7559 - accuracy: 0.7668\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4829 - precision: 0.7772 - recall: 0.7523 - f1: 0.7585 - accuracy: 0.7686\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4802 - precision: 0.7796 - recall: 0.7544 - f1: 0.7606 - accuracy: 0.7708\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4785 - precision: 0.7804 - recall: 0.7543 - f1: 0.7609 - accuracy: 0.7713\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4769 - precision: 0.7807 - recall: 0.7556 - f1: 0.7618 - accuracy: 0.7719\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4750 - precision: 0.7816 - recall: 0.7558 - f1: 0.7621 - accuracy: 0.7723\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 16s 155us/step - loss: 0.4735 - precision: 0.7828 - recall: 0.7584 - f1: 0.7644 - accuracy: 0.7742\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4721 - precision: 0.7844 - recall: 0.7582 - f1: 0.7648 - accuracy: 0.7750\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 17s 164us/step - loss: 0.4704 - precision: 0.7835 - recall: 0.7586 - f1: 0.7648 - accuracy: 0.7750\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.4708 - precision: 0.7821 - recall: 0.7594 - f1: 0.7643 - accuracy: 0.7742\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4679 - precision: 0.7848 - recall: 0.7634 - f1: 0.7678 - accuracy: 0.7773\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4672 - precision: 0.7856 - recall: 0.7644 - f1: 0.7688 - accuracy: 0.7781\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4658 - precision: 0.7854 - recall: 0.7649 - f1: 0.7693 - accuracy: 0.7786\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 18s 166us/step - loss: 0.4648 - precision: 0.7862 - recall: 0.7658 - f1: 0.7700 - accuracy: 0.7791\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4637 - precision: 0.7874 - recall: 0.7661 - f1: 0.7704 - accuracy: 0.7794\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4613 - precision: 0.7893 - recall: 0.7679 - f1: 0.7726 - accuracy: 0.7822\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4630 - precision: 0.7888 - recall: 0.7653 - f1: 0.7708 - accuracy: 0.7803\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4612 - precision: 0.7910 - recall: 0.7671 - f1: 0.7726 - accuracy: 0.7822\n",
      "53213/53213 [==============================] - 2s 34us/step\n",
      "[CV]  layer_3_sz=128, layer_2_sz=128, layer_1_sz=64, drp_2=0.5, drp_1=0.3, score=0.775, total= 8.4min\n",
      "[CV] layer_3_sz=128, layer_2_sz=128, layer_1_sz=64, drp_2=0.5, drp_1=0.3 \n",
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.6012 - precision: 0.6803 - recall: 0.6428 - f1: nan - accuracy: 0.6721\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.5368 - precision: 0.7429 - recall: 0.7170 - f1: 0.7224 - accuracy: 0.7342\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.5232 - precision: 0.7494 - recall: 0.7265 - f1: 0.7309 - accuracy: 0.7429\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.5156 - precision: 0.7544 - recall: 0.7312 - f1: 0.7361 - accuracy: 0.7471\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.5083 - precision: 0.7592 - recall: 0.7371 - f1: 0.7411 - accuracy: 0.7518\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.5046 - precision: 0.7616 - recall: 0.7407 - f1: 0.7445 - accuracy: 0.7549\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.4998 - precision: 0.7639 - recall: 0.7449 - f1: 0.7478 - accuracy: 0.7575\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4971 - precision: 0.7672 - recall: 0.7455 - f1: 0.7495 - accuracy: 0.7599\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.4941 - precision: 0.7685 - recall: 0.7496 - f1: 0.7524 - accuracy: 0.7621\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 17s 164us/step - loss: 0.4913 - precision: 0.7698 - recall: 0.7509 - f1: 0.7540 - accuracy: 0.7640\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 17s 164us/step - loss: 0.4882 - precision: 0.7712 - recall: 0.7529 - f1: 0.7556 - accuracy: 0.7656\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4857 - precision: 0.7726 - recall: 0.7541 - f1: 0.7571 - accuracy: 0.7666\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4842 - precision: 0.7748 - recall: 0.7556 - f1: 0.7587 - accuracy: 0.7679\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4818 - precision: 0.7746 - recall: 0.7600 - f1: 0.7609 - accuracy: 0.7697\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4801 - precision: 0.7772 - recall: 0.7610 - f1: 0.7627 - accuracy: 0.7716\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 16s 155us/step - loss: 0.4785 - precision: 0.7756 - recall: 0.7620 - f1: 0.7627 - accuracy: 0.7714\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4768 - precision: 0.7779 - recall: 0.7623 - f1: 0.7638 - accuracy: 0.7723\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4742 - precision: 0.7792 - recall: 0.7642 - f1: 0.7654 - accuracy: 0.7744\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4737 - precision: 0.7777 - recall: 0.7664 - f1: 0.7658 - accuracy: 0.7738\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 16s 155us/step - loss: 0.4714 - precision: 0.7797 - recall: 0.7647 - f1: 0.7659 - accuracy: 0.7749\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4713 - precision: 0.7793 - recall: 0.7655 - f1: 0.7662 - accuracy: 0.7744\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4686 - precision: 0.7833 - recall: 0.7703 - f1: 0.7706 - accuracy: 0.7792\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4682 - precision: 0.7825 - recall: 0.7686 - f1: 0.7694 - accuracy: 0.7779\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 16s 148us/step - loss: 0.4670 - precision: 0.7828 - recall: 0.7729 - f1: 0.7719 - accuracy: 0.7797\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4661 - precision: 0.7838 - recall: 0.7716 - f1: 0.7718 - accuracy: 0.7795\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4646 - precision: 0.7845 - recall: 0.7709 - f1: 0.7715 - accuracy: 0.7802\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.4632 - precision: 0.7847 - recall: 0.7731 - f1: 0.7729 - accuracy: 0.7807\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4626 - precision: 0.7862 - recall: 0.7750 - f1: 0.7743 - accuracy: 0.7824\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.4605 - precision: 0.7855 - recall: 0.7757 - f1: 0.7750 - accuracy: 0.7827\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4605 - precision: 0.7887 - recall: 0.7745 - f1: 0.7755 - accuracy: 0.7838\n",
      "53213/53213 [==============================] - 2s 32us/step\n",
      "[CV]  layer_3_sz=128, layer_2_sz=128, layer_1_sz=64, drp_2=0.5, drp_1=0.3, score=0.773, total= 8.3min\n",
      "[CV] layer_3_sz=32, layer_2_sz=64, layer_1_sz=256, drp_2=0.3, drp_1=0.0 \n",
      "Epoch 1/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.5592 - precision: 0.7233 - recall: 0.6802 - f1: nan - accuracy: 0.7131\n",
      "Epoch 2/30\n",
      "106426/106426 [==============================] - 16s 151us/step - loss: 0.5097 - precision: 0.7596 - recall: 0.7317 - f1: 0.7377 - accuracy: 0.7508\n",
      "Epoch 3/30\n",
      "106426/106426 [==============================] - 16s 150us/step - loss: 0.4991 - precision: 0.7669 - recall: 0.7390 - f1: 0.7455 - accuracy: 0.7579\n",
      "Epoch 4/30\n",
      "106426/106426 [==============================] - 16s 150us/step - loss: 0.4927 - precision: 0.7703 - recall: 0.7440 - f1: 0.7499 - accuracy: 0.7614\n",
      "Epoch 5/30\n",
      "106426/106426 [==============================] - 16s 150us/step - loss: 0.4864 - precision: 0.7746 - recall: 0.7489 - f1: 0.7544 - accuracy: 0.7657\n",
      "Epoch 6/30\n",
      "106426/106426 [==============================] - 16s 149us/step - loss: 0.4813 - precision: 0.7781 - recall: 0.7517 - f1: 0.7577 - accuracy: 0.7685\n",
      "Epoch 7/30\n",
      "106426/106426 [==============================] - 16s 153us/step - loss: 0.4775 - precision: 0.7785 - recall: 0.7551 - f1: 0.7594 - accuracy: 0.7710\n",
      "Epoch 8/30\n",
      "106426/106426 [==============================] - 16s 152us/step - loss: 0.4723 - precision: 0.7832 - recall: 0.7575 - f1: 0.7633 - accuracy: 0.7745\n",
      "Epoch 9/30\n",
      "106426/106426 [==============================] - 16s 146us/step - loss: 0.4683 - precision: 0.7851 - recall: 0.7566 - f1: 0.7638 - accuracy: 0.7755\n",
      "Epoch 10/30\n",
      "106426/106426 [==============================] - 16s 149us/step - loss: 0.4641 - precision: 0.7886 - recall: 0.7599 - f1: 0.7671 - accuracy: 0.7782\n",
      "Epoch 11/30\n",
      "106426/106426 [==============================] - 16s 148us/step - loss: 0.4604 - precision: 0.7916 - recall: 0.7638 - f1: 0.7706 - accuracy: 0.7821\n",
      "Epoch 12/30\n",
      "106426/106426 [==============================] - 16s 153us/step - loss: 0.4566 - precision: 0.7938 - recall: 0.7639 - f1: 0.7722 - accuracy: 0.7837\n",
      "Epoch 13/30\n",
      "106426/106426 [==============================] - 17s 155us/step - loss: 0.4524 - precision: 0.7972 - recall: 0.7673 - f1: 0.7756 - accuracy: 0.7868\n",
      "Epoch 14/30\n",
      "106426/106426 [==============================] - 17s 155us/step - loss: 0.4483 - precision: 0.8007 - recall: 0.7689 - f1: 0.7777 - accuracy: 0.7888\n",
      "Epoch 15/30\n",
      "106426/106426 [==============================] - 17s 158us/step - loss: 0.4454 - precision: 0.8020 - recall: 0.7689 - f1: 0.7787 - accuracy: 0.7903\n",
      "Epoch 16/30\n",
      "106426/106426 [==============================] - 17s 155us/step - loss: 0.4407 - precision: 0.8057 - recall: 0.7724 - f1: 0.7824 - accuracy: 0.7932\n",
      "Epoch 17/30\n",
      "106426/106426 [==============================] - 17s 158us/step - loss: 0.4371 - precision: 0.8076 - recall: 0.7763 - f1: 0.7851 - accuracy: 0.7962\n",
      "Epoch 18/30\n",
      "106426/106426 [==============================] - 16s 154us/step - loss: 0.4327 - precision: 0.8095 - recall: 0.7767 - f1: 0.7866 - accuracy: 0.7981\n",
      "Epoch 19/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.4291 - precision: 0.8117 - recall: 0.7796 - f1: 0.7890 - accuracy: 0.8004\n",
      "Epoch 20/30\n",
      "106426/106426 [==============================] - 16s 153us/step - loss: 0.4255 - precision: 0.8144 - recall: 0.7797 - f1: 0.7901 - accuracy: 0.8011\n",
      "Epoch 21/30\n",
      "106426/106426 [==============================] - 16s 155us/step - loss: 0.4226 - precision: 0.8175 - recall: 0.7828 - f1: 0.7934 - accuracy: 0.8040\n",
      "Epoch 22/30\n",
      "106426/106426 [==============================] - 16s 148us/step - loss: 0.4182 - precision: 0.8198 - recall: 0.7845 - f1: 0.7954 - accuracy: 0.8063\n",
      "Epoch 23/30\n",
      "106426/106426 [==============================] - 16s 150us/step - loss: 0.4143 - precision: 0.8206 - recall: 0.7878 - f1: 0.7978 - accuracy: 0.8086\n",
      "Epoch 24/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.4107 - precision: 0.8235 - recall: 0.7894 - f1: 0.8002 - accuracy: 0.8109\n",
      "Epoch 25/30\n",
      "106426/106426 [==============================] - 16s 154us/step - loss: 0.4068 - precision: 0.8258 - recall: 0.7915 - f1: 0.8023 - accuracy: 0.8130\n",
      "Epoch 26/30\n",
      "106426/106426 [==============================] - 17s 155us/step - loss: 0.4027 - precision: 0.8291 - recall: 0.7932 - f1: 0.8047 - accuracy: 0.8158\n",
      "Epoch 27/30\n",
      "106426/106426 [==============================] - 16s 148us/step - loss: 0.3996 - precision: 0.8290 - recall: 0.7960 - f1: 0.8061 - accuracy: 0.8170\n",
      "Epoch 28/30\n",
      "106426/106426 [==============================] - 16s 150us/step - loss: 0.3968 - precision: 0.8336 - recall: 0.7980 - f1: 0.8092 - accuracy: 0.8188\n",
      "Epoch 29/30\n",
      "106426/106426 [==============================] - 16s 151us/step - loss: 0.3926 - precision: 0.8337 - recall: 0.7987 - f1: 0.8099 - accuracy: 0.8201\n",
      "Epoch 30/30\n",
      "106426/106426 [==============================] - 16s 153us/step - loss: 0.3885 - precision: 0.8395 - recall: 0.8024 - f1: 0.8147 - accuracy: 0.8245\n",
      "53214/53214 [==============================] - 2s 33us/step\n",
      "[CV]  layer_3_sz=32, layer_2_sz=64, layer_1_sz=256, drp_2=0.3, drp_1=0.0, score=0.763, total= 8.2min\n",
      "[CV] layer_3_sz=32, layer_2_sz=64, layer_1_sz=256, drp_2=0.3, drp_1=0.0 \n",
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.5597 - precision: 0.7209 - recall: 0.6882 - f1: nan - accuracy: 0.7129\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.5121 - precision: 0.7583 - recall: 0.7310 - f1: 0.7373 - accuracy: 0.7492\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.5021 - precision: 0.7630 - recall: 0.7394 - f1: 0.7439 - accuracy: 0.7544\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 16s 146us/step - loss: 0.4950 - precision: 0.7672 - recall: 0.7416 - f1: 0.7469 - accuracy: 0.7586\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4896 - precision: 0.7715 - recall: 0.7477 - f1: 0.7525 - accuracy: 0.7631\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4843 - precision: 0.7747 - recall: 0.7488 - f1: 0.7544 - accuracy: 0.7653\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 16s 148us/step - loss: 0.4803 - precision: 0.7792 - recall: 0.7531 - f1: 0.7591 - accuracy: 0.7690\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.4756 - precision: 0.7826 - recall: 0.7551 - f1: 0.7616 - accuracy: 0.7720\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 16s 148us/step - loss: 0.4713 - precision: 0.7859 - recall: 0.7561 - f1: 0.7641 - accuracy: 0.7746\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.4672 - precision: 0.7866 - recall: 0.7585 - f1: 0.7657 - accuracy: 0.7765\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 16s 147us/step - loss: 0.4627 - precision: 0.7920 - recall: 0.7615 - f1: 0.7697 - accuracy: 0.7804\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4582 - precision: 0.7933 - recall: 0.7622 - f1: 0.7710 - accuracy: 0.7818\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.4535 - precision: 0.7973 - recall: 0.7642 - f1: 0.7736 - accuracy: 0.7847\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4499 - precision: 0.7994 - recall: 0.7655 - f1: 0.7751 - accuracy: 0.7866\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 16s 148us/step - loss: 0.4451 - precision: 0.8026 - recall: 0.7698 - f1: 0.7792 - accuracy: 0.7899\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4412 - precision: 0.8057 - recall: 0.7687 - f1: 0.7801 - accuracy: 0.7915\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 16s 147us/step - loss: 0.4379 - precision: 0.8086 - recall: 0.7700 - f1: 0.7821 - accuracy: 0.7937\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4329 - precision: 0.8124 - recall: 0.7753 - f1: 0.7873 - accuracy: 0.7978\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4290 - precision: 0.8136 - recall: 0.7775 - f1: 0.7889 - accuracy: 0.7998\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.4251 - precision: 0.8168 - recall: 0.7803 - f1: 0.7920 - accuracy: 0.8023\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4216 - precision: 0.8190 - recall: 0.7808 - f1: 0.7932 - accuracy: 0.8038\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4173 - precision: 0.8213 - recall: 0.7846 - f1: 0.7966 - accuracy: 0.8072\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 16s 148us/step - loss: 0.4134 - precision: 0.8246 - recall: 0.7833 - f1: 0.7972 - accuracy: 0.8084\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 16s 148us/step - loss: 0.4096 - precision: 0.8259 - recall: 0.7858 - f1: 0.7994 - accuracy: 0.8103\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.4049 - precision: 0.8303 - recall: 0.7887 - f1: 0.8029 - accuracy: 0.8135\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4020 - precision: 0.8319 - recall: 0.7899 - f1: 0.8039 - accuracy: 0.8145\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 16s 147us/step - loss: 0.3969 - precision: 0.8332 - recall: 0.7960 - f1: 0.8081 - accuracy: 0.8181\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.3944 - precision: 0.8368 - recall: 0.7951 - f1: 0.8093 - accuracy: 0.8194\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.3901 - precision: 0.8376 - recall: 0.7975 - f1: 0.8112 - accuracy: 0.8213\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.3862 - precision: 0.8423 - recall: 0.8000 - f1: 0.8147 - accuracy: 0.8246\n",
      "53213/53213 [==============================] - 2s 34us/step\n",
      "[CV]  layer_3_sz=32, layer_2_sz=64, layer_1_sz=256, drp_2=0.3, drp_1=0.0, score=0.769, total= 8.0min\n",
      "[CV] layer_3_sz=32, layer_2_sz=64, layer_1_sz=256, drp_2=0.3, drp_1=0.0 \n",
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 16s 155us/step - loss: 0.5620 - precision: 0.7220 - recall: 0.6884 - f1: 0.6944 - accuracy: 0.7129\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.5115 - precision: 0.7586 - recall: 0.7296 - f1: 0.7363 - accuracy: 0.7490\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.5005 - precision: 0.7642 - recall: 0.7384 - f1: 0.7441 - accuracy: 0.7558\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4943 - precision: 0.7686 - recall: 0.7432 - f1: 0.7487 - accuracy: 0.7603\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4880 - precision: 0.7749 - recall: 0.7470 - f1: 0.7532 - accuracy: 0.7646\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4836 - precision: 0.7772 - recall: 0.7524 - f1: 0.7577 - accuracy: 0.7684\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 16s 155us/step - loss: 0.4786 - precision: 0.7794 - recall: 0.7540 - f1: 0.7597 - accuracy: 0.7705\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4733 - precision: 0.7829 - recall: 0.7576 - f1: 0.7635 - accuracy: 0.7738\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.4694 - precision: 0.7864 - recall: 0.7591 - f1: 0.7658 - accuracy: 0.7766\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4647 - precision: 0.7885 - recall: 0.7623 - f1: 0.7685 - accuracy: 0.7788\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4599 - precision: 0.7905 - recall: 0.7663 - f1: 0.7715 - accuracy: 0.7821\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4553 - precision: 0.7949 - recall: 0.7669 - f1: 0.7743 - accuracy: 0.7842\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4519 - precision: 0.7971 - recall: 0.7715 - f1: 0.7774 - accuracy: 0.7874\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4472 - precision: 0.7992 - recall: 0.7745 - f1: 0.7799 - accuracy: 0.7897\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4428 - precision: 0.8020 - recall: 0.7744 - f1: 0.7819 - accuracy: 0.7921\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.4387 - precision: 0.8031 - recall: 0.7791 - f1: 0.7847 - accuracy: 0.7943\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4352 - precision: 0.8069 - recall: 0.7800 - f1: 0.7867 - accuracy: 0.7966\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.4309 - precision: 0.8082 - recall: 0.7831 - f1: 0.7889 - accuracy: 0.7985\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4263 - precision: 0.8117 - recall: 0.7857 - f1: 0.7924 - accuracy: 0.8015\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4226 - precision: 0.8154 - recall: 0.7876 - f1: 0.7951 - accuracy: 0.8049\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4196 - precision: 0.8148 - recall: 0.7909 - f1: 0.7967 - accuracy: 0.8056\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4149 - precision: 0.8192 - recall: 0.7919 - f1: 0.7992 - accuracy: 0.8086\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4104 - precision: 0.8222 - recall: 0.7947 - f1: 0.8020 - accuracy: 0.8109\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4079 - precision: 0.8225 - recall: 0.7966 - f1: 0.8033 - accuracy: 0.8124\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4039 - precision: 0.8245 - recall: 0.7986 - f1: 0.8052 - accuracy: 0.8139\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.3991 - precision: 0.8277 - recall: 0.7996 - f1: 0.8073 - accuracy: 0.8165\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.3965 - precision: 0.8293 - recall: 0.8028 - f1: 0.8097 - accuracy: 0.8185\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.3927 - precision: 0.8317 - recall: 0.8048 - f1: 0.8119 - accuracy: 0.8206\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.3873 - precision: 0.8337 - recall: 0.8061 - f1: 0.8137 - accuracy: 0.8224\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.3850 - precision: 0.8365 - recall: 0.8088 - f1: 0.8164 - accuracy: 0.8245\n",
      "53213/53213 [==============================] - 2s 33us/step\n",
      "[CV]  layer_3_sz=32, layer_2_sz=64, layer_1_sz=256, drp_2=0.3, drp_1=0.0, score=0.767, total= 8.3min\n",
      "[CV] layer_3_sz=32, layer_2_sz=128, layer_1_sz=128, drp_2=0.5, drp_1=0.0 \n",
      "Epoch 1/30\n",
      "106426/106426 [==============================] - 16s 152us/step - loss: 0.5723 - precision: 0.7128 - recall: 0.6775 - f1: 0.6845 - accuracy: 0.7018\n",
      "Epoch 2/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.5160 - precision: 0.7598 - recall: 0.7218 - f1: 0.7332 - accuracy: 0.7480\n",
      "Epoch 3/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.5049 - precision: 0.7651 - recall: 0.7320 - f1: 0.7411 - accuracy: 0.7544\n",
      "Epoch 4/30\n",
      "106426/106426 [==============================] - 16s 150us/step - loss: 0.4971 - precision: 0.7674 - recall: 0.7400 - f1: 0.7466 - accuracy: 0.7587\n",
      "Epoch 5/30\n",
      "106426/106426 [==============================] - 16s 152us/step - loss: 0.4920 - precision: 0.7715 - recall: 0.7435 - f1: 0.7500 - accuracy: 0.7622\n",
      "Epoch 6/30\n",
      "106426/106426 [==============================] - 16s 151us/step - loss: 0.4872 - precision: 0.7751 - recall: 0.7486 - f1: 0.7546 - accuracy: 0.7660\n",
      "Epoch 7/30\n",
      "106426/106426 [==============================] - 16s 147us/step - loss: 0.4836 - precision: 0.7749 - recall: 0.7516 - f1: 0.7561 - accuracy: 0.7673\n",
      "Epoch 8/30\n",
      "106426/106426 [==============================] - 16s 147us/step - loss: 0.4804 - precision: 0.7778 - recall: 0.7519 - f1: 0.7578 - accuracy: 0.7695\n",
      "Epoch 9/30\n",
      "106426/106426 [==============================] - 16s 150us/step - loss: 0.4769 - precision: 0.7796 - recall: 0.7566 - f1: 0.7613 - accuracy: 0.7722\n",
      "Epoch 10/30\n",
      "106426/106426 [==============================] - 15s 145us/step - loss: 0.4731 - precision: 0.7816 - recall: 0.7577 - f1: 0.7629 - accuracy: 0.7741\n",
      "Epoch 11/30\n",
      "106426/106426 [==============================] - 16s 149us/step - loss: 0.4707 - precision: 0.7829 - recall: 0.7596 - f1: 0.7645 - accuracy: 0.7756\n",
      "Epoch 12/30\n",
      "106426/106426 [==============================] - 16s 150us/step - loss: 0.4668 - precision: 0.7853 - recall: 0.7615 - f1: 0.7666 - accuracy: 0.7772\n",
      "Epoch 13/30\n",
      "106426/106426 [==============================] - 15s 145us/step - loss: 0.4644 - precision: 0.7862 - recall: 0.7624 - f1: 0.7676 - accuracy: 0.7784\n",
      "Epoch 14/30\n",
      "106426/106426 [==============================] - 16s 147us/step - loss: 0.4612 - precision: 0.7891 - recall: 0.7648 - f1: 0.7703 - accuracy: 0.7811\n",
      "Epoch 15/30\n",
      "106426/106426 [==============================] - 16s 147us/step - loss: 0.4584 - precision: 0.7917 - recall: 0.7672 - f1: 0.7729 - accuracy: 0.7836\n",
      "Epoch 16/30\n",
      "106426/106426 [==============================] - 16s 147us/step - loss: 0.4557 - precision: 0.7925 - recall: 0.7688 - f1: 0.7740 - accuracy: 0.7842\n",
      "Epoch 17/30\n",
      "106426/106426 [==============================] - 16s 147us/step - loss: 0.4527 - precision: 0.7927 - recall: 0.7706 - f1: 0.7752 - accuracy: 0.7854\n",
      "Epoch 18/30\n",
      "106426/106426 [==============================] - 16s 152us/step - loss: 0.4497 - precision: 0.7953 - recall: 0.7706 - f1: 0.7766 - accuracy: 0.7872\n",
      "Epoch 19/30\n",
      "106426/106426 [==============================] - 16s 146us/step - loss: 0.4474 - precision: 0.7969 - recall: 0.7745 - f1: 0.7793 - accuracy: 0.7888\n",
      "Epoch 20/30\n",
      "106426/106426 [==============================] - 16s 147us/step - loss: 0.4451 - precision: 0.7986 - recall: 0.7742 - f1: 0.7800 - accuracy: 0.7902\n",
      "Epoch 21/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.4415 - precision: 0.8001 - recall: 0.7765 - f1: 0.7819 - accuracy: 0.7918\n",
      "Epoch 22/30\n",
      "106426/106426 [==============================] - 16s 153us/step - loss: 0.4396 - precision: 0.8022 - recall: 0.7778 - f1: 0.7837 - accuracy: 0.7938\n",
      "Epoch 23/30\n",
      "106426/106426 [==============================] - 16s 152us/step - loss: 0.4375 - precision: 0.8044 - recall: 0.7773 - f1: 0.7844 - accuracy: 0.7947\n",
      "Epoch 24/30\n",
      "106426/106426 [==============================] - 16s 150us/step - loss: 0.4345 - precision: 0.8053 - recall: 0.7822 - f1: 0.7872 - accuracy: 0.7970\n",
      "Epoch 25/30\n",
      "106426/106426 [==============================] - 16s 150us/step - loss: 0.4317 - precision: 0.8057 - recall: 0.7821 - f1: 0.7876 - accuracy: 0.7975\n",
      "Epoch 26/30\n",
      "106426/106426 [==============================] - 16s 146us/step - loss: 0.4300 - precision: 0.8084 - recall: 0.7829 - f1: 0.7891 - accuracy: 0.7990\n",
      "Epoch 27/30\n",
      "106426/106426 [==============================] - 16s 152us/step - loss: 0.4272 - precision: 0.8100 - recall: 0.7870 - f1: 0.7921 - accuracy: 0.8011\n",
      "Epoch 28/30\n",
      "106426/106426 [==============================] - 16s 150us/step - loss: 0.4257 - precision: 0.8109 - recall: 0.7855 - f1: 0.7919 - accuracy: 0.8019\n",
      "Epoch 29/30\n",
      "106426/106426 [==============================] - 16s 148us/step - loss: 0.4227 - precision: 0.8110 - recall: 0.7893 - f1: 0.7940 - accuracy: 0.8032\n",
      "Epoch 30/30\n",
      "106426/106426 [==============================] - 16s 151us/step - loss: 0.4202 - precision: 0.8141 - recall: 0.7901 - f1: 0.7959 - accuracy: 0.8051\n",
      "53214/53214 [==============================] - 2s 32us/step\n",
      "[CV]  layer_3_sz=32, layer_2_sz=128, layer_1_sz=128, drp_2=0.5, drp_1=0.0, score=0.770, total= 8.0min\n",
      "[CV] layer_3_sz=32, layer_2_sz=128, layer_1_sz=128, drp_2=0.5, drp_1=0.0 \n",
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.5717 - precision: 0.7126 - recall: 0.6711 - f1: nan - accuracy: 0.7012\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.5177 - precision: 0.7563 - recall: 0.7265 - f1: 0.7338 - accuracy: 0.7457\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 16s 148us/step - loss: 0.5065 - precision: 0.7623 - recall: 0.7357 - f1: 0.7417 - accuracy: 0.7524\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.5000 - precision: 0.7655 - recall: 0.7403 - f1: 0.7456 - accuracy: 0.7567\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 16s 155us/step - loss: 0.4936 - precision: 0.7677 - recall: 0.7430 - f1: 0.7486 - accuracy: 0.7591\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4893 - precision: 0.7705 - recall: 0.7469 - f1: 0.7516 - accuracy: 0.7620\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.4850 - precision: 0.7732 - recall: 0.7485 - f1: 0.7539 - accuracy: 0.7641\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4812 - precision: 0.7764 - recall: 0.7503 - f1: 0.7564 - accuracy: 0.7670\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.4774 - precision: 0.7785 - recall: 0.7516 - f1: 0.7577 - accuracy: 0.7687\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4740 - precision: 0.7820 - recall: 0.7546 - f1: 0.7614 - accuracy: 0.7720\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4706 - precision: 0.7842 - recall: 0.7559 - f1: 0.7630 - accuracy: 0.7741\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4673 - precision: 0.7863 - recall: 0.7578 - f1: 0.7649 - accuracy: 0.7759\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4645 - precision: 0.7870 - recall: 0.7584 - f1: 0.7659 - accuracy: 0.7769\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4608 - precision: 0.7904 - recall: 0.7601 - f1: 0.7686 - accuracy: 0.7794\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4577 - precision: 0.7932 - recall: 0.7624 - f1: 0.7711 - accuracy: 0.7818\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4552 - precision: 0.7944 - recall: 0.7636 - f1: 0.7723 - accuracy: 0.7829\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4525 - precision: 0.7973 - recall: 0.7646 - f1: 0.7740 - accuracy: 0.7849\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4494 - precision: 0.7989 - recall: 0.7674 - f1: 0.7763 - accuracy: 0.7870\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4472 - precision: 0.8003 - recall: 0.7676 - f1: 0.7770 - accuracy: 0.7879\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4439 - precision: 0.8024 - recall: 0.7712 - f1: 0.7799 - accuracy: 0.7905\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4423 - precision: 0.8055 - recall: 0.7698 - f1: 0.7809 - accuracy: 0.7920\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4383 - precision: 0.8066 - recall: 0.7738 - f1: 0.7836 - accuracy: 0.7935\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4361 - precision: 0.8084 - recall: 0.7746 - f1: 0.7850 - accuracy: 0.7954\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4333 - precision: 0.8093 - recall: 0.7764 - f1: 0.7865 - accuracy: 0.7969\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4307 - precision: 0.8123 - recall: 0.7777 - f1: 0.7884 - accuracy: 0.7984\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4290 - precision: 0.8137 - recall: 0.7795 - f1: 0.7899 - accuracy: 0.7998\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4267 - precision: 0.8135 - recall: 0.7791 - f1: 0.7897 - accuracy: 0.7994\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4241 - precision: 0.8153 - recall: 0.7839 - f1: 0.7928 - accuracy: 0.8023\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4224 - precision: 0.8169 - recall: 0.7813 - f1: 0.7927 - accuracy: 0.8031\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4201 - precision: 0.8195 - recall: 0.7814 - f1: 0.7938 - accuracy: 0.8046\n",
      "53213/53213 [==============================] - 2s 43us/step\n",
      "[CV]  layer_3_sz=32, layer_2_sz=128, layer_1_sz=128, drp_2=0.5, drp_1=0.0, score=0.770, total= 8.3min\n",
      "[CV] layer_3_sz=32, layer_2_sz=128, layer_1_sz=128, drp_2=0.5, drp_1=0.0 \n",
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.5774 - precision: 0.7049 - recall: 0.6949 - f1: 0.6909 - accuracy: 0.6988\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.5187 - precision: 0.7555 - recall: 0.7267 - f1: 0.7336 - accuracy: 0.7456\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.5075 - precision: 0.7603 - recall: 0.7345 - f1: 0.7403 - accuracy: 0.7522\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.5000 - precision: 0.7651 - recall: 0.7429 - f1: 0.7471 - accuracy: 0.7581\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4947 - precision: 0.7690 - recall: 0.7463 - f1: 0.7505 - accuracy: 0.7611\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.4898 - precision: 0.7726 - recall: 0.7489 - f1: 0.7535 - accuracy: 0.7642\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4853 - precision: 0.7741 - recall: 0.7512 - f1: 0.7559 - accuracy: 0.7668\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4814 - precision: 0.7774 - recall: 0.7534 - f1: 0.7585 - accuracy: 0.7691\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4781 - precision: 0.7803 - recall: 0.7546 - f1: 0.7608 - accuracy: 0.7716\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4752 - precision: 0.7822 - recall: 0.7579 - f1: 0.7631 - accuracy: 0.7731\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4712 - precision: 0.7859 - recall: 0.7594 - f1: 0.7657 - accuracy: 0.7759\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4683 - precision: 0.7855 - recall: 0.7602 - f1: 0.7661 - accuracy: 0.7767\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4643 - precision: 0.7879 - recall: 0.7611 - f1: 0.7679 - accuracy: 0.7789\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4612 - precision: 0.7926 - recall: 0.7645 - f1: 0.7717 - accuracy: 0.7817\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4582 - precision: 0.7927 - recall: 0.7665 - f1: 0.7728 - accuracy: 0.7828\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4546 - precision: 0.7952 - recall: 0.7682 - f1: 0.7753 - accuracy: 0.7853\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4522 - precision: 0.7976 - recall: 0.7731 - f1: 0.7786 - accuracy: 0.7880\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4487 - precision: 0.7980 - recall: 0.7722 - f1: 0.7788 - accuracy: 0.7886\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4465 - precision: 0.7986 - recall: 0.7739 - f1: 0.7798 - accuracy: 0.7900\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4440 - precision: 0.8007 - recall: 0.7758 - f1: 0.7817 - accuracy: 0.7916\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 16s 148us/step - loss: 0.4407 - precision: 0.8040 - recall: 0.7761 - f1: 0.7835 - accuracy: 0.7936\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4387 - precision: 0.8051 - recall: 0.7780 - f1: 0.7849 - accuracy: 0.7944\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4358 - precision: 0.8064 - recall: 0.7790 - f1: 0.7863 - accuracy: 0.7962\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4332 - precision: 0.8080 - recall: 0.7817 - f1: 0.7886 - accuracy: 0.7978\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.4302 - precision: 0.8094 - recall: 0.7836 - f1: 0.7900 - accuracy: 0.7992\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.4281 - precision: 0.8113 - recall: 0.7853 - f1: 0.7919 - accuracy: 0.8012\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 15s 144us/step - loss: 0.4256 - precision: 0.8117 - recall: 0.7873 - f1: 0.7933 - accuracy: 0.8027\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 16s 146us/step - loss: 0.4225 - precision: 0.8156 - recall: 0.7866 - f1: 0.7948 - accuracy: 0.8043\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 16s 148us/step - loss: 0.4207 - precision: 0.8169 - recall: 0.7894 - f1: 0.7970 - accuracy: 0.8058\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 17s 164us/step - loss: 0.4180 - precision: 0.8157 - recall: 0.7893 - f1: 0.7961 - accuracy: 0.8055\n",
      "53213/53213 [==============================] - 2s 37us/step\n",
      "[CV]  layer_3_sz=32, layer_2_sz=128, layer_1_sz=128, drp_2=0.5, drp_1=0.0, score=0.770, total= 8.2min\n",
      "[CV] layer_3_sz=64, layer_2_sz=64, layer_1_sz=128, drp_2=0.5, drp_1=0.0 \n",
      "Epoch 1/30\n",
      "106426/106426 [==============================] - 18s 165us/step - loss: 0.5765 - precision: 0.7067 - recall: 0.6910 - f1: 0.6901 - accuracy: 0.7000\n",
      "Epoch 2/30\n",
      "106426/106426 [==============================] - 17s 164us/step - loss: 0.5178 - precision: 0.7580 - recall: 0.7242 - f1: 0.7332 - accuracy: 0.7474\n",
      "Epoch 3/30\n",
      "106426/106426 [==============================] - 18s 164us/step - loss: 0.5071 - precision: 0.7624 - recall: 0.7337 - f1: 0.7410 - accuracy: 0.7536\n",
      "Epoch 4/30\n",
      "106426/106426 [==============================] - 17s 155us/step - loss: 0.5008 - precision: 0.7661 - recall: 0.7369 - f1: 0.7441 - accuracy: 0.7569\n",
      "Epoch 5/30\n",
      "106426/106426 [==============================] - 17s 161us/step - loss: 0.4958 - precision: 0.7684 - recall: 0.7415 - f1: 0.7477 - accuracy: 0.7600\n",
      "Epoch 6/30\n",
      "106426/106426 [==============================] - 17s 162us/step - loss: 0.4917 - precision: 0.7730 - recall: 0.7440 - f1: 0.7514 - accuracy: 0.7636\n",
      "Epoch 7/30\n",
      "106426/106426 [==============================] - 16s 152us/step - loss: 0.4876 - precision: 0.7763 - recall: 0.7475 - f1: 0.7547 - accuracy: 0.7664\n",
      "Epoch 8/30\n",
      "106426/106426 [==============================] - 17s 164us/step - loss: 0.4845 - precision: 0.7781 - recall: 0.7485 - f1: 0.7560 - accuracy: 0.7682\n",
      "Epoch 9/30\n",
      "106426/106426 [==============================] - 16s 153us/step - loss: 0.4803 - precision: 0.7810 - recall: 0.7484 - f1: 0.7579 - accuracy: 0.7700\n",
      "Epoch 10/30\n",
      "106426/106426 [==============================] - 17s 155us/step - loss: 0.4777 - precision: 0.7814 - recall: 0.7498 - f1: 0.7585 - accuracy: 0.7711\n",
      "Epoch 11/30\n",
      "106426/106426 [==============================] - 16s 154us/step - loss: 0.4743 - precision: 0.7847 - recall: 0.7504 - f1: 0.7606 - accuracy: 0.7735\n",
      "Epoch 12/30\n",
      "106426/106426 [==============================] - 17s 160us/step - loss: 0.4711 - precision: 0.7874 - recall: 0.7526 - f1: 0.7630 - accuracy: 0.7761\n",
      "Epoch 13/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.4681 - precision: 0.7897 - recall: 0.7564 - f1: 0.7664 - accuracy: 0.7782\n",
      "Epoch 14/30\n",
      "106426/106426 [==============================] - 17s 158us/step - loss: 0.4645 - precision: 0.7918 - recall: 0.7562 - f1: 0.7670 - accuracy: 0.7795\n",
      "Epoch 15/30\n",
      "106426/106426 [==============================] - 17s 159us/step - loss: 0.4627 - precision: 0.7920 - recall: 0.7570 - f1: 0.7676 - accuracy: 0.7805\n",
      "Epoch 16/30\n",
      "106426/106426 [==============================] - 17s 157us/step - loss: 0.4598 - precision: 0.7968 - recall: 0.7565 - f1: 0.7698 - accuracy: 0.7829\n",
      "Epoch 17/30\n",
      "106426/106426 [==============================] - 16s 154us/step - loss: 0.4573 - precision: 0.7985 - recall: 0.7598 - f1: 0.7726 - accuracy: 0.7848\n",
      "Epoch 18/30\n",
      "106426/106426 [==============================] - 17s 161us/step - loss: 0.4545 - precision: 0.7997 - recall: 0.7603 - f1: 0.7729 - accuracy: 0.7856\n",
      "Epoch 19/30\n",
      "106426/106426 [==============================] - 17s 163us/step - loss: 0.4519 - precision: 0.8029 - recall: 0.7608 - f1: 0.7750 - accuracy: 0.7881\n",
      "Epoch 20/30\n",
      "106426/106426 [==============================] - 16s 154us/step - loss: 0.4498 - precision: 0.8030 - recall: 0.7625 - f1: 0.7759 - accuracy: 0.7884\n",
      "Epoch 21/30\n",
      "106426/106426 [==============================] - 16s 153us/step - loss: 0.4465 - precision: 0.8055 - recall: 0.7653 - f1: 0.7786 - accuracy: 0.7908\n",
      "Epoch 22/30\n",
      "106426/106426 [==============================] - 17s 157us/step - loss: 0.4439 - precision: 0.8090 - recall: 0.7641 - f1: 0.7796 - accuracy: 0.7929\n",
      "Epoch 23/30\n",
      "106426/106426 [==============================] - 16s 153us/step - loss: 0.4422 - precision: 0.8096 - recall: 0.7646 - f1: 0.7799 - accuracy: 0.7931\n",
      "Epoch 24/30\n",
      "106426/106426 [==============================] - 17s 160us/step - loss: 0.4392 - precision: 0.8111 - recall: 0.7673 - f1: 0.7823 - accuracy: 0.7950\n",
      "Epoch 25/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.4365 - precision: 0.8132 - recall: 0.7684 - f1: 0.7838 - accuracy: 0.7967\n",
      "Epoch 26/30\n",
      "106426/106426 [==============================] - 17s 160us/step - loss: 0.4350 - precision: 0.8153 - recall: 0.7694 - f1: 0.7855 - accuracy: 0.7981\n",
      "Epoch 27/30\n",
      "106426/106426 [==============================] - 16s 153us/step - loss: 0.4322 - precision: 0.8175 - recall: 0.7710 - f1: 0.7871 - accuracy: 0.7998\n",
      "Epoch 28/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.4296 - precision: 0.8175 - recall: 0.7708 - f1: 0.7871 - accuracy: 0.8002\n",
      "Epoch 29/30\n",
      "106426/106426 [==============================] - 16s 150us/step - loss: 0.4284 - precision: 0.8188 - recall: 0.7701 - f1: 0.7876 - accuracy: 0.8006\n",
      "Epoch 30/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.4254 - precision: 0.8213 - recall: 0.7735 - f1: 0.7905 - accuracy: 0.8028\n",
      "53214/53214 [==============================] - 2s 33us/step\n",
      "[CV]  layer_3_sz=64, layer_2_sz=64, layer_1_sz=128, drp_2=0.5, drp_1=0.0, score=0.769, total= 8.4min\n",
      "[CV] layer_3_sz=64, layer_2_sz=64, layer_1_sz=128, drp_2=0.5, drp_1=0.0 \n",
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.5816 - precision: 0.7033 - recall: 0.6655 - f1: 0.6732 - accuracy: 0.6927\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.5213 - precision: 0.7529 - recall: 0.7243 - f1: 0.7313 - accuracy: 0.7435\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.5100 - precision: 0.7594 - recall: 0.7319 - f1: 0.7383 - accuracy: 0.7499\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.5019 - precision: 0.7634 - recall: 0.7385 - f1: 0.7440 - accuracy: 0.7551\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4968 - precision: 0.7678 - recall: 0.7425 - f1: 0.7481 - accuracy: 0.7585\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4919 - precision: 0.7704 - recall: 0.7431 - f1: 0.7498 - accuracy: 0.7609\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4886 - precision: 0.7731 - recall: 0.7450 - f1: 0.7521 - accuracy: 0.7630\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4846 - precision: 0.7759 - recall: 0.7477 - f1: 0.7545 - accuracy: 0.7654\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4817 - precision: 0.7792 - recall: 0.7494 - f1: 0.7576 - accuracy: 0.7684\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 17s 164us/step - loss: 0.4780 - precision: 0.7797 - recall: 0.7492 - f1: 0.7577 - accuracy: 0.7691\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4752 - precision: 0.7827 - recall: 0.7504 - f1: 0.7593 - accuracy: 0.7709\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.4724 - precision: 0.7837 - recall: 0.7533 - f1: 0.7619 - accuracy: 0.7732\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4702 - precision: 0.7885 - recall: 0.7528 - f1: 0.7635 - accuracy: 0.7751\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4667 - precision: 0.7877 - recall: 0.7573 - f1: 0.7656 - accuracy: 0.7764\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4646 - precision: 0.7917 - recall: 0.7571 - f1: 0.7673 - accuracy: 0.7780\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4611 - precision: 0.7925 - recall: 0.7600 - f1: 0.7696 - accuracy: 0.7805\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4586 - precision: 0.7953 - recall: 0.7601 - f1: 0.7712 - accuracy: 0.7822\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4562 - precision: 0.7968 - recall: 0.7589 - f1: 0.7707 - accuracy: 0.7823\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4538 - precision: 0.7979 - recall: 0.7627 - f1: 0.7737 - accuracy: 0.7847\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4514 - precision: 0.8002 - recall: 0.7636 - f1: 0.7751 - accuracy: 0.7862\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4479 - precision: 0.8010 - recall: 0.7665 - f1: 0.7773 - accuracy: 0.7881\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4452 - precision: 0.8032 - recall: 0.7674 - f1: 0.7788 - accuracy: 0.7900\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 17s 164us/step - loss: 0.4428 - precision: 0.8027 - recall: 0.7699 - f1: 0.7800 - accuracy: 0.7907\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4405 - precision: 0.8053 - recall: 0.7714 - f1: 0.7816 - accuracy: 0.7920\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4387 - precision: 0.8072 - recall: 0.7716 - f1: 0.7829 - accuracy: 0.7936\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4367 - precision: 0.8087 - recall: 0.7751 - f1: 0.7851 - accuracy: 0.7957\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4348 - precision: 0.8114 - recall: 0.7742 - f1: 0.7859 - accuracy: 0.7964\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4320 - precision: 0.8124 - recall: 0.7765 - f1: 0.7878 - accuracy: 0.7983\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4300 - precision: 0.8152 - recall: 0.7769 - f1: 0.7892 - accuracy: 0.7998\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4280 - precision: 0.8139 - recall: 0.7765 - f1: 0.7887 - accuracy: 0.7992\n",
      "53213/53213 [==============================] - 2s 36us/step\n",
      "[CV]  layer_3_sz=64, layer_2_sz=64, layer_1_sz=128, drp_2=0.5, drp_1=0.0, score=0.767, total= 8.5min\n",
      "[CV] layer_3_sz=64, layer_2_sz=64, layer_1_sz=128, drp_2=0.5, drp_1=0.0 \n",
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.5837 - precision: 0.7069 - recall: 0.6192 - f1: nan - accuracy: 0.6901\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 18s 167us/step - loss: 0.5208 - precision: 0.7509 - recall: 0.7301 - f1: 0.7334 - accuracy: 0.7446\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 18s 164us/step - loss: 0.5088 - precision: 0.7584 - recall: 0.7408 - f1: 0.7427 - accuracy: 0.7530\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.5020 - precision: 0.7605 - recall: 0.7462 - f1: 0.7466 - accuracy: 0.7558\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4968 - precision: 0.7643 - recall: 0.7492 - f1: 0.7500 - accuracy: 0.7593\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4919 - precision: 0.7666 - recall: 0.7577 - f1: 0.7556 - accuracy: 0.7637\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4886 - precision: 0.7695 - recall: 0.7583 - f1: 0.7573 - accuracy: 0.7651\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4852 - precision: 0.7699 - recall: 0.7598 - f1: 0.7580 - accuracy: 0.7666\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4815 - precision: 0.7714 - recall: 0.7632 - f1: 0.7608 - accuracy: 0.7691\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4784 - precision: 0.7748 - recall: 0.7659 - f1: 0.7639 - accuracy: 0.7718\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4753 - precision: 0.7760 - recall: 0.7675 - f1: 0.7654 - accuracy: 0.7737\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.4722 - precision: 0.7774 - recall: 0.7712 - f1: 0.7678 - accuracy: 0.7754\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4688 - precision: 0.7798 - recall: 0.7707 - f1: 0.7687 - accuracy: 0.7769\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4656 - precision: 0.7798 - recall: 0.7756 - f1: 0.7712 - accuracy: 0.7786\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4627 - precision: 0.7831 - recall: 0.7760 - f1: 0.7733 - accuracy: 0.7810\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4600 - precision: 0.7851 - recall: 0.7774 - f1: 0.7748 - accuracy: 0.7824\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4575 - precision: 0.7861 - recall: 0.7784 - f1: 0.7758 - accuracy: 0.7833\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4540 - precision: 0.7894 - recall: 0.7827 - f1: 0.7801 - accuracy: 0.7871\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4513 - precision: 0.7905 - recall: 0.7834 - f1: 0.7808 - accuracy: 0.7880\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4489 - precision: 0.7912 - recall: 0.7865 - f1: 0.7827 - accuracy: 0.7896\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 18s 165us/step - loss: 0.4458 - precision: 0.7950 - recall: 0.7876 - f1: 0.7855 - accuracy: 0.7929\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4432 - precision: 0.7964 - recall: 0.7899 - f1: 0.7872 - accuracy: 0.7935\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4405 - precision: 0.7983 - recall: 0.7904 - f1: 0.7885 - accuracy: 0.7956\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4379 - precision: 0.7992 - recall: 0.7930 - f1: 0.7898 - accuracy: 0.7964\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4363 - precision: 0.7982 - recall: 0.7944 - f1: 0.7903 - accuracy: 0.7964\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4333 - precision: 0.8004 - recall: 0.7967 - f1: 0.7926 - accuracy: 0.7992\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4307 - precision: 0.8026 - recall: 0.7978 - f1: 0.7942 - accuracy: 0.8008\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4286 - precision: 0.8056 - recall: 0.7973 - f1: 0.7954 - accuracy: 0.8021\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4265 - precision: 0.8040 - recall: 0.7979 - f1: 0.7949 - accuracy: 0.8017\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4238 - precision: 0.8069 - recall: 0.8029 - f1: 0.7989 - accuracy: 0.8050\n",
      "53213/53213 [==============================] - 2s 35us/step\n",
      "[CV]  layer_3_sz=64, layer_2_sz=64, layer_1_sz=128, drp_2=0.5, drp_1=0.0, score=0.771, total= 8.6min\n",
      "[CV] layer_3_sz=32, layer_2_sz=64, layer_1_sz=256, drp_2=0.3, drp_1=0.5 \n",
      "Epoch 1/30\n",
      "106426/106426 [==============================] - 17s 162us/step - loss: 0.5889 - precision: 0.6947 - recall: 0.6546 - f1: 0.6636 - accuracy: 0.6861\n",
      "Epoch 2/30\n",
      "106426/106426 [==============================] - 17s 162us/step - loss: 0.5283 - precision: 0.7446 - recall: 0.7217 - f1: 0.7259 - accuracy: 0.7382\n",
      "Epoch 3/30\n",
      "106426/106426 [==============================] - 17s 160us/step - loss: 0.5137 - precision: 0.7550 - recall: 0.7313 - f1: 0.7362 - accuracy: 0.7479\n",
      "Epoch 4/30\n",
      "106426/106426 [==============================] - 17s 160us/step - loss: 0.5062 - precision: 0.7616 - recall: 0.7365 - f1: 0.7421 - accuracy: 0.7541\n",
      "Epoch 5/30\n",
      "106426/106426 [==============================] - 17s 163us/step - loss: 0.5012 - precision: 0.7624 - recall: 0.7407 - f1: 0.7450 - accuracy: 0.7565\n",
      "Epoch 6/30\n",
      "106426/106426 [==============================] - 18s 166us/step - loss: 0.4963 - precision: 0.7680 - recall: 0.7437 - f1: 0.7489 - accuracy: 0.7601\n",
      "Epoch 7/30\n",
      "106426/106426 [==============================] - 17s 160us/step - loss: 0.4931 - precision: 0.7688 - recall: 0.7466 - f1: 0.7510 - accuracy: 0.7624\n",
      "Epoch 8/30\n",
      "106426/106426 [==============================] - 17s 162us/step - loss: 0.4887 - precision: 0.7708 - recall: 0.7499 - f1: 0.7537 - accuracy: 0.7648\n",
      "Epoch 9/30\n",
      "106426/106426 [==============================] - 17s 163us/step - loss: 0.4861 - precision: 0.7723 - recall: 0.7489 - f1: 0.7543 - accuracy: 0.7653\n",
      "Epoch 10/30\n",
      "106426/106426 [==============================] - 17s 160us/step - loss: 0.4831 - precision: 0.7744 - recall: 0.7531 - f1: 0.7571 - accuracy: 0.7686\n",
      "Epoch 11/30\n",
      "106426/106426 [==============================] - 17s 158us/step - loss: 0.4799 - precision: 0.7756 - recall: 0.7543 - f1: 0.7582 - accuracy: 0.7694\n",
      "Epoch 12/30\n",
      "106426/106426 [==============================] - 17s 158us/step - loss: 0.4773 - precision: 0.7784 - recall: 0.7586 - f1: 0.7619 - accuracy: 0.7719\n",
      "Epoch 13/30\n",
      "106426/106426 [==============================] - 17s 157us/step - loss: 0.4756 - precision: 0.7803 - recall: 0.7598 - f1: 0.7633 - accuracy: 0.7735\n",
      "Epoch 14/30\n",
      "106426/106426 [==============================] - 16s 155us/step - loss: 0.4720 - precision: 0.7818 - recall: 0.7605 - f1: 0.7648 - accuracy: 0.7751\n",
      "Epoch 15/30\n",
      "106426/106426 [==============================] - 17s 159us/step - loss: 0.4703 - precision: 0.7831 - recall: 0.7600 - f1: 0.7653 - accuracy: 0.7759\n",
      "Epoch 16/30\n",
      "106426/106426 [==============================] - 18s 166us/step - loss: 0.4680 - precision: 0.7837 - recall: 0.7616 - f1: 0.7664 - accuracy: 0.7772\n",
      "Epoch 17/30\n",
      "106426/106426 [==============================] - 17s 161us/step - loss: 0.4658 - precision: 0.7867 - recall: 0.7628 - f1: 0.7682 - accuracy: 0.7791\n",
      "Epoch 18/30\n",
      "106426/106426 [==============================] - 17s 160us/step - loss: 0.4643 - precision: 0.7887 - recall: 0.7657 - f1: 0.7707 - accuracy: 0.7805\n",
      "Epoch 19/30\n",
      "106426/106426 [==============================] - 17s 158us/step - loss: 0.4615 - precision: 0.7897 - recall: 0.7685 - f1: 0.7729 - accuracy: 0.7828\n",
      "Epoch 20/30\n",
      "106426/106426 [==============================] - 17s 160us/step - loss: 0.4599 - precision: 0.7890 - recall: 0.7687 - f1: 0.7727 - accuracy: 0.7823\n",
      "Epoch 21/30\n",
      "106426/106426 [==============================] - 17s 157us/step - loss: 0.4581 - precision: 0.7895 - recall: 0.7681 - f1: 0.7728 - accuracy: 0.7828\n",
      "Epoch 22/30\n",
      "106426/106426 [==============================] - 17s 159us/step - loss: 0.4565 - precision: 0.7926 - recall: 0.7690 - f1: 0.7744 - accuracy: 0.7847\n",
      "Epoch 23/30\n",
      "106426/106426 [==============================] - 17s 157us/step - loss: 0.4550 - precision: 0.7908 - recall: 0.7713 - f1: 0.7749 - accuracy: 0.7850\n",
      "Epoch 24/30\n",
      "106426/106426 [==============================] - 17s 161us/step - loss: 0.4535 - precision: 0.7925 - recall: 0.7705 - f1: 0.7755 - accuracy: 0.7858\n",
      "Epoch 25/30\n",
      "106426/106426 [==============================] - 17s 157us/step - loss: 0.4510 - precision: 0.7938 - recall: 0.7739 - f1: 0.7777 - accuracy: 0.7871\n",
      "Epoch 26/30\n",
      "106426/106426 [==============================] - 17s 161us/step - loss: 0.4490 - precision: 0.7940 - recall: 0.7752 - f1: 0.7790 - accuracy: 0.7886\n",
      "Epoch 27/30\n",
      "106426/106426 [==============================] - 17s 163us/step - loss: 0.4493 - precision: 0.7952 - recall: 0.7774 - f1: 0.7803 - accuracy: 0.7895\n",
      "Epoch 28/30\n",
      "106426/106426 [==============================] - 17s 163us/step - loss: 0.4466 - precision: 0.7965 - recall: 0.7758 - f1: 0.7801 - accuracy: 0.7897\n",
      "Epoch 29/30\n",
      "106426/106426 [==============================] - 17s 158us/step - loss: 0.4446 - precision: 0.7985 - recall: 0.7770 - f1: 0.7817 - accuracy: 0.7916\n",
      "Epoch 30/30\n",
      "106426/106426 [==============================] - 17s 164us/step - loss: 0.4430 - precision: 0.7984 - recall: 0.7794 - f1: 0.7829 - accuracy: 0.7921\n",
      "53214/53214 [==============================] - 2s 35us/step\n",
      "[CV]  layer_3_sz=32, layer_2_sz=64, layer_1_sz=256, drp_2=0.3, drp_1=0.5, score=0.773, total= 8.6min\n",
      "[CV] layer_3_sz=32, layer_2_sz=64, layer_1_sz=256, drp_2=0.3, drp_1=0.5 \n",
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.5910 - precision: 0.6920 - recall: 0.6644 - f1: 0.6691 - accuracy: 0.6845\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.5312 - precision: 0.7447 - recall: 0.7171 - f1: 0.7236 - accuracy: 0.7359\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.5174 - precision: 0.7527 - recall: 0.7274 - f1: 0.7328 - accuracy: 0.7444\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.5105 - precision: 0.7572 - recall: 0.7330 - f1: 0.7381 - accuracy: 0.7487\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 17s 164us/step - loss: 0.5041 - precision: 0.7601 - recall: 0.7396 - f1: 0.7431 - accuracy: 0.7540\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4998 - precision: 0.7655 - recall: 0.7411 - f1: 0.7465 - accuracy: 0.7574\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4956 - precision: 0.7687 - recall: 0.7437 - f1: 0.7495 - accuracy: 0.7599\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4920 - precision: 0.7715 - recall: 0.7445 - f1: 0.7512 - accuracy: 0.7623\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4885 - precision: 0.7724 - recall: 0.7471 - f1: 0.7529 - accuracy: 0.7638\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 18s 164us/step - loss: 0.4864 - precision: 0.7745 - recall: 0.7506 - f1: 0.7560 - accuracy: 0.7660\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4822 - precision: 0.7765 - recall: 0.7520 - f1: 0.7576 - accuracy: 0.7674\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4803 - precision: 0.7757 - recall: 0.7538 - f1: 0.7581 - accuracy: 0.7681\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4773 - precision: 0.7787 - recall: 0.7563 - f1: 0.7613 - accuracy: 0.7712\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4754 - precision: 0.7813 - recall: 0.7571 - f1: 0.7626 - accuracy: 0.7727\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4722 - precision: 0.7841 - recall: 0.7595 - f1: 0.7650 - accuracy: 0.7753\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4717 - precision: 0.7835 - recall: 0.7591 - f1: 0.7647 - accuracy: 0.7748\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4678 - precision: 0.7871 - recall: 0.7612 - f1: 0.7678 - accuracy: 0.7779\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4660 - precision: 0.7855 - recall: 0.7623 - f1: 0.7676 - accuracy: 0.7776\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4645 - precision: 0.7871 - recall: 0.7633 - f1: 0.7690 - accuracy: 0.7788\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4628 - precision: 0.7894 - recall: 0.7639 - f1: 0.7701 - accuracy: 0.7795\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4601 - precision: 0.7911 - recall: 0.7663 - f1: 0.7726 - accuracy: 0.7825\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4578 - precision: 0.7910 - recall: 0.7687 - f1: 0.7740 - accuracy: 0.7832\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4581 - precision: 0.7907 - recall: 0.7672 - f1: 0.7727 - accuracy: 0.7824\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4553 - precision: 0.7947 - recall: 0.7689 - f1: 0.7755 - accuracy: 0.7852\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4542 - precision: 0.7946 - recall: 0.7699 - f1: 0.7763 - accuracy: 0.7853\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4520 - precision: 0.7952 - recall: 0.7727 - f1: 0.7778 - accuracy: 0.7869\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4512 - precision: 0.7946 - recall: 0.7732 - f1: 0.7776 - accuracy: 0.7873\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4477 - precision: 0.7972 - recall: 0.7732 - f1: 0.7794 - accuracy: 0.7888\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4468 - precision: 0.7989 - recall: 0.7748 - f1: 0.7807 - accuracy: 0.7900\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4458 - precision: 0.7976 - recall: 0.7764 - f1: 0.7810 - accuracy: 0.7902\n",
      "53213/53213 [==============================] - 2s 36us/step\n",
      "[CV]  layer_3_sz=32, layer_2_sz=64, layer_1_sz=256, drp_2=0.3, drp_1=0.5, score=0.775, total= 8.5min\n",
      "[CV] layer_3_sz=32, layer_2_sz=64, layer_1_sz=256, drp_2=0.3, drp_1=0.5 \n",
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.5935 - precision: 0.6850 - recall: 0.6710 - f1: 0.6683 - accuracy: 0.6790\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.5313 - precision: 0.7450 - recall: 0.7171 - f1: 0.7234 - accuracy: 0.7365\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.5177 - precision: 0.7541 - recall: 0.7287 - f1: 0.7343 - accuracy: 0.7463\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.5101 - precision: 0.7584 - recall: 0.7362 - f1: 0.7404 - accuracy: 0.7512\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.5025 - precision: 0.7630 - recall: 0.7402 - f1: 0.7449 - accuracy: 0.7553\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4995 - precision: 0.7666 - recall: 0.7443 - f1: 0.7485 - accuracy: 0.7591\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4946 - precision: 0.7695 - recall: 0.7477 - f1: 0.7518 - accuracy: 0.7618\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4905 - precision: 0.7705 - recall: 0.7488 - f1: 0.7532 - accuracy: 0.7638\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4875 - precision: 0.7723 - recall: 0.7522 - f1: 0.7559 - accuracy: 0.7657\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.4851 - precision: 0.7735 - recall: 0.7531 - f1: 0.7565 - accuracy: 0.7662\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4819 - precision: 0.7757 - recall: 0.7558 - f1: 0.7593 - accuracy: 0.7689\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4794 - precision: 0.7769 - recall: 0.7584 - f1: 0.7613 - accuracy: 0.7708\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4757 - precision: 0.7787 - recall: 0.7595 - f1: 0.7627 - accuracy: 0.7723\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4735 - precision: 0.7805 - recall: 0.7598 - f1: 0.7638 - accuracy: 0.7738\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4718 - precision: 0.7814 - recall: 0.7612 - f1: 0.7649 - accuracy: 0.7747\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4694 - precision: 0.7840 - recall: 0.7643 - f1: 0.7679 - accuracy: 0.7769\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4676 - precision: 0.7834 - recall: 0.7652 - f1: 0.7679 - accuracy: 0.7769\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4649 - precision: 0.7855 - recall: 0.7684 - f1: 0.7710 - accuracy: 0.7804\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4631 - precision: 0.7867 - recall: 0.7664 - f1: 0.7704 - accuracy: 0.7797\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4610 - precision: 0.7873 - recall: 0.7694 - f1: 0.7724 - accuracy: 0.7812\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4588 - precision: 0.7894 - recall: 0.7709 - f1: 0.7741 - accuracy: 0.7832\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4572 - precision: 0.7892 - recall: 0.7734 - f1: 0.7755 - accuracy: 0.7841\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4558 - precision: 0.7903 - recall: 0.7715 - f1: 0.7748 - accuracy: 0.7841\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 16s 155us/step - loss: 0.4534 - precision: 0.7916 - recall: 0.7735 - f1: 0.7765 - accuracy: 0.7854\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4517 - precision: 0.7933 - recall: 0.7729 - f1: 0.7770 - accuracy: 0.7868\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4502 - precision: 0.7942 - recall: 0.7759 - f1: 0.7793 - accuracy: 0.7881\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 18s 165us/step - loss: 0.4490 - precision: 0.7950 - recall: 0.7761 - f1: 0.7796 - accuracy: 0.7884\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4474 - precision: 0.7954 - recall: 0.7786 - f1: 0.7809 - accuracy: 0.7894\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 17s 163us/step - loss: 0.4469 - precision: 0.7957 - recall: 0.7783 - f1: 0.7809 - accuracy: 0.7901\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4447 - precision: 0.7982 - recall: 0.7768 - f1: 0.7815 - accuracy: 0.7911\n",
      "53213/53213 [==============================] - 2s 33us/step\n",
      "[CV]  layer_3_sz=32, layer_2_sz=64, layer_1_sz=256, drp_2=0.3, drp_1=0.5, score=0.775, total= 8.4min\n",
      "[CV] layer_3_sz=32, layer_2_sz=128, layer_1_sz=256, drp_2=0.5, drp_1=0.0 \n",
      "Epoch 1/30\n",
      "106426/106426 [==============================] - 17s 162us/step - loss: 0.5604 - precision: 0.7206 - recall: 0.6897 - f1: 0.6961 - accuracy: 0.7131\n",
      "Epoch 2/30\n",
      "106426/106426 [==============================] - 17s 159us/step - loss: 0.5097 - precision: 0.7596 - recall: 0.7345 - f1: 0.7396 - accuracy: 0.7517\n",
      "Epoch 3/30\n",
      "106426/106426 [==============================] - 16s 153us/step - loss: 0.4996 - precision: 0.7662 - recall: 0.7403 - f1: 0.7459 - accuracy: 0.7581\n",
      "Epoch 4/30\n",
      "106426/106426 [==============================] - 16s 154us/step - loss: 0.4923 - precision: 0.7699 - recall: 0.7455 - f1: 0.7505 - accuracy: 0.7620\n",
      "Epoch 5/30\n",
      "106426/106426 [==============================] - 16s 150us/step - loss: 0.4867 - precision: 0.7742 - recall: 0.7480 - f1: 0.7538 - accuracy: 0.7655\n",
      "Epoch 6/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.4807 - precision: 0.7776 - recall: 0.7530 - f1: 0.7582 - accuracy: 0.7694\n",
      "Epoch 7/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.4759 - precision: 0.7819 - recall: 0.7542 - f1: 0.7606 - accuracy: 0.7723\n",
      "Epoch 8/30\n",
      "106426/106426 [==============================] - 16s 153us/step - loss: 0.4715 - precision: 0.7839 - recall: 0.7587 - f1: 0.7644 - accuracy: 0.7754\n",
      "Epoch 9/30\n",
      "106426/106426 [==============================] - 16s 153us/step - loss: 0.4663 - precision: 0.7862 - recall: 0.7600 - f1: 0.7662 - accuracy: 0.7775\n",
      "Epoch 10/30\n",
      "106426/106426 [==============================] - 16s 150us/step - loss: 0.4621 - precision: 0.7905 - recall: 0.7604 - f1: 0.7685 - accuracy: 0.7804\n",
      "Epoch 11/30\n",
      "106426/106426 [==============================] - 17s 158us/step - loss: 0.4572 - precision: 0.7935 - recall: 0.7639 - f1: 0.7716 - accuracy: 0.7835\n",
      "Epoch 12/30\n",
      "106426/106426 [==============================] - 16s 153us/step - loss: 0.4522 - precision: 0.7966 - recall: 0.7662 - f1: 0.7745 - accuracy: 0.7860\n",
      "Epoch 13/30\n",
      "106426/106426 [==============================] - 16s 151us/step - loss: 0.4480 - precision: 0.8003 - recall: 0.7670 - f1: 0.7766 - accuracy: 0.7884\n",
      "Epoch 14/30\n",
      "106426/106426 [==============================] - 16s 149us/step - loss: 0.4424 - precision: 0.8029 - recall: 0.7703 - f1: 0.7798 - accuracy: 0.7913\n",
      "Epoch 15/30\n",
      "106426/106426 [==============================] - 16s 155us/step - loss: 0.4377 - precision: 0.8077 - recall: 0.7732 - f1: 0.7834 - accuracy: 0.7947\n",
      "Epoch 16/30\n",
      "106426/106426 [==============================] - 16s 155us/step - loss: 0.4344 - precision: 0.8076 - recall: 0.7743 - f1: 0.7841 - accuracy: 0.7956\n",
      "Epoch 17/30\n",
      "106426/106426 [==============================] - 16s 152us/step - loss: 0.4292 - precision: 0.8124 - recall: 0.7781 - f1: 0.7884 - accuracy: 0.7997\n",
      "Epoch 18/30\n",
      "106426/106426 [==============================] - 16s 147us/step - loss: 0.4250 - precision: 0.8138 - recall: 0.7795 - f1: 0.7901 - accuracy: 0.8015\n",
      "Epoch 19/30\n",
      "106426/106426 [==============================] - 16s 150us/step - loss: 0.4204 - precision: 0.8172 - recall: 0.7822 - f1: 0.7930 - accuracy: 0.8039\n",
      "Epoch 20/30\n",
      "106426/106426 [==============================] - 16s 148us/step - loss: 0.4156 - precision: 0.8225 - recall: 0.7842 - f1: 0.7967 - accuracy: 0.8081\n",
      "Epoch 21/30\n",
      "106426/106426 [==============================] - 17s 155us/step - loss: 0.4114 - precision: 0.8233 - recall: 0.7862 - f1: 0.7981 - accuracy: 0.8094\n",
      "Epoch 22/30\n",
      "106426/106426 [==============================] - 16s 151us/step - loss: 0.4070 - precision: 0.8263 - recall: 0.7889 - f1: 0.8009 - accuracy: 0.8121\n",
      "Epoch 23/30\n",
      "106426/106426 [==============================] - 16s 148us/step - loss: 0.4027 - precision: 0.8282 - recall: 0.7938 - f1: 0.8046 - accuracy: 0.8149\n",
      "Epoch 24/30\n",
      "106426/106426 [==============================] - 16s 151us/step - loss: 0.3986 - precision: 0.8306 - recall: 0.7941 - f1: 0.8059 - accuracy: 0.8167\n",
      "Epoch 25/30\n",
      "106426/106426 [==============================] - 16s 149us/step - loss: 0.3947 - precision: 0.8334 - recall: 0.7967 - f1: 0.8088 - accuracy: 0.8193\n",
      "Epoch 26/30\n",
      "106426/106426 [==============================] - 16s 152us/step - loss: 0.3907 - precision: 0.8355 - recall: 0.7975 - f1: 0.8101 - accuracy: 0.8207\n",
      "Epoch 27/30\n",
      "106426/106426 [==============================] - 15s 145us/step - loss: 0.3859 - precision: 0.8393 - recall: 0.8022 - f1: 0.8146 - accuracy: 0.8251\n",
      "Epoch 28/30\n",
      "106426/106426 [==============================] - 16s 149us/step - loss: 0.3831 - precision: 0.8398 - recall: 0.8008 - f1: 0.8138 - accuracy: 0.8243\n",
      "Epoch 29/30\n",
      "106426/106426 [==============================] - 15s 145us/step - loss: 0.3791 - precision: 0.8429 - recall: 0.8048 - f1: 0.8179 - accuracy: 0.8282\n",
      "Epoch 30/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.3745 - precision: 0.8456 - recall: 0.8081 - f1: 0.8207 - accuracy: 0.8304\n",
      "53214/53214 [==============================] - 2s 40us/step\n",
      "[CV]  layer_3_sz=32, layer_2_sz=128, layer_1_sz=256, drp_2=0.5, drp_1=0.0, score=0.763, total= 8.1min\n",
      "[CV] layer_3_sz=32, layer_2_sz=128, layer_1_sz=256, drp_2=0.5, drp_1=0.0 \n",
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.5637 - precision: 0.7228 - recall: 0.6680 - f1: nan - accuracy: 0.7094\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.5134 - precision: 0.7563 - recall: 0.7360 - f1: 0.7388 - accuracy: 0.7490\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.5025 - precision: 0.7629 - recall: 0.7401 - f1: 0.7441 - accuracy: 0.7548\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4950 - precision: 0.7668 - recall: 0.7473 - f1: 0.7500 - accuracy: 0.7597\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.4898 - precision: 0.7697 - recall: 0.7484 - f1: 0.7520 - accuracy: 0.7625\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 16s 147us/step - loss: 0.4842 - precision: 0.7752 - recall: 0.7503 - f1: 0.7558 - accuracy: 0.7666\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4793 - precision: 0.7791 - recall: 0.7545 - f1: 0.7598 - accuracy: 0.7701\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4744 - precision: 0.7815 - recall: 0.7574 - f1: 0.7624 - accuracy: 0.7725\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 16s 147us/step - loss: 0.4687 - precision: 0.7825 - recall: 0.7621 - f1: 0.7654 - accuracy: 0.7748\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 16s 155us/step - loss: 0.4647 - precision: 0.7865 - recall: 0.7632 - f1: 0.7681 - accuracy: 0.7779\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 15s 144us/step - loss: 0.4603 - precision: 0.7903 - recall: 0.7662 - f1: 0.7718 - accuracy: 0.7814\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.4553 - precision: 0.7944 - recall: 0.7699 - f1: 0.7753 - accuracy: 0.7848\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.4512 - precision: 0.7955 - recall: 0.7684 - f1: 0.7755 - accuracy: 0.7859\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 16s 147us/step - loss: 0.4472 - precision: 0.7986 - recall: 0.7723 - f1: 0.7787 - accuracy: 0.7886\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4423 - precision: 0.8017 - recall: 0.7749 - f1: 0.7817 - accuracy: 0.7912\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4384 - precision: 0.8050 - recall: 0.7781 - f1: 0.7851 - accuracy: 0.7943\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.4336 - precision: 0.8068 - recall: 0.7804 - f1: 0.7873 - accuracy: 0.7966\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.4303 - precision: 0.8080 - recall: 0.7824 - f1: 0.7888 - accuracy: 0.7985\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4265 - precision: 0.8116 - recall: 0.7861 - f1: 0.7920 - accuracy: 0.8009\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4209 - precision: 0.8138 - recall: 0.7887 - f1: 0.7949 - accuracy: 0.8040\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4174 - precision: 0.8168 - recall: 0.7910 - f1: 0.7975 - accuracy: 0.8063\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4140 - precision: 0.8192 - recall: 0.7917 - f1: 0.7991 - accuracy: 0.8077\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 16s 148us/step - loss: 0.4098 - precision: 0.8205 - recall: 0.7951 - f1: 0.8016 - accuracy: 0.8100\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4051 - precision: 0.8241 - recall: 0.7977 - f1: 0.8046 - accuracy: 0.8135\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4014 - precision: 0.8263 - recall: 0.8004 - f1: 0.8073 - accuracy: 0.8157\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.3968 - precision: 0.8276 - recall: 0.8022 - f1: 0.8088 - accuracy: 0.8171\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.3939 - precision: 0.8297 - recall: 0.8059 - f1: 0.8118 - accuracy: 0.8198\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 16s 155us/step - loss: 0.3901 - precision: 0.8315 - recall: 0.8068 - f1: 0.8130 - accuracy: 0.8210\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.3856 - precision: 0.8334 - recall: 0.8102 - f1: 0.8158 - accuracy: 0.8237\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.3824 - precision: 0.8346 - recall: 0.8112 - f1: 0.8172 - accuracy: 0.8251\n",
      "53213/53213 [==============================] - 2s 32us/step\n",
      "[CV]  layer_3_sz=32, layer_2_sz=128, layer_1_sz=256, drp_2=0.5, drp_1=0.0, score=0.766, total= 8.1min\n",
      "[CV] layer_3_sz=32, layer_2_sz=128, layer_1_sz=256, drp_2=0.5, drp_1=0.0 \n",
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.5608 - precision: 0.7201 - recall: 0.6936 - f1: 0.6972 - accuracy: 0.7119\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.5119 - precision: 0.7579 - recall: 0.7360 - f1: 0.7399 - accuracy: 0.7503\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.5017 - precision: 0.7629 - recall: 0.7451 - f1: 0.7468 - accuracy: 0.7564\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.4945 - precision: 0.7678 - recall: 0.7474 - f1: 0.7505 - accuracy: 0.7606\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4891 - precision: 0.7717 - recall: 0.7501 - f1: 0.7536 - accuracy: 0.7639\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 16s 155us/step - loss: 0.4825 - precision: 0.7748 - recall: 0.7554 - f1: 0.7588 - accuracy: 0.7686\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4781 - precision: 0.7774 - recall: 0.7591 - f1: 0.7612 - accuracy: 0.7710\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4725 - precision: 0.7817 - recall: 0.7592 - f1: 0.7638 - accuracy: 0.7735\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4681 - precision: 0.7847 - recall: 0.7617 - f1: 0.7666 - accuracy: 0.7766\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.4638 - precision: 0.7867 - recall: 0.7675 - f1: 0.7702 - accuracy: 0.7797\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 16s 147us/step - loss: 0.4585 - precision: 0.7902 - recall: 0.7693 - f1: 0.7731 - accuracy: 0.7824\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 16s 147us/step - loss: 0.4535 - precision: 0.7920 - recall: 0.7707 - f1: 0.7747 - accuracy: 0.7847\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.4490 - precision: 0.7956 - recall: 0.7747 - f1: 0.7784 - accuracy: 0.7877\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 16s 148us/step - loss: 0.4448 - precision: 0.7992 - recall: 0.7776 - f1: 0.7817 - accuracy: 0.7914\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 16s 148us/step - loss: 0.4396 - precision: 0.8014 - recall: 0.7827 - f1: 0.7859 - accuracy: 0.7949\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 15s 145us/step - loss: 0.4353 - precision: 0.8045 - recall: 0.7857 - f1: 0.7888 - accuracy: 0.7970\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4295 - precision: 0.8088 - recall: 0.7868 - f1: 0.7918 - accuracy: 0.8009\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 15s 145us/step - loss: 0.4258 - precision: 0.8112 - recall: 0.7913 - f1: 0.7949 - accuracy: 0.8029\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.4223 - precision: 0.8122 - recall: 0.7930 - f1: 0.7964 - accuracy: 0.8048\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.4175 - precision: 0.8155 - recall: 0.7954 - f1: 0.7994 - accuracy: 0.8075\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4132 - precision: 0.8174 - recall: 0.7992 - f1: 0.8025 - accuracy: 0.8105\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 15s 145us/step - loss: 0.4092 - precision: 0.8206 - recall: 0.7988 - f1: 0.8038 - accuracy: 0.8120\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 15s 144us/step - loss: 0.4043 - precision: 0.8230 - recall: 0.8027 - f1: 0.8067 - accuracy: 0.8148\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 16s 146us/step - loss: 0.4001 - precision: 0.8254 - recall: 0.8042 - f1: 0.8088 - accuracy: 0.8168\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.3966 - precision: 0.8271 - recall: 0.8067 - f1: 0.8110 - accuracy: 0.8187\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.3932 - precision: 0.8292 - recall: 0.8083 - f1: 0.8126 - accuracy: 0.8207\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 16s 148us/step - loss: 0.3876 - precision: 0.8324 - recall: 0.8111 - f1: 0.8160 - accuracy: 0.8240\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.3846 - precision: 0.8341 - recall: 0.8131 - f1: 0.8178 - accuracy: 0.8255\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.3801 - precision: 0.8376 - recall: 0.8159 - f1: 0.8210 - accuracy: 0.8283\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.3765 - precision: 0.8380 - recall: 0.8168 - f1: 0.8216 - accuracy: 0.8291\n",
      "53213/53213 [==============================] - 2s 33us/step\n",
      "[CV]  layer_3_sz=32, layer_2_sz=128, layer_1_sz=256, drp_2=0.5, drp_1=0.0, score=0.767, total= 8.0min\n",
      "[CV] layer_3_sz=32, layer_2_sz=128, layer_1_sz=128, drp_2=0.3, drp_1=0.5 \n",
      "Epoch 1/30\n",
      "106426/106426 [==============================] - 16s 154us/step - loss: 0.5943 - precision: 0.6854 - recall: 0.6557 - f1: 0.6608 - accuracy: 0.6774\n",
      "Epoch 2/30\n",
      "106426/106426 [==============================] - 16s 154us/step - loss: 0.5325 - precision: 0.7439 - recall: 0.7108 - f1: 0.7198 - accuracy: 0.7339\n",
      "Epoch 3/30\n",
      "106426/106426 [==============================] - 16s 152us/step - loss: 0.5187 - precision: 0.7516 - recall: 0.7269 - f1: 0.7321 - accuracy: 0.7447\n",
      "Epoch 4/30\n",
      "106426/106426 [==============================] - 16s 148us/step - loss: 0.5117 - precision: 0.7551 - recall: 0.7311 - f1: 0.7361 - accuracy: 0.7489\n",
      "Epoch 5/30\n",
      "106426/106426 [==============================] - 16s 150us/step - loss: 0.5063 - precision: 0.7602 - recall: 0.7342 - f1: 0.7402 - accuracy: 0.7528\n",
      "Epoch 6/30\n",
      "106426/106426 [==============================] - 16s 151us/step - loss: 0.5014 - precision: 0.7631 - recall: 0.7406 - f1: 0.7449 - accuracy: 0.7568\n",
      "Epoch 7/30\n",
      "106426/106426 [==============================] - 16s 154us/step - loss: 0.4981 - precision: 0.7650 - recall: 0.7425 - f1: 0.7471 - accuracy: 0.7585\n",
      "Epoch 8/30\n",
      "106426/106426 [==============================] - 16s 148us/step - loss: 0.4940 - precision: 0.7664 - recall: 0.7470 - f1: 0.7500 - accuracy: 0.7606\n",
      "Epoch 9/30\n",
      "106426/106426 [==============================] - 16s 151us/step - loss: 0.4905 - precision: 0.7702 - recall: 0.7458 - f1: 0.7513 - accuracy: 0.7631\n",
      "Epoch 10/30\n",
      "106426/106426 [==============================] - 17s 155us/step - loss: 0.4885 - precision: 0.7715 - recall: 0.7496 - f1: 0.7542 - accuracy: 0.7650\n",
      "Epoch 11/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.4860 - precision: 0.7705 - recall: 0.7507 - f1: 0.7545 - accuracy: 0.7655\n",
      "Epoch 12/30\n",
      "106426/106426 [==============================] - 16s 153us/step - loss: 0.4839 - precision: 0.7730 - recall: 0.7516 - f1: 0.7558 - accuracy: 0.7668\n",
      "Epoch 13/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.4823 - precision: 0.7735 - recall: 0.7543 - f1: 0.7576 - accuracy: 0.7680\n",
      "Epoch 14/30\n",
      "106426/106426 [==============================] - 16s 153us/step - loss: 0.4794 - precision: 0.7756 - recall: 0.7575 - f1: 0.7602 - accuracy: 0.7705\n",
      "Epoch 15/30\n",
      "106426/106426 [==============================] - 16s 155us/step - loss: 0.4771 - precision: 0.7783 - recall: 0.7565 - f1: 0.7611 - accuracy: 0.7720\n",
      "Epoch 16/30\n",
      "106426/106426 [==============================] - 16s 151us/step - loss: 0.4756 - precision: 0.7780 - recall: 0.7595 - f1: 0.7625 - accuracy: 0.7731\n",
      "Epoch 17/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.4730 - precision: 0.7806 - recall: 0.7597 - f1: 0.7639 - accuracy: 0.7738\n",
      "Epoch 18/30\n",
      "106426/106426 [==============================] - 16s 155us/step - loss: 0.4723 - precision: 0.7807 - recall: 0.7632 - f1: 0.7656 - accuracy: 0.7756\n",
      "Epoch 19/30\n",
      "106426/106426 [==============================] - 16s 151us/step - loss: 0.4712 - precision: 0.7816 - recall: 0.7599 - f1: 0.7645 - accuracy: 0.7752\n",
      "Epoch 20/30\n",
      "106426/106426 [==============================] - 16s 154us/step - loss: 0.4693 - precision: 0.7815 - recall: 0.7633 - f1: 0.7662 - accuracy: 0.7763\n",
      "Epoch 21/30\n",
      "106426/106426 [==============================] - 16s 152us/step - loss: 0.4680 - precision: 0.7829 - recall: 0.7639 - f1: 0.7671 - accuracy: 0.7776\n",
      "Epoch 22/30\n",
      "106426/106426 [==============================] - 16s 153us/step - loss: 0.4659 - precision: 0.7848 - recall: 0.7645 - f1: 0.7681 - accuracy: 0.7782\n",
      "Epoch 23/30\n",
      "106426/106426 [==============================] - 17s 158us/step - loss: 0.4658 - precision: 0.7838 - recall: 0.7667 - f1: 0.7692 - accuracy: 0.7791\n",
      "Epoch 24/30\n",
      "106426/106426 [==============================] - 16s 154us/step - loss: 0.4639 - precision: 0.7850 - recall: 0.7673 - f1: 0.7698 - accuracy: 0.7796\n",
      "Epoch 25/30\n",
      "106426/106426 [==============================] - 16s 150us/step - loss: 0.4625 - precision: 0.7873 - recall: 0.7683 - f1: 0.7713 - accuracy: 0.7811\n",
      "Epoch 26/30\n",
      "106426/106426 [==============================] - 16s 151us/step - loss: 0.4612 - precision: 0.7874 - recall: 0.7654 - f1: 0.7704 - accuracy: 0.7815\n",
      "Epoch 27/30\n",
      "106426/106426 [==============================] - 16s 155us/step - loss: 0.4600 - precision: 0.7896 - recall: 0.7670 - f1: 0.7721 - accuracy: 0.7823\n",
      "Epoch 28/30\n",
      "106426/106426 [==============================] - 17s 155us/step - loss: 0.4587 - precision: 0.7893 - recall: 0.7691 - f1: 0.7731 - accuracy: 0.7828\n",
      "Epoch 29/30\n",
      "106426/106426 [==============================] - 17s 156us/step - loss: 0.4583 - precision: 0.7893 - recall: 0.7695 - f1: 0.7733 - accuracy: 0.7834\n",
      "Epoch 30/30\n",
      "106426/106426 [==============================] - 16s 151us/step - loss: 0.4569 - precision: 0.7905 - recall: 0.7695 - f1: 0.7740 - accuracy: 0.7838\n",
      "53214/53214 [==============================] - 2s 32us/step\n",
      "[CV]  layer_3_sz=32, layer_2_sz=128, layer_1_sz=128, drp_2=0.3, drp_1=0.5, score=0.770, total= 8.2min\n",
      "[CV] layer_3_sz=32, layer_2_sz=128, layer_1_sz=128, drp_2=0.3, drp_1=0.5 \n",
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.6001 - precision: 0.6857 - recall: 0.6273 - f1: nan - accuracy: 0.6741\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.5365 - precision: 0.7409 - recall: 0.7150 - f1: 0.7208 - accuracy: 0.7324\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.5238 - precision: 0.7478 - recall: 0.7264 - f1: 0.7301 - accuracy: 0.7409\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.5158 - precision: 0.7549 - recall: 0.7287 - f1: 0.7345 - accuracy: 0.7459\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.5096 - precision: 0.7577 - recall: 0.7354 - f1: 0.7398 - accuracy: 0.7503\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.5053 - precision: 0.7601 - recall: 0.7386 - f1: 0.7428 - accuracy: 0.7532\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.5016 - precision: 0.7637 - recall: 0.7419 - f1: 0.7463 - accuracy: 0.7563\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4990 - precision: 0.7662 - recall: 0.7422 - f1: 0.7474 - accuracy: 0.7580\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4960 - precision: 0.7666 - recall: 0.7439 - f1: 0.7486 - accuracy: 0.7587\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4933 - precision: 0.7685 - recall: 0.7457 - f1: 0.7508 - accuracy: 0.7610\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.4900 - precision: 0.7709 - recall: 0.7488 - f1: 0.7533 - accuracy: 0.7634\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4879 - precision: 0.7711 - recall: 0.7487 - f1: 0.7535 - accuracy: 0.7634\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4846 - precision: 0.7753 - recall: 0.7536 - f1: 0.7582 - accuracy: 0.7676\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4835 - precision: 0.7749 - recall: 0.7527 - f1: 0.7574 - accuracy: 0.7671\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4817 - precision: 0.7752 - recall: 0.7539 - f1: 0.7586 - accuracy: 0.7686\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4794 - precision: 0.7765 - recall: 0.7568 - f1: 0.7604 - accuracy: 0.7701\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4769 - precision: 0.7795 - recall: 0.7559 - f1: 0.7613 - accuracy: 0.7713\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 16s 155us/step - loss: 0.4763 - precision: 0.7775 - recall: 0.7580 - f1: 0.7610 - accuracy: 0.7704\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 16s 155us/step - loss: 0.4750 - precision: 0.7810 - recall: 0.7563 - f1: 0.7622 - accuracy: 0.7723\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4728 - precision: 0.7803 - recall: 0.7598 - f1: 0.7637 - accuracy: 0.7732\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4719 - precision: 0.7828 - recall: 0.7603 - f1: 0.7651 - accuracy: 0.7751\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4693 - precision: 0.7817 - recall: 0.7614 - f1: 0.7656 - accuracy: 0.7752\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4686 - precision: 0.7834 - recall: 0.7611 - f1: 0.7659 - accuracy: 0.7755\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4666 - precision: 0.7839 - recall: 0.7635 - f1: 0.7675 - accuracy: 0.7765\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4663 - precision: 0.7871 - recall: 0.7639 - f1: 0.7691 - accuracy: 0.7786\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4654 - precision: 0.7852 - recall: 0.7650 - f1: 0.7689 - accuracy: 0.7782\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4638 - precision: 0.7869 - recall: 0.7655 - f1: 0.7701 - accuracy: 0.7800\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 16s 149us/step - loss: 0.4617 - precision: 0.7876 - recall: 0.7667 - f1: 0.7709 - accuracy: 0.7803\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4612 - precision: 0.7878 - recall: 0.7665 - f1: 0.7713 - accuracy: 0.7801\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 16s 147us/step - loss: 0.4615 - precision: 0.7886 - recall: 0.7662 - f1: 0.7716 - accuracy: 0.7809\n",
      "53213/53213 [==============================] - 2s 34us/step\n",
      "[CV]  layer_3_sz=32, layer_2_sz=128, layer_1_sz=128, drp_2=0.3, drp_1=0.5, score=0.776, total= 8.2min\n",
      "[CV] layer_3_sz=32, layer_2_sz=128, layer_1_sz=128, drp_2=0.3, drp_1=0.5 \n",
      "Epoch 1/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.6025 - precision: 0.6771 - recall: 0.6560 - f1: 0.6578 - accuracy: 0.6708\n",
      "Epoch 2/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.5376 - precision: 0.7396 - recall: 0.7110 - f1: 0.7180 - accuracy: 0.7313\n",
      "Epoch 3/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.5220 - precision: 0.7504 - recall: 0.7263 - f1: 0.7312 - accuracy: 0.7426\n",
      "Epoch 4/30\n",
      "106427/106427 [==============================] - 16s 155us/step - loss: 0.5152 - precision: 0.7528 - recall: 0.7326 - f1: 0.7358 - accuracy: 0.7468\n",
      "Epoch 5/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.5090 - precision: 0.7565 - recall: 0.7371 - f1: 0.7401 - accuracy: 0.7502\n",
      "Epoch 6/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.5042 - precision: 0.7612 - recall: 0.7416 - f1: 0.7447 - accuracy: 0.7551\n",
      "Epoch 7/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.5002 - precision: 0.7621 - recall: 0.7430 - f1: 0.7460 - accuracy: 0.7561\n",
      "Epoch 8/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4975 - precision: 0.7634 - recall: 0.7445 - f1: 0.7474 - accuracy: 0.7577\n",
      "Epoch 9/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4949 - precision: 0.7681 - recall: 0.7449 - f1: 0.7496 - accuracy: 0.7602\n",
      "Epoch 10/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4918 - precision: 0.7678 - recall: 0.7494 - f1: 0.7522 - accuracy: 0.7620\n",
      "Epoch 11/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4897 - precision: 0.7706 - recall: 0.7511 - f1: 0.7543 - accuracy: 0.7641\n",
      "Epoch 12/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4876 - precision: 0.7716 - recall: 0.7518 - f1: 0.7551 - accuracy: 0.7651\n",
      "Epoch 13/30\n",
      "106427/106427 [==============================] - 17s 162us/step - loss: 0.4847 - precision: 0.7738 - recall: 0.7542 - f1: 0.7576 - accuracy: 0.7674\n",
      "Epoch 14/30\n",
      "106427/106427 [==============================] - 17s 160us/step - loss: 0.4830 - precision: 0.7742 - recall: 0.7542 - f1: 0.7581 - accuracy: 0.7679\n",
      "Epoch 15/30\n",
      "106427/106427 [==============================] - 17s 155us/step - loss: 0.4807 - precision: 0.7763 - recall: 0.7572 - f1: 0.7602 - accuracy: 0.7698\n",
      "Epoch 16/30\n",
      "106427/106427 [==============================] - 16s 150us/step - loss: 0.4787 - precision: 0.7767 - recall: 0.7574 - f1: 0.7610 - accuracy: 0.7709\n",
      "Epoch 17/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4772 - precision: 0.7783 - recall: 0.7598 - f1: 0.7628 - accuracy: 0.7718\n",
      "Epoch 18/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4763 - precision: 0.7783 - recall: 0.7592 - f1: 0.7624 - accuracy: 0.7723\n",
      "Epoch 19/30\n",
      "106427/106427 [==============================] - 17s 158us/step - loss: 0.4746 - precision: 0.7800 - recall: 0.7627 - f1: 0.7650 - accuracy: 0.7739\n",
      "Epoch 20/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4723 - precision: 0.7801 - recall: 0.7613 - f1: 0.7646 - accuracy: 0.7741\n",
      "Epoch 21/30\n",
      "106427/106427 [==============================] - 16s 155us/step - loss: 0.4716 - precision: 0.7827 - recall: 0.7630 - f1: 0.7666 - accuracy: 0.7761\n",
      "Epoch 22/30\n",
      "106427/106427 [==============================] - 16s 151us/step - loss: 0.4704 - precision: 0.7828 - recall: 0.7646 - f1: 0.7675 - accuracy: 0.7771\n",
      "Epoch 23/30\n",
      "106427/106427 [==============================] - 17s 161us/step - loss: 0.4686 - precision: 0.7822 - recall: 0.7649 - f1: 0.7675 - accuracy: 0.7767\n",
      "Epoch 24/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4666 - precision: 0.7838 - recall: 0.7685 - f1: 0.7702 - accuracy: 0.7784\n",
      "Epoch 25/30\n",
      "106427/106427 [==============================] - 17s 159us/step - loss: 0.4656 - precision: 0.7855 - recall: 0.7678 - f1: 0.7705 - accuracy: 0.7798\n",
      "Epoch 26/30\n",
      "106427/106427 [==============================] - 16s 152us/step - loss: 0.4648 - precision: 0.7876 - recall: 0.7682 - f1: 0.7715 - accuracy: 0.7805\n",
      "Epoch 27/30\n",
      "106427/106427 [==============================] - 17s 156us/step - loss: 0.4635 - precision: 0.7853 - recall: 0.7665 - f1: 0.7698 - accuracy: 0.7792\n",
      "Epoch 28/30\n",
      "106427/106427 [==============================] - 16s 153us/step - loss: 0.4620 - precision: 0.7878 - recall: 0.7701 - f1: 0.7728 - accuracy: 0.7817\n",
      "Epoch 29/30\n",
      "106427/106427 [==============================] - 17s 157us/step - loss: 0.4617 - precision: 0.7884 - recall: 0.7714 - f1: 0.7735 - accuracy: 0.7822\n",
      "Epoch 30/30\n",
      "106427/106427 [==============================] - 16s 154us/step - loss: 0.4598 - precision: 0.7881 - recall: 0.7719 - f1: 0.7741 - accuracy: 0.7825\n",
      "53213/53213 [==============================] - 2s 35us/step\n",
      "[CV]  layer_3_sz=32, layer_2_sz=128, layer_1_sz=128, drp_2=0.3, drp_1=0.5, score=0.774, total= 8.4min\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=1)]: Done  30 out of  30 | elapsed: 253.8min finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/30\n",
      "159640/159640 [==============================] - 25s 160us/step - loss: 0.5589 - precision: 0.7210 - recall: 0.6883 - f1: 0.6941 - accuracy: 0.7110\n",
      "Epoch 2/30\n",
      "159640/159640 [==============================] - 26s 164us/step - loss: 0.5149 - precision: 0.7532 - recall: 0.7293 - f1: 0.7342 - accuracy: 0.7457\n",
      "Epoch 3/30\n",
      "159640/159640 [==============================] - 25s 158us/step - loss: 0.5041 - precision: 0.7603 - recall: 0.7366 - f1: 0.7414 - accuracy: 0.7528\n",
      "Epoch 4/30\n",
      "159640/159640 [==============================] - 25s 156us/step - loss: 0.4973 - precision: 0.7640 - recall: 0.7423 - f1: 0.7464 - accuracy: 0.7571\n",
      "Epoch 5/30\n",
      "159640/159640 [==============================] - 25s 156us/step - loss: 0.4919 - precision: 0.7677 - recall: 0.7454 - f1: 0.7498 - accuracy: 0.7605\n",
      "Epoch 6/30\n",
      "159640/159640 [==============================] - 24s 153us/step - loss: 0.4863 - precision: 0.7699 - recall: 0.7510 - f1: 0.7540 - accuracy: 0.7639\n",
      "Epoch 7/30\n",
      "159640/159640 [==============================] - 25s 157us/step - loss: 0.4832 - precision: 0.7735 - recall: 0.7506 - f1: 0.7551 - accuracy: 0.7659\n",
      "Epoch 8/30\n",
      "159640/159640 [==============================] - 25s 156us/step - loss: 0.4798 - precision: 0.7764 - recall: 0.7534 - f1: 0.7583 - accuracy: 0.7686\n",
      "Epoch 9/30\n",
      "159640/159640 [==============================] - 24s 153us/step - loss: 0.4757 - precision: 0.7776 - recall: 0.7544 - f1: 0.7595 - accuracy: 0.7705\n",
      "Epoch 10/30\n",
      "159640/159640 [==============================] - 25s 154us/step - loss: 0.4734 - precision: 0.7789 - recall: 0.7570 - f1: 0.7612 - accuracy: 0.7715\n",
      "Epoch 11/30\n",
      "159640/159640 [==============================] - 25s 154us/step - loss: 0.4706 - precision: 0.7812 - recall: 0.7586 - f1: 0.7635 - accuracy: 0.7738\n",
      "Epoch 12/30\n",
      "159640/159640 [==============================] - 25s 155us/step - loss: 0.4686 - precision: 0.7830 - recall: 0.7601 - f1: 0.7650 - accuracy: 0.7752\n",
      "Epoch 13/30\n",
      "159640/159640 [==============================] - 25s 157us/step - loss: 0.4661 - precision: 0.7845 - recall: 0.7606 - f1: 0.7663 - accuracy: 0.7764\n",
      "Epoch 14/30\n",
      "159640/159640 [==============================] - 24s 153us/step - loss: 0.4637 - precision: 0.7853 - recall: 0.7630 - f1: 0.7679 - accuracy: 0.7782\n",
      "Epoch 15/30\n",
      "159640/159640 [==============================] - 24s 153us/step - loss: 0.4618 - precision: 0.7866 - recall: 0.7649 - f1: 0.7694 - accuracy: 0.7792\n",
      "Epoch 16/30\n",
      "159640/159640 [==============================] - 24s 152us/step - loss: 0.4595 - precision: 0.7886 - recall: 0.7656 - f1: 0.7707 - accuracy: 0.7809\n",
      "Epoch 17/30\n",
      "159640/159640 [==============================] - 24s 153us/step - loss: 0.4583 - precision: 0.7897 - recall: 0.7663 - f1: 0.7716 - accuracy: 0.7815\n",
      "Epoch 18/30\n",
      "159640/159640 [==============================] - 25s 154us/step - loss: 0.4552 - precision: 0.7914 - recall: 0.7680 - f1: 0.7735 - accuracy: 0.7837\n",
      "Epoch 19/30\n",
      "159640/159640 [==============================] - 24s 153us/step - loss: 0.4544 - precision: 0.7925 - recall: 0.7681 - f1: 0.7742 - accuracy: 0.7840\n",
      "Epoch 20/30\n",
      "159640/159640 [==============================] - 24s 153us/step - loss: 0.4520 - precision: 0.7927 - recall: 0.7695 - f1: 0.7749 - accuracy: 0.7845\n",
      "Epoch 21/30\n",
      "159640/159640 [==============================] - 24s 153us/step - loss: 0.4504 - precision: 0.7941 - recall: 0.7719 - f1: 0.7769 - accuracy: 0.7866\n",
      "Epoch 22/30\n",
      "159640/159640 [==============================] - 24s 152us/step - loss: 0.4494 - precision: 0.7945 - recall: 0.7723 - f1: 0.7773 - accuracy: 0.7872\n",
      "Epoch 23/30\n",
      "159640/159640 [==============================] - 25s 155us/step - loss: 0.4494 - precision: 0.7957 - recall: 0.7700 - f1: 0.7766 - accuracy: 0.7868\n",
      "Epoch 24/30\n",
      "159640/159640 [==============================] - 24s 151us/step - loss: 0.4465 - precision: 0.7969 - recall: 0.7736 - f1: 0.7791 - accuracy: 0.7890\n",
      "Epoch 25/30\n",
      "159640/159640 [==============================] - 25s 154us/step - loss: 0.4457 - precision: 0.7973 - recall: 0.7740 - f1: 0.7795 - accuracy: 0.7890\n",
      "Epoch 26/30\n",
      "159640/159640 [==============================] - 25s 155us/step - loss: 0.4444 - precision: 0.7977 - recall: 0.7746 - f1: 0.7801 - accuracy: 0.7895\n",
      "Epoch 27/30\n",
      "159640/159640 [==============================] - 25s 157us/step - loss: 0.4428 - precision: 0.7992 - recall: 0.7747 - f1: 0.7808 - accuracy: 0.7909\n",
      "Epoch 28/30\n",
      "159640/159640 [==============================] - 25s 156us/step - loss: 0.4405 - precision: 0.8004 - recall: 0.7768 - f1: 0.7825 - accuracy: 0.7924\n",
      "Epoch 29/30\n",
      "159640/159640 [==============================] - 25s 156us/step - loss: 0.4401 - precision: 0.8001 - recall: 0.7764 - f1: 0.7823 - accuracy: 0.7924\n",
      "Epoch 30/30\n",
      "159640/159640 [==============================] - 25s 156us/step - loss: 0.4391 - precision: 0.8008 - recall: 0.7773 - f1: 0.7829 - accuracy: 0.7922\n",
      "Best: 0.775762 using {'layer_3_sz': 128, 'layer_2_sz': 256, 'layer_1_sz': 256, 'drp_2': 0.3, 'drp_1': 0.5}\n",
      "0.774583 (0.002296) with: {'layer_3_sz': 64, 'layer_2_sz': 128, 'layer_1_sz': 256, 'drp_2': 0.5, 'drp_1': 0.5}\n",
      "0.775762 (0.002092) with: {'layer_3_sz': 128, 'layer_2_sz': 256, 'layer_1_sz': 256, 'drp_2': 0.3, 'drp_1': 0.5}\n",
      "0.774333 (0.002559) with: {'layer_3_sz': 32, 'layer_2_sz': 128, 'layer_1_sz': 128, 'drp_2': 0.0, 'drp_1': 0.5}\n",
      "0.771766 (0.002942) with: {'layer_3_sz': 128, 'layer_2_sz': 128, 'layer_1_sz': 64, 'drp_2': 0.5, 'drp_1': 0.3}\n",
      "0.766242 (0.002628) with: {'layer_3_sz': 32, 'layer_2_sz': 64, 'layer_1_sz': 256, 'drp_2': 0.3, 'drp_1': 0.0}\n",
      "0.769926 (0.000299) with: {'layer_3_sz': 32, 'layer_2_sz': 128, 'layer_1_sz': 128, 'drp_2': 0.5, 'drp_1': 0.0}\n",
      "0.769011 (0.001519) with: {'layer_3_sz': 64, 'layer_2_sz': 64, 'layer_1_sz': 128, 'drp_2': 0.5, 'drp_1': 0.0}\n",
      "0.774496 (0.000869) with: {'layer_3_sz': 32, 'layer_2_sz': 64, 'layer_1_sz': 256, 'drp_2': 0.3, 'drp_1': 0.5}\n",
      "0.765431 (0.001832) with: {'layer_3_sz': 32, 'layer_2_sz': 128, 'layer_1_sz': 256, 'drp_2': 0.5, 'drp_1': 0.0}\n",
      "0.773108 (0.002271) with: {'layer_3_sz': 32, 'layer_2_sz': 128, 'layer_1_sz': 128, 'drp_2': 0.3, 'drp_1': 0.5}\n"
     ]
    }
   ],
   "source": [
    "import warnings\n",
    "import sklearn.exceptions\n",
    "warnings.filterwarnings(\"ignore\", category=sklearn.exceptions.UndefinedMetricWarning)\n",
    "warnings.simplefilter(action='ignore', category=FutureWarning)\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout\n",
    "from keras.optimizers import Adam\n",
    "from keras import backend as K\n",
    "\n",
    "def my_model(layer_1_sz, layer_2_sz, layer_3_sz, drp_1, drp_2):\n",
    "  model = Sequential()\n",
    "  model.add(Dense(layer_1_sz, input_dim=X_train.shape[1] , activation='relu'))\n",
    "  model.add(Dropout(drp_1))\n",
    "  model.add(Dense(layer_2_sz,  activation='relu'))\n",
    "  model.add(Dropout(drp_2))\n",
    "  model.add(Dense(layer_3_sz,  activation='relu'))\n",
    "  model.add(Dense(1,  activation='sigmoid'))\n",
    "  model.compile(loss='binary_crossentropy',\n",
    "                  optimizer = Adam(lr=0.0001),\n",
    "                  metrics=[precision, recall, f1, accuracy])\n",
    "  return model\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from keras.wrappers.scikit_learn import KerasClassifier\n",
    "from sklearn.model_selection import RandomizedSearchCV\n",
    "\n",
    "layer_1_sz = [256, 128, 64]\n",
    "layer_2_sz = [256, 128, 64]\n",
    "layer_3_sz =  [128, 64, 32]\n",
    "drp_1 = [0.0, 0.3, 0.5]\n",
    "drp_2 = [0.0, 0.3, 0.5]\n",
    "\n",
    "model = KerasClassifier(build_fn = my_model, epochs=30, batch_size=32, verbose=1)\n",
    "\n",
    "param_grid = dict(layer_1_sz = layer_1_sz, layer_2_sz = layer_2_sz, layer_3_sz = layer_3_sz, drp_1=drp_1,drp_2=drp_2)\n",
    "\n",
    "#grid = GridSearchCV(estimator=model, param_grid=param_grid, n_jobs=-1, cv=3, verbose=10, scoring='f1_macro')\n",
    "grid = RandomizedSearchCV(estimator=model\n",
    "                          , param_distributions = param_grid\n",
    "                          , n_jobs=1, cv=3, verbose=10\n",
    "                          , scoring='f1_macro')\n",
    "grid_result = grid.fit(X_train, y_train)\n",
    "\n",
    "# summarize results\n",
    "print(\"Best: %f using %s\" % (grid_result.best_score_, grid_result.best_params_))\n",
    "means = grid_result.cv_results_['mean_test_score']\n",
    "stds = grid_result.cv_results_['std_test_score']\n",
    "params = grid_result.cv_results_['params']\n",
    "for mean, stdev, param in zip(means, stds, params):\n",
    "    print(\"%f (%f) with: %r\" % (mean, stdev, param))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "3DUeficR4Gs4"
   ],
   "include_colab_link": true,
   "name": "Galanos_Kyrkos_Saltos_TA_4th_Assignment.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}