prediction_Window_21.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "# Library\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import keras\n",
    "import csv\n",
    "import matplotlib.pyplot as plt\n",
    "from keras.layers.core import Dense, Activation, Flatten, Dropout\n",
    "from keras.layers import Conv2D, Embedding, BatchNormalization, Input, Add\n",
    "from keras.models import Sequential\n",
    "from keras.preprocessing.text import Tokenizer\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn import preprocessing\n",
    "from keras.utils import to_categorical\n",
    "from sklearn.metrics import f1_score, roc_auc_score, recall_score, confusion_matrix\n",
    "from keras.callbacks import TensorBoard, ModelCheckpoint, CSVLogger, LearningRateScheduler\n",
    "from keras import regularizers\n",
    "from keras.models import Model\n",
    "from keras.optimizers import Adam, SGD\n",
    "from time import time\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PELM Dataset, S positive shape:  (1941, 21)\n",
      "PELM Dataset, T positive shape:  (833, 21)\n",
      "PELM Dataset, Y positive shape:  (423, 21)\n",
      "PELM Dataset, S negative shape:  (1459, 21)\n",
      "PELM Dataset, T negative shape:  (1258, 21)\n",
      "PELM Dataset, Y negative shape:  (423, 21)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Read sample from Dataset\n",
    "\n",
    "with open('fixed_sequences_length_21_PELM/Group_Phos_S_w21_PELM_pos.fasta', 'r') as f:\n",
    "    PELM_s_positif_txt = f.readlines()\n",
    "with open('fixed_sequences_length_21_PELM/Group_Phos_T_w21_PELM_pos.fasta', 'r') as f:\n",
    "    PELM_t_positif_txt = f.readlines()\n",
    "with open('fixed_sequences_length_21_PELM/Group_Phos_Y_w21_PELM_pos.fasta', 'r') as f:\n",
    "    PELM_y_positif_txt = f.readlines()\n",
    "with open('fixed_sequences_length_21_PELM/Group_Phos_S_w21_PELM_neg.fasta', 'r') as f:\n",
    "    PELM_s_negatif_txt = f.readlines()\n",
    "with open('fixed_sequences_length_21_PELM/Group_Phos_T_w21_PELM_neg.fasta', 'r') as f:\n",
    "    PELM_t_negatif_txt = f.readlines()\n",
    "with open('fixed_sequences_length_21_PELM/Group_Phos_Y_w21_PELM_neg.fasta', 'r') as f:\n",
    "    PELM_y_negatif_txt = f.readlines()\n",
    "    \n",
    "# with open('fixed_sequences_length_21_PPA/S_IDS_pos.fasta', 'r') as f:\n",
    "#     PPA_s_positif_txt = f.readlines()\n",
    "# with open('fixed_sequences_length_21_PPA/T_IDS_pos.fasta', 'r') as f:\n",
    "#     PPA_t_positif_txt = f.readlines()\n",
    "# with open('fixed_sequences_length_21_PPA/Y_IDS_pos.fasta', 'r') as f:\n",
    "#     PPA_y_positif_txt = f.readlines()\n",
    "# with open('fixed_sequences_length_21_PPA/S_IDS_neg.fasta', 'r') as f:\n",
    "#     PPA_s_negatif_txt = f.readlines()\n",
    "# with open('fixed_sequences_length_21_PPA/T_IDS_neg.fasta', 'r') as f:\n",
    "#     PPA_t_negatif_txt = f.readlines()\n",
    "# with open('fixed_sequences_length_21_PPA/Y_IDS_neg.fasta', 'r') as f:\n",
    "#     PPA_y_negatif_txt = f.readlines()\n",
    "\n",
    "# Pick the window 9\n",
    "\n",
    "PELM_s_positif = np.array([])\n",
    "for i in range(1,len(PELM_s_positif_txt),2):\n",
    "    temp = PELM_s_positif_txt[i]\n",
    "    temp1 = temp[0:21]\n",
    "    temp2 = list(temp1)\n",
    "    PELM_s_positif = np.append(PELM_s_positif, temp2)\n",
    "print('PELM Dataset, S positive shape: ', PELM_s_positif.reshape(int(len(PELM_s_positif)/21),21).shape)\n",
    "\n",
    "PELM_t_positif = np.array([])\n",
    "for i in range(1,len(PELM_t_positif_txt),2):\n",
    "    temp = PELM_t_positif_txt[i]\n",
    "    temp1 = temp[0:21]\n",
    "    temp2 = list(temp1)\n",
    "    PELM_t_positif = np.append(PELM_t_positif, temp2)\n",
    "print('PELM Dataset, T positive shape: ', PELM_t_positif.reshape(int(len(PELM_t_positif)/21),21).shape)\n",
    "    \n",
    "PELM_y_positif = np.array([])\n",
    "for i in range(1,len(PELM_y_positif_txt),2):\n",
    "    temp = PELM_y_positif_txt[i]\n",
    "    temp1 = temp[0:21]\n",
    "    temp2 = list(temp1)\n",
    "    PELM_y_positif = np.append(PELM_y_positif, temp2)\n",
    "print('PELM Dataset, Y positive shape: ', PELM_y_positif.reshape(int(len(PELM_y_positif)/21),21).shape)\n",
    "\n",
    "PELM_s_negatif = np.array([])\n",
    "for i in range(1,len(PELM_s_negatif_txt),2):\n",
    "    temp = PELM_s_negatif_txt[i]\n",
    "    temp1 = temp[0:21]\n",
    "    temp2 = list(temp1)\n",
    "    PELM_s_negatif = np.append(PELM_s_negatif, temp2)\n",
    "print('PELM Dataset, S negative shape: ', PELM_s_negatif.reshape(int(len(PELM_s_negatif)/21),21).shape)\n",
    "\n",
    "PELM_t_negatif = np.array([])\n",
    "for i in range(1,len(PELM_t_negatif_txt),2):\n",
    "    temp = PELM_t_negatif_txt[i]\n",
    "    temp1 = temp[0:21]\n",
    "    temp2 = list(temp1)\n",
    "    PELM_t_negatif = np.append(PELM_t_negatif, temp2)\n",
    "print('PELM Dataset, T negative shape: ', PELM_t_negatif.reshape(int(len(PELM_t_negatif)/21),21).shape)\n",
    "    \n",
    "PELM_y_negatif = np.array([])\n",
    "for i in range(1,len(PELM_y_negatif_txt),2):\n",
    "    temp = PELM_y_negatif_txt[i]\n",
    "    temp1 = temp[0:21]\n",
    "    temp2 = list(temp1)\n",
    "    PELM_y_negatif = np.append(PELM_y_negatif, temp2)\n",
    "print('PELM Dataset, Y negative shape: ', PELM_y_negatif.reshape(int(len(PELM_y_negatif)/21),21).shape)\n",
    "\n",
    "print()\n",
    "\n",
    "# PPA_s_positif = np.array([])\n",
    "# for i in range(1,len(PPA_s_positif_txt),2):\n",
    "#     temp = PPA_s_positif_txt[i]\n",
    "#     temp1 = temp[0:21]\n",
    "#     temp2 = list(temp1)\n",
    "#     PPA_s_positif = np.append(PPA_s_positif, temp2)\n",
    "# print('PPA Dataset, S positive shape: ', PPA_s_positif.reshape(int(len(PPA_s_positif)/21),21).shape)\n",
    "\n",
    "# PPA_t_positif = np.array([])\n",
    "# for i in range(1,len(PPA_t_positif_txt),2):\n",
    "#     temp = PPA_t_positif_txt[i]\n",
    "#     temp1 = temp[0:21]\n",
    "#     temp2 = list(temp1)\n",
    "#     PPA_t_positif = np.append(PPA_t_positif, temp2)\n",
    "# print('PPA Dataset, T positive shape: ', PPA_t_positif.reshape(int(len(PPA_t_positif)/21),21).shape)\n",
    "    \n",
    "# PPA_y_positif = np.array([])\n",
    "# for i in range(1,len(PPA_y_positif_txt),2):\n",
    "#     temp = PPA_y_positif_txt[i]\n",
    "#     temp1 = temp[0:21]\n",
    "#     temp2 = list(temp1)\n",
    "#     PPA_y_positif = np.append(PPA_y_positif, temp2)\n",
    "# print('PPA Dataset, Y positive shape: ', PPA_y_positif.reshape(int(len(PPA_y_positif)/21),21).shape)\n",
    "\n",
    "\n",
    "# PPA_s_negatif = np.array([])\n",
    "# for i in range(1,len(PPA_s_negatif_txt),2):\n",
    "#     temp = PPA_s_negatif_txt[i]\n",
    "#     temp1 = temp[0:21]\n",
    "#     temp2 = list(temp1)\n",
    "#     PPA_s_negatif = np.append(PPA_s_negatif, temp2)\n",
    "# print('PPA Dataset, S negative shape: ', PPA_s_negatif.reshape(int(len(PPA_s_negatif)/21),21).shape)\n",
    "\n",
    "# PPA_t_negatif = np.array([])\n",
    "# for i in range(1,len(PPA_t_negatif_txt),2):\n",
    "#     temp = PPA_t_negatif_txt[i]\n",
    "#     temp1 = temp[0:21]\n",
    "#     temp2 = list(temp1)\n",
    "#     PPA_t_negatif = np.append(PPA_t_negatif, temp2)\n",
    "# print('PPA Dataset, T negative shape: ', PPA_t_negatif.reshape(int(len(PPA_t_negatif)/21),21).shape)\n",
    "    \n",
    "# PPA_y_negatif = np.array([])\n",
    "# for i in range(1,len(PPA_y_negatif_txt),2):\n",
    "#     temp = PPA_y_negatif_txt[i]\n",
    "#     temp1 = temp[0:21]\n",
    "#     temp2 = list(temp1)\n",
    "#     PPA_y_negatif = np.append(PPA_y_negatif, temp2)\n",
    "# print('PPA Dataset, Y negative shape: ', PPA_y_negatif.reshape(int(len(PPA_y_negatif)/21),21).shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Positive Dataset shape:  (1941, 21)\n",
      "Positive Label shape:  (1941, 1)\n",
      "Negative Dataset shape:  (1459, 21)\n",
      "Negative Label shape:  (1459, 1)\n"
     ]
    }
   ],
   "source": [
    "# Choose Dataset to train, make sure correspond with negative dataset\n",
    "\n",
    "dataset_pos = PELM_s_positif\n",
    "dataset_neg = PELM_s_negatif\n",
    "string_name = 'PELM_s'\n",
    "\n",
    "# Expand dimension, Reshape and Create Label\n",
    "\n",
    "sequenceLP = int(len(dataset_pos)/21)\n",
    "dataset_pos = np.expand_dims(dataset_pos, axis=0)\n",
    "dataset_pos = dataset_pos.reshape(sequenceLP,21)\n",
    "label_pos = np.ones((sequenceLP,), dtype=int)\n",
    "label_pos = np.expand_dims(label_pos, axis=0)\n",
    "label_pos = label_pos.reshape(sequenceLP,1)\n",
    "\n",
    "sequenceLN = int(len(dataset_neg)/21)\n",
    "dataset_neg = np.expand_dims(dataset_neg, axis=0)\n",
    "dataset_neg = dataset_neg.reshape(sequenceLN,21)\n",
    "label_neg = np.zeros((sequenceLN,), dtype=int)\n",
    "label_neg = np.expand_dims(label_neg, axis=0)\n",
    "label_neg = label_neg.reshape(sequenceLN,1)\n",
    "\n",
    "# Validate\n",
    "\n",
    "print('Positive Dataset shape: ', dataset_pos.shape)\n",
    "print('Positive Label shape: ', label_pos.shape)\n",
    "print('Negative Dataset shape: ', dataset_neg.shape)\n",
    "print('Negative Label shape: ', label_neg.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training sample shape:  (2720, 21)\n",
      "Training label shape:  (2720, 2)\n",
      "Validation sample shape:  (680, 21)\n",
      "Validation label shape:  (680, 2)\n"
     ]
    }
   ],
   "source": [
    "# Dataset preparation\n",
    "\n",
    "dataset_X = np.concatenate((dataset_pos, dataset_neg), axis=0, out=None)\n",
    "dataset_Y = np.concatenate((label_pos, label_neg), axis=0, out=None)\n",
    "\n",
    "# Tokenizing, Unique character got its own number\n",
    "\n",
    "asam = ['A','R','N','D','C','Q','E','G','H','I','L','K','M','F','P','S','T','W','Y','V']\n",
    "tokenizer = Tokenizer(char_level=True)\n",
    "tokenizer.fit_on_texts(asam)\n",
    "dataset_X_token = []\n",
    "for i in range(len(dataset_X)):\n",
    "    temp = tokenizer.texts_to_sequences(dataset_X[i])\n",
    "    dataset_X_token = np.append(dataset_X_token, temp)\n",
    "\n",
    "dataset_X_token = dataset_X_token-1\n",
    "dataset_X_token = dataset_X_token.reshape(len(dataset_X),21)\n",
    "\n",
    "# Onehot\n",
    "\n",
    "dataset_X_token_onehot = to_categorical(dataset_X_token)\n",
    "dataset_X_token_onehot = np.expand_dims(dataset_X_token_onehot, axis=3)\n",
    "dataset_X_token_onehot = dataset_X_token_onehot.reshape(len(dataset_X),21,20,1)\n",
    "\n",
    "dataset_Y_onehot = to_categorical(dataset_Y)\n",
    "\n",
    "# Spliting Dataset\n",
    "\n",
    "validation_size = 0.2\n",
    "randomS = 16\n",
    "train_X, valid_X, train_Y, valid_Y = train_test_split(dataset_X_token, dataset_Y_onehot, \n",
    "                                                      test_size=validation_size, random_state=randomS)\n",
    "\n",
    "# Validation\n",
    "\n",
    "print('Training sample shape: ', train_X.shape)\n",
    "print('Training label shape: ', train_Y.shape)\n",
    "print('Validation sample shape: ', valid_X.shape)\n",
    "print('Validation label shape: ', valid_Y.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_8 (Embedding)      (None, 21, 128)           2560      \n",
      "_________________________________________________________________\n",
      "flatten_8 (Flatten)          (None, 2688)              0         \n",
      "_________________________________________________________________\n",
      "dense_41 (Dense)             (None, 128)               344192    \n",
      "_________________________________________________________________\n",
      "dropout_34 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_42 (Dense)             (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "dropout_35 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_43 (Dense)             (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "dropout_36 (Dropout)         (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "dense_44 (Dense)             (None, 2)                 258       \n",
      "=================================================================\n",
      "Total params: 380,034\n",
      "Trainable params: 380,034\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Neural Network Modeling\n",
    "\n",
    "epochs = 100\n",
    "\n",
    "model = Sequential()\n",
    "model.add(Embedding(20, 128, input_length=21))\n",
    "model.add(Flatten(data_format=None))\n",
    "model.add(Dense(128, activation='elu', kernel_initializer='he_uniform'))\n",
    "model.add(Dropout(0.8, noise_shape=None, seed=None))\n",
    "model.add(Dense(128, activation='elu', kernel_initializer='he_uniform'))\n",
    "model.add(Dropout(0.8, noise_shape=None, seed=None))\n",
    "model.add(Dense(128, activation='elu', kernel_initializer='he_uniform'))\n",
    "model.add(Dropout(0.8, noise_shape=None, seed=None))\n",
    "model.add(Dense(2, activation='softmax'))\n",
    "model.compile(loss='categorical_crossentropy', optimizer='SGD', metrics=['accuracy'])\n",
    "\n",
    "def scheduler(epoch):\n",
    "    if epoch > 60:\n",
    "        return 0.00001\n",
    "    elif epoch > 30:\n",
    "        return 0.0001\n",
    "    else:\n",
    "        return 0.001 \n",
    "\n",
    "lr_schedule= LearningRateScheduler(scheduler)\n",
    "checkpoint = ModelCheckpoint('weight_best.hdf5', monitor='val_loss', verbose=0, save_best_only=True, \n",
    "                             save_weights_only=False, mode='auto', period=1)\n",
    "callback_list = [checkpoint, lr_schedule]\n",
    "\n",
    "model.summary()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 2720 samples, validate on 680 samples\n",
      "Epoch 1/100\n",
      "2720/2720 [==============================] - 1s 452us/step - loss: 0.8752 - acc: 0.5048 - val_loss: 0.6906 - val_acc: 0.5235\n",
      "Epoch 2/100\n",
      "2720/2720 [==============================] - 0s 79us/step - loss: 0.8522 - acc: 0.5136 - val_loss: 0.6879 - val_acc: 0.5456\n",
      "Epoch 3/100\n",
      "2720/2720 [==============================] - 0s 80us/step - loss: 0.8330 - acc: 0.5246 - val_loss: 0.6862 - val_acc: 0.5426\n",
      "Epoch 4/100\n",
      "2720/2720 [==============================] - 0s 80us/step - loss: 0.8321 - acc: 0.5199 - val_loss: 0.6846 - val_acc: 0.5500\n",
      "Epoch 5/100\n",
      "2720/2720 [==============================] - 0s 80us/step - loss: 0.8378 - acc: 0.5048 - val_loss: 0.6833 - val_acc: 0.5471\n",
      "Epoch 6/100\n",
      "2720/2720 [==============================] - 0s 81us/step - loss: 0.7922 - acc: 0.5305 - val_loss: 0.6823 - val_acc: 0.5412\n",
      "Epoch 7/100\n",
      "2720/2720 [==============================] - 0s 81us/step - loss: 0.7919 - acc: 0.5272 - val_loss: 0.6810 - val_acc: 0.5426\n",
      "Epoch 8/100\n",
      "2720/2720 [==============================] - 0s 81us/step - loss: 0.8019 - acc: 0.5165 - val_loss: 0.6801 - val_acc: 0.5426\n",
      "Epoch 9/100\n",
      "2720/2720 [==============================] - 0s 81us/step - loss: 0.7963 - acc: 0.5154 - val_loss: 0.6786 - val_acc: 0.5412\n",
      "Epoch 10/100\n",
      "2720/2720 [==============================] - 0s 96us/step - loss: 0.7862 - acc: 0.5110 - val_loss: 0.6776 - val_acc: 0.5412\n",
      "Epoch 11/100\n",
      "2720/2720 [==============================] - 0s 85us/step - loss: 0.7762 - acc: 0.5235 - val_loss: 0.6768 - val_acc: 0.5426\n",
      "Epoch 12/100\n",
      "2720/2720 [==============================] - 0s 88us/step - loss: 0.7710 - acc: 0.5232 - val_loss: 0.6762 - val_acc: 0.5412\n",
      "Epoch 13/100\n",
      "2720/2720 [==============================] - 0s 90us/step - loss: 0.7546 - acc: 0.5408 - val_loss: 0.6754 - val_acc: 0.5412\n",
      "Epoch 14/100\n",
      "2720/2720 [==============================] - 0s 79us/step - loss: 0.7557 - acc: 0.5368 - val_loss: 0.6748 - val_acc: 0.5412\n",
      "Epoch 15/100\n",
      "2720/2720 [==============================] - 0s 81us/step - loss: 0.7502 - acc: 0.5379 - val_loss: 0.6743 - val_acc: 0.5456\n",
      "Epoch 16/100\n",
      "2720/2720 [==============================] - 0s 91us/step - loss: 0.7528 - acc: 0.5276 - val_loss: 0.6735 - val_acc: 0.5441\n",
      "Epoch 17/100\n",
      "2720/2720 [==============================] - 0s 87us/step - loss: 0.7547 - acc: 0.5294 - val_loss: 0.6732 - val_acc: 0.5441\n",
      "Epoch 18/100\n",
      "2720/2720 [==============================] - 0s 81us/step - loss: 0.7429 - acc: 0.5511 - val_loss: 0.6724 - val_acc: 0.5426\n",
      "Epoch 19/100\n",
      "2720/2720 [==============================] - 0s 86us/step - loss: 0.7333 - acc: 0.5423 - val_loss: 0.6718 - val_acc: 0.5426\n",
      "Epoch 20/100\n",
      "2720/2720 [==============================] - 0s 88us/step - loss: 0.7349 - acc: 0.5507 - val_loss: 0.6709 - val_acc: 0.5426\n",
      "Epoch 21/100\n",
      "2720/2720 [==============================] - 0s 81us/step - loss: 0.7293 - acc: 0.5375 - val_loss: 0.6703 - val_acc: 0.5412\n",
      "Epoch 22/100\n",
      "2720/2720 [==============================] - 0s 87us/step - loss: 0.7302 - acc: 0.5434 - val_loss: 0.6697 - val_acc: 0.5426\n",
      "Epoch 23/100\n",
      "2720/2720 [==============================] - 0s 84us/step - loss: 0.7314 - acc: 0.5419 - val_loss: 0.6691 - val_acc: 0.5412\n",
      "Epoch 24/100\n",
      "2720/2720 [==============================] - 0s 87us/step - loss: 0.7387 - acc: 0.5305 - val_loss: 0.6683 - val_acc: 0.5412\n",
      "Epoch 25/100\n",
      "2720/2720 [==============================] - 0s 84us/step - loss: 0.7273 - acc: 0.5353 - val_loss: 0.6678 - val_acc: 0.5412\n",
      "Epoch 26/100\n",
      "2720/2720 [==============================] - 0s 82us/step - loss: 0.7293 - acc: 0.5445 - val_loss: 0.6675 - val_acc: 0.5426\n",
      "Epoch 27/100\n",
      "2720/2720 [==============================] - 0s 88us/step - loss: 0.7112 - acc: 0.5658 - val_loss: 0.6668 - val_acc: 0.5426\n",
      "Epoch 28/100\n",
      "2720/2720 [==============================] - 0s 102us/step - loss: 0.7073 - acc: 0.5566 - val_loss: 0.6659 - val_acc: 0.5412\n",
      "Epoch 29/100\n",
      "2720/2720 [==============================] - 0s 101us/step - loss: 0.7221 - acc: 0.5540 - val_loss: 0.6650 - val_acc: 0.5426\n",
      "Epoch 30/100\n",
      "2720/2720 [==============================] - 0s 92us/step - loss: 0.7092 - acc: 0.5621 - val_loss: 0.6644 - val_acc: 0.5426\n",
      "Epoch 31/100\n",
      "2720/2720 [==============================] - 0s 90us/step - loss: 0.6999 - acc: 0.5673 - val_loss: 0.6639 - val_acc: 0.5441\n",
      "Epoch 32/100\n",
      "2720/2720 [==============================] - 0s 109us/step - loss: 0.7031 - acc: 0.5563 - val_loss: 0.6638 - val_acc: 0.5441\n",
      "Epoch 33/100\n",
      "2720/2720 [==============================] - 0s 97us/step - loss: 0.7121 - acc: 0.5507 - val_loss: 0.6638 - val_acc: 0.5426\n",
      "Epoch 34/100\n",
      "2720/2720 [==============================] - 0s 95us/step - loss: 0.7037 - acc: 0.5511 - val_loss: 0.6638 - val_acc: 0.5441\n",
      "Epoch 35/100\n",
      "2720/2720 [==============================] - 0s 111us/step - loss: 0.7161 - acc: 0.5478 - val_loss: 0.6637 - val_acc: 0.5441\n",
      "Epoch 36/100\n",
      "2720/2720 [==============================] - 0s 85us/step - loss: 0.7060 - acc: 0.5537 - val_loss: 0.6636 - val_acc: 0.5441\n",
      "Epoch 37/100\n",
      "2720/2720 [==============================] - 0s 77us/step - loss: 0.7064 - acc: 0.5493 - val_loss: 0.6636 - val_acc: 0.5441\n",
      "Epoch 38/100\n",
      "2720/2720 [==============================] - 0s 90us/step - loss: 0.7050 - acc: 0.5526 - val_loss: 0.6635 - val_acc: 0.5441\n",
      "Epoch 39/100\n",
      "2720/2720 [==============================] - 0s 76us/step - loss: 0.7109 - acc: 0.5537 - val_loss: 0.6635 - val_acc: 0.5441\n",
      "Epoch 40/100\n",
      "2720/2720 [==============================] - 0s 86us/step - loss: 0.7095 - acc: 0.5500 - val_loss: 0.6634 - val_acc: 0.5441\n",
      "Epoch 41/100\n",
      "2720/2720 [==============================] - 0s 79us/step - loss: 0.7094 - acc: 0.5614 - val_loss: 0.6634 - val_acc: 0.5441\n",
      "Epoch 42/100\n",
      "2720/2720 [==============================] - 0s 74us/step - loss: 0.6955 - acc: 0.5640 - val_loss: 0.6634 - val_acc: 0.5441\n",
      "Epoch 43/100\n",
      "2720/2720 [==============================] - 0s 76us/step - loss: 0.7124 - acc: 0.5500 - val_loss: 0.6633 - val_acc: 0.5441\n",
      "Epoch 44/100\n",
      "2720/2720 [==============================] - 0s 81us/step - loss: 0.7038 - acc: 0.5540 - val_loss: 0.6632 - val_acc: 0.5441\n",
      "Epoch 45/100\n",
      "2720/2720 [==============================] - 0s 107us/step - loss: 0.7091 - acc: 0.5570 - val_loss: 0.6631 - val_acc: 0.5441\n",
      "Epoch 46/100\n",
      "2720/2720 [==============================] - 0s 93us/step - loss: 0.7000 - acc: 0.5629 - val_loss: 0.6631 - val_acc: 0.5441\n",
      "Epoch 47/100\n",
      "2720/2720 [==============================] - 0s 78us/step - loss: 0.6989 - acc: 0.5647 - val_loss: 0.6630 - val_acc: 0.5441\n",
      "Epoch 48/100\n",
      "2720/2720 [==============================] - 0s 97us/step - loss: 0.7035 - acc: 0.5555 - val_loss: 0.6629 - val_acc: 0.5441\n",
      "Epoch 49/100\n",
      "2720/2720 [==============================] - 0s 112us/step - loss: 0.7001 - acc: 0.5710 - val_loss: 0.6629 - val_acc: 0.5441\n",
      "Epoch 50/100\n",
      "2720/2720 [==============================] - 0s 97us/step - loss: 0.6911 - acc: 0.5776 - val_loss: 0.6628 - val_acc: 0.5441\n",
      "Epoch 51/100\n",
      "2720/2720 [==============================] - 0s 87us/step - loss: 0.6967 - acc: 0.5713 - val_loss: 0.6627 - val_acc: 0.5441\n",
      "Epoch 52/100\n",
      "2720/2720 [==============================] - 0s 103us/step - loss: 0.6962 - acc: 0.5607 - val_loss: 0.6627 - val_acc: 0.5441\n",
      "Epoch 53/100\n",
      "2720/2720 [==============================] - 0s 99us/step - loss: 0.6976 - acc: 0.5695 - val_loss: 0.6626 - val_acc: 0.5441\n",
      "Epoch 54/100\n",
      "2720/2720 [==============================] - 0s 88us/step - loss: 0.7008 - acc: 0.5588 - val_loss: 0.6625 - val_acc: 0.5441\n",
      "Epoch 55/100\n",
      "2720/2720 [==============================] - 0s 82us/step - loss: 0.6992 - acc: 0.5596 - val_loss: 0.6625 - val_acc: 0.5441\n",
      "Epoch 56/100\n",
      "2720/2720 [==============================] - 0s 83us/step - loss: 0.7097 - acc: 0.5426 - val_loss: 0.6624 - val_acc: 0.5441\n",
      "Epoch 57/100\n",
      "2720/2720 [==============================] - 0s 99us/step - loss: 0.6927 - acc: 0.5618 - val_loss: 0.6624 - val_acc: 0.5441\n",
      "Epoch 58/100\n",
      "2720/2720 [==============================] - 0s 82us/step - loss: 0.7089 - acc: 0.5496 - val_loss: 0.6624 - val_acc: 0.5441\n",
      "Epoch 59/100\n",
      "2720/2720 [==============================] - 0s 88us/step - loss: 0.7004 - acc: 0.5577 - val_loss: 0.6623 - val_acc: 0.5441\n",
      "Epoch 60/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2720/2720 [==============================] - 0s 88us/step - loss: 0.6998 - acc: 0.5607 - val_loss: 0.6623 - val_acc: 0.5441\n",
      "Epoch 61/100\n",
      "2720/2720 [==============================] - 0s 90us/step - loss: 0.6978 - acc: 0.5540 - val_loss: 0.6622 - val_acc: 0.5441\n",
      "Epoch 62/100\n",
      "2720/2720 [==============================] - 0s 89us/step - loss: 0.6887 - acc: 0.5680 - val_loss: 0.6622 - val_acc: 0.5441\n",
      "Epoch 63/100\n",
      "2720/2720 [==============================] - 0s 89us/step - loss: 0.7118 - acc: 0.5426 - val_loss: 0.6622 - val_acc: 0.5441\n",
      "Epoch 64/100\n",
      "2720/2720 [==============================] - 0s 79us/step - loss: 0.6955 - acc: 0.5669 - val_loss: 0.6621 - val_acc: 0.5441\n",
      "Epoch 65/100\n",
      "2720/2720 [==============================] - 0s 79us/step - loss: 0.6943 - acc: 0.5643 - val_loss: 0.6621 - val_acc: 0.5441\n",
      "Epoch 66/100\n",
      "2720/2720 [==============================] - 0s 85us/step - loss: 0.7006 - acc: 0.5585 - val_loss: 0.6621 - val_acc: 0.5441\n",
      "Epoch 67/100\n",
      "2720/2720 [==============================] - 0s 91us/step - loss: 0.7031 - acc: 0.5592 - val_loss: 0.6621 - val_acc: 0.5441\n",
      "Epoch 68/100\n",
      "2720/2720 [==============================] - 0s 86us/step - loss: 0.6924 - acc: 0.5801 - val_loss: 0.6621 - val_acc: 0.5441\n",
      "Epoch 69/100\n",
      "2720/2720 [==============================] - 0s 89us/step - loss: 0.6948 - acc: 0.5699 - val_loss: 0.6621 - val_acc: 0.5441\n",
      "Epoch 70/100\n",
      "2720/2720 [==============================] - 0s 88us/step - loss: 0.7063 - acc: 0.5603 - val_loss: 0.6621 - val_acc: 0.5441\n",
      "Epoch 71/100\n",
      "2720/2720 [==============================] - 0s 83us/step - loss: 0.6985 - acc: 0.5570 - val_loss: 0.6621 - val_acc: 0.5441\n",
      "Epoch 72/100\n",
      "2720/2720 [==============================] - 0s 80us/step - loss: 0.7033 - acc: 0.5640 - val_loss: 0.6621 - val_acc: 0.5441\n",
      "Epoch 73/100\n",
      "2720/2720 [==============================] - 0s 81us/step - loss: 0.6954 - acc: 0.5728 - val_loss: 0.6621 - val_acc: 0.5441\n",
      "Epoch 74/100\n",
      "2720/2720 [==============================] - 0s 80us/step - loss: 0.6962 - acc: 0.5713 - val_loss: 0.6621 - val_acc: 0.5441\n",
      "Epoch 75/100\n",
      "2720/2720 [==============================] - 0s 88us/step - loss: 0.7096 - acc: 0.5563 - val_loss: 0.6621 - val_acc: 0.5441\n",
      "Epoch 76/100\n",
      "2720/2720 [==============================] - 0s 86us/step - loss: 0.6997 - acc: 0.5463 - val_loss: 0.6621 - val_acc: 0.5441\n",
      "Epoch 77/100\n",
      "2720/2720 [==============================] - 0s 82us/step - loss: 0.7022 - acc: 0.5574 - val_loss: 0.6621 - val_acc: 0.5441\n",
      "Epoch 78/100\n",
      "2720/2720 [==============================] - 0s 85us/step - loss: 0.7046 - acc: 0.5651 - val_loss: 0.6621 - val_acc: 0.5441\n",
      "Epoch 79/100\n",
      "2720/2720 [==============================] - 0s 88us/step - loss: 0.6997 - acc: 0.5629 - val_loss: 0.6621 - val_acc: 0.5441\n",
      "Epoch 80/100\n",
      "2720/2720 [==============================] - 0s 81us/step - loss: 0.7001 - acc: 0.5625 - val_loss: 0.6620 - val_acc: 0.5441\n",
      "Epoch 81/100\n",
      "2720/2720 [==============================] - 0s 90us/step - loss: 0.6988 - acc: 0.5629 - val_loss: 0.6620 - val_acc: 0.5441\n",
      "Epoch 82/100\n",
      "2720/2720 [==============================] - 0s 82us/step - loss: 0.7038 - acc: 0.5581 - val_loss: 0.6620 - val_acc: 0.5441\n",
      "Epoch 83/100\n",
      "2720/2720 [==============================] - 0s 81us/step - loss: 0.6985 - acc: 0.5687 - val_loss: 0.6620 - val_acc: 0.5441\n",
      "Epoch 84/100\n",
      "2720/2720 [==============================] - 0s 83us/step - loss: 0.6939 - acc: 0.5702 - val_loss: 0.6620 - val_acc: 0.5441\n",
      "Epoch 85/100\n",
      "2720/2720 [==============================] - 0s 88us/step - loss: 0.6974 - acc: 0.5599 - val_loss: 0.6620 - val_acc: 0.5441\n",
      "Epoch 86/100\n",
      "2720/2720 [==============================] - 0s 88us/step - loss: 0.7004 - acc: 0.5610 - val_loss: 0.6620 - val_acc: 0.5441\n",
      "Epoch 87/100\n",
      "2720/2720 [==============================] - 0s 88us/step - loss: 0.7097 - acc: 0.5434 - val_loss: 0.6620 - val_acc: 0.5441\n",
      "Epoch 88/100\n",
      "2720/2720 [==============================] - 0s 91us/step - loss: 0.6923 - acc: 0.5643 - val_loss: 0.6620 - val_acc: 0.5441\n",
      "Epoch 89/100\n",
      "2720/2720 [==============================] - 0s 88us/step - loss: 0.7085 - acc: 0.5507 - val_loss: 0.6620 - val_acc: 0.5441\n",
      "Epoch 90/100\n",
      "2720/2720 [==============================] - 0s 86us/step - loss: 0.6958 - acc: 0.5640 - val_loss: 0.6620 - val_acc: 0.5441\n",
      "Epoch 91/100\n",
      "2720/2720 [==============================] - 0s 87us/step - loss: 0.6906 - acc: 0.5647 - val_loss: 0.6620 - val_acc: 0.5441\n",
      "Epoch 92/100\n",
      "2720/2720 [==============================] - 0s 87us/step - loss: 0.6875 - acc: 0.5754 - val_loss: 0.6620 - val_acc: 0.5441\n",
      "Epoch 93/100\n",
      "2720/2720 [==============================] - 0s 85us/step - loss: 0.6959 - acc: 0.5750 - val_loss: 0.6619 - val_acc: 0.5441\n",
      "Epoch 94/100\n",
      "2720/2720 [==============================] - 0s 91us/step - loss: 0.6947 - acc: 0.5588 - val_loss: 0.6619 - val_acc: 0.5441\n",
      "Epoch 95/100\n",
      "2720/2720 [==============================] - 0s 86us/step - loss: 0.7009 - acc: 0.5654 - val_loss: 0.6619 - val_acc: 0.5441\n",
      "Epoch 96/100\n",
      "2720/2720 [==============================] - 0s 83us/step - loss: 0.7016 - acc: 0.5577 - val_loss: 0.6619 - val_acc: 0.5441\n",
      "Epoch 97/100\n",
      "2720/2720 [==============================] - 0s 82us/step - loss: 0.6916 - acc: 0.5743 - val_loss: 0.6619 - val_acc: 0.5441\n",
      "Epoch 98/100\n",
      "2720/2720 [==============================] - 0s 87us/step - loss: 0.7068 - acc: 0.5522 - val_loss: 0.6619 - val_acc: 0.5441\n",
      "Epoch 99/100\n",
      "2720/2720 [==============================] - 0s 86us/step - loss: 0.7012 - acc: 0.5687 - val_loss: 0.6619 - val_acc: 0.5441\n",
      "Epoch 100/100\n",
      "2720/2720 [==============================] - 0s 82us/step - loss: 0.6950 - acc: 0.5636 - val_loss: 0.6619 - val_acc: 0.5441\n"
     ]
    }
   ],
   "source": [
    "# Train The Model\n",
    "\n",
    "model_train = model.fit(train_X, train_Y, epochs=epochs, batch_size=32, \n",
    "                        validation_data=(valid_X, valid_Y), callbacks=callback_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEICAYAAACzliQjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJztnXmYFdW16H+LZhIQQcAEQWhUwtyNTYsTDqgoJorzU4MDJITEiPiMVx8GXzR+MV6jiZqr1xf0aRxQ4Jmo4ERETYyJRhoEFBBBwACiNoPNpELb6/2x63QfDmeoc/oMdeqs3/fVd6p27aq9dlWdtVetvWttUVUMwzCM0qBFoQUwDMMw8ocpfcMwjBLClL5hGEYJYUrfMAyjhDClbxiGUUKY0jcMwyghTOmXICJSJiI7RKRXNvMWEhE5XESyPv5YRE4VkbVR2ytE5Hg/eTMo6yER+XmmxxuGH1oWWgAjNSKyI2qzHfA18I23/WNVnZ7O+VT1G6BDtvOWAqraLxvnEZEJwKWqelLUuSdk49yGkQxT+kWAqjYqXc+SnKCq8xLlF5GWqlqfD9kMIxX2PAYLc++EABH5lYjMFJGnRGQ7cKmIHCMib4vIFyKyUUR+LyKtvPwtRURFpNzbfsLb/5KIbBeRt0SkT7p5vf1niMiHIlInIv8lIv8QkXEJ5PYj449FZJWIbBWR30cdWyYid4vIZhFZDYxOcn2misiMmLT7ReR33voEEVnu1ecjzwpPdK71InKSt95ORB73ZFsKDIvJe5OIrPbOu1RExnjpQ4D7gOM919mmqGt7S9TxP/HqvllEnhWR7n6uTTrXOSKPiMwTkS0i8qmI3BBVzv/2rsk2EakRkYPjudJE5M3Iffau5xteOVuAm0Skr4i87pWxybtuB0Qd39urY623/14RaevJPCAqX3cR2SUiXRLV10iBqtpSRAuwFjg1Ju1XwG7gLFxDvh9wJHAU7m3uUOBDYJKXvyWgQLm3/QSwCagGWgEzgScyyHsQsB0429v3M2APMC5BXfzI+BxwAFAObInUHZgELAV6Al2AN9zjHLecQ4EdQPuoc38OVHvbZ3l5BDgZ+BKo8PadCqyNOtd64CRv/S7gr0BnoDewLCbv/wC6e/fk+54M3/L2TQD+GiPnE8At3vppnoxDgbbAfwOv+bk2aV7nA4DPgGuANkBHYLi370ZgMdDXq8NQ4EDg8NhrDbwZuc9e3eqBK4Ey3PP4HeAUoLX3nPwDuCuqPu9717O9l/84b9804Laocq4Dnin0/7CYl4ILYEuaNyyx0n8txXH/Afw/bz2eIv8/UXnHAO9nkPcHwN+j9gmwkQRK36eMR0ft/zPwH976Gzg3V2Tfd2MVUcy53wa+762fAaxIkvd54CpvPZnS/3f0vQB+Gp03znnfB77nradS+o8Cv47a1xHXj9Mz1bVJ8zpfBsxPkO+jiLwx6X6U/uoUMlwQKRc4HvgUKIuT7zhgDSDe9iLgvGz/r0ppMfdOeFgXvSEi/UXkBe91fRtwK9A1yfGfRq3vInnnbaK8B0fLoe5fuj7RSXzK6Kss4OMk8gI8CVzirX/f247IcaaI/MtzPXyBs7KTXasI3ZPJICLjRGSx56L4Aujv87zg6td4PlXdBmwFekTl8XXPUlznQ3DKPR7J9qUi9nn8tojMEpENngx/jJFhrbpBA3uhqv/AvTWMEJHBQC/ghQxlMjCffpiIHa74B5xlebiqdgR+gbO8c8lGnCUKgIgIeyupWJoj40acsoiQakjpLOBUEemBcz896cm4H/A0cDvO9dIJ+ItPOT5NJIOIHAo8gHNxdPHO+0HUeVMNL/0E5zKKnG9/nBtpgw+5Ykl2ndcBhyU4LtG+nZ5M7aLSvh2TJ7Z+d+BGnQ3xZBgXI0NvESlLIMdjwKW4t5JZqvp1gnyGD0zph5f9gTpgp9cR9uM8lPk8UCUiZ4lIS5yfuFuOZJwF/E8R6eF16v2vZJlV9VOcC+KPONfOSm9XG5yfuRb4RkTOxPme/crwcxHpJO47hklR+zrgFF8trv37Ec7Sj/AZ0DO6QzWGp4AfikiFiLTBNUp/V9WEb05JSHadZwO9RGSSiLQRkY4iMtzb9xDwKxE5TBxDReRAXGP3KW7AQJmITCSqgUoiw06gTkQOwbmYIrwFbAZ+La5zfD8ROS5q/+M4d9D3cQ2A0QxM6YeX64ArcB2rf8B1uOYUVf0MuAj4He5PfBjwLs7Cy7aMDwCvAu8B83HWeiqexPnoG107qvoFcC3wDK4z9AJc4+WHm3FvHGuBl4hSSKq6BPgv4B0vTz/gX1HHvgKsBD4TkWg3TeT4l3FumGe843sBY33KFUvC66yqdcAo4HxcQ/QhcKK3+07gWdx13obrVG3rue1+BPwc16l/eEzd4nEzMBzX+MwG/hQlQz1wJjAAZ/X/G3cfIvvX4u7z16r6zzTrbsQQ6RwxjKzjva5/Alygqn8vtDxG8SIij+E6h28ptCzFjn2cZWQVERmNGynzJW7I3x6ctWsYGeH1j5wNDCm0LGHA3DtGthkBrMb5sk8HzrWONyNTROR23LcCv1bVfxdanjBg7h3DMIwSwix9wzCMEiJwPv2uXbtqeXl5ocUwDMMoKhYsWLBJVZMNkQYCqPTLy8upqakptBiGYRhFhYik+iodMPeOYRhGSWFK3zAMo4QwpW8YhlFCBM6nH489e/awfv16vvrqq0KLYgSItm3b0rNnT1q1ShS+xjCMWIpC6a9fv57999+f8vJyXOBGo9RRVTZv3sz69evp06dP6gMMwwB8undEZLSIrPCmZpsSZ/84b5qzRd4yIWrfb8RNFbfcmz4tba391Vdf0aVLF1P4RiMiQpcuXeztr5lMnw7l5dCihfudPr3QEhm5JqWl7wXNuh8XiW89MF9EZqvqspisM1V1Usyxx+Jmvqnwkt7ERfD7a7qCmsI3YrFnonlMnw4TJ8KuXW7744/dNsDYTON5GoHHj6U/HFilqqtVdTcwAxf8yA+Km9+zNS5ueStc+FbDMArM1KlNCj/Crl0u3QgvfpR+D/ae+mw98WdDOl9ElojI094kCajqW8DruHjgG4G5qrq8mTLnnc2bNzN06FCGDh3Kt7/9bXr06NG4vXv3bl/nGD9+PCtWrEia5/7772e6vV8beeLfCcKXJUo3wkG2hmzOwU2cXYGbHOJRABE5HDcxQk9cQ3GyiBwfe7CITBSRGhGpqa2tbbYw2fZTdunShUWLFrFo0SJ+8pOfcO211zZut27dGnAdiw0NDQnP8cgjj9CvX7+k5Vx11VWMLbL36vr6+kKLYGRIrwQTTCZKN8KBH6W/gb3nAe1JzDydqro5KnzuQ8Awb/1c4G1V3aGqO3CzCx0TW4CqTlPValWt7tYtZeiIpET8lB9/DKpNfspcGNCrVq1i4MCBjB07lkGDBrFx40YmTpxIdXU1gwYN4tZbb23MO2LECBYtWkR9fT2dOnViypQpVFZWcswxx/D5558DcNNNN3HPPfc05p8yZQrDhw+nX79+/POfbsKgnTt3cv755zNw4EAuuOACqqurWbRo0T6y3XzzzRx55JEMHjyYn/zkJ0SiqX744YecfPLJVFZWUlVVxdq1awH49a9/zZAhQ6isrGSq934fkRng008/5fDDDwfgoYce4pxzzmHkyJGcfvrpbNu2jZNPPpmqqioqKip4/vmmiaceeeQRKioqqKysZPz48dTV1XHooYc2NhZbt27da9vIH7fdBu3a7Z3Wrp1LN0KMqiZdcJ29q4E+ON/8YmBQTJ7uUesRRQ9u6rx53jla4aZdOytZecOGDdNYli1btk9aInr3VnXqfu+ld2/fp0jKzTffrHfeeaeqqq5cuVJFROfPn9+4f/PmzaqqumfPHh0xYoQuXbpUVVWPO+44fffdd3XPnj0K6Isvvqiqqtdee63efvvtqqo6depUvfvuuxvz33DDDaqq+txzz+npp5+uqqq33367/vSnP1VV1UWLFmmLFi303Xff3UfOiBwNDQ168cUXN5ZXVVWls2fPVlXVL7/8Unfu3KmzZ8/WESNG6K5du/Y6NiKzqurGjRv1sMMOU1XVBx98UHv16qVbtmxRVdXdu3drXV2dqqp+9tlnevjhhzfK169fv8bzRX4vvfRSnTNnjqqq3n///Y31zIR0ng1jX554wv03RNzvE08UWiIjU4AaTaHPVTW1pa9u/spJwFxgOW42+qUicquIjPGyTfaGZS4GJuNmugc3b+lHuPktFwOLVXVO5k1UavLtpzzssMOorq5u3H7qqaeoqqqiqqqK5cuXs2xZ7CAn2G+//TjjjDMAGDZsWKO1Hct55523T54333yTiy++GIDKykoGDRoU99hXX32V4cOHU1lZyd/+9jeWLl3K1q1b2bRpE2eddRbgPm5q164d8+bN4wc/+AH77bcfAAceeGDKep922ml07twZcIbDlClTqKio4LTTTmPdunVs2rSJ1157jYsuuqjxfJHfCRMm8MgjjwDuTWD8+PEpyysUYR/SOHYsrF0LDQ3ut8i8i0YG+Po4S1VfBF6MSftF1PqNuKnxYo/7BvhxM2VMi169nEsnXnouaN++feP6ypUruffee3nnnXfo1KkTl156adxx5JF+AICysrKEro02bdqkzBOPXbt2MWnSJBYuXEiPHj246aabMhrP3rJly8Z+itjjo+v92GOPUVdXx8KFC2nZsiU9e/ZMWt6JJ57IpEmTeP3112nVqhX9+/dPW7Z8YEMajTASutg7hfRTbtu2jf3335+OHTuyceNG5s6dm/UyjjvuOGbNmgXAe++9F/dN4ssvv6RFixZ07dqV7du386c//QmAzp07061bN+bMcS9bX331Fbt27WLUqFE8/PDDfPnllwBs2bIFcGGuFyxYAMDTTz+dUKa6ujoOOuggWrZsySuvvMKGDa7L5+STT2bmzJmN54v8Alx66aWMHTs20Fa+DWk0wkjolP7YsTBtGvTuDSLud9q0/FhmVVVVDBw4kP79+3P55Zdz3HHHZb2Mq6++mg0bNjBw4EB++ctfMnDgQA444IC98nTp0oUrrriCgQMHcsYZZ3DUUUc17ps+fTq//e1vqaioYMSIEdTW1nLmmWcyevRoqqurGTp0KHfffTcA119/Pffeey9VVVVs3bo1oUyXXXYZ//znPxkyZAgzZsygb9++gHM/3XDDDZxwwgkMHTqU66+/vvGYsWPHUldXx0UXXZTNy5NVbEijEUYCN0dudXW1xk6isnz5cgYMGFAgiYJFfX099fX1tG3blpUrV3LaaaexcuVKWrYsijBKjcyYMYO5c+c2+vYzJZfPRnl5fFdh797O/20YQUJEFqhqdap8xaUpDHbs2MEpp5xCfX09qsof/vCHolP4V155JfPmzePll18utChJue22vX36YEMajeKnuLSFQadOnRr97MXKAw88UGgRfBFxCU6d6lw6vXo5hW+duKXF9OnhegZC59M3jGxiQxpLm3gfe44fD127Fu8wXrP0DcMwEhBvBNeePbB5s1svxmG8ZukbhmEkwM9IrVTDeIP2gZ8pfcMwjAT4/agzUeOQz1hgfjGl74ORI0fu86HVPffcw5VXXpn0uA4dOgDwySefcMEFF8TNc9JJJxE7RDWWe+65h11R75jf/e53+eKLL/yIbhhGM4j3sWc8EjUOQfzAz5S+Dy655BJmzJixV9qMGTO45JJLfB1/8MEHJ/2iNRWxSv/FF1+kU6dOGZ8v32iKsNOGEVRiP/bs0gWioqgAyYfxBvEDP1P6Prjgggt44YUXGidMWbt2LZ988gnHH39847j5qqoqhgwZwnPPPbfP8WvXrmXw4MGAC5Fw8cUXM2DAAM4999zG0Afgxq9HwjLffPPNAPz+97/nk08+YeTIkYwcORJw4RE2bdoEwO9+9zsGDx7M4MGDG8Myr127lgEDBvCjH/2IQYMGcdppp+1VToQ5c+Zw1FFHccQRR3Dqqafy2WduUrMdO3Ywfvx4hgwZQkVFRWMYh5dffpmqqioqKys55ZRTALjlllu46667Gs85ePBg1q5dy9q1a+nXrx+XX345gwcPZt26dXHrBzB//nyOPfZYKisrGT58ONu3b+eEE07YK2T0iBEjWLx4cVr3zcgtQfNV54roEVybNsHDD/v/4j+Qcxb4CcWZzyVVaOVrrlE98cTsLtdckyJmqap+73vf02effVZVXXjj6667TlVdCOVIWOHa2lo97LDDtKGhQVVV27dvr6qqa9as0UGDBqmq6m9/+1sdP368qqouXrxYy8rKGkMzR0IP19fX64knnqiLFy9WVdXevXtrbW1toyyR7ZqaGh08eLDu2LFDt2/frgMHDtSFCxfqmjVrtKysrDEs8oUXXqiPP/74PnXasmVLo6wPPvig/uxnP1NV1RtuuEGvibooW7Zs0c8//1x79uypq1ev3kvW6FDTqqqDBg3SNWvW6Jo1a1RE9K233mrcF69+X3/9tfbp00ffeecdVVWtq6vTPXv26B//+MdGGVasWKHxngtVC61cKJ54QrVdu73Dl7drZ6GZY8nndSJboZUNR7SLJ9q1o6r8/Oc/p6KiglNPPZUNGzY0WszxeOONN7j00ksBqKiooKKionHfrFmzqKqq4ogjjmDp0qVxg6lF8+abb3LuuefSvn17OnTowHnnncff//53APr06cPQoUOBxOGb169fz+mnn86QIUO48847Wbp0KQDz5s3jqquuaszXuXNn3n77bU444QT69OkD+Au/3Lt3b44++uik9VuxYgXdu3fnyCOPBKBjx460bNmSCy+8kOeff549e/bw8MMPM27cuJTlGfkjiL7qIFLIWGCJKLpx+p4HI++cffbZXHvttSxcuJBdu3YxbJibHGz69OnU1tayYMECWrVqRXl5eUZhjNesWcNdd93F/Pnz6dy5M+PGjcvoPBEiYZnBhWaO5965+uqr+dnPfsaYMWP461//yi233JJ2OdHhl2HvEMzR4ZfTrV+7du0YNWoUzz33HLNmzQrEV8hh+zKzOSTySX/8sXP1lPK1iWXs2GBdC7P0fdKhQwdGjhzJD37wg706cCNhhVu1asXrr7/Ox/EidEVxwgkn8OSTTwLw/vvvs2TJEsCFZW7fvj0HHHAAn332GS+99FLjMfvvvz/bt2/f51zHH388zz77LLt27WLnzp0888wzHH/8PlMQJ6Suro4ePdwc948++mhj+qhRo7j//vsbt7du3crRRx/NG2+8wZo1a4C9wy8vXLgQgIULFzbujyVR/fr168fGjRuZP38+ANu3b2+cO2DChAlMnjyZI488snHClnwQz1cdxKF3hSSZT7rUr03QMaWfBpdccgmLFy/eS+mPHTuWmpoahgwZwmOPPZZyQpArr7ySHTt2MGDAAH7xi180vjFUVlZyxBFH0L9/f77//e/vFZZ54sSJjB49urEjN0JVVRXjxo1j+PDhHHXUUUyYMIEjjjjCd31uueUWLrzwQoYNG0bXrl0b02+66Sa2bt3K4MGDqays5PXXX6dbt25MmzaN8847j8rKysaQyOeffz5btmxh0KBB3HfffXznO9+JW1ai+rVu3ZqZM2dy9dVXU1lZyahRoxrfAIYNG0bHjh3zGnM/kXK/5hpzZ0STaihjKV+bwOPH8Z/Ppblz5BrhYcOGDdq3b1/95ptvEubJ9rORaI7lRItIVosvKiLz6xb62tg8vw6y2ZErIqNFZIWIrBKRKXH2jxORWhFZ5C0TvPSRUWmLROQrETkny+2WEUIee+wxjjrqKG677TZatMjfC2m646cLOvQuy6Q7BDMylLF37/j783FtzO2WAalaBaAMN7n5oUBr3ATnA2PyjAPuS3GeA4EtQLtk+czSN9IhX5Z+ly7Jh94Vu7XZnKGFhRy+meh+9e6d+7KDBlm09IcDq1R1taruBmYAZ2fQvlwAvKSqu1LmjIOrk2E0kYtnItEcy/fem3joXRiszeYMwSzksMQgfvEaeFK1Cjhl/VDU9mXEWPU4S38jsAR4GjgkznleA85MUMZEoAao6dWr1z4t2OrVq7W2trbxQyLDaGho0Nra2saPxbJJulZ7GKxNkeLqs0jVn1AM1z7bb4f4tPSzNU5/DvCUqn4tIj8GHgVOjuwUke7AEGBuvINVdRowDdwcubH7e/bsyfr166mtrc2SuEYYaNu2LT179sz6edMdVx0Ga7NXr/jzAQexzyLyZhX7ZhKhGKa0jK1DXuPyp2oVgGOAuVHbNwI3JslfBtTFpF0DTPPTCiX63N4wgkq+Lf1c9B8UU1iFVBZ+EGWOJRfPDD4tfT9KvyWwGuhDU0fuoJg83aPWzwXejtn/NjDSj0Cm9I1iI58KM5dlFUtndLG5ouKRizr4VfopO3JVtR6YhHPNLAdmqepSEblVRMZ42SaLyFIRWQxMxvn4ARCRcuAQ4G9pv4YYgaVUIiz6IZ8dmbmMeVMs8wEHMnJlmhS0Dn5ahnwuZukHn2JyBYSNZBZisVjqzSUMz18u6kC23Dv5XkzpB58wjFYpVjL9jiBshKGBK9ToHXF5g0N1dbWmmj7QKCwtWji1EouIcw0YuSPeyJV27WC//WDz5n3z9+7tXDVG+BGRBapanSqfBVwzgPR89GHwqRYrifoPvKCn+1BMw0aN/GBK30j7i9JEX60GfWx0WIjX4WoNceEplsENpvSNtEeEBHE2oFKn2BriYlGQfimqUBx+HP/5XKwjN/+EYdxzJoShMzCaYqlPMYy+KcZQHNjoHcMvQXhg800xKJ7mEOQGoNDPW6prk8mzEQTDyZS+4ZtiVoCZKrdCK55cEvSvdgupIJNdm+YEcQvC82RK30iLIFuGiWiOcguCZZYrcqWAstWYFFJBpvOdQzrPRnOvTTb+f6b0jdDTHOURBMssV+SqQcvWNSvkm2Wia+NnSVXPTBV3tq6HKX0j9GSi3KJf4WOPLxaXVipy1aBlszGJVpBdurglnrLM9htounMg5+PZyNb9MqVvZJ2guYDS/bPEs6giiiwI9ckWubKkc9GYxJO1VSvXCOSiYU50bSLlJapfLp+NbDWmpvSNrBLEzt50ZQqSSyfXDWixxNzPxPLORt9E7LUJwzy/pvSNrBJUhZnMNRBLUDpvg9iA+iXbjUkmPvZc3a9CvcmaT9+UfiAJg8IMSsMVFDmCQC4t/aC5I5Nho3eMwBEURdUcObJpYTfnT+q3AS0mpZUp8e5JssXv/Srmt6lMMaVvZJWg/Ima+8aRDUXa3Gvhp+Eq9PXOZ4MT665r3Xrfexu5PvbxXWKyqvSB0cAKYBUwJc7+cUAtsMhbJkTt6wX8BTfV4jKgPFlZpvSDSxAszyD8mZsrgx+FXsh6FnODk+qr2jB8fJeIrCl9oAz4CDiUponRB8bkGQfcl+D4vwKjvPUOQLtk5ZnSN5JRaIWkmt7bRiIFlkqxFbIPJQgNayb4cRUFvQ7Nwa/S9xNaeTiwSlVXq+puYAZwtp8IniIyEGipqq94ET13qOquFIcZRkKCENbZb+z6ZOF2U01Cnsv4+KnCGieaeCXoE7LECxEeTZBDTeeVVK0CcAHwUNT2ZcRY9ThLfyOwBHgaOMRLPwd4Hvgz8C5wJ1AWp4yJQA1Q06tXr9w3iYbRDFK9bTQncJffMnIlu2rxWvrJhn/myh0ZBJdnBLLo3vGj9LsAbbz1HwOvRR1bh3MNtQT+BPwwWXnm3jGKgWRum+YE7vJTRnPk9dMQBcGFlgn5bqyCdp2yqfSPAeZGbd8I3JgkfxlQ560fDfwtat9lwP3JyjOlbxQzfsad59tizqQhCpIFGyGVTPlWwkF7I8qm0m8JrAb60NSROygmT/eo9XOBt7WpAVgMdPO2HwGuSlaeKX2jmEn1hWkqJZQLZRvEhihd/Cr0fDZWyTrbC9FoZk3pu3PxXeBD3CieqV7arcAYb/12YKmn4F8H+kcdO8rz9b8H/BFonawsU/pGMZPKhZJK4efCUm1uQ5QtmqMIg2ZVJ5MpXmz+fFzjrCr9fC6m9I1sk+8PjYIWJqI5DVGkToX+oC0oYUCiSTdiZ64bKFP6RsYE0Z+bKYXobMv0+uVKsTXnGmTr+jW3QQuipa8a/14XqoEypW9kRCZ/8iA3EkFVFvHIpayZ3qNMZMqFIgzaSJlkFOqZM6VvZES6D2zQ/4xBdAskIojXMt3rl0uXR5CNi2gKdR9N6RsZke6fPOiWtF/5gqJQgiJHhHTvb9A6NwtF0Y/eyediSr+wpPsnD7ol7cfqCqKFHRTSvTZBG8ZYSpjSNzIi3T950C191dTKphjqUEjSUdZ2LQuHKX0jY9L5k4fBSg7620oxEYbnoVjxq/T9RNk0SozoCJC33eaiFyaKyBiEqJfxSBVJMppcRrTMFunUp5AE9XkwovDTMuRzMUs/OOQy0mOqN4nmTqSRjtxBt06DLp8RDDD3jtFccuGfzUfHarbGlgcF85MbfvCr9MXlDQ7V1dVaU1NTaDEMnCsh3uMh4lw/mVBe7iYTiaV3b+dS8psnGbmQu5CErT5GbhCRBapanSqf+fSNhOTC151sVqaI3zqewk92bCzF4KNPh7DVxygspvSNhNx2m5tiLprmTjmXSFEdeGDT1ILpHhtLLuQuJGGrTykTiA55Pz6gfC7m0w8W2fZ1p/uZfqYdl0H20WdC2OpTiuS6Qx7ryC1uwvAnT1SHdAJyRTosi7H+hhFNrjvkTekXMcksgqA3BhH5Ih83henLXiN8BGWmrWxgSr+IKdagVfEaK79K3MaiG/km389cUVn6wGhgBbAKmBJn/zigFljkLROi9n0TlT47VVmm9FNPbxdUa9jPXKzJrJpkVleuP+gySo98v10WjU8fN7n5R8ChNE2MPjAmzzjgvgTH7/AjSGQxpe9Peebi9bC5+GmsMvlDhT1SpjVWhaEQMZdyea+zqfSPAeZGbd8I3BiTp2iU/ttvq65aldMimk3Q5t70S6rGKp4S9vMn8GORFWufQDE3VsVOsT4zicim0r8AeChq+7JYBe8p/Y3AEuBp4JCoffVADfA2cE6CMiZ6eWp69eqV0wvTo4fq0UfntIisEE8ZBl1BxJMvYk3FU+h+6+PHIivWSJlhUzzFRND/T+mSb6XfBWg9Wr/fAAAWaklEQVTjrf8YeC1qXw/v91BgLXBYsvJyael/8UXTzV2yJGfF5JSguwJyEXs9zJZ+sTZWYSHo/6d0yKt7JyZ/GVCXYN8fgQuSlZdLpf/WW01/qquvzlkxhk/8Krww+/SLtbEygodfpe8nDMN8oK+I9BGR1sDFwOzoDCLSPWpzDLDcS+8sIm289a7AccAyH2XmhOXL3e+RR8Ljj8OuXYWSpPSI9/m535gyfmK0F2scdwuxYOQdPy0D8F3gQ9wonqle2q3AGG/9dmApbmTP60B/L/1Y4D0v/T3gh6nKyqWlf/31qq1bq77yirOmHn00Z0UZUSSywq+8sjit82wTJheDUTiw0Mr7ctZZLjTvkiXQrx8cdBC8+WZOijKiSBYqOTIz17//7Sz8224LvnVuGEHEQivHYflyGDDAvf5PnAj/+AcsXVpoqcJPsnDK0VMzrl1rCt8wck3JKP2vvoI1a5zSB7jiCmjVCh58sLBylQIWD750CUQoYWMvSkbpr1zprMmI0u/WDU45BebNK6xcpYB1VpYm06c3zZGg6n4nTjTFX2hKRulHRu7079+UVl3t0nfuLIxMpUKxjqwxmsfUqfuOkNu1y6UbhaNloQXIF8uXO4XTr19T2rBhzvpfvBiOPbZwspUCY8eaki81kvXlGIUjlJa+Kpx7LrzwQlPa8uXOp7jffk1pw4a53wUL8ioee/bAaafBrFn5Ldcw8on15QSTUCr9nTvh2Wfhllua0iIjd6Lp2dP59vOt9P/1L3jlFdeZvHBhfsuOxjrZjFxifTnBJJRKf/t291tT45TqN9/AihX7Kn0RZ+3nW+nPm+fK7tLFvZFs2pTf8sE62YzcY305wSSUPv1t25rWH3wQrr8evv56707cCMOGwV/+4jqYYq2SXDFvnutE/u//hhEj4KKL4KWXoKV3N1rkoSlO1snm908Z+Y42QiK5Gxoyk9Eofi65xC3R2POQnJz///18tpvPJRthGObPd+ro299W3X9/1Zkz3fabb+6b989/dvv++c9mF+uLujrVsjLVG290248+um+wrZ/+VLWhIb3zpvspf3OjO27f7kJURx977rmqe/Y05WloUP3Rj+KXY4sttuy7HHVUev/7aPAZhiHUlv7kyfDzn8Ott7rtWPcOOIsbnIvnmGNyL9sbbzh306mnuu3LL3dvGMu8MHQffODeAAYMgEmT/J0z4qqJWO4RVw0kttp79YofGsFPJ5sqjB8P77wDN9wA7dvDp5/CAw+4N4U77nD57r7bvWldein07euvLoaRbZYsgdmz3QCKCK1awZgxUFFROLni0aNHHgrx0zLkc8mGpf/MM67VXLBAddAgt37QQfHzNjSoduumOm5cs4v1xTXXqLZtq/rll/H3f/ON6plnqrZsqfq3v/k7ZybheZsTivg//9Pl/81v9k6/8kqXPnOm6quvqrZooXreeem/tYSVXARWs2BtqSmV8NVkc2L0fC7ZUPqPPeZqtnKl6r33uvUTT0ycf/Ro1YqKpu36etUPPmha6upSl7ltmz/ZBg1SHTUqeZ4vvlD9zndcQ/XOO065Hnywq8fBB7vtnTub8mfqqnniCTeTWOS899wTP9/WrU3XYuZMp8wvumhfZf7116rHHts0teOAAf6vS9jJRbz/Yp1DIN+UykQ1Ja3077vP1eyzz1S3bFHdbz/VSZMS55861fnZd+1S3b1bdeTIvR+OLl2Sz6v7n//pLPPnnksu1yefuPPdcUfqOixbptqhQ/yHFdzbyeefu7zJ5qZNZv098cS+f4jbbts7z+LFrl8kOs+QIao7diSuY/fuqh07qq5YkbqepUIurM1SsWCbS6lcp5JW+rff7mq2a5fbfu891c2bE+ePdOa+/bbq5Mlu/Ze/VH3ySdfReuCBqoMHu87LWF56ySnONm2covvgg8TlPP64Nrqd/LBihWrXrvEfWBHVk05yHadPPKHaqlVixR/P+lu40LmZRoxw9XzySeeKEVF94QWXZ/Nm1UMPdUr88cddnqeeSn4tVVXXrXNvWUYTubA2S8WCbS6l8kZU0kr/xhud5e3Xl/zxx+5KRCz8a6/de/9f/uJcGhdeuPc5V61S7dRJtbLSWeZdu6r279/kDor1tx5/vGtAvvnGf10S/bEjy7XXuklhRPZ9sBNZNbW1brtnT/c2FGHnTtUjjlA94ADXeJ1+umtM3nrLv7xGfMzSLyyl0PdR0kr/qquccvVLQ0OTRR2xnmO54w63//rrVefMUZ0927k5DjxQdfVql+e115yb6KyzVK+7zln/sRbY8OHp1SXZHzvyVtK+vesr2L49eSMxZ45bTj7ZyfbOO/uWt2aNc2e1b++OmTYtPXmN+JhP38g1WVX6wGhgBbAKmBJn/zigFljkLRNi9ncE1gP3pSorG0r/8svTt3bOOUe1V68mP3ksDQ2u8zL6D1ZWpjp37t757r47sdKN9A+kQ7I/9u7drpHq1En1ww9d/mT+/ejlkUcSl/nKK65uEyemJ6uRHBu9Y+QSv0o/5XSJIlKGmx93lKe45wOXqOqyqDzjgGpVjTuyXETuBboBWxLliZCN6RLPPRc++siNz/XLjh1QXw+dOiXO09Dgzllf77a/9S045JB983344d7RPGNJccn3Yfr0xFMK7tnjwk4ceGBT3ugx+wBt2rjjzzjDbXfuDIcdlrzMzz93cYlE0pPVMIzC4He6RD8fZw0HVqnqau/EM4CzgWVJj2oSZBjwLeBlIKVA2WD7dth///SO6dAhdZ4WLWDo0NT5vvMdF2ck0byw6ZIsLHGrVk0KP5IXmj/v7EEHpS+nYRjBx0+Uhx7Auqjt9V5aLOeLyBIReVpEDgEQkRbAb4H/aLakabBtG3TsmM8S96WQEQZt3lnDMBKRrdA+c4ByVa0AXgEe9dJ/CryoquuTHSwiE0WkRkRqamtrmy1MEJS+RRg0DCOI+HHvbACiPdc9vbRGVHVz1OZDwG+89WOA40Xkp0AHoLWI7FDVKTHHTwOmgfPpp1WDOGTi3skFNluUYRhBw4+lPx/oKyJ9RKQ1cDEwOzqDiHSP2hwDLAdQ1bGq2ktVy3EunsdiFX4uCIKlnwybvMQwjEKRUumraj0wCZiLU+azVHWpiNwqImO8bJNFZKmILAYm44ZwFoSGBjcSJwiWfjyyOXmJNR6GYaRLyiGb+aa5Qza3bYMDDoC77oLrrsuiYFmivDzxqJ61a/2fJ97QzHbtrN/AMEoVv0M2QzddYiSWftDcOxGrPJ7CBze8Mh2SzXxlGIaRiNBNohKZHzdI7p14VnksfiYviSZRI5Fu42EYRmlhln4eiGeVR5PJ+P1EjUS6jYdhGKVF6JR+EC39ZNZ3puP3C/nxl2EYxUvolH4QLf1E1nek8zaTjtdMPv6y0T6GYYRW6QfJ0s+VVZ5OuIVsDhU1DKN4CZ3Sj7h3gmTpByEkg432MQwDQjh6J4iWPhQ+JION9jEMA0Jq6bdu7WLIG03YaB/DMCCESj/ocXcKhY32MQwDQqr0g+baCQJB6FcwDKPwhM6nv327WfqJKHS/gmEYhSeUlr4pfcMwjPiETukHZQIVwzCMIBI6pW+WvmEYRmJCqfTN0jcMw4hP6JR+Jh25YY5JE+a6GYaRPqEavVNf70ILpKP0Y2PdR2LSQPGPdAlz3QzDyAxflr6IjBaRFSKySkT2mdhcRMaJSK2ILPKWCV56bxFZ6KUtFZGfZLsC0ezY4X7Tce+EOSZNmOtmGEZmpFT6IlIG3A+cAQwELhGRgXGyzlTVod7ykJe2EThGVYcCRwFTROTgLMm+D5mEVQ5STJpsu2KCVDfDMIKBH0t/OLBKVVer6m5gBnC2n5Or6m5V/drbbOOzvIzJJNhaUGLS5CL0cVDqZhhGcPCjhHsA66K213tpsZwvIktE5GkROSSSKCKHiMgS7xx3qOonsQeKyEQRqRGRmtra2jSr0EQmYZWDEpMmF66YoNTNMIzgkC3Lew5QrqoVwCvAo5EdqrrOSz8cuEJEvhV7sKpOU9VqVa3u1q1bxkJkYukHJSZNLlwxQambYRjBwc/onQ3AIVHbPb20RlR1c9TmQ8BvYk+iqp+IyPvA8cDT6YuamkwnUAlCTJpevZxLJ156cwhC3QzDCA5+LP35QF8R6SMirYGLgdnRGUSke9TmGGC5l95TRPbz1jsDI4AV2RA8HkGcHzcVkc7bjz921ng05ooxDCPbpFT6qloPTALm4pT5LFVdKiK3isgYL9tkb0jmYmAyMM5LHwD8y0v/G3CXqr6X7UpEyMasWfn8mCm68xZcB25E8ZsrxjCMXODr4yxVfRF4MSbtF1HrNwI3xjnuFaCimTL6JuLeyVTp5/tjpnidt6pO4a9dm/3yDMMwQhWGYds2aNsWWrXK7Ph8f8xk4+gNw8g3oVL6zZ1AJd9K2MbRG4aRb0Kl9JsbVjnfStjG0RuGkW9Cp/Sb04mbbyVs4+gNw8g3oYqy2Vz3TkTZTp3qXDq9ejmFn0slbOPoDcPIJ6G39NMdgjl2rBs509Dgfk0hG4YRJkJt6Vs8ecMwjL0JnaUfrfQtnrxhGMbehE7pR7t38jEE06YjNAyjmAiN0t+9G77+em9LP9dDMOPFwL/sMjcSxxoAwzCCSGiUfrwQDLkegpkojAJkZxIUwzCMbBMapd+6NfzqV3DccU1puR4Hn8pNZP0HhmEEDdGIaRoQqqurtaamptBi+CISEjkZIm74p2EYRi4RkQWqWp0qX2gs/UIQz30Ui8XRMQwjSJjSbwbR7iOwSVAMwwg+pvSbSeQLXlV4/HGLo2MYRrAJ1Re5hcbi6BiGEXR8WfoiMlpEVojIKhGZEmf/OBGpFZFF3jLBSx8qIm95UykuEZGLsl2BdLAPqQzDKHVSWvoiUgbcD4wC1gPzRWS2qi6LyTpTVSfFpO0CLlfVlSJyMLBAROaq6hfZED4dLA6PYRiGP0t/OLBKVVer6m5gBnC2n5Or6oequtJb/wT4HOiWqbDNweLwGIZh+FP6PYB1UdvrvbRYzvdcOE+LyCGxO0VkONAa+CjOvokiUiMiNbW1tT5FTw+bj9YwDCN7o3fmAOWqWgG8AjwavVNEugOPA+NVdZ9PlVR1mqpWq2p1t265eRGw+WgNwzD8Kf0NQLTl3tNLa0RVN6vq197mQ8CwyD4R6Qi8AExV1bebJ27m2Hy0hmEY/pT+fKCviPQRkdbAxcDs6AyeJR9hDLDcS28NPAM8pqpPZ0fkzLD5aA3DMHyM3lHVehGZBMwFyoCHVXWpiNwK1KjqbGCyiIwB6oEtwDjv8P8BnAB0EZFI2jhVXZTdavjDxtEbhlHqWMA1wzCMEGAB17KAfcxlGEbYsDAMCbCPuQzDCCNm6SfAPuYyDCOMmNJPQKKPtj7+2Fw9hmEUL6b0E5Dsoy2b/9YwjGLFlH4CUs2KZa4ewzCKEVP6CYidFSseFrfHMIxiw5R+EiKzYiVS/Ba3xzCMYiOUSj/b4+stbo9hGGEhdEo/Mr7+44/dvLXZ6HS1uD2GYYSF0IVhKC93ij6W3r2dq8YwDCOMlGwYBpssxTAMIzGhU/o2WYphGEZiQqf0rdPVMAwjMaFT+tbpahiGkZjQKX1oGl/f0OB+s63wLeSyYRjFioVWThMLuWwYRjHjy9IXkdEiskJEVonIlDj7x4lIrYgs8pYJUfteFpEvROT5bApeKCzksmEYxUxKS19EyoD7gVHAemC+iMxW1WUxWWeq6qQ4p7gTaAf8uLnCBgEbEmoYRjHjx9IfDqxS1dWquhuYAZzttwBVfRXYnqF8gcOGhBqGUcz4Ufo9gHVR2+u9tFjOF5ElIvK0iBySjhAiMlFEakSkpra2Np1D844NCTUMo5jJ1uidOUC5qlYArwCPpnOwqk5T1WpVre7WrVuWRMoNNiTUMIxixs/onQ1AtOXe00trRFU3R20+BPym+aIFl7FjTckbhlGc+LH05wN9RaSPiLQGLgZmR2cQke5Rm2OA5dkT0TAMw8gWKS19Va0XkUnAXKAMeFhVl4rIrUCNqs4GJovIGKAe2AKMixwvIn8H+gMdRGQ98ENVnZv9qhiGYRipCF1oZcMwjFKkZEMrG4ZhGIkxpW8YhlFCmNI3DMMoIUzpG4ZhlBCm9A3DMEqI0Cj9dGPcW0x8wzBKkVDE0083xr3FxDcMo1QJxTj98nKnuGPp3dvNnNXc/IZhGEGnpMbppxvj3mLiG4ZRqoRC6acb495i4huGUaqEQumnG+PeYuIbhlGqhELppxvj3mLiG4ZRqoSiI9cwDKPUKamOXMMwDMMfpvQNwzBKCFP6hmEYJYQpfcMwjBLClL5hGEYJEbjROyJSC8QJkuCbrsCmLIlTLJRinaE0612KdYbSrHe6de6tqt1SZQqc0m8uIlLjZ9hSmCjFOkNp1rsU6wylWe9c1dncO4ZhGCWEKX3DMIwSIoxKf1qhBSgApVhnKM16l2KdoTTrnZM6h86nbxiGYSQmjJa+YRiGkQBT+oZhGCVEaJS+iIwWkRUiskpEphRanlwhIoeIyOsiskxElorINV76gSLyiois9H47F1rWbCMiZSLyrog87233EZF/efd8poi0LrSM2UZEOonI0yLygYgsF5Fjwn6vReRa79l+X0SeEpG2YbzXIvKwiHwuIu9HpcW9t+L4vVf/JSJSlWm5oVD6IlIG3A+cAQwELhGRgYWVKmfUA9ep6kDgaOAqr65TgFdVtS/wqrcdNq4Blkdt3wHcraqHA1uBHxZEqtxyL/CyqvYHKnH1D+29FpEewGSgWlUHA2XAxYTzXv8RGB2TlujengH09ZaJwAOZFhoKpQ8MB1ap6mpV3Q3MAM4usEw5QVU3qupCb307Tgn0wNX3US/bo8A5hZEwN4hIT+B7wEPetgAnA097WcJY5wOAE4D/C6Cqu1X1C0J+r4GWwH4i0hJoB2wkhPdaVd8AtsQkJ7q3ZwOPqeNtoJOIdM+k3LAo/R7Auqjt9V5aqBGRcuAI4F/At1R1o7frU+BbBRIrV9wD3AA0eNtdgC9Utd7bDuM97wPUAo94bq2HRKQ9Ib7XqroBuAv4N07Z1wELCP+9jpDo3mZNx4VF6ZccItIB+BPwP1V1W/Q+deNwQzMWV0TOBD5X1QWFliXPtASqgAdU9QhgJzGunBDe6844q7YPcDDQnn1dICVBru5tWJT+BuCQqO2eXlooEZFWOIU/XVX/7CV/Fnnd834/L5R8OeA4YIyIrMW57k7G+bo7eS4ACOc9Xw+sV9V/edtP4xqBMN/rU4E1qlqrqnuAP+Puf9jvdYRE9zZrOi4sSn8+0Nfr4W+N6/iZXWCZcoLny/6/wHJV/V3UrtnAFd76FcBz+ZYtV6jqjaraU1XLcff2NVUdC7wOXOBlC1WdAVT1U2CdiPTzkk4BlhHie41z6xwtIu28Zz1S51Df6ygS3dvZwOXeKJ6jgbooN1B6qGooFuC7wIfAR8DUQsuTw3qOwL3yLQEWect3cT7uV4GVwDzgwELLmqP6nwQ8760fCrwDrAL+H9Cm0PLloL5DgRrvfj8LdA77vQZ+CXwAvA88DrQJ470GnsL1W+zBvdX9MNG9BQQ3QvEj4D3c6KaMyrUwDIZhGCVEWNw7hmEYhg9M6RuGYZQQpvQNwzBKCFP6hmEYJYQpfcMwjBLClL5hGEYJYUrfMAyjhPj/IfcycxFqGcwAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEICAYAAACzliQjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4xLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvDW2N/gAAIABJREFUeJzt3X28VVW97/HPj+e2IM/lA8LGtGCDCLhFvUSImqGlHo084NbUfL6Vp8x7I7Uyi3Os61XT4/GVeTITlLh6LCuVTslJPaayMUUREVRABBUwUETDvfndP8Zcm8VyPT/stfaa3/frNV97rbnGnHPMNff6zTHHGHNMc3dERCQeulU7AyIi0nkU9EVEYkRBX0QkRhT0RURiREFfRCRGFPRFRGJEQV8KYmbdzWybmQ0vZ9pqMrMDzKzsfZfN7BgzW530foWZTcknbRHbutXMLit2+Szr/aGZ/aLc65Xq6VHtDEhlmdm2pLcNwN+B9uj9Be4+r5D1uXs70LfcaePA3T9ZjvWY2bnA6e5+ZNK6zy3HuqX+KejXOXfvCLpRSfJcd/9jpvRm1sPd2zojbyLS+VS9E3PR5fuvzOwuM3sHON3MjjCzx81si5ltMLMbzKxnlL6HmbmZNUbv50afP2Bm75jZX8xsZKFpo8+PM7MXzWyrmd1oZv9tZmdlyHc+ebzAzFaZ2d/M7IakZbub2XVmttnMXgamZ/l+Ljez+SnzbjKza6PX55rZ8mh/XopK4ZnWtc7MjoxeN5jZHVHelgGHpKS9wsxejta7zMxOjOYfBPwrMCWqOtuU9N1embT8hdG+bzazX5vZ3vl8N7mY2clRfraY2UNm9smkzy4zs/Vm9raZvZC0r4eb2VPR/DfM7P/kuz2pAHfXFJMJWA0ckzLvh8AO4ARCIeAjwKHAYYQrwf2BF4GvRul7AA40Ru/nApuAZqAn8CtgbhFpPwq8A5wUfXYJ8AFwVoZ9ySePvwH6A43AW4l9B74KLAOGAYOBh8NPIe129ge2AXskrftNoDl6f0KUxoCjgPeAcdFnxwCrk9a1Djgyen0N8F/AQGAE8HxK2lOBvaNjclqUh49Fn50L/FdKPucCV0avj43yOB7oA/wb8FA+302a/f8h8Ivo9egoH0dFx+gyYEX0egywBtgrSjsS2D96vRiYFb3uBxxW7d9CnCeV9AXgUXf/rbvvdPf33H2xuz/h7m3u/jJwCzA1y/J3u3uru38AzCMEm0LTfh542t1/E312HeEEkVaeefwXd9/q7qsJATaxrVOB69x9nbtvBq7Osp2XgecIJyOAzwB/c/fW6PPfuvvLHjwE/AlI21ib4lTgh+7+N3dfQyi9J293gbtviI7JnYQTdnMe6wVoAW5196fd/X1gNjDVzIYlpcn03WQzE7jP3R+KjtHVhBPHYUAb4QQzJqoifCX67iCcvA80s8Hu/o67P5HnfkgFKOgLwKvJb8xslJn93sxeN7O3gauAIVmWfz3p9XayN95mSrtPcj7c3Qkl47TyzGNe2yKUULO5E5gVvT4tep/Ix+fN7Akze8vMthBK2dm+q4S9s+XBzM4ys2eiapQtwKg81wth/zrW5+5vA38D9k1KU8gxy7TenYRjtK+7rwC+STgOb0bVhXtFSc8GmoAVZvakmR2f535IBSjoC4TL/WQ/JZRuD3D3PYHvEqovKmkDoboFADMzdg9SqUrJ4wZgv6T3ubqULgCOMbN9CSX+O6M8fgS4G/gXQtXLAOAPeebj9Ux5MLP9gZuBi4DB0XpfSFpvru6l6wlVRon19SNUI72WR74KWW83wjF7DcDd57r7ZELVTnfC94K7r3D3mYQqvP8L3GNmfUrMixRJQV/S6QdsBd41s9HABZ2wzd8BE83sBDPrAfwTMLRCeVwAfN3M9jWzwcC3siV299eBR4FfACvcfWX0UW+gF7ARaDezzwNHF5CHy8xsgIX7GL6a9FlfQmDfSDj/nUco6Se8AQxLNFyncRdwjpmNM7PehOD7iLtnvHIqIM8nmtmR0bb/F6Ed5gkzG21m06LtvRdNOwk7cIaZDYmuDLZG+7azxLxIkRT0JZ1vAmcSftA/JTS4VpS7vwH8I3AtsBn4OPBXwn0F5c7jzYS692cJjYx357HMnYSG2Y6qHXffAnwDuJfQGDqDcPLKx/cIVxyrgQeAXyatdylwI/BklOaTQHI9+H8CK4E3zCy5miax/IOEapZ7o+WHE+r5S+Luywjf+c2EE9J04MSofr838GNCO8zrhCuLy6NFjweWW+gddg3wj+6+o9T8SHEsVJ2K1BYz606oTpjh7o9UOz8i9UIlfakZZjY9qu7oDXyH0OvjySpnS6SuKOhLLfkU8DKh6uCzwMnunql6R0SKoOodEZEYUUlfRCRGam7AtSFDhnhjY2O1syEi0qUsWbJkk7tn6+YM1GDQb2xspLW1tdrZEBHpUsws153lgKp3RERiRUFfRCRGFPRFRGKk5ur0RaRzffDBB6xbt47333+/2lmRPPTp04dhw4bRs2emoZeyU9AXibl169bRr18/GhsbCYObSq1ydzZv3sy6desYOXJk7gXSqJvqnXnzoLERunULf+cV9Lhvkfh6//33GTx4sAJ+F2BmDB48uKSrsroo6c+bB+efD9u3h/dr1oT3AC0ljy0oUv8U8LuOUo9VXZT0L798V8BP2L49zBcRkV3qIuivXVvYfBGpHZs3b2b8+PGMHz+evfbai3333bfj/Y4d+Q27f/bZZ7NixYqsaW666Sbmlane91Of+hRPP/10WdbV2eqiemf48FClk26+iJTXvHnhKnrt2vAbmzOntGrUwYMHdwTQK6+8kr59+3LppZfulsbdcXe6dUtfTr3ttttybucrX/lK8ZmsI3VR0p8zBxoadp/X0BDmi0j5JNrP1qwB913tZ5XoOLFq1SqamppoaWlhzJgxbNiwgfPPP5/m5mbGjBnDVVdd1ZE2UfJua2tjwIABzJ49m4MPPpgjjjiCN998E4ArrriC66+/viP97NmzmTRpEp/85Cd57LHHAHj33Xf5whe+QFNTEzNmzKC5uTlniX7u3LkcdNBBjB07lssuuwyAtrY2zjjjjI75N9xwAwDXXXcdTU1NjBs3jtNPP73s31k+6qKknyhllLP0ISIflq39rBK/txdeeIFf/vKXNDc3A3D11VczaNAg2tramDZtGjNmzKCpqWm3ZbZu3crUqVO5+uqrueSSS/j5z3/O7NmzP7Rud+fJJ5/kvvvu46qrruLBBx/kxhtvZK+99uKee+7hmWeeYeLEiVnzt27dOq644gpaW1vp378/xxxzDL/73e8YOnQomzZt4tlnnwVgy5YtAPz4xz9mzZo19OrVq2NeZ6uLkj6Ef7jVq2HnzvBXAV+k/Dq7/ezjH/94R8AHuOuuu5g4cSITJ05k+fLlPP/88x9a5iMf+QjHHXccAIcccgirV69Ou+5TTjnlQ2keffRRZs6cCcDBBx/MmDFjsubviSee4KijjmLIkCH07NmT0047jYcffpgDDjiAFStWcPHFF7Nw4UL69+8PwJgxYzj99NOZN29e0TdXlapugr6IVF6mdrJKtZ/tscceHa9XrlzJT37yEx566CGWLl3K9OnT0/ZX79WrV8fr7t2709bWlnbdvXv3zpmmWIMHD2bp0qVMmTKFm266iQsuuACAhQsXcuGFF7J48WImTZpEe3t7WbebDwV9EclbNdvP3n77bfr168eee+7Jhg0bWLhwYdm3MXnyZBYsWADAs88+m/ZKItlhhx3GokWL2Lx5M21tbcyfP5+pU6eyceNG3J0vfvGLXHXVVTz11FO0t7ezbt06jjrqKH784x+zadMmtqfWlXWCuqjTF5HOUc32s4kTJ9LU1MSoUaMYMWIEkydPLvs2vva1r/GlL32JpqamjilRNZPOsGHD+MEPfsCRRx6Ju3PCCSfwuc99jqeeeopzzjkHd8fM+NGPfkRbWxunnXYa77zzDjt37uTSSy+lX79+Zd+HXGruGbnNzc2uh6iIdJ7ly5czevToamejJrS1tdHW1kafPn1YuXIlxx57LCtXrqRHj9oqH6c7Zma2xN2bMyzSobb2RESkirZt28bRRx9NW1sb7s5Pf/rTmgv4paqvvRERKcGAAQNYsmRJtbNRUWrIFRGJEQV9EZEYUdAXEYkRBX0RkRhR0BeRqpo2bdqHbrS6/vrrueiii7Iu17dvXwDWr1/PjBkz0qY58sgjydUF/Prrr9/tJqnjjz++LOPiXHnllVxzzTUlr6fcFPRFpKpmzZrF/Pnzd5s3f/58Zs2aldfy++yzD3fffXfR208N+vfffz8DBgwoen21TkFfRKpqxowZ/P73v+94YMrq1atZv349U6ZM6eg3P3HiRA466CB+85vffGj51atXM3bsWADee+89Zs6cyejRozn55JN57733OtJddNFFHcMyf+973wPghhtuYP369UybNo1p06YB0NjYyKZNmwC49tprGTt2LGPHju0Ylnn16tWMHj2a8847jzFjxnDsscfutp10nn76aQ4//HDGjRvHySefzN/+9reO7SeGWk4M9PbnP/+54yEyEyZM4J133in6u01H/fRFpMPXvw7lfiDU+PEQxcu0Bg0axKRJk3jggQc46aSTmD9/PqeeeipmRp8+fbj33nvZc8892bRpE4cffjgnnnhixufE3nzzzTQ0NLB8+XKWLl2629DIc+bMYdCgQbS3t3P00UezdOlSLr74Yq699loWLVrEkCFDdlvXkiVLuO2223jiiSdwdw477DCmTp3KwIEDWblyJXfddRc/+9nPOPXUU7nnnnuyjo//pS99iRtvvJGpU6fy3e9+l+9///tcf/31XH311bzyyiv07t27o0rpmmuu4aabbmLy5Mls27aNPn36FPBt51aXJf1586CxEbp1C38r8YAHESmf5Cqe5Kodd+eyyy5j3LhxHHPMMbz22mu88cYbGdfz8MMPdwTfcePGMW7cuI7PFixYwMSJE5kwYQLLli3LOZjao48+ysknn8wee+xB3759OeWUU3jkkUcAGDlyJOPHjweyD98MYXz/LVu2MHXqVADOPPNMHn744Y48trS0MHfu3I47fydPnswll1zCDTfcwJYtW8p+R3BeazOz6cBPgO7Are5+dcrnw4HbgQFRmtnufr+ZNQLLgcTDKx939wvLk/X0Ek/2SVTRJZ7sAxpjXySXbCXySjrppJP4xje+wVNPPcX27ds55JBDAJg3bx4bN25kyZIl9OzZk8bGxrTDKefyyiuvcM0117B48WIGDhzIWWedVdR6EhLDMkMYmjlX9U4mv//973n44Yf57W9/y5w5c3j22WeZPXs2n/vc57j//vuZPHkyCxcuZNSoUUXnNVXOkr6ZdQduAo4DmoBZZtaUkuwKYIG7TwBmAv+W9NlL7j4+mioa8CH7k31EpDb17duXadOm8eUvf3m3BtytW7fy0Y9+lJ49e7Jo0SLWpHsYdpJPf/rT3HnnnQA899xzLF26FAjDMu+xxx7079+fN954gwceeKBjmX79+qWtN58yZQq//vWv2b59O++++y733nsvU6ZMKXjf+vfvz8CBAzuuEu644w6mTp3Kzp07efXVV5k2bRo/+tGP2Lp1K9u2beOll17ioIMO4lvf+haHHnooL7zwQsHbzCafkv4kYJW7vwxgZvOBk4DkayMH9oxe9wfWlzOThejsJ/uISHnMmjWLk08+ebeePC0tLZxwwgkcdNBBNDc35yzxXnTRRZx99tmMHj2a0aNHd1wxHHzwwUyYMIFRo0ax33777TYs8/nnn8/06dPZZ599WLRoUcf8iRMnctZZZzFp0iQAzj33XCZMmJC1KieT22+/nQsvvJDt27ez//77c9ttt9He3s7pp5/O1q1bcXcuvvhiBgwYwHe+8x0WLVpEt27dGDNmTMdTwMol59DKZjYDmO7u50bvzwAOc/evJqXZG/gDMBDYAzjG3ZdE1TvLgBeBt4Er3P2RbNsrdWjlxsZQpZNqxIjwGEUR2Z2GVu56ShlauVwNubOAX7j7MOB44A4z6wZsAIZH1T6XAHea2Z6pC5vZ+WbWamatGzduLCkj1Xyyj4hIrcsn6L8G7Jf0flg0L9k5wAIAd/8L0AcY4u5/d/fN0fwlwEvAJ1I34O63uHuzuzcPHTq08L1I0tICt9wSSvZm4e8tt6gRV0QE8gv6i4EDzWykmfUiNNTel5JmLXA0gJmNJgT9jWY2NGoIxsz2Bw4EXi5X5jNpaQlVOTt3hr8K+CLZ1doT9CSzUo9VzqDv7m3AV4GFhO6XC9x9mZldZWYnRsm+CZxnZs8AdwFnecjZp4GlZvY0cDdwobu/VVKORaSs+vTpw+bNmxX4uwB3Z/PmzSXdsKVn5IrE3AcffMC6detK6rcunadPnz4MGzaMnj177jZfz8gVkbz07NmTkSNHVjsb0knqchgGERFJT0FfRCRGYhv0NSibiMRRLOv0NSibiMRVLEv6GpRNROIqlkFfg7KJSFzVfdBPV3c/fHj6tJnmi4jUi7qu009Xd3/GGeAexuVJvi9Ng7KJSBzUdUk/Xd19ItAnAj9oUDYRiY+6LunnqqN31zj7IhIvdV3Sz6eOXo23IhIndR300z1QJZUab0UkTuo66Cc/UAV21eEnqPFWROKmroM+7HqgijvccYeeqCUi8VbXDbmpWloU5EUk3uq+pC8iIrso6IuIxIiCvohIjCjoi4jEiIK+iEiMKOiLiMSIgr6ISIwo6IuIxIiCvohIjCjoi4jEiIK+iEiMKOiLiMSIgr6ISIwo6IuIxIiCvohIjCjoi4jEiIK+iEiMKOiLiMSIgr6ISIwo6IuIxIiCfhbz5kFjI3TrFv7Om1ftHImIlKZHtTNQq+bNg/PPh+3bw/s1a8J7gJaW6uVLRKQUeZX0zWy6ma0ws1VmNjvN58PNbJGZ/dXMlprZ8UmffTtaboWZfbacma+kyy/fFfATtm8P80VEuqqcJX0z6w7cBHwGWAcsNrP73P35pGRXAAvc/WYzawLuBxqj1zOBMcA+wB/N7BPu3l7uHSm3tWsLmy8i0hXkU9KfBKxy95fdfQcwHzgpJY0De0av+wPro9cnAfPd/e/u/gqwKlpfzRs+vLD5IiJdQT5Bf1/g1aT366J5ya4ETjezdYRS/tcKWBYzO9/MWs2sdePGjXlmvbLmzIGGht3nmYW6fTXqikhXVa7eO7OAX7j7MOB44A4zy3vd7n6Luze7e/PQoUPLlKXStLTALbfAiBHhvRm4h9eJRl0FfhHpavIJzK8B+yW9HxbNS3YOsADA3f8C9AGG5LlszWppgdWrQ+BPBPwENeqKSFeUT9BfDBxoZiPNrBehYfa+lDRrgaMBzGw0IehvjNLNNLPeZjYSOBB4slyZ7yxq1BWRepEz6Lt7G/BVYCGwnNBLZ5mZXWVmJ0bJvgmcZ2bPAHcBZ3mwjHAF8DzwIPCVWuy5k3wT1pAhYUq+IUuNuiJSL8xT6y2qrLm52VtbWztte6k3YaVqaIAzz4Tbb989TUNDqPPXjVoiUgvMbIm7N+dKF/thGNLdhJVs+3a4//5djbpm4a8Cvoh0RbEv6Xfr9uFG2lRmsHNn5+RHRKQYKunnKZ96edXdi0i9iH3QT3cTVrKGhpBGRKQexD7oJ9+EZQaDB4cpW929hlwWka5KQysTgnq+jbIacllEurLYl/QLpSGXRaQrU9AvkO7OFZGuTEG/QLo7V0S6MgX9AqXr7aMePiLSVSjoFyi1t4/uzhWRrkS9d4pQSG8fEZFaopK+iEiMKOiXSDdqiUhXouqdEuhGLRHpalTSL4Fu1BKRrkZBvwS6UUtEuhoF/RLoRi0R6WoU9EuQ7kYts1C3r0ZdEalFCvolSL5RC0LATzyFK9Goq8AvIrVEQb9ELS2wenUI/KmPXVSjrojUGgX9MlGjroh0BQr6ZaJGXRHpChT0y0Sjb4pIV6CgXyYafVNEugINw1BGGn1TRGqdSvoiIjGioC8iEiMK+iIiMaKgLyISIwr6IiIxoqAvIhIjCvqdRI9VFJFaoKBfQYlAbwZnnBFG3nTPPAKnTgwiUmkK+hWSeH7umjXhfa4ROJPTZzsxiIiUQkG/QtI9PzfV2rW7Svenn67n7YpI5WkYhgrJZ0jlQYNCaT7byUFDM4tIOeVV0jez6Wa2wsxWmdnsNJ9fZ2ZPR9OLZrYl6bP2pM/uK2fma1muIZXNYPPm3FcDGppZRMopZ0nfzLoDNwGfAdYBi83sPnd/PpHG3b+RlP5rwISkVbzn7uPLl+WuYc6cD5fiE49TTH6sYjYamllEyi2fkv4kYJW7v+zuO4D5wElZ0s8C7ipH5rqydEMt33FH+scqpqOhmUWkEvIJ+vsCrya9XxfN+xAzGwGMBB5Kmt3HzFrN7HEz+4cMy50fpWnduHFjnlmvfYnn5+7cGf62tOSuo29ogLlzd6UXESmncvfemQnc7e7tSfNGuHszcBpwvZl9PHUhd7/F3ZvdvXno0KFlzlJtyVZHr9K9iFRaPkH/NWC/pPfDonnpzCSlasfdX4v+vgz8F7vX98dOpscqqnQvIp0hn6C/GDjQzEaaWS9CYP9QLxwzGwUMBP6SNG+gmfWOXg8BJgPPpy4bJ3qsoohUU86g7+5twFeBhcByYIG7LzOzq8zsxKSkM4H57rs1U44GWs3sGWARcHVyr5+4SlfXn0pDMohIJZjn05WkEzU3N3tra2u1s1FViSEZkrt7NjToikBEMjOzJVH7aVYahqEGpRvCQUMyiEg5KOjXoEzdOjUkg4iUSkG/BmXq1lnMkAxqGxCRZAr6NShTt85Ch2TQcM0ikkpBvwaVq1un2gZEJJWCfo3K1K2zkOoatQ2ISCoF/S6k0OqacrYNiEh9UNDvAop9ula52gZEpH4o6Ne41GftppOpukZDPohIKj0uscbl86zd1OqaefPCcmvXhs/mzFGgF5FAJf0al6vR1SxcBQwZEiYzOOOM/Or91YdfJH4U9GtctkbX5Mcubt4cJvjwk7nS1furD79IPCno17hMjbGDB+f32MWEtWt3L9mfeab68IvEkYJ+jcvUGPvWW4WtZ9Cg3Uv27e3p06kPv0h9U9DvAtLdqFVIX/vElUKuBmFQH36Reqeg30Wlq/ZJZhb+FnJloD78IvVPQb+LSq32GTw4TIkqoDvuCNU4ua4MundXH36ROFHQ78KSq302bQpTpkcwZmoQvv12PbZRJE4U9GOimLtz1a1TpP7oGbmSUWNj+uEfRowIVwYiUjv0jFwpmYZmFqk/CvqSkYZmFqk/CvqSkYZmFqk/CvqSUTmHZlYvIJHaoKGVJauWltL77id6ASXuCE70AkqsX0Q6j0r6UnF6QLtI7VDQl7JKV43TWb2AVIUkkpuCvgDlCZiZbuYaNCh9+nL2AorrjWQ60UnB3L2mpkMOOcSlc82d697Q4B7CZZgaGsL8QowYsfs6EtPgwYWvf+7csD6z8DdXXjJte8SIwvahKynXcZP6ALR6HjG26kE+dVLQ73z5Bsxcgdgs/XrMCgvi6YJZz57h5GEW/iZeJ9ZVrm1XUqZ8FJu/Wj3R1cr3nawW81RuCvqSt2wBMyGfUmW5glCm9WSaGhrCSaBcVxmVkOn7u+ii4vOXz3HrbLV49ZEtT/V0MlDQl7xlC7KJH0I+Ab1cP/hMwSzblCm4ZzoZdHZpONP317178fmrxZJ+V8pTNQoElTzJKOhL3tIF69QfQqbPUkuV+fxTJ6dJV1VTaEk/kY90266V0nChJ7J88lfJUnWxwalWvu988pStoFMJlb4KUtCXguQKtqWUSFO3k+skkvy31B9qrZQ8K1HSd69MybGU4FSu9qFyKrQQUakTVKX/FxX0pSjZgm05euAU8gNM5GXwYPdevbLnKzUfydtK3ads6SsVhCpRp18ppQSnfE4YxZ5Uij1GmbbX2VV/lb4KUtCXomT7wRfS+yTTD62QElfyDzBXlVCydNtO/ODyTV+JwFvu3jvlzkdCqT2hcqUp5qRS6jEq5H+0Ut+/SvoK+jWp0B9CoaWoTNUZ2Ur7hSr0x1UrVUCdoZReWKU2fOa60st2rCt1jCp1wq3GSaasQR+YDqwAVgGz03x+HfB0NL0IbEn67ExgZTSdmWtbCvrVV8gPoZhG10JK/MX8qAu9jC73ZXcpgaTSQSif77kS1SG52nKS11OuBvlqXUEl8pUuuBdyxVqosgV9oDvwErA/0At4BmjKkv5rwM+j14OAl6O/A6PXA7NtT0G/ayi2l01qNVHiHz/bD6VQhVZR5dNltZDvpZCqpXItW2iecgXPcgfeXP8X2dpvijnhdHbVTSEntUrlr5xB/whgYdL7bwPfzpL+MeAz0etZwE+TPvspMCvb9hT0a18+/+CdMfRCIfnL1GiaT2+hQn6MuQJctnWVsmwpecoWPN0Lu0pIXS6fq7p8e2qlpst2QuzsKrt8T2yVzF85g/4M4Nak92cA/5oh7QhgA9A9en8pcEXS598BLs22PQX92pdvcOrsy+tkxfQcyhZ8cv0YC7nyybSufINfod9lrvUW2mZTrpMYFNfGk+5Ypf7P5RN4yymfY5d83CvRk6daQf9bwI1J7/MK+sD5QCvQOnz48OL3WjpFruBYze6G2RRzp2/yjzFTfWyuLqX5/rALqS4r19VHKW02uXog5XOyKeZYZDpRpLu6zBZ4i1VMgaJSQ5Ykq0r1DvBX4H8kvVf1Th2qpd4ulW50LiSg5DuVWh2SaT3ZuoTmU9VWaD1+robLTPXwySeMUo5JoVM5qheLqTosR7fifJQz6PeIGmBHJjXkjkmTbhSwGrCkeYOAV6JG3IHR60HZtqegX/s6u5GsXPkopS0iWwDLNhXaOJ0tkKZbd7Z9Sw4iF12UPaAV2mMnn5NgPu066babGFE103oLrRLKJ5Dm+7+U7SRVaG+cfHr7FKLcXTaPj7pivgRcHs27CjgxKc2VwNVplv0yoavnKuDsXNtS0O8aqllfn1DsTT6FdKvLt6oiW6DJVPrOp1SZbwNqqY3AhfbNz+ckWOrNXIWUqnMdh1zy/V8qpX2k1G3nUtag35mTgr7kq9TGsEpWDRVzQ1sp6QttSEz9DgoN3MVur1BVVY1rAAAK5ElEQVS5TprFHIdSuqMW0xMq1/9ZuRp1FfSl7nVG20K+1S3JD3kp9gSSzxVKoUMbZAsihfYtL2R7nVXdV2ingkKrsVLXU+g9D/mcsFXSV9CXPFW6bSFbPXkpd1NWorteMQG8lMBdiYbIYhQaMAupxsr0XZSj2i05Tbn+j2MX9HfudP/Od9xffbWoxaWLqmTbQqWuJEpdbz5VHvk0DJba9bYW2nUKDZj59kbK5/iUUu2Wzx3QhYpd0H/hBfd+/dw/9jH3//7volYhsptKlMjdSyvZldL9MlUtdb0tRTnaZpL3uZCTYbHVbpX4jmMX9N3dly1zP+CAUL/6s58VvRoRd6/sD7bYkl0581QrXW87Uyl17MV8R535Hccy6Lu7v/WW+7HHhj075ZRwBSBSjFoMirU0ImhXlWufS2ngLmZ75RLboO/u/sEH7j/4gXvfvuFGjgsuUF2/FKfWgmK9VMnUuny6staafIN+N+pQjx5wxRXw0kvwP/8n/PznsP/+cPbZ8Pzz1c6ddCUtLbB6NezcGf62tFQ3P3PmQEPD7vMaGsJ8KZ/EcR8xIv3nw4d3anbKqi6DfsJHPwo33AAvvggXXgi/+hWMGQOf/SwsWAB//3u1cyhSmJYWuOWWEIzMwt9bbqn+yahe1eNJ1sJVQe1obm721tbWiqx70ya4+Wa49VZYuxYGD4aZM+ELX4ApU8IVgohIsnnz4PLLQ8wYPjwE/Fo8yZrZEndvzpkuTkE/ob0d/vjHUO3z29/Ce+/BkCEwY0aoAjr00FCKEhHpKvIN+nVdvZNJ9+6hiudXv4KNG+Huu+GYY+D22+Gww2DsWPjnf4YlS0JdrohIvYhl0E+2xx6heueuu+D110P9aP/+4XKuuRk+9jE480xYvLjaORURKV3sg36yPfeE886Dxx6DDRvgjjvguOPg3nth0iSYPDlcHWzfXu2ciogUR0E/g732gtNPh1/+Etatg+uuCyeCmTNh6FA49VSYPx9efTX03BUR6Qpi2ZBbrPZ2+POfQxvAPffAm2+G+XvvHRp/J06E8eNhwgTYbz81BotI51HvnQprb4ennoInngjT4sXhfoDE17nvvqEb6JQp4UQwdmyoPhIRqQQF/Sp491149llobYVHH4VHHoH163d9PmJECP5NTeEmsUMPhVGjoJsq2USkRAr6NcAd1qwJJ4LEtGwZrFgBO3aENAMHwhFHhOC/996wzz7hxDBmTOhaKiKSj3yDvu5BrSAzaGwM0wkn7Jrf1gYrV8Ljj4eeQn/5CyxaFG4SS+jfP/QWOvzw0FZwyCGhcVlEpBQq6dcId3j77dBT6KmndlUPvfDCrnaC4cPhqKPCNGXKrvFXRERUvVMn3nkHnnkm3B386KPhimDz5vDZwIG7eguNGxempibo3bu6eRaRzqegX6d27gwngccfh7/+NUzPPQfvvx8+79EDRo8OJ4OJE0N7wYQJ0KtXdfMtIpWlOv061a1bCOITJuya19YGq1aFhuKnnw7Tn/4U7igG6NMnnAA+8Qn4+MfDtM8+oY1gn32gX7/q7IuIdD4F/TrQo0fo/TNqFHzxi7vmr18fGokfeyzcR/CHP+zehTThgAPCMBOTJsFnPhOuFNRWIFKfVL0TM9u3wyuvhCElXn89jBHe2gpPPgmvvRbSjBgBxx8fRiI98sjQk0hEapvq9KVga9fCgw/C/feH5w28+26oTjr00DCNGROmpqbwABoRqR0K+lKSHTtCY/Ef/xjaB5YuhW3bdn0+dGioBho6NNxE1qNHqBJK/DuZQc+eu6YePcLUs2doVO7dO7zu1i0s3717mN+r1640iWW7d9+VLrGeHj12zevW7cPrT7cu3ewm9UxBX8rKPYwoumxZeLj88uVh2rIljEPU1rYrrVnoZfTBB7um9vZdrxN3I3e25BNR8okhcRJK/E2cPHr02P3kkzw/eUosl1hHIn23bmGbqdtNnKgSJ63kk2MiTfJJrUePXetPnFyT9ynxPvE6eUrMT06fulw+aVLTp/tus20v9bWUn3rvSFmZhZvDhg8PzxgohXs4SezYEU4O7e27Tgo7doQH1iefMJLTJE4wifmpyyefWJLXs2PHhz9PTvf3v+963dYWpm3bdl8mMT9xEku8TizX3l6e7zoOsp24UudnOkHlOnFlOvllOplle50pr+XO37hxYcj2SlLQl06XXPKtJ8knrsRJyX33k1ziRJfpZJVYPvFZ4oopcWJKvqJKrDv5dfKUmJ+cPnW5fNKkpk+Va3u50mSbnymvufYh0/eQab+yvc6U10rkb//9qTgFfZEySbRN9OlT7ZyIZKZBfUVEYkRBX0QkRhT0RURiREFfRCRGFPRFRGJEQV9EJEYU9EVEYkRBX0QkRmpu7B0z2wisKWEVQ4BNZcpOVxHHfYZ47ncc9xniud+F7vMIdx+aK1HNBf1SmVlrPoMO1ZM47jPEc7/juM8Qz/2u1D6rekdEJEYU9EVEYqQeg/4t1c5AFcRxnyGe+x3HfYZ47ndF9rnu6vRFRCSzeizpi4hIBgr6IiIxUjdB38ymm9kKM1tlZrOrnZ9KMbP9zGyRmT1vZsvM7J+i+YPM7D/NbGX0d2C181puZtbdzP5qZr+L3o80syeiY/4rM+tV7TyWm5kNMLO7zewFM1tuZkfU+7E2s29E/9vPmdldZtanHo+1mf3czN40s+eS5qU9thbcEO3/UjObWOx26yLom1l34CbgOKAJmGVmTdXNVcW0Ad909ybgcOAr0b7OBv7k7gcCf4re15t/ApYnvf8RcJ27HwD8DTinKrmqrJ8AD7r7KOBgwv7X7bE2s32Bi4Fmdx8LdAdmUp/H+hfA9JR5mY7tccCB0XQ+cHOxG62LoA9MAla5+8vuvgOYD5xU5TxVhLtvcPenotfvEILAvoT9vT1KdjvwD9XJYWWY2TDgc8Ct0XsDjgLujpLU4z73Bz4N/DuAu+9w9y3U+bEmPMb1I2bWA2gANlCHx9rdHwbeSpmd6dieBPzSg8eBAWa2dzHbrZegvy/watL7ddG8umZmjcAE4AngY+6+IfrodeBjVcpWpVwP/G9gZ/R+MLDF3ROPCq/HYz4S2AjcFlVr3Wpme1DHx9rdXwOuAdYSgv1WYAn1f6wTMh3bssW4egn6sWNmfYF7gK+7+9vJn3noh1s3fXHN7PPAm+6+pNp56WQ9gInAze4+AXiXlKqcOjzWAwml2pHAPsAefLgKJBYqdWzrJei/BuyX9H5YNK8umVlPQsCf5+7/Ec1+I3G5F/19s1r5q4DJwIlmtppQdXcUoa57QFQFAPV5zNcB69z9iej93YSTQD0f62OAV9x9o7t/APwH4fjX+7FOyHRsyxbj6iXoLwYOjFr4exEafu6rcp4qIqrL/ndgubtfm/TRfcCZ0eszgd90dt4qxd2/7e7D3L2RcGwfcvcWYBEwI0pWV/sM4O6vA6+a2SejWUcDz1PHx5pQrXO4mTVE/+uJfa7rY50k07G9D/hS1IvncGBrUjVQYdy9LibgeOBF4CXg8mrnp4L7+SnCJd9S4OloOp5Qx/0nYCXwR2BQtfNaof0/Evhd9Hp/4ElgFfD/gN7Vzl8F9nc80Bod718DA+v9WAPfB14AngPuAHrX47EG7iK0W3xAuKo7J9OxBYzQQ/El4FlC76aitqthGEREYqReqndERCQPCvoiIjGioC8iEiMK+iIiMaKgLyISIwr6IiIxoqAvIhIj/x9SYcCyafBcYAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot The Training Accuracy\n",
    "\n",
    "accuracy = model_train.history['acc']\n",
    "val_accuracy = model_train.history['val_acc']\n",
    "loss = model_train.history['loss']\n",
    "val_loss = model_train.history['val_loss']\n",
    "epochs = range(len(accuracy))\n",
    "plt.plot(epochs, accuracy, 'bo', label='Training accuracy')\n",
    "plt.plot(epochs, val_accuracy, 'b', label='Validation accuracy')\n",
    "plt.title('Training and validation accuracy')\n",
    "plt.legend()\n",
    "plt.savefig('results/Acc_result_{}.png'.format(string_name))\n",
    "plt.figure()\n",
    "plt.plot(epochs, loss, 'bo', label='Training loss')\n",
    "plt.plot(epochs, val_loss, 'b', label='Validation loss')\n",
    "plt.title('Training and validation loss')\n",
    "plt.legend()\n",
    "plt.savefig('results/Loss_result_{}.png'.format(string_name))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Model Score Summary\n",
    "\n",
    "model.load_weights(\"weight_best.hdf5\")\n",
    "y_pred = np.argmax(model.predict(valid_X), axis=1)\n",
    "y_true = np.argmax(valid_Y, axis = 1)\n",
    "\n",
    "tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()\n",
    "f1 = f1_score(y_true, y_pred, labels=None, pos_label=1, average='binary', sample_weight=None)\n",
    "auc = roc_auc_score(y_true, y_pred, average='macro', sample_weight=None, max_fpr=None)\n",
    "sensi = tp/(tp+fn)\n",
    "specificity = tn/(tn+fp)\n",
    "accu = (tn + tp)/(tn + tp + fn + fp)\n",
    "mcc = ((tp*tn)-(fp*fn))/np.sqrt((tp+fp)*(tp+fn)*(tn+fp)*(tn+fn))\n",
    "\n",
    "print('{} Result'.format(string_name))\n",
    "print('Accuracy :', accu)\n",
    "print('AUC :', auc)\n",
    "print('F1 :', f1)\n",
    "print('Sensitivity :', sensi)\n",
    "print('Specificity :', specificity)\n",
    "print('MCC :', mcc)\n",
    "\n",
    "with open('results/summary_{}.csv'.format(string_name), mode='w') as summary_file:\n",
    "    employee_writer = csv.writer(summary_file, delimiter=',', quotechar='\"', quoting=csv.QUOTE_MINIMAL)\n",
    "    employee_writer.writerow(['Accuracy :', accu])\n",
    "    employee_writer.writerow(['AUC :', auc])\n",
    "    employee_writer.writerow(['F1 :', f1])\n",
    "    employee_writer.writerow(['Sensitivity :', sensi])\n",
    "    employee_writer.writerow(['Specificity :', specificity])\n",
    "    employee_writer.writerow(['MCC :', mcc])\n",
    "    \n",
    "    employee_writer.writerow([''])\n",
    "    \n",
    "    employee_writer.writerow(['TP :', tp])\n",
    "    employee_writer.writerow(['FP :', fp])\n",
    "    employee_writer.writerow(['TN :', tn])\n",
    "    employee_writer.writerow(['FN :', fn])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}