LSTM_High_Dim_timeseries.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Classification of a time series using Keras"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this example, we use time series data for classification. The following tasks are already completed:\n",
    "\n",
    "    1. Data is clean (data gaps imputed, NaN columns deleted)\n",
    "    2. Non-numeric data is encoded into numeric data types.\n",
    "    3. Dimensionality reduction is alredy done.\n",
    "    4. Sequence lengths have been determined through domain knowledge and signal processing."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import some libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np \n",
    "import pandas as pd\n",
    "\n",
    "import keras as K\n",
    "from keras.models import *\n",
    "\n",
    "from keras.layers.core import *\n",
    "from keras.optimizers import RMSprop, Adam\n",
    "from keras.layers import LSTM\n",
    "from keras.models import load_model\n",
    "from keras.callbacks import EarlyStopping\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.wrappers.scikit_learn import KerasRegressor, KerasClassifier\n",
    "\n",
    "\n",
    "# from sklearn.preprocessing import MinMaxScaler\n",
    "# from sklearn.metrics import mean_squared_error\n",
    "import matplotlib.pyplot as plt\n",
    "import pickle\n",
    "\n",
    "np.random.seed(500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.7/dist-packages/sklearn/externals/joblib/__init__.py:15: DeprecationWarning: sklearn.externals.joblib is deprecated in 0.21 and will be removed in 0.23. Please import this functionality directly from joblib, which can be installed with: pip install joblib. If this warning is raised when loading pickled models, you may need to re-serialize those models with scikit-learn 0.21+.\n",
      "  warnings.warn(msg, category=DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import expon, randint\n",
    "from keras.utils import np_utils\n",
    "from sklearn.utils.testing import ignore_warnings\n",
    "from sklearn.exceptions import ConvergenceWarning\n",
    "from sklearn.externals import joblib\n",
    "from sklearn.metrics import confusion_matrix\n",
    "from sklearn.model_selection import train_test_split, GridSearchCV, RandomizedSearchCV\n",
    "\n",
    "from sklearn.model_selection import learning_curve\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Read the data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read files\n",
    "X_train_df = pd.read_parquet('X_train_df.parquet')\n",
    "X_test_df = pd.read_parquet('X_test_df.parquet')\n",
    "y_train_df = pd.read_parquet('y_train_df.parquet')\n",
    "y_test_df = pd.read_parquet('y_test_df.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(214171, 448)\n",
      "(91805, 448)\n",
      "(3511, 1)\n",
      "(1505, 1)\n"
     ]
    }
   ],
   "source": [
    "print(X_train_df.shape)\n",
    "print(X_test_df.shape)\n",
    "print(y_train_df.shape)\n",
    "print(y_test_df.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Convert input to 3D tensor for LSTM. If you are using a CNN, remember to embed the input accordingly. \n",
    "### We'll have a CNN in a later post."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3511, 61, 448)\n",
      "(1505, 61, 448)\n",
      "(3511, 9)\n",
      "(1505, 9)\n"
     ]
    }
   ],
   "source": [
    "# reshape the data for LSTM\n",
    "time_steps = 61  # sequence length\n",
    "num_classes= 9\n",
    "X_train = np.reshape(X_train_df.values, (int(X_train_df.shape[0]/time_steps), time_steps, X_train_df.shape[1]))\n",
    "X_test = np.reshape(X_test_df.values, (int(X_test_df.shape[0]/time_steps), time_steps, X_test_df.shape[1]))\n",
    "\n",
    "y_train = np_utils.to_categorical(y_train_df, num_classes, dtype='float32')\n",
    "y_test = np_utils.to_categorical(y_test_df, num_classes, dtype='float32')\n",
    "print(X_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_train.shape)\n",
    "print(y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Build a keras model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.utils import plot_model\n",
    "from keras.models import Model\n",
    "from keras.layers import Input\n",
    "from keras.layers import Dense\n",
    "from keras.layers.recurrent import LSTM\n",
    "\n",
    "\n",
    "def build_model_lstm(time_steps, feats_dim, num_classes, optimizer='rmsprop', \n",
    "        init='glorot_uniform'):\n",
    "    x_in = Input(shape=(time_steps, feats_dim))\n",
    "    h1 = LSTM(128,return_sequences=True)(x_in)\n",
    "    h1 = Dropout(0.5)(h1)\n",
    "    h2 = LSTM(64,return_sequences=True)(h1)\n",
    "    h2 = Dropout(0.5)(h2)\n",
    "    h3 = LSTM(32,return_sequences=True)(h2)\n",
    "    h4 = LSTM(32)(h3)\n",
    "    h4 = Dropout(0.5)(h4)\n",
    "    out = Dense(num_classes, kernel_initializer=init, \n",
    "                   activation='softmax')(h4)\n",
    "    model = Model(inputs=x_in, outputs=out)\n",
    "    model.compile(optimizer=optimizer,\n",
    "                  loss='categorical_crossentropy',\n",
    "                  metrics=['accuracy'])\n",
    "    return model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, X_train, y_train, batch_size = 32, \n",
    "        epochs=2000, shuffle = True):\n",
    "    es = EarlyStopping(monitor='val_loss', verbose=1, patience=5, min_delta=10e-6)\n",
    "    mc = ModelCheckpoint('best_model_LSTM_example.h5', monitor='val_loss', verbose=1, save_best_only=True)\n",
    "    \n",
    "    if len(X_train) % 2 == 1:  # if len(X_train) is odd\n",
    "        X_train = X_train[:-1,:]\n",
    "        y_train = y_train[:-1,:]\n",
    "    x_val = X_train[:int(len(X_train)*0.2)]\n",
    "    y_val = y_train[:int(len(X_train)*0.2)]\n",
    "    \n",
    "    \n",
    "    \n",
    "    history = model.fit(X_train, y_train, batch_size, epochs, shuffle, validation_data=(x_val, y_val) ,callbacks=[es, mc])\n",
    "    \n",
    "    plt.plot(history.history['loss'])\n",
    "    plt.title('model loss')\n",
    "    plt.ylabel('loss')\n",
    "    plt.xlabel('epoch')\n",
    "    plt.legend(['train', 'test'], loc='upper left')\n",
    "    plt.show()\n",
    "    trained_model = model\n",
    "    return trained_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/keras/backend/tensorflow_backend.py:3445: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_1 (InputLayer)         (None, 61, 448)           0         \n",
      "_________________________________________________________________\n",
      "lstm_1 (LSTM)                (None, 61, 128)           295424    \n",
      "_________________________________________________________________\n",
      "dropout_1 (Dropout)          (None, 61, 128)           0         \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (None, 61, 64)            49408     \n",
      "_________________________________________________________________\n",
      "dropout_2 (Dropout)          (None, 61, 64)            0         \n",
      "_________________________________________________________________\n",
      "lstm_3 (LSTM)                (None, 61, 32)            12416     \n",
      "_________________________________________________________________\n",
      "lstm_4 (LSTM)                (None, 32)                8320      \n",
      "_________________________________________________________________\n",
      "dropout_3 (Dropout)          (None, 32)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 9)                 297       \n",
      "=================================================================\n",
      "Total params: 365,865\n",
      "Trainable params: 365,865\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "feats_dim = X_train.shape[2]\n",
    "model = build_model_lstm(time_steps, feats_dim, num_classes)\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/math_ops.py:3066: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "Train on 3510 samples, validate on 702 samples\n",
      "Epoch 1/200\n",
      "3510/3510 [==============================] - 42s 12ms/step - loss: 0.8590 - acc: 0.7957 - val_loss: 0.4433 - val_acc: 0.8960\n",
      "\n",
      "Epoch 00001: val_loss improved from inf to 0.44334, saving model to best_model_LSTM_example.h5\n",
      "Epoch 2/200\n",
      "3510/3510 [==============================] - 38s 11ms/step - loss: 0.4538 - acc: 0.8912 - val_loss: 0.3318 - val_acc: 0.9103\n",
      "\n",
      "Epoch 00002: val_loss improved from 0.44334 to 0.33176, saving model to best_model_LSTM_example.h5\n",
      "Epoch 3/200\n",
      "3510/3510 [==============================] - 40s 11ms/step - loss: 0.3714 - acc: 0.9097 - val_loss: 0.2765 - val_acc: 0.9274\n",
      "\n",
      "Epoch 00003: val_loss improved from 0.33176 to 0.27654, saving model to best_model_LSTM_example.h5\n",
      "Epoch 4/200\n",
      "3510/3510 [==============================] - 39s 11ms/step - loss: 0.3000 - acc: 0.9259 - val_loss: 0.2433 - val_acc: 0.9444\n",
      "\n",
      "Epoch 00004: val_loss improved from 0.27654 to 0.24328, saving model to best_model_LSTM_example.h5\n",
      "Epoch 5/200\n",
      "3510/3510 [==============================] - 39s 11ms/step - loss: 0.2754 - acc: 0.9373 - val_loss: 0.2419 - val_acc: 0.9473\n",
      "\n",
      "Epoch 00005: val_loss improved from 0.24328 to 0.24191, saving model to best_model_LSTM_example.h5\n",
      "Epoch 6/200\n",
      "3510/3510 [==============================] - 38s 11ms/step - loss: 0.2311 - acc: 0.9453 - val_loss: 0.1605 - val_acc: 0.9615\n",
      "\n",
      "Epoch 00006: val_loss improved from 0.24191 to 0.16054, saving model to best_model_LSTM_example.h5\n",
      "Epoch 7/200\n",
      "3510/3510 [==============================] - 40s 11ms/step - loss: 0.2060 - acc: 0.9519 - val_loss: 0.1877 - val_acc: 0.9416\n",
      "\n",
      "Epoch 00007: val_loss did not improve from 0.16054\n",
      "Epoch 8/200\n",
      "3510/3510 [==============================] - 40s 11ms/step - loss: 0.1943 - acc: 0.9507 - val_loss: 0.1820 - val_acc: 0.9501\n",
      "\n",
      "Epoch 00008: val_loss did not improve from 0.16054\n",
      "Epoch 9/200\n",
      "3510/3510 [==============================] - 40s 11ms/step - loss: 0.1888 - acc: 0.9556 - val_loss: 0.1321 - val_acc: 0.9644\n",
      "\n",
      "Epoch 00009: val_loss improved from 0.16054 to 0.13209, saving model to best_model_LSTM_example.h5\n",
      "Epoch 10/200\n",
      "3510/3510 [==============================] - 40s 11ms/step - loss: 0.1655 - acc: 0.9607 - val_loss: 0.1130 - val_acc: 0.9658\n",
      "\n",
      "Epoch 00010: val_loss improved from 0.13209 to 0.11299, saving model to best_model_LSTM_example.h5\n",
      "Epoch 11/200\n",
      "3510/3510 [==============================] - 41s 12ms/step - loss: 0.1677 - acc: 0.9615 - val_loss: 0.1347 - val_acc: 0.9658\n",
      "\n",
      "Epoch 00011: val_loss did not improve from 0.11299\n",
      "Epoch 12/200\n",
      "3510/3510 [==============================] - 40s 12ms/step - loss: 0.1579 - acc: 0.9618 - val_loss: 0.1033 - val_acc: 0.9729\n",
      "\n",
      "Epoch 00012: val_loss improved from 0.11299 to 0.10333, saving model to best_model_LSTM_example.h5\n",
      "Epoch 13/200\n",
      "3510/3510 [==============================] - 39s 11ms/step - loss: 0.1421 - acc: 0.9650 - val_loss: 0.0851 - val_acc: 0.9744\n",
      "\n",
      "Epoch 00013: val_loss improved from 0.10333 to 0.08513, saving model to best_model_LSTM_example.h5\n",
      "Epoch 14/200\n",
      "3510/3510 [==============================] - 38s 11ms/step - loss: 0.1270 - acc: 0.9647 - val_loss: 0.0813 - val_acc: 0.9758\n",
      "\n",
      "Epoch 00014: val_loss improved from 0.08513 to 0.08130, saving model to best_model_LSTM_example.h5\n",
      "Epoch 15/200\n",
      "3510/3510 [==============================] - 38s 11ms/step - loss: 0.1304 - acc: 0.9670 - val_loss: 0.0797 - val_acc: 0.9772\n",
      "\n",
      "Epoch 00015: val_loss improved from 0.08130 to 0.07975, saving model to best_model_LSTM_example.h5\n",
      "Epoch 16/200\n",
      "3510/3510 [==============================] - 38s 11ms/step - loss: 0.1277 - acc: 0.9695 - val_loss: 0.0803 - val_acc: 0.9744\n",
      "\n",
      "Epoch 00016: val_loss did not improve from 0.07975\n",
      "Epoch 17/200\n",
      "3510/3510 [==============================] - 38s 11ms/step - loss: 0.1224 - acc: 0.9701 - val_loss: 0.0763 - val_acc: 0.9758\n",
      "\n",
      "Epoch 00017: val_loss improved from 0.07975 to 0.07625, saving model to best_model_LSTM_example.h5\n",
      "Epoch 18/200\n",
      "3510/3510 [==============================] - 38s 11ms/step - loss: 0.1077 - acc: 0.9718 - val_loss: 0.0622 - val_acc: 0.9786\n",
      "\n",
      "Epoch 00018: val_loss improved from 0.07625 to 0.06216, saving model to best_model_LSTM_example.h5\n",
      "Epoch 19/200\n",
      "3510/3510 [==============================] - 38s 11ms/step - loss: 0.1129 - acc: 0.9726 - val_loss: 0.0766 - val_acc: 0.9758\n",
      "\n",
      "Epoch 00019: val_loss did not improve from 0.06216\n",
      "Epoch 20/200\n",
      "3510/3510 [==============================] - 38s 11ms/step - loss: 0.0964 - acc: 0.9761 - val_loss: 0.0728 - val_acc: 0.9843\n",
      "\n",
      "Epoch 00020: val_loss did not improve from 0.06216\n",
      "Epoch 21/200\n",
      "3510/3510 [==============================] - 38s 11ms/step - loss: 0.0978 - acc: 0.9726 - val_loss: 0.0504 - val_acc: 0.9829\n",
      "\n",
      "Epoch 00021: val_loss improved from 0.06216 to 0.05043, saving model to best_model_LSTM_example.h5\n",
      "Epoch 22/200\n",
      "3510/3510 [==============================] - 38s 11ms/step - loss: 0.0843 - acc: 0.9783 - val_loss: 0.0384 - val_acc: 0.9886\n",
      "\n",
      "Epoch 00022: val_loss improved from 0.05043 to 0.03841, saving model to best_model_LSTM_example.h5\n",
      "Epoch 23/200\n",
      "3510/3510 [==============================] - 38s 11ms/step - loss: 0.0996 - acc: 0.9744 - val_loss: 0.0485 - val_acc: 0.9858\n",
      "\n",
      "Epoch 00023: val_loss did not improve from 0.03841\n",
      "Epoch 24/200\n",
      "3510/3510 [==============================] - 38s 11ms/step - loss: 0.0810 - acc: 0.9778 - val_loss: 0.0574 - val_acc: 0.9858\n",
      "\n",
      "Epoch 00024: val_loss did not improve from 0.03841\n",
      "Epoch 25/200\n",
      "3510/3510 [==============================] - 38s 11ms/step - loss: 0.0802 - acc: 0.9778 - val_loss: 0.0826 - val_acc: 0.9815\n",
      "\n",
      "Epoch 00025: val_loss did not improve from 0.03841\n",
      "Epoch 26/200\n",
      "3510/3510 [==============================] - 38s 11ms/step - loss: 0.1068 - acc: 0.9744 - val_loss: 0.0470 - val_acc: 0.9858\n",
      "\n",
      "Epoch 00026: val_loss did not improve from 0.03841\n",
      "Epoch 27/200\n",
      "3510/3510 [==============================] - 38s 11ms/step - loss: 0.0761 - acc: 0.9792 - val_loss: 0.0833 - val_acc: 0.9715\n",
      "\n",
      "Epoch 00027: val_loss did not improve from 0.03841\n",
      "Epoch 00027: early stopping\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvnQurowAAIABJREFUeJzt3Xl8XHW9//HXJ5M9TZM2Tdss3SilGy1FSgFRLLLYFlm0igVBUa/V3wVFUa7listFuXLxXteLCigP0SubLFKgWAEpi2wtWEr3llLapFuaNl2yL5/fH3MSpmmapm1OJpl5Px+PmDnLzHyOQ+ed8/2e8/2auyMiIgKQEu8CRESk91AoiIhIG4WCiIi0USiIiEgbhYKIiLRRKIiISBuFgkgXmdnvzeyHXdx3o5mde6yvI9LTFAoiItJGoSAiIm0UCpJQgmab681smZlVm9nvzGyImT1pZvvM7GkzGxCz/0VmtsLMqsxskZmNj9l2spm9ETzvfiCz3Xt91MyWBs99ycwmH2XNXzSz9Wa2y8zmm1lxsN7M7KdmtsPM9prZW2Z2YrBtlpmtDGorN7NvHtX/YSLtKBQkEc0GzgNOAC4EngT+HSgk+t/8VwHM7ATgXuBrwbYFwGNmlm5m6cBfgD8CA4E/B69L8NyTgbuALwEFwO3AfDPLOJJCzezDwI+AS4Ei4F3gvmDz+cBZwXHkBftUBtt+B3zJ3XOBE4G/H8n7ihyKQkES0S/dfbu7lwMvAK+6+z/dvQ54BDg52O9TwBPu/pS7NwL/DWQB7wdOB9KAn7l7o7s/CCyOeY+5wO3u/qq7N7v73UB98Lwj8WngLnd/w93rgRuAM8xsJNAI5ALjAHP3Ve6+NXheIzDBzPq7+253f+MI31ekQwoFSUTbYx7XdrDcL3hcTPQvcwDcvQXYDJQE28r9wBEj3415PAL4RtB0VGVmVcCw4HlHon0N+4meDZS4+9+B/wVuA3aY2R1m1j/YdTYwC3jXzJ4zszOO8H1FOqRQkGS2heiXOxBtwyf6xV4ObAVKgnWthsc83gzc7O75MT/Z7n7vMdaQQ7Q5qhzA3X/h7qcAE4g2I10frF/s7hcDg4k2cz1whO8r0iGFgiSzB4ALzOwcM0sDvkG0Cegl4GWgCfiqmaWZ2ceBaTHPvRP4spmdFnQI55jZBWaWe4Q13At8zsymBP0R/0m0uWujmZ0avH4aUA3UAS1Bn8enzSwvaPbaC7Qcw/8PIm0UCpK03H0NcAXwS2An0U7pC929wd0bgI8DVwG7iPY/PBzz3CXAF4k27+wG1gf7HmkNTwPfAR4ienYyGpgTbO5PNHx2E21iqgR+HGy7EthoZnuBLxPtmxA5ZqZJdkREpJXOFEREpE2ooWBmM8xsTXBjzrwOto8ws2eCG40WmVlpmPWIiEjnQms+MrMIsJboTURlRK/xvszdV8bs82fgcXe/O7iJ53PufmUoBYmIyGGFeaYwDVjv7huCTrv7gIvb7TOB9+7EfLaD7SIi0oNSQ3ztEqLXcrcqA05rt8+bRK/w+DnwMSDXzArcvTJ2JzObS/QOUnJyck4ZN25caEWLiCSi119/fae7Fx5uvzBDoSu+CfyvmV0FPE/0hp3m9ju5+x3AHQBTp071JUuW9GSNIiJ9npm9e/i9wg2FcqJ3h7YqDda1cfctRM8UMLN+wGx3rwqxJhER6USYfQqLgTFmNioYcXIOMD92BzMbZGatNdxAdNRJERGJk9BCwd2bgGuAhcAq4AF3X2FmN5nZRcFu04E1ZrYWGALcHFY9IiJyeH3ujuaO+hQaGxspKyujrq4uTlX1jMzMTEpLS0lLS4t3KSLSx5jZ6+4+9XD7xbujuVuUlZWRm5vLyJEjOXBQy8Th7lRWVlJWVsaoUaPiXY6IJKiEGOairq6OgoKChA0EADOjoKAg4c+GRCS+EiIUgIQOhFbJcIwiEl8JEwqHU13fxNY9tfS1PhQRkZ6UNKFQ09BMxb56mlu6PxSqqqr41a9+dcTPmzVrFlVVui1DRHqPpAmF9Ei06aWxufsnqDpUKDQ1NXX6vAULFpCfn9/t9YiIHK2EuPqoK9JSo/nX2OxkdfNrz5s3j7fffpspU6aQlpZGZmYmAwYMYPXq1axdu5ZLLrmEzZs3U1dXx7XXXsvcuXMBGDlyJEuWLGH//v3MnDmTD3zgA7z00kuUlJTw6KOPkpXV3ZWKiHQu4ULhPx5bwcotew9a70BNfRPpqSmkRY7sBGlCcX++d+HEQ26/5ZZbWL58OUuXLmXRokVccMEFLF++vO3S0bvuuouBAwdSW1vLqaeeyuzZsykoKDjgNdatW8e9997LnXfeyaWXXspDDz3EFVdccUR1iogcq4QLhUOx4H96opt52rRpB9xL8Itf/IJHHnkEgM2bN7Nu3bqDQmHUqFFMmTIFgFNOOYWNGzf2QKUiIgdKuFDo7C/61dv2kp2WyvCC7FBryMnJaXu8aNEinn76aV5++WWys7OZPn16h/caZGRktD2ORCLU1taGWqOISEeSpqMZIC2SEkpHc25uLvv27etw2549exgwYADZ2dmsXr2aV155pdvfX0SkuyTcmUJn0iMpVNd3fkXQ0SgoKODMM8/kxBNPJCsriyFDhrRtmzFjBr/5zW8YP348Y8eO5fTTT+/29xcR6S4JMSDeqlWrGD9+/GGfu21PLRX7GjixpH+fvTu4q8cqIhKrqwPiJV3zkeM0NfetIBQR6SlJFwoADSH0K4iIJIKECYWuNIO1hkIYnc09oa819YlI3xNqKJjZDDNbY2brzWxeB9uHm9mzZvZPM1tmZrOO5n0yMzOprKw87JdmWmrrUBd978u1dT6FzMzMeJciIgkstKuPzCwC3AacB5QBi81svruvjNntRqLTdP7azCYAC4CRR/pepaWllJWVUVFRcdh9K6pqqd6eys7svjd7WevMayIiYQnzktRpwHp33wBgZvcBFwOxoeBA/+BxHrDlaN4oLS2ty7ORfeUnzzG6MIfbr5x8NG8lIpLQwmw+KgE2xyyXBetifR+4wszKiJ4lfKWjFzKzuWa2xMyWdOVsoDPF+Vls3aPZy0REOhLvjubLgN+7eykwC/ijmR1Uk7vf4e5T3X1qYWHhMb1hcV4mW6oUCiIiHQkzFMqBYTHLpcG6WF8AHgBw95eBTGBQiDVRlJfFzv311Dc1h/k2IiJ9UpihsBgYY2ajzCwdmAPMb7fPJuAcADMbTzQUjq196DCK8qNX72xTE5KIyEFCCwV3bwKuARYCq4heZbTCzG4ys4uC3b4BfNHM3gTuBa7ykC/GL8mPTlyjJiQRkYOFOiCeuy8g2oEcu+67MY9XAmeGWUN7RXnRM4UtVRqaWkSkvXh3NPe4orzomcLWPQoFEZH2ki4UstIjDMxJZ4v6FEREDpJ0oQDRJqStaj4SETlIkoZCljqaRUQ6kJShUJyfyRb1KYiIHCRJQyGLfXVN7KtrjHcpIiK9SlKGQutlqRoDSUTkQEkZCsVtN7CpCUlEJFZSh4LOFEREDpSUoTAkN4MUQ5elioi0k5ShkBpJYXBuJuW6LFVE5ABJGQoQvSxVQ12IiBwoaUOhSDOwiYgcJGlDIToDWy0hj9QtItKnJG0oFOVlUd/Uwq7qhniXIiLSayRtKOiyVBGRg4UaCmY2w8zWmNl6M5vXwfafmtnS4GetmVWFWU+s4nxNtiMi0l5oM6+ZWQS4DTgPKAMWm9n8YLY1ANz96zH7fwU4Oax62mudbEehICLynjDPFKYB6919g7s3APcBF3ey/2VE52nuEQU56aSnpqj5SEQkRpihUAJsjlkuC9YdxMxGAKOAvx9i+1wzW2JmSyoqKrqluJQUoygvUzOwiYjE6C0dzXOAB929uaON7n6Hu09196mFhYXd9qZFwWWpIiISFWYolAPDYpZLg3UdmUMPNh21Ks7P0vhHIiIxwgyFxcAYMxtlZulEv/jnt9/JzMYBA4CXQ6ylQ8V5WWzfV09Tc0tPv7WISK8UWii4exNwDbAQWAU84O4rzOwmM7soZtc5wH0eh1uLi/IzaW5xduyr7+m3FhHplUK7JBXA3RcAC9qt+2675e+HWUNnivNab2CrbbuZTUQkmfWWjua4eG8GNl2BJCICSR4KRfmtczWrs1lEBJI8FPpnptEvI1VnCiIigaQOBYiOgaR7FUREopI+FIryNNmOiEirpA8FnSmIiLxHoZCXRWV1A3WNHY6wISKSVJI+FIqCy1K3qQlJREShUJwXTLajy1JFRBQKRbqBTUSkjUIhOFPQaKkiIgoFMtMiFOSka7IdEREUCkB0uAtdlioiolAAopelavwjERGFAtA6A5uaj0REFApEO5v31Text64x3qWIiMRVqKFgZjPMbI2ZrTezeYfY51IzW2lmK8zsnjDrOZTWy1J1tiAiyS60mdfMLALcBpwHlAGLzWy+u6+M2WcMcANwprvvNrPBYdXTmZL8925gGzs0Nx4liIj0CmGeKUwD1rv7BndvAO4DLm63zxeB29x9N4C77wixnkMqytOZgogIhBsKJcDmmOWyYF2sE4ATzOwfZvaKmc0IsZ5DGpybQYqhy1JFJOmF1nx0BO8/BpgOlALPm9kkd6+K3cnM5gJzAYYPH979RURSGNo/U+MfiUjSC/NMoRwYFrNcGqyLVQbMd/dGd38HWEs0JA7g7ne4+1R3n1pYWBhKsUW6LFVEJNRQWAyMMbNRZpYOzAHmt9vnL0TPEjCzQUSbkzaEWNMhFeXpTEFEJLRQcPcm4BpgIbAKeMDdV5jZTWZ2UbDbQqDSzFYCzwLXu3tlWDV1piQ/Oi2nu8fj7UVEeoVQ+xTcfQGwoN2678Y8duC64CeuivIyaWhqobK6gUH9MuJdjohIXOiO5oBuYBMRUSi0KQ7uVSjXZakiksQUCoHi4K5mjZYqIslMoRAYmJNORmoKWzXZjogkMYVCwMwoystU85GIJDWFQozovAoKBRFJXgqFGEV5WWo+EpGkplCIUZyfyfa9dTQ1t8S7FBGRuFAoxCjOz6LFYfu++niXIiISFwqFGEV5wWWp6lcQkSSlUIhRHNzVvEX9CiKSpBQKMVrPFDTZjogkK4VCjNzMNHIzU9V8JCJJS6HQTnFelpqPRCRpKRTaKcrP1PhHIpK0FArtFOdnsUXDZ4tIklIotFOcl8mu6gbqGpvjXYqISI8LNRTMbIaZrTGz9WY2r4PtV5lZhZktDX7+Jcx6uqIomFdBw12ISDIKbTpOM4sAtwHnAWXAYjOb7+4r2+16v7tfE1YdR6rtXoWqWkYNyolzNSIiPSvMM4VpwHp33+DuDcB9wMUhvl+3aJ1sR/cqiEgyCjMUSoDNMctlwbr2ZpvZMjN70MyGdfRCZjbXzJaY2ZKKioowam0ztHWoCzUfiUgSindH82PASHefDDwF3N3RTu5+h7tPdfephYWFoRaUkRphUL90nSmISFLqUiiY2bVm1t+ifmdmb5jZ+Yd5WjkQ+5d/abCujbtXunvrkKS/BU7pauFhKs7XDWwikpy6eqbweXffC5wPDACuBG45zHMWA2PMbJSZpQNzgPmxO5hZUcziRcCqLtYTqqK8TA11ISJJqatXH1nwexbwR3dfYWbW2RPcvcnMrgEWAhHgruB5NwFL3H0+8FUzuwhoAnYBVx3NQXS3orwsXly3E3fnMIcpIpJQuhoKr5vZ34BRwA1mlgscdnoyd18ALGi37rsxj28Abuh6uT2jJD+L6oZm9tY1kZeVFu9yRER6TFdD4QvAFGCDu9eY2UDgc+GVFV9F+a1XINUqFEQkqXS1T+EMYI27V5nZFcCNwJ7wyoqvtruaNQaSiCSZrobCr4EaMzsJ+AbwNvCH0KqKs9Yb2MrV2SwiSaarodDk7k70juT/dffbgNzwyoqvwbmZRFJMQ2iLSNLpap/CPjO7geilqB80sxQgYRvbIynG0P6Zaj4SkaTT1TOFTwH1RO9X2Eb0RrQfh1ZVLzBsYBZLy6pobvF4lyIi0mO6FApBEPwJyDOzjwJ17p6wfQoAnz1jJBsqqnlgyebD7ywikiC6OszFpcBrwCeBS4FXzewTYRYWbzNOHMqpIwfwP39bw/76pniXIyLSI7rafPRt4FR3/6y7f4bosNjfCa+s+DMzbrxgAjv3N/DrRevjXY6ISI/oaiikuPuOmOXKI3hun3XSsHw+dnIJd77wDmW7a+JdjohI6Lr6xf5XM1sYTJ95FfAE7YavSFTXf2QsKQa3/nVNvEsREQldVzuarwfuACYHP3e4+7fCLKy3KM7PYu4Hj2P+m1t4Y9PueJcjIhKqLjcBuftD7n5d8PNImEX1Nl/60GgKczP44eMrid7DJyKSmDoNBTPbZ2Z7O/jZZ2Z7e6rIeMvJSOX688fyxqYqnnhra7zLEREJTaeh4O657t6/g59cd+/fU0X2BrNPKWV8UX9ueXI1dY3N8S5HRCQUCX8FUXeJpBg3XjCest21/P6ljfEuR0QkFKGGgpnNMLM1ZrbezOZ1st9sM3MzmxpmPcfqzOMHce74wdz29/Xs3F9/+CeIiPQxoYWCmUWA24CZwATgMjOb0MF+ucC1wKth1dKdbpg1ntrGZn729Np4lyIi0u3CPFOYBqx39w3u3gDcR3To7fZ+APwX0CeGJB1d2I8rTh/BPa9uYu32ffEuR0SkW4UZCiVA7GhyZcG6Nmb2PmCYuz/R2QuZ2VwzW2JmSyoqKrq/0iN07Tlj6JeRyn8uWBXvUkREulXcOpqDORl+QnQmt065+x3uPtXdpxYWFoZf3GEMyEnnq+eMYdGaCp5bG/+QEhHpLmGGQjkwLGa5NFjXKhc4EVhkZhuB04H5vb2zudWVZ4xgREE2Nz+xkqbmlniXIyLSLcIMhcXAGDMbZWbpwBxgfutGd9/j7oPcfaS7jwReAS5y9yUh1tRtMlIj3DBzHGu37+eBJWXxLkdEpFuEFgru3gRcAywEVgEPuPsKM7vJzC4K63170kcmDmXayIH85Kk17KtrjHc5IiLHLNQ+BXdf4O4nuPtod785WPddd5/fwb7T+8pZQisz48aPjmfn/gZ+tejteJcjInLMdEfzMZpcms/HTy7hdy++w+ZdmnNBRPo2hUI3uH7GWCJmfG/+Co2iKiJ9mkKhGxTlZfGN80/g76t3MP/NLfEuR0TkqCkUusnnzhzFScPy+Y/HVlKpcZFEpI9SKHSTSIpx6+zJ7Ktr5KbHV8a7HBGRo6JQ6EZjh+Zy9dnH8+jSLTyzanu8yxEROWIKhW72r9OPZ+yQXL79yHL26t4FEeljFArdLD01hf/6xGR27KvjlidXx7scEZEjolAIwZRh+XzhA6O459VNvPx2ZbzLERHpMoVCSK47byzDB2Zzw8PLqG3QnM4i0jcoFEKSlR7hltmT2FhZo1naRKTPUCiE6P2jB3HZtGHc+cIGlpVVxbscEZHDUiiEbN7M8RTmZvBvDy6joUnzLohI76ZQCFleVho/vGQSq7ft4zfPaSRVEendFAo94LwJQ7jwpGJ++fd1rNu+L97liIgckkKhh3zvwgn0y0jl3x5aRnOLRlIVkd4p1FAwsxlmtsbM1pvZvA62f9nM3jKzpWb2oplNCLOeeBrUL4PvXTiRf26q4u6XNsa7HBGRDoUWCmYWAW4DZgITgMs6+NK/x90nufsU4FbgJ2HV0xtcPKWYs8cW8uOFazQhj4j0SmGeKUwD1rv7BndvAO4DLo7dwd33xizmAAndrmJm3PyxSaQYzHtYVyOJSO8TZiiUAJtjlsuCdQcws6vN7G2iZwpf7eiFzGyumS0xsyUVFRWhFNtTivOz+PYFE/jH+ko+/D+LePiNMvUxiEivEfeOZne/zd1HA98CbjzEPne4+1R3n1pYWNizBYbg8tOG84fPTyM/O43rHniTmT9/nr+t2KapPEUk7sIMhXJgWMxyabDuUO4DLgmxnl7lrBMKmX/1B7jt8vfR1OzM/ePrfPzXL2kAPRGJqzBDYTEwxsxGmVk6MAeYH7uDmY2JWbwAWBdiPb1OSopxweQi/vb1s7jl45PYWlXHZXe+wmfueo3l5XviXZ6IJKHQQsHdm4BrgIXAKuABd19hZjeZ2UXBbteY2QozWwpcB3w2rHp6s9RICnOmDWfR9dP59qzxLCur4qO/fJGr//QGb1fsj3d5IpJErK+1Y0+dOtWXLFkS7zJCtbeukd8+v4HfvvgO9U0tfPKUUq49dwxFeVnxLk1E+igze93dpx5uv7h3NMvB+memcd35Y3n+387mytNH8PAb5cz8+QsaaVVEQqdQ6MUG9cvg+xdNZOHXz6JfRiqX3/kqizfuindZIpLAFAp9wKhBOfz5y2cwODeDz/zuNV5ctzPeJYlIglIo9BFFeVnc/6UzGFGQzefvXswzq7bHuyQRSUAKhT6kMDeDe794OuOG5vKlP77OE8u2xrskEUkwCoU+ZkBOOv/3L6dx8vB8vnLvGzz4elm8SxKRBKJQ6IP6Z6Zx9+en8f7Rg/jmn9/kj6+8G++SRCRBKBT6qOz0VH772amcM24w3/nLcu58fkO8SxKRBKBQ6MMy0yL85spTuGByETcvWMXPn16nQfVE5JikxrsAOTZpkRR+MedkstIi/PTptdQ0NjFvxjjMLN6liUgfpFBIAJEU49bZk8lKi3D7cxuobWjm+xdOJCVFwSAiR0ahkCBSUoybLp5IVnqEO57fwIaKam6YNY6JxXnxLk1E+hD1KSQQM+OGmeP4wcUTWb5lDx/95Yt8/f6lmg9aRLpMo6QmqD21jfzmube568V3cIfPnDGCq88+ngE56fEuTUTioKujpCoUEtzWPbX89Km1PPh6GTkZqfy/6aP5/JmjyEyLxLs0EelBCgU5wJpt+7j1r6t5ZvUOhvbP5LrzTmD2KaVE1BktkhR6xXwKZjbDzNaY2Xozm9fB9uvMbKWZLTOzZ8xsRJj1JLOxQ3P53VWncv/c0xmal8m/PbSMGT97nqdXbte9DSLSJrQzBTOLAGuB84AyonM2X+buK2P2ORt41d1rzOz/AdPd/VOdva7OFI6du/PX5du4deEa3tlZzakjB3DhScWcflwBYwb30z0OIgmoq2cKYV6SOg1Y7+4bgoLuAy4G2kLB3Z+N2f8V4IoQ65GAmTFzUhHnThjCfYs3c/tzb/PdR1cAUJCTzunHFXD6cQM5Y3QBowsVEiLJJMxQKAE2xyyXAad1sv8XgCdDrEfaSYukcOXpI7jitOFs3lXLKxsqeWVDJS9vqOSJt6LDcg/ql85pxxVwxnEFnH5cAaMLcxQSIgmsV9y8ZmZXAFOBDx1i+1xgLsDw4cN7sLLkYGYML8hmeEE2l546DHdn066aICR28fLblW1zNwzql8EpI/IZNiCb4vwsSgZkUZKfRemALPKy0hQYIn1cmKFQDgyLWS4N1h3AzM4Fvg18yN3rO3ohd78DuAOifQrdX6rEMjNGFOQwoiCHT506HHfn3cqatjOJt8r38NzaCuoaWw54Xk56pC0oivPfC4upIwdSkp8Vp6MRkSMRZigsBsaY2SiiYTAHuDx2BzM7GbgdmOHuO0KsRY6BmTFyUA4jB+UwZ1r0TM3d2V3TSPnuWsqraijbXcuWqjrKq2oor6rlzc1V7K5pBCA1xfjk1FL+dfrxDBuYHc9DEZHDCC0U3L3JzK4BFgIR4C53X2FmNwFL3H0+8GOgH/DnoNlhk7tfFFZN0n3MjIE56QzMSWdSacfjK9U0NLFpVw33vLqJ+17bzJ+XlPHJqcO4+uzRlA5QOIj0Rrp5TXrE1j21/HrR29z32mYc55NTh/Gv0xUOIj1FdzRLr7SlKhoO9y9+LxyuPvt49TmIhEyhIL3alqpafrVoPfcvjl61fGkQDsUKB5FQKBSkTyivquVXz67ngSXRcPjEKcMYPjCbmoYmquubo78bmqmpb6K6oYmahmaq66PbqhuaAJhQ1J8pw/I5KfgpzsvUpbEi7SgUpE+JDYfGZifFICc9leyMSNvv7PRUctIjZGek0i9Y19TsLN+yhxVb9tLQFL1EdlC/dE4qfS8kTirNIz9bQ4ZLclMoSJ9U29AMQGZayhH9td/Q1MKabftYWlbFm5ujP+sr9tP6n/fIgmxOGpbPtFEDmTFxKAX9MsIoX6TXUihI0ttX18hb5Xt4c/Me3txcxdLNVWzbW0ckxXj/6AIuPKmYj0wcSl5WWrxLFQmdQkGkHXdn1dZ9PL5sC48t28LmXbWkRYwPnVDIRycXc+6EIfTL6BUjv4h0O4WCSCfcnWVle3jszS08vmwr2/bWkZGawofHDebCk4o5e+xgstI1O50kDoWCSBe1tDivb9rNY29uYcFbW9m5v4Hs9AjnTRjCrElFfOiEQk1fKn2eQkHkKDQ1t/DqO7t47M0t/HXFNqpqGslOj3D2uMHMPHEoZ48dTI6amKQPUiiIHKPG5hZefruSJ5dv428rtlFZ3UBGagpnnVDIrElDOWf8EPpnqpNa+gaFgkg3am5xFm/cxV+Xb+PJ5VvZvreetIhx5vGDmHniUM6bMJSBOekHPWdvbSNVtY3srmlgT0309+6aRvbUNFDf3EJuRiq5mWn0y0glNzP6OPr7vfXpqaFOpS5JQqEgEpKWFuefm6v46/KtPLl8G2W7a4mkGCeV5tHiUFXTQFVtI3tqGznUPy+z6Mx3rTfcdSYjNYXczDQG9UunOD+L4vzM6O+8rLblIf0zSYsoPOTQFAoiPcDdWbFlL08u38qrG3aRlR4hPzud/Kw0BmSnRR9npzEg+J2fnc6A7DRyM9OIpBj1Tc3sr2tiX10T++ub2FvXGH1c18S+1sfB+op99WypqmPLnlqqgrkqWqUYDM7NpDg/k6L8LErzszhjdAHvHz1IZxoCKBREElpNQ1M0IKpqoz97oo+37gkmO9pdS0NzC7mZqXx43GA+MnEoHzqhUJ3kSayroaD/QkT6oOz0VI4f3I/jB/frcHtdYzP/WL+ThSu28dTK7Ty6dAsZqSl8cMwgPjJxKOeOH8KAnCMfD6qlxamsjl6yq4BJTPpURRJQZlqEc8YP4ZzxQ2hqbmHxxt0sXBG9iurpVTuIpBjTRg7kIxOHcP7EoRTnZ+HuVNU0smVPLVur6qJnHXvq2BpvP1TWAAAKSUlEQVSciWzdU8v2PfU0NLeQkZrCueOHcPGUYqaPHawmqgQSavORmc0Afk50Os7fuvst7bafBfwMmAzMcfcHD/eaaj4SOXruzvLyvSxcsY2FK7axbsd+AErys9hV3UBtY/MB+6emGEPzMinOy6IoP5OivCyK8jJ5u2I/jy/byq7qBvKz05g1qYhLppQwdcQAUlI0bPnh1Dc1c++rm+iflcbHTi7pkaHe496nYGYRYC1wHlAGLAYuc/eVMfuMBPoD3wTmKxREetaGiv0sXLGdlVv3Mjg3I7iqKdpZXZyXyaB+GYf8km9sbuHFdTv5y9Jy/rZiO7WNzZTkZ3HRlGIumVLC2KG5PXw0vZ+789TK7dy8YBXvVtYAcNYJhdzy8UmhTzDVG0LhDOD77v6RYPkGAHf/UQf7/h54XKEg0jdV1zfx1Mrt/GVpOS+s20lzizNuaC6XnFzCRScVM7R/JvVNLTQ0tVDf1Ex98LuusYWG5hbqG99bHzFj6sgB3ToHxp6aRp5atZ1nVm2nKC+Ly08bxvGDeza0Vm/byw8eX8k/1ldy/OB+3HjBeDbtquGWJ1eTYsa3LxjPnFOHhXbW0BtC4RPADHf/l2D5SuA0d7+mg31/TyehYGZzgbkAw4cPP+Xdd98NpWYROXY799fz+Jtb+MvSLSzdXHVUr5FicMqIAZw9bjBnjx3MuKG5R/xluau6gadWbmPBW9v4x/qdNLU4Q/pnsKu6gcZmZ9rIgVx22jBmnlgU6thWu6ob+MlTa7jn1U3kZqZx3XkncPlpw9vuK9lUWcO3HlrGyxsq+eCYQfzo45MoHZDd7XUkVCjE0pmCSN+xcWc1C1dso7axmYzUCBmpKWSkpZAeSSEjLVhOTSE9NaVte01DMy+sq+DZNTtYXr4XgKK8TKaPHcyHxw3mzOMLyE7v+BqZnfvrWbhiG0++tY2XN1TS3OIMH5jNzElDmXViEZNL86isbuDB18u497VNvFtZQ352GrPfV8pl04Yf8mquo9HY3MIfXn6Xnz+9luqGZq48fQRfO3dMh2dALS3OPa9t4kcLVgHw7xeM5/Jpw7v1rKE3hIKaj0TkmGzfW8eiNTt4dnUFL6yroLqhmfRICqcdN5APB2cR2ekRFq7YxhNvbeW1d3bR4jBqUA6zJg1l5olFTCzu3+GXa0uL8/KGSu55dRMLV2yjqcWZNmognz5tODNOHEpG6tGfPTy7egc/eGIlGyqqOeuEQr5zwXjGDDl8c9XmXTXMe3gZ/1hfyftHF/BfsyczbGD3nDX0hlBIJdrRfA5QTrSj+XJ3X9HBvr9HoSAinWhoamHxxl08u3oHf1+zgw0V1QdsP35wP2ZNKmLWpKGMHXJkzU0V++rbzh427aphQHD2cOmpwxial0lqihFJMdJSUjq9umr9jn384PFVPLe2guMG5XDjR8dz9tjBR1SLu3Pva5v5zwWraHHnhpnj+PRpI475qq64h0JQxCyil5xGgLvc/WYzuwlY4u7zzexU4BFgAFAHbHP3iZ29pkJBRADerazm2dU7qG5o5vwJQ7r0l/jhtLQ4/3h7J/e8uomnVm6nqeXg70ez6KW6qSkp0bCIvPe4Yn892ekRrj1nDJ85Y+Qx3b9RXlXLvIeW8cK6nZx+3EBunX0SwwuO/qyhV4RCGBQKItITduyr4+mVO6hpaKKpxWlucZqaneaWFhrbLTcFjwv6pfOFD4yioF9Gt9Tg7jywZDM/fHwVTS3OrZ+YzIUnFR/Va2mYCxGRYzA4N5PLTxse1xrMjE+dOpwPjinku48uZ9SgnNDfU6EgItLLFedn8dvPntoj76UBS0REpI1CQURE2igURESkjUJBRETaKBRERKSNQkFERNooFEREpI1CQURE2vS5YS7MrAI42gkVBgE7u7Gc3ixZjjVZjhOS51iT5TihZ491hLsXHm6nPhcKx8LMlnRl7I9EkCzHmizHCclzrMlynNA7j1XNRyIi0kahICIibZItFO6IdwE9KFmONVmOE5LnWJPlOKEXHmtS9SmIiEjnku1MQUREOqFQEBGRNkkTCmY2w8zWmNl6M5sX73rCYmYbzewtM1tqZgk1b6mZ3WVmO8xsecy6gWb2lJmtC34PiGeN3eUQx/p9MysPPtulwRzofZqZDTOzZ81spZmtMLNrg/UJ9bl2cpy97jNNij4FM4sAa4HzgDJgMXCZu6+Ma2EhMLONwFR3T7ibf8zsLGA/8Ad3PzFYdyuwy91vCcJ+gLt/K551dodDHOv3gf3u/t/xrK07mVkRUOTub5hZLvA6cAlwFQn0uXZynJfSyz7TZDlTmAasd/cN7t4A3AdcHOea5Ai5+/PArnarLwbuDh7fTfQfWp93iGNNOO6+1d3fCB7vA1YBJSTY59rJcfY6yRIKJcDmmOUyeukH0g0c+JuZvW5mc+NdTA8Y4u5bg8fbgCHxLKYHXGNmy4LmpT7dpNKemY0ETgZeJYE/13bHCb3sM02WUEgmH3D39wEzgauDZoik4NG20ERuD/01MBqYAmwF/ie+5XQfM+sHPAR8zd33xm5LpM+1g+PsdZ9psoRCOTAsZrk0WJdw3L08+L0DeIRo01ki2x6017a22+6Icz2hcfft7t7s7i3AnSTIZ2tmaUS/KP/k7g8HqxPuc+3oOHvjZ5osobAYGGNmo8wsHZgDzI9zTd3OzHKCTizMLAc4H1je+bP6vPnAZ4PHnwUejWMtoWr9kgx8jAT4bM3MgN8Bq9z9JzGbEupzPdRx9sbPNCmuPgIILvX6GRAB7nL3m+NcUrczs+OInh0ApAL3JNJxmtm9wHSiww1vB74H/AV4ABhOdEj1S929z3fQHuJYpxNtZnBgI/ClmHb3PsnMPgC8ALwFtASr/51oe3vCfK6dHOdl9LLPNGlCQUREDi9Zmo9ERKQLFAoiItJGoSAiIm0UCiIi0kahICIibRQKIj3IzKab2ePxrkPkUBQKIiLSRqEg0gEzu8LMXgvGuL/dzCJmtt/MfhqMh/+MmRUG+04xs1eCQc0eaR3UzMyON7OnzexNM3vDzEYHL9/PzB40s9Vm9qfgbleRXkGhINKOmY0HPgWc6e5TgGbg00AOsMTdJwLPEb3LGOAPwLfcfTLRO1Zb1/8JuM3dTwLeT3TAM4iOkPk1YAJwHHBm6Acl0kWp8S5ApBc6BzgFWBz8EZ9FdEC2FuD+YJ//Ax42szwg392fC9bfDfw5GIOqxN0fAXD3OoDg9V5z97JgeSkwEngx/MMSOTyFgsjBDLjb3W84YKXZd9rtd7RjxNTHPG5G/w6lF1HzkcjBngE+YWaDoW2+4BFE/718ItjncuBFd98D7DazDwbrrwSeC2bXKjOzS4LXyDCz7B49CpGjoL9QRNpx95VmdiPRGexSgEbgaqAamBZs20G03wGiQzv/JvjS3wB8Llh/JXC7md0UvMYne/AwRI6KRkkV6SIz2+/u/eJdh0iY1HwkIiJtdKYgIiJtdKYgIiJtFAoiItJGoSAiIm0UCiIi0kahICIibf4/Dp3tY0IyArEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<keras.engine.training.Model at 0x7fcf4cdb1a20>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_model(model, X_train, y_train, 32,200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Inspect prediction accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3511/3511 [==============================] - 14s 4ms/step\n",
      "[0.08269726071049897, 0.9737966391341498]\n"
     ]
    }
   ],
   "source": [
    "lstm_score_train = model.evaluate(X_train,y_train)\n",
    "print(lstm_score_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1505/1505 [==============================] - 6s 4ms/step\n",
      "[0.2143229364015261, 0.9534883720930233]\n"
     ]
    }
   ],
   "source": [
    "lstm_score_test = model.evaluate(X_test,y_test)\n",
    "print(lstm_score_test)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}