edge_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Simple AI model for Embedded AI & Edge computing [+info](https://blog.crespum.eu/2018/05/28/spaceup-bcn-18-edge-computing/)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook generates a model using TensorFlow using [notMNIST](http://yaroslavvb.blogspot.com/2011/09/notmnist-dataset.html) to identify letters from A to J. It's based on [RoboND-NN-Lab](https://github.com/udacity/RoboND-NN-Lab)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "# Load the modules\n",
    "import math\n",
    "import hashlib\n",
    "import os\n",
    "from urllib.request import urlretrieve\n",
    "\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "from sklearn.utils import resample\n",
    "from tqdm import tqdm\n",
    "from zipfile import ZipFile\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data pre-processing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We'll be using 200000 images\n"
     ]
    }
   ],
   "source": [
    "# The notMNIST dataset is too large for many computers to handle\n",
    "# We'll be using a subset of this data\n",
    "labels_count = 10 # number of letters we'll be trying to identify\n",
    "size_limit = 20000 * labels_count # totaly number of images\n",
    "\n",
    "print(\"We'll be using {} images\".format(size_limit))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All files downloaded.\n"
     ]
    }
   ],
   "source": [
    "def download(url, file):\n",
    "    \"\"\"\n",
    "    Download file from <url>\n",
    "    :param url: URL to file\n",
    "    :param file: Local file path\n",
    "    \"\"\"\n",
    "    if not os.path.isfile(file):\n",
    "        print('Downloading ' + file + '...')\n",
    "        urlretrieve(url, file)\n",
    "        print('Download Finished')\n",
    "\n",
    "# Download the training and test dataset.\n",
    "download('https://s3.amazonaws.com/udacity-sdc/notMNIST_train.zip', 'notMNIST_train.zip')\n",
    "download('https://s3.amazonaws.com/udacity-sdc/notMNIST_test.zip', 'notMNIST_test.zip')\n",
    "\n",
    "# Make sure the files aren't corrupted\n",
    "assert hashlib.md5(open('notMNIST_train.zip', 'rb').read()).hexdigest() == 'c8673b3f28f489e9cdf3a3d74e2ac8fa',\\\n",
    "        'notMNIST_train.zip file is corrupted.  Remove the file and try again.'\n",
    "assert hashlib.md5(open('notMNIST_test.zip', 'rb').read()).hexdigest() == '5d3c7e653e63471c88df796156a9dfa9',\\\n",
    "        'notMNIST_test.zip file is corrupted.  Remove the file and try again.'\n",
    "\n",
    "# Wait until you see that all files have been downloaded.\n",
    "print('All files downloaded.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 210001/210001 [00:40<00:00, 5224.85files/s]\n",
      "100%|██████████| 10001/10001 [00:01<00:00, 5294.66files/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All features and labels uncompressed.\n"
     ]
    }
   ],
   "source": [
    "def uncompress_features_labels(file):\n",
    "    \"\"\"\n",
    "    Uncompress features and labels from a zip file\n",
    "    :param file: The zip file to extract the data from\n",
    "    \"\"\"\n",
    "    features = []\n",
    "    labels = []\n",
    "\n",
    "    with ZipFile(file) as zipf:\n",
    "        # Progress Bar\n",
    "        filenames_pbar = tqdm(zipf.namelist(), unit='files')\n",
    "        \n",
    "        # Get features and labels from all files\n",
    "        for filename in filenames_pbar:\n",
    "            # Check if the file is a directory\n",
    "            if not filename.endswith('/'):\n",
    "                with zipf.open(filename) as image_file:\n",
    "                    image = Image.open(image_file)\n",
    "                    image.load()\n",
    "                    # Load image data as 1 dimensional array\n",
    "                    # We're using float32 to save on memory space\n",
    "                    feature = np.array(image, dtype=np.float32).flatten()\n",
    "\n",
    "                # Get the the letter from the filename.  This is the letter of the image.\n",
    "                label = os.path.split(filename)[1][0]\n",
    "\n",
    "                features.append(feature)\n",
    "                labels.append(label)\n",
    "    return np.array(features), np.array(labels)\n",
    "\n",
    "# Get the features and labels from the zip files\n",
    "train_features, train_labels = uncompress_features_labels('notMNIST_train.zip')\n",
    "test_features, test_labels = uncompress_features_labels('notMNIST_test.zip')\n",
    "\n",
    "# Limit the amount of data to work with\n",
    "train_features, train_labels = resample(train_features, train_labels, n_samples=size_limit)\n",
    "\n",
    "# Set flags for feature engineering.  This will prevent us from skipping an important step.\n",
    "is_features_normal = False\n",
    "is_labels_encod = False\n",
    "\n",
    "# Wait until you see that all features and labels have been uncompressed.\n",
    "print('All features and labels uncompressed.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Features normalized\n"
     ]
    }
   ],
   "source": [
    "def normalize_grayscale(image_data):\n",
    "    \"\"\"\n",
    "    Normalize the image data with Min-Max scaling to a range of [0.1, 0.9]\n",
    "    :param image_data: The image data to be normalized\n",
    "    :return: Normalized image data\n",
    "    \"\"\"\n",
    "    a = 0.1\n",
    "    b = 0.9\n",
    "    return a + image_data*(b-a)/(255)\n",
    "\n",
    "train_features = normalize_grayscale(train_features)\n",
    "test_features = normalize_grayscale(test_features)\n",
    "is_features_normal = True\n",
    "\n",
    "print('Features normalized')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Labels One-Hot Encoded\n"
     ]
    }
   ],
   "source": [
    "# Turn labels into numbers and apply One-Hot Encoding\n",
    "encoder = LabelBinarizer()\n",
    "encoder.fit(train_labels)\n",
    "train_labels = encoder.transform(train_labels)\n",
    "test_labels = encoder.transform(test_labels)\n",
    "\n",
    "# Change to float32, so it can be multiplied against the features in TensorFlow, which are float32\n",
    "train_labels = train_labels.astype(np.float32)\n",
    "test_labels = test_labels.astype(np.float32)\n",
    "is_labels_encod = True\n",
    "\n",
    "print('Labels One-Hot Encoded')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training features and labels randomized and split.\n"
     ]
    }
   ],
   "source": [
    "assert is_features_normal, 'You skipped the step to normalize the features'\n",
    "assert is_labels_encod, 'You skipped the step to One-Hot Encode the labels'\n",
    "\n",
    "# Get randomized datasets for training and validation\n",
    "train_features, valid_features, train_labels, valid_labels = train_test_split(\n",
    "    train_features,\n",
    "    train_labels,\n",
    "    test_size=0.05,\n",
    "    random_state=832289)\n",
    "\n",
    "print('Training features and labels randomized and split.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create TensorFlow model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_count = 784\n",
    "\n",
    "features = tf.placeholder(tf.float32, name='netin')\n",
    "labels = tf.placeholder(tf.float32, name='labels')\n",
    "\n",
    "weights = tf.Variable(tf.truncated_normal((features_count, labels_count)), name='weights')\n",
    "biases = tf.Variable(tf.zeros(labels_count), name='biases')\n",
    "\n",
    "# Feed dicts for training, validation, and test session\n",
    "train_feed_dict = {features: train_features, labels: train_labels}\n",
    "valid_feed_dict = {features: valid_features, labels: valid_labels}\n",
    "test_feed_dict = {features: test_features, labels: test_labels}\n",
    "\n",
    "# Linear Function WX + b\n",
    "logits = tf.add(tf.matmul(features, weights), biases)\n",
    "\n",
    "prediction = tf.nn.softmax(logits, name='netout')\n",
    "\n",
    "# Cross entropy\n",
    "cross_entropy = -tf.reduce_sum(labels * tf.log(prediction), axis=1)\n",
    "\n",
    "# Training loss\n",
    "loss = tf.reduce_mean(cross_entropy)\n",
    "\n",
    "# Create an operation that initializes all variables\n",
    "init = tf.global_variables_initializer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy function created.\n"
     ]
    }
   ],
   "source": [
    "# Determine if the predictions are correct\n",
    "is_correct_prediction = tf.equal(tf.argmax(prediction, 1), tf.argmax(labels, 1))\n",
    "# Calculate the accuracy of the predictions\n",
    "accuracy = tf.reduce_mean(tf.cast(is_correct_prediction, tf.float32))\n",
    "\n",
    "print('Accuracy function created.')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train & save model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define hyper-parameters\n",
    "epochs = 4\n",
    "batch_size = 100\n",
    "learning_rate = 0.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch  1/4: 100%|██████████| 1900/1900 [00:12<00:00, 158.31batches/s]\n",
      "Epoch  2/4: 100%|██████████| 1900/1900 [00:11<00:00, 161.30batches/s]\n",
      "Epoch  3/4: 100%|██████████| 1900/1900 [00:11<00:00, 162.13batches/s]\n",
      "Epoch  4/4: 100%|██████████| 1900/1900 [00:11<00:00, 161.52batches/s]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzs3Xl4VNX5wPHvOzPZCWsg7ISwLyIBDBCQRagCiixqRcWVClJprFRbtVqtrdb2Z7FGFKEoaFlUVBBRFsOqRAiByL4lGEwCBEhCwpKQzMz5/TGTIYFAAoZkIu/neebJ3HO3cyYz573n3HPvFWMMSimllLexVHUGlFJKqdJogFJKKeWVNEAppZTyShqglFJKeSUNUEoppbySBiillFJeSQOUUkopr6QBSqkKJiIpIjK4qvOhVHWnAUoppZRX0gClVCURkUdFJElEskRksYg0dqeLiLwhIkdFJFdEtotIZ/e8YSKyS0ROiki6iDxVtaVQqvJogFKqEojITcA/gF8DjYCDwEfu2TcD/YC2QC33Mpnuee8BE4wxwUBnYFUlZlupKmWr6gwodY24D3jfGLMFQESeBbJFJAwoBIKB9kC8MWZ3sfUKgY4istUYkw1kV2qulapC2oJSqnI0xtVqAsAYcwpXK6mJMWYVMBV4GzgqIjNEpKZ70TuAYcBBEVkrIr0rOd9KVRkNUEpVjkNAi6IJEQkC6gHpAMaYGGNMd6Ajrq6+p93pm4wxI4AGwCLgk0rOt1JVRgOUUleHj4j4F72A+cDDItJVRPyAV4GNxpgUEblBRHqKiA9wGsgHnCLiKyL3iUgtY0whkAs4q6xESlUyDVBKXR1fA3nFXgOAF4DPgMNAK2CMe9mawH9xnV86iKvr7//c8+4HUkQkF3gM17kspa4Jog8sVEop5Y20BaWUUsoraYBSSinllTRAKaWU8koaoJRSSnklr7uTREhIiAkLC6vqbCillLpKNm/efNwYU7+s5bwuQIWFhZGQkFDV2VBKKXWViMjBspfSLj6llFJeyusClNM4sTvtVZ0NpZRSVczrAlTi4US2Z2yv6mwopZSqYl4XoADOFJ6p6iwopZSqYhqglFJKeaUKC1Ai8r77kdU7iqW95H5M9Q/u17DybCvPnldR2VJKKVVNVWQLajYwpJT0N4wxXd2vr8uzIW1BKaWUqrAAZYxZB2RVxLY0QCmllKqMc1CTRGSbuwuwTmkLiMh4EUkQkQTQAKWUUurqB6hpuB7M1hXXQ9r+XdpCxpgZxpgexpgeoAFKKaXUVQ5QxpgMY4zDGOPE9cTQyPKsl1eogySUUupad1UDlIg0KjY5CthxsWWLraMtKKWUUhV3s1gRmQ8MAEJEJA14ERggIl0BA6QAE8rajkUsGqCUUkpVXIAyxtxTSvJ7l7sdDVBKKaXAC+8kYRGLXqirlFLKOwOUtqCUUkppgFJKKeWVNEAppZTyShqglFJKeSWvDFA6SEIppZRXBihtQSmllNIApZRSyitpgFJKKeWVvDJA5RXmYYyp6qwopZSqQl4ZoAyGs46zVZ0VpZRSVcgrAxToM6GUUupapwFKKaWUV/LaAKUPLVRKqWub1wYobUEppdS1TQOUUkopr6QBSimllFfSAKWUUsoreW2A0hvGKqXUta3CApSIvC8iR0VkR7G0uiLyjYjsd/+tU47tANqCUkqpa11FtqBmA0POS3sGWGmMaQOsdE9fOkPaxaeUUooKDFDGmHVA1nnJI4AP3O8/AEaWtR2rWAENUEopda272uegQo0xh93vjwChpS0kIuNFJEFEEjKPZwJ6oa5SSl3rKm2QhHHdnrzUW5QbY2YYY3oYY3rUr18fX6uvtqCUUuoad7UDVIaINAJw/z1anpUCbAEaoJRS6hp3tQPUYuBB9/sHgS/Ks1KgT6AGKKWUusZV5DDz+cD3QDsRSRORccBrwK9EZD8w2D1dpkCfQL0OSimlrnG2itqQMeaei8wadLnb0haUUkopr7uTBGiAUkop5aUBKsBHB0kopdS1zisDlLaglFJKeW2A0kESSil1bfPaAKUtKKWUurZ5ZYDSC3WVUkp5ZYDSFpRSSimvDVB5hXm4bt+nlFLqWuS1AcphHBQ6C6s6K0oppaqI1wYo0GdCKaXUtcwrA1SALQDQAKWUUtcyrwxQRS0ofWihUkpdu7w6QGkLSimlrl0aoJRSSnklrwxQAT56Dkoppa51XhmgtAWllFLKqwOU3jBWKaWuXV4doLQFpZRS1y4NUEoppbySrTJ2IiIpwEnAAdiNMT0utbxeqKuUUqpSApTbQGPM8fIsqBfqKqWU8souPh+rDzaLTVtQSil1DausAGWAFSKyWUTGnz9TRMaLSIKIJBw7dgzQZ0IppdS1rrICVF9jTDdgKPC4iPQrPtMYM8MY08MY06N+/fqAPlVXKaWudZUSoIwx6e6/R4GFQGRZ6wT6BOp1UEopdQ276gFKRIJEJLjoPXAzsKOs9bSLTymlrm2VMYovFFgoIkX7m2eMWVbWSs1rNSc+PZ5CRyE+Vp+rnUellFJe5qq3oIwxB4wx17tfnYwxr5Rnvd/e8FtSc1NZsGvB1c6iUkopL+SVw8wBhrUZRvuQ9rwe9zrGmKrOjlJKqUrmtQHKIhae6v0UiUcSWZ2yuqqzo5RSqpJ5bYACuK/LfYQGhfJ63OtVnRWllFKVzKsDlL/Nn99F/o6lSUsJfCWQ0NdDeWnNS1WdLaWUUpWgMu/Fd0We6PUEFrGQmZdJ4pFE/rr2rwxpPYReTXtVddaUUkpdReJtAxB69OhhEhISSp13quAU7ae2p0FQAzY9ugmrxVrJuVNKKfVzicjmsp5qAV7exXe+Gr41eOOWN0g8ksi0hGk/e3vZedmcPHuyAnKmlFKqolWrAAVwZ8c7+VX4r3h+1fPsPb73irdjjGHgBwMZ+fHICsydUkqpilLtApSI8M6t7+Bn8yPq/SjiUuOuaDtxqXFszdjKqh9XseNomXdeUkopVcmqXYACaF23Nd+P+556AfUY9OEgnlrxFH9b+zf+s+E/rDu4rlz38JuxZQY1fGvgZ/Xj3YR3KyHXl/by2pd1OL1SShVTrQZJnO/Y6WP8+tNf8+3Bb3EYhyfdZrER0TCCqGZR9GvRjxHtRpQYUJGdl03jKY156PqHOFV4ii/2fMGhPxyihm+NCi9PeWxK30TkzEgsYmHbY9vo1KBTpeeh0FHI6pTVDA4fjEWq5XGLUqqa+EUOkjhf/aD6rH5wNfa/2Cl8oZCMpzL48p4veTrqaQJ9ApmxeQZ3fHIH/Wb3Y1/mPs96c7fPJd+ez/ju45nYYyInC04yb/u8K8qD0zhJz02/4tsxGWP4Y+wfCQkMIdg3mD/F/gkAu9POpK8n8fd1f7+i7V6uP6/6M7fMuYX3trxXKfvzJtsytpF4OLGqs6GUOk+1bkGVpdBRyEc7PuKJZU+QZ8/jd5G/Y3jb4Tz+9eP42fzY9OgmjDF0nd4Vi1jYMn4L7ruuY4whKy+LeoH1St12Tn4Os36Yxdub3iYpK4kR7UYwddhUmtZsijGG1NxUEg8nsjVjK5FNIhnSeohnu29ufJO8wjye6PUEa1LWcOu8W4kZEkOePY8/xf6Jb+7/hvcS3+OjHR9hEQtbH9tK5wadK+QzAViTsoa/r/s7L/Z/kRtb3Mi3B7+l/+z+WMRCWO0w9kzag81ydS+R+2jHRxhjuOe6e67qfspy7PQx2r/dHrvTzs7f7qRpzaZVko+jp4/SIKhBley7Km1K38TJgpPc1PKmqs6KqkTlbUH9ogNUkcMnDxO9LJpFexZhd9oBmHHbDB7t/igA0xOm89hXjxEaFEpUsygKnYV8n/o9mXmZtKrTiiGthzC09VAGthyIr9WX6QnT+cuav5CVl0VUsyiimkbx9qa3sVls9Gjcg60ZW8nKy/LsXxA+HPUhY7uM5ZV1r/D86ucBaFazGT5WHwRh1+O7cBon7aa2I+NUBmcdZ3n+xueZumkqvZv25uv7vr6gXIdOHqJRjUaeoFoeM7fMZOJXE3EaJxax8PqvXufNjW8iIrzY/0UeXPQg8++Yz5jOYwBXkP9w64e88q3rJvRPRT3FIxGP4G/zv7J/BrD3+F6um3YdTuNk06ObiGgUccXb+rkeXPQg87bPw8fiw8CWA1lyzxKcxsnf1v2NGr41eCrqqQrfpzGmxP/s7fi3mbR0ErNHzObBrg9W+P681eGTh+n4TkfOFJ5hy/gtVdK1DVyzj/T5w/I/sDdzLwvvXljp5dcAVYqc/BxiD8Sy69gunu7ztKeSdRons3+YzZqUNcSlxmGz2IhqFkXbem357qfvWPXjKvLsefhZ/WgQ1IDU3FQGhg3kn4P/yQ1NbgDgQPYB/rDiD6TlphHRMML1ahRBqzqtGPPZGNakrGFM5zHM2z6PsV3G8puI3/Dk8idJPJLIJ3d+wl2d7gLg4x0fM+azMfx94N/5c78/83/r/48/xv6R2PtjGRQ+CHBVcC+ueZG/rfsbYbXDuKPDHfRs0pMavjWoE1CHiIYR+Nn8POW2O+2sTVnL7K2zmbNtDje3upkZt83gsa8eY1nSMixiYd1D6+jdrDed3umEn9WPxAmJrEheweNfP05ydjI3NL4Bq8XKhrQNhAaFclvb27i51c1ENomkcXBjfCw+7M3cS1xqHGcKz9CiVgvC64TToX6HEue0jDEMmTuEDWkbCPQJJDQolPhH4/G1+l7W/9IYw/rU9Szdv5S7Ot1F14ZdS8zPK8zj0MlDNAhqQLBfcKnbWPXjKgZ9OIjn+j5Hg6AG/H7575k6dCpf7f+KpUlLAVj5wMoKO7ovcBTwyrpXmLJhCpN7TeaF/i8QeyCWW+fdilWs1PSryd5Jey9ote/P3M+ZwjN0btC5Qi5OX7x3MSfPnuS+Lvf97G1dKWMMoz8ZzdL9S6nhW4MWtVuwYdwGbBYb7ye+D8C4buMqfL9ZeVnUDajrmV6wcwEPffEQ026dxgPXP1Dh+/NWX+37itvm3wbAS/1f4sUBL1bq/jVAVaB8ez7fHvyWpUlL2XlsJxN7TGREuxHlbrmcKTzD8PnDWfXjKkZ3GM3Hd36MzWLD4XSwN3MvHet3LLF88e6efHs+7ae2p25AXZaPXU5IYAh/XvVn/vHdPxjdYTRn7WdZkbyCQmehZ/0AWwD9WvSjQVADUk6ksOvYLjLzMgnyCWJij4n8Y/A/PPv/1/p/US+wHuO7jwfgw60f8uCiBxkYNpDVKatpH9Kefw3+F7e1dX2Z16Ss4e1NbxN7IJacszmefV7sCcihQaEMaT2E0R1GM6zNML7c+yWjPxnNf275D2G1wxj58UheHvAyL/R/wbNOyokUvt7/NcPaDCOsdhjgqtB2H99NUlYSB7IP8NGOj9iYvhFwtVDv63IfkY0jWZWyivU/refYmWMA1Auox9RhU7m7093EpcbxhxV/IDU3lRsa38DWjK1YxMKOiTvwtfrSb3Y/4lLjsIqVKbdM4a34t7A77WyfuL3EAJrUnFS+OfANNzS+gc4NOnu+B3anvdSuUWMM36d9z2+/+i1bM7bStWFXfjjyAz2b9GT38d20rN2SabdO48ZZN/JIxCPMGD6D42eO886md1iwa4HnMohg32CimkUxqv0o7uh4ByGBIZ59nMg/wfKk5Zx1nGVw+GAaBzcu9bsYlxpH/9n9sTvtvHvru0zoMaHU5cqyPWM7C/csZHSH0Rd0P9uddjLPZFLTryYBPgGedIfTQYGjgACfAD7Z+Ql3f3o3/xr8L1rVbcUdn9zBk72eJCkriS/3fQnAwrsXMrJ9xVynmJyVzLMrn2XBrgWM6TyGt4e9TXx6PLfPvx0AH6sPCY8m0KF+B886B08cZFnSMrLzs+kQ0oHODToTXif8snoszjdn2xxSc1L5U98/VdlgpOy8bDq904mQwBA6NejEgp0L+H7c94TWCGXy8snk2/NZcNeCEv+7n2vL4S0s2LmAOzveSffG3TVAeZszhWdYsm8JI9qNKNG6KY+PdnzEPZ+5ztU0q9mM1NxUJnSfwDu3voNFLOSezeXgiYOcKjjFkVNHWJ2ymtgDsZwsOEnL2i1pXbc1t7a5laFthhLoE3jJfRU6CmnzVhvSctN4pu8zvNDvhVLza3fa2ZS+iZ3HdpKem05WXhZdQrvQp3kfavvX5uCJg+w5vodlyctYnrSc7PxsGtVohMM4CA0KZcuELdgsNu757B4+2/UZI9uPpHfT3mzN2MqcbXNwGAdWsXLPdffQuEZjPtn1CSknUjz7b123NU/2epKR7Ufy1sa3+M/G/5Bvz6dFrRYMCBtAm7ptaFijIf/d8l82pm+kS2gXtmVso0lwE/qH9SfhUAI/Zv/I1/d9zeDwwQDsy9zH418/zjN9nmFQ+CDPubnHejzGhO4T2HJ4C5/u/pRlSctwGicA4XXCaVqzKXuP7+Xo6aOMaD+Cp3o/RUSjCLZnbGd96npm/TCLHUd30LBGQ6bfNp3b293O/O3zeeyrxwiwBRD/aDzNazXnqRVP8e/v/82kGybx4bYPOXn2JH2b9+WODq5gtD51PSt/XMm+zH1YxUqnBp2o6VcTp3ESnx7v6b4G6FS/Ex3rdySsdhi9mvZieNvhZOVl0W1GN/xt/rQPac/S/UuZO3puifOAe4/vZfHexfRq2os+zfsgCLuP72blgZUYDP42f5YmLWXRnkUAWMXKb2/4Lf1b9OeLvV+wInkFR08fxWCoG1CXZ/o8w4QeE/hk5yf8bd3f+CnnJ5rXak52XjbtQ9oTN87VYzH287HM3T4XX6sv/xj0D+bvmM++zH0kPJpAm3ptOF1wmlU/ruLr/V8Tfyienk16clfHu+hQvwMHTxzk6Omj9A/rT02/mp6yGGPYmL6R97a8xwdbP8DH6sOIdiNYsGsB9QPrcyL/BO1C2jH/jvn0m9WPhjUasurBVXy842OmJUxj57GdF3zvuzXqxsQeExnSeghWsXKm8AyxB2JZsn8JpwpOMaDFAPqH9adZzWbU9q9NnYA6nkA0f/t87v38XgB+3enXfDjyw3LVBYWOQmwWGyJCUlYSr8e9zme7P+OODnfwyk2vUC+wHqcKTrEmZQ1puWkcO32MhjUacl+X+wj0CcTutPPprk/Ze3wvLWq3YMm+JSzas4j4R+MJrxPOddOuwxjDifwTOI2TfHs+I9qP4NO7PvW02E8XnGbljyuJS42jd9PeDG0ztMxeD4fTweqU1Uz5foqnR8JmsfFCvxd4ccCLGqB+STalb/J8Qbo36s4L/V+4akdgB7IPYHfaaVuvbYVsr9BRyNKkpczcMpNVP65i2dhl9G3eF4DMM5n8YcUfWJ2ymp9yfiLAFsD47uMZ22Us87fPZ/rm6Z5WwZ0d7uS60OtoWbslIYEhJY5kj50+xqmCU4TVDiuRbnfamfL9FKZ8P4XfdPsNz/Z9liDfIADPebhLeWLpE8TEx3imG9VoxCMRjzC6w2gSDiWweO9iV0VXrx3BfsH8b9v/yMrLQhAMrt9WZJNIxkWM457O95Tobjx6+ihO46RhjYbAuXtNpp9MZ0S7Ebw66NULWtfGGLZlbOOjHR+x89hOThWcosBRQN/mfRnRbgQBPgEsT1rOmoNrSM5K5mDOQQocBTQJbkLdgLrsz9rP9+O+p129dgydO5TvfvqO3s1606NRD/Zn7eer/V959tW0ZlMCfQJLjIAFqO1fmyd6PsF9193HGxveYPrm6TiNk7oBdRnWZhit6rSiXkA9liYtZWnSUmwWG3anncgmkQxtPZSkrCQOnTzE28Pe9rRYsvOy+evav/JQ14fo2rArB08cpNuMbtQPrE+zWs1Yd3AdBY4CgnyC6NaoGwmHEsiz55XIV02/mozvNp6IRhHEpcYReyCWvZl7CfQJ5P4u9/OX/n+hcXBjEg8n8sCiB3A4Hax5aA0NghqwLGkZQ+cOLZHXMZ3GMLTNUBoHN2bP8T1sSNvAzC0z2X50+wXfk5a1W1InoA6JhxM9//ei78vjNzxOm3ptGPv5WKKaRTGk9RCeXfksA8IG8HTU057yvr3pbT7f/Tlt6rVhQIsB+Nv8WZ2yms2HN2MVK/UC63H09FFsFhs3tbyJb5K/oZZ/LXo26cmqH1dx1nG2RJ5CAkMY02kMS/YvKXFwB/BCvxd4eeDLgKur+5Y5t3BLq1t4e9jbfLnvS3639HeMixjH9aHX89X+r1iTsoazjrOe73W9gHp0a9SNzLxMcs/mckurW5jYYyKt6rYiLjWOZUnLmLd9Hukn0wkJDOHJXk9y73X38vyq55m7fS68hAYoVb2k56YT6BNInYA6nrSTZ0/iMA5q+9eukjydKTzDO5veoWnNpkQ0jKB13daXPA90uuA0c7bN4fCpw3Rt2JWIhhG0qN2i3PtLykoiJz+H7o27V0T2cTgdfL3/a6ZumkrsgVhmjZjlOdeSezaXv6/7O+tT17Pl8BZq+tXktz1+y/3X38+GtA18vPNj19F0uxHc1vY2gnyCyLfnUyegTomW+J7je8g4lUGf5n0u6OJcd3Adc7fNZXi74dza5tbL6h5bkbyC2+bdRpt6bRjaeihDWw+lb/O++Nn8OF1wmqVJSzl88jBhtcMI9Ankv1v+y6e7PsVhHAT5BNG7WW/u7nQ3d3e6+4LzkMYYHMZRIr//jvs3mw9vZlLkJKKaRZWaJ2MMG9I2sC1jGwaDVaz0ad6HDiEdEBGy87LZkLaBY2eOkZ2XzddJX7MieQUA14dez9qH1lLLvxZzt81l3OJxJYJKTb+a3NHhDlJzU1n/03rsTju9mvaib/O+OI2TzDOZNKzRkN/e8FsaBTdie8Z2nlz+JCknUhjedjgj2o+gTd02hASGEJ8ez7/i/sWSfUvo3bQ3z/R9hptb3UxabhoZpzLo1bRXie9x7tlcgn2DPf+fZ2Kf4Z/r/wlAu3rtGNZmGLe2uZXezXqzJmUN/9v2P5KzkgkJDMFmsbE8eTkFjgJ8rb4UOAqwWWwMaT2E+7vcz/C2w0t0F36661Pu6nSX9wQoERkCvAlYgZnGmNcutqwGKKWujrzCvIueV7A77QjidU8IOGs/e1ld4qk5qRw7c4wuoV2u+qUS5bXz6E4+3/05j3Z/1NNaBldQ2HpkKz8c+YFAn0Du7ny351xngaMAh9Pxs88DnS44TaBP4GWfNzPG8NX+r2gf0p7WdVuXufzxM8eZ/cNsjpw6woCwAfRr0a9Ed+v5vOYclIhYgX3Ar4A0YBNwjzFmV2nLa4BSSqlfNm+6k0QkkGSMOWCMKQA+AkZUwn6VUkpVY5URoJoAqcWm09xpSiml1EV5RSetiIwHxrsnT4nIlT/oqeqFAMerOhMVQMvhXbQc3kXL8fOUa+RQZQSodKBZsemm7jQPY8wMYEYl5OWqE5GE8vStejsth3fRcngXLUflqIwuvk1AGxFpKSK+wBhgcSXsVymlVDV21VtQxhi7iEwCluMaZv6+MebCS7SVUkqpYirlHJQx5mvgwttx/zL9Iroq0XJ4Gy2Hd9FyVAKvu5OEUkopBdX8ibpKKaV+uTRAKaWU8koaoMpBRN4XkaMisqNYWl0R+UZE9rv/1nGni4jEiEiSiGwTkW7F1nnQvfx+EanUR6eKSDMRWS0iu0Rkp4g8UU3L4S8i8SKy1V2Ov7rTW4rIRnd+P3aPGEVE/NzTSe75YcW29aw7fa+I3FKZ5SiWB6uIJIrIkupaDhFJEZHtIvKDiCS406rV98q9/9oi8qmI7BGR3SLSu7qVQ0Tauf8PRa9cEfl9dSuHhzFGX2W8gH5AN2BHsbR/Ac+43z8D/NP9fhiwFBCgF7DRnV4XOOD+W8f9vk4llqER0M39PhjX/RE7VsNyCFDD/d4H2OjO3yfAGHf6u8BE9/vfAu+6348BPna/7whsBfyAlkAyYK2C79ZkYB6wxD1d7coBpAAh56VVq++VOw8fAL9xv/cFalfHchQrjxU4guui2GpZjkr/0KrrCwijZIDaCzRyv28E7HW/n47rZrgllgPuAaYXSy+xXBWU5wtcN/CttuUAAoEtQE9cV8Pb3Om9geXu98uB3u73NvdyAjwLPFtsW57lKjH/TYGVwE3AEne+qmM5UrgwQFWr7xVQC/gR98Cx6lqO8/J+M7C+OpdDu/iuXKgx5rD7/REg1P3+Yvce9Jp7Erq7hyJwtT6qXTnc3WI/AEeBb3C1Gk4YY4oeKVs8T578uufnAPXwgnIA/wH+CDjd0/WonuUwwAoR2Syu25ZB9ftetQSOAbPcXa4zRSSI6leO4sYA893vq2U5NEBVAOM6xKgW4/VFpAbwGfB7Y0xu8XnVpRzGGIcxpiuuFkgk0L6Ks3TZROQ24KgxZnNV56UC9DXGdAOGAo+LSL/iM6vJ98qGqxt/mjEmAjiNqyvMo5qUAwD3ucvbgQXnz6tO5dAAdeUyRKQRgPvvUXf6xe49WOY9Ca82EfHBFZzmGmM+dydXu3IUMcacAFbj6gqrLSJFF54Xz5Mnv+75tYBMqr4cfYDbRSQF1yNobsL1UM/qVg6MMenuv0eBhbgOGqrb9yoNSDPGbHRPf4orYFW3chQZCmwxxmS4p6tlOTRAXbnFQNHIlgdxndMpSn/APTqmF5DjblovB24WkTruETQ3u9MqhYgI8B6w2xgzpdis6laO+iJS2/0+ANd5tN24AtWdFylHUfnuBFa5jyAXA2Pco+NaAm2A+MopBRhjnjXGNDXGhOHqillljLmPalYOEQkSkeCi97i+DzuoZt8rY8wRIFVE2rmTBgG7qls5irmHc917UF3LURUn76rbC9c/+jBQiOtIaxyu/v+VwH4gFqjrXlaAt3GdF9kO9Ci2nUeAJPfr4UouQ19czfptwA/u17BqWI4uQKK7HDuAv7jTw3FVzEm4ujX83On+7ukk9/zwYtv6s7t8e4GhVfj9GsC5UXzVqhzu/G51v3YCf3anV6vvlXv/XYEE93drEa7Ra9WxHEG4Wte1iqVVu3IYY/RWR0oppbyTdvEppZTyShqglFJKeSUNUEoppbySBiillFJeSQOUUkopr6QBSimllFfSAKWUUsoraYBSSinllTRAKaWU8koaoJRSSnklDVBKKaW8kgYopZRSXkkDlFJKKa+kAUqpMojEs/HHAAAgAElEQVTIGhHJFhG/qs6LUtcSDVBKXYKIhAE34nqW1u2VuF9b2Usp9cumAUqpS3sA2ADM5twTSRGRABH5t4gcFJEcEfnO/YRfRKSviMSJyAkRSRWRh9zpa0TkN8W28ZCIfFds2ojI4yKyH9eD5RCRN93byBWRzSJyY7HlrSLynIgki8hJ9/xmIvK2iPy7eCFEZLGIPHk1PiClrhYNUEpd2gPAXPfrFhEJdae/DnQHooC6wB8Bp4i0AJYCbwH1cT2l9YfL2N9IoCfQ0T29yb2NusA8YIGI+LvnTcb1aO9hQE1cT0A9A3wA3CMiFgARCQEGu9dXqtrQAKXURYhIX6AF8IkxZjOux2Lf6674HwGeMMakG2Mcxpg4Y8xZ4F4g1hgz3xhTaIzJNMZcToD6hzEmyxiTB2CMmePeht0Y82/AD2jnXvY3wPPGmL3GZat72XggBxjkXm4MsMYYk/EzPxKlKpUGKKUu7kFghTHmuHt6njstBPDHFbDO1+wi6eWVWnxCRJ4Skd3ubsQTQC33/sva1wfAWPf7scD/fkaelKoSeiJWqVK4zyf9GrCKyBF3sh9QG2gE5AOtgK3nrZoKRF5ks6eBwGLTDUtZxhTLw424ug4HATuNMU4RyQak2L5aATtK2c4cYIeIXA90ABZdJE9KeS1tQSlVupGAA9e5oK7uVwfgW1znpd4HpohIY/dghd7uYehzgcEi8msRsYlIPRHp6t7mD8BoEQkUkdbAuDLyEAzYgWOATUT+gutcU5GZwN9EpI24dBGRegDGmDRc56/+B3xW1GWoVHWiAUqp0j0IzDLG/GSMOVL0AqYC9wHPANtxBYEs4J+AxRjzE65BC39wp/8AXO/e5htAAZCBqwtubhl5WA4sA/YBB3G12op3AU4BPgFWALnAe0BAsfkfANeh3XuqmhJjTNlLKaWqHRHph6urr4XRH7qqhrQFpdQvkIj4AE8AMzU4qeqqzAAlIu+LyFERKe1ELO6+7xgRSRKRbSLSrdi8B0Vkv/v1YGnrK6Uqloh0AE7gGszxnyrOjlJXrMwuPnc3wSngQ2NM51LmDwN+h6vfvSfwpjGmp4jUBRKAHrhGJm0Guhtjsiu2CEoppX6JymxBGWPW4TrZezEjcAUvY4zZANQWkUbALcA37osOs4FvgCEVkWmllFK/fBVxHVQTSo4sSnOnXSz9AiIyHhgPEBQU1L19+/YVkC2llFLeaPPmzceNMfXLWs4rLtQ1xswAZgD06NHDJCQkVHGOlFJKXS0icrA8y1XEKL50XLdcKdLUnXaxdKWUUqpMFRGgFgMPuEfz9QJyjDGHcV1keLOI1BGROsDN7jSllFKqTGV28YnIfGAAECIiacCLgA+AMeZd4GtcI/iScN3q/2H3vCwR+RuuK+0BXjbGXGqwhVJKKeVRZoAyxtxTxnwDPH6Ree/jumeZUkopdVn0ThJKKaW8kgYopZRSXkkDlFJKKa+kAUoppZRX0gCllFLKK2mAUkop5ZU0QCmllPJKGqCUUkp5JQ1QSimlvJIGKKWUUl5JA5RSSimPd9cmE5d8vERaXPJx3l2bfFXXLY1XPA9KKaWKe3dtMl2a1iKqVYgn7dnPtwHwj9FdLjstLvk429JyeKx/qwv2sS0tx7OvouWK0ktbPqpViOf9l1sPefZVtN8il5NWtN+DmacvWZ6i/QKe/BXNb1Ev6IJylLa9S30WUa1C6NK0FpPmJdK1WS1Ca/oz/PrGjJudwOSb25TIS2nlsFpg3OwEereoRWidIIZ3LX1d8QsMphw0QKkKVVrFUtoP4mruu7wVzpVs71IVzvDrG19QaZRWMQBXraLzprTSKslLVbDFP6uiim5kC3+Gd6gPDeqzZNthADJy8z0V50XTnE4yduwjNLQOwwddz7gPLqwki/Yxultj3ozdx+TONZmyPYfR3ZvyZux+Jt/cBjgXIMJDghg3axPv9a6JNf0EDy8zWDDYLEJ47lGWJJ3F4TQY4047dYwl+4vSwCaG8KMHWZJqwSEWjOBazlLAlISjjO7cgCW7j4MxDM9P5ctDdjKsAWzKcoDA8JoFWH/K4uFlBRhjeLqtH3GNarjK63Bye0Aub+YGMtn/MFPONmL0Dc1Zsu0IAMOb+vPlzqNkiC8/pOYw9d6IEmUbfn1jJs3dwtTugbBjB10PO1l1qhm+TidLNiQzOfAYMUsLsRvBJhD+426WZAbiQDAINqsQLvlM23yU0elbmVvQBV9nBks2/cjk62oR880+7E7353L6GLZaDcMpBw1QXqyijiLLe9RVERVUeP0gxs1O4L2HevDl1kNk5ObzfXLWBT/20tYdfn1jZqw7wPh+4VdUYZescFwVzJQV+0tMF89DWQGlaHuTWwqTVjmZ2K8lS7YdPlcJWS2E1w/ypH22JZ2nb2lLXPJxd6XhICNxF6F1gxk+6DrGLdjlyUPRZ2QFHn5vIxbjxIYhfMcmluTVx2GxYiwWbBYL4TV9WPJDOg4nrv3iJHx7PEvOhOAw4q4QIXzbRpbkh+IwYIzTVZFsWscS08xVkRjj2sfaZSwJaOfah4ir0shKZ8kBBw7c+zBOwr/+nCV1rsdhdS8HhCd8xxJ7o3PbswjhKbtZcjwQh8OJcTpd686ZyZTWNzG6dTBLDtldlW7qFqwn4eG8Bvj6WJl+f/dznxVAVhbDe7aCwACmrUpicvp6ppy+ngV7cwgo3MH07/7LzobhvJJ3O77GwZLvk5h+aCU7gxvySl4EvjhZEv8j05MWs/OnLF4Z+Ai++3NZsnc1k8/uJ+arAuwirs9gfSzTpDmjD21lXkEkfVJ+4NWCrvRJ+YF5hQ7uzdnLtC9OwT//yZI2v8JhsWB12Jm8fj4TTt1DodWGxRhsTgeDkzbyqn0gI3euJrZ1T4Ay0la604TBSRt4NX+ge79deW7Ve3Q6eoAJo/5Mvs0Xu7WA51bNdKXl/tmzX4B/7yzAZ/ta1+dCEK/eNM5VjjB3OZzCc8HH6LQ7gQnz+pNv88Fu9eG5oAyi6vcs8T0dPvM1pu7dw4SsP1ForY+Pv2HUmYMsDGqJxX6WkyvXQffh2IrK0amUshW40pa2jmQUR1lobYilMJ+T784sue7ZgZiCMzmUgxh3Yb2FNz3y/XIDxJVU9pc6ytx5KIcpK/aXqOw3pWQDMP3+7uVO25mew+sr9mGAp29pS6fGtZjwv80A3H59Iz7fcojJvRoSE5furqDAZrMQPagtMauTSlTI0X2aEbPuIA5jME6DTQzRAceJOVUHBxasFiG6UQExh2zkG7Bj4V7JYCkhTKyZS8zJ2q5K1X3kGd0thJhtJ3A4DVYM0S2EmAN219GWcRJ9eCMxoTecq0wxRDtTiKF5iUoyulYu007WYujxPcyr25E+h3ezvlEH+pxMZX3N5twb0ZClOzKYWHiAGNMMh9WGXSw8PaTduc/D4eQGk01oi8YMv6kzk+ZsZuLelcQ06E7zE0fYFRrOyL3fEduyOwCDf9zMonZ9GblvPbFh3bCLFWO14ANM3zqfndkFvHLTOHwddvzsBUSnriemzSCwWIm2phKTH0qhE8Q4PRXdolJ++BdN27felRcRBh9IYFHbPueWs1gYnLKFRW2izqWJuNLa9mHk7rXEht9wiX3Eu9J+SiA2tEPJ5ZI3ENsiwrW95HgWte3rWrdNL7BaGZzzI4vqtqdP6nbWN+vMc3uW0mnHRiYM/yOFVhtiDA6rjVtDrazN82fqr7vAG1MYF3gDhVYfAnAw/Zs34eQpHrnrRfKx4o+D8af2MicgnP65B1lYpx3+jgLGH/iWOc170j9tGwtbReFfmM/4vauY0/lX9A+rxcKUM/g7Cxm/fRmz2g0sWY60RNY26kh/ZyYLfZsSGeQg/rSVUaeSWesTSv+sZBY17MLIwjRifRoCwsPBucw4XYd8I0R3rglWKzFbs4msBfE5EN0pGPz9idl87FxauA18fIjZm0dkiA/xxwuJrpUDOTnE0JzIgALi83wZ5Z/D2vxAxtbJZ0ZOMPlOGFXfsDbT6UrLCnTtN6Keax/fp+Nv7IzP3sGckOvo37I2C5NPEtm8FvE/5TAqPZG1dcIZu3cNM7reRr5YGXVkK8vqtmVI6g+sbdeLqU1Pwb9fZ8LwP3Gd71m2SG3yjTAqojFr9x1nbM/mzPj2APmFTqJvCIX8fGK25xDZKJD4w2eI7lgDzp4lJrmQyIBC4vN8GNW1MWv3u9ddm0y+wxBd77Rr3dP1iKwJX7z1e84e3i9l1cHlClAiMgR4E7ACM40xr503/w1goHsyEGhgjKntnucAtrvn/WSMuf1S+6qqAFVad85/v01myor99G5V19N9UFSx3xBW54I0T2V/cxtiVp5XsQ9qXWpa0RH+4q2uo8jp93dn50/ZvL4yCV+bhegeDYiJSyPfYbCLhedaW+kUEsCE+JPkOw12LDzHATrlHmFCUA/ysWK3WnmuTxM6+RUwIfYQhQ4nYgyIYCxWfCwwvUkOOw/l8KoJp8+hnaxv3LH0SjA9kdgGHcEYBu/fwKJOA8qsQB/e/CUzIkeR7+PPqJ2rWduyG/0PJJSr8n1485fM6j7cNf9goqvyS08ktn47Vx6KV5LNI0BgcPImV6DYuZq1bXrS336MhUEtiTx7lHi/BozasZK14T3O5WH3OmLDe2C3WDFWGz5WC9Mz17Fza7LrqNthx08M01e/AydO8MidL5KPhUjLSeKdwUQ7fgRjiLGFE+nMJt5Sh+jCZHA4iPFvi7/9LOMPxjGnTT/6twlh4d5sVwX70/fMatDVVc4tS5gROZp8my/RIWegcWNitp0gsllN4lNziW5UCMePE1PYiEjbaeLtQUTXzgWbjZjjgUQ2DSY+7STRN7UGIGZV0rl1i6eF1SE+Jbv0tE7Brn1k+BFZ10p8loPo62q59pGYeW65fmGuynRrNpEt6hB/8LztNfAj/uhZogeEg8VSYh+jfE+w9pQPYy0ZzPALJ98B0cHZpO7+kYVNuzHq1AHe2PMFcfuP8siYv5MvVvztBYxP28CsNv3BauXhqDBPJVlaxVnetOi+zSE31/U5F+XPvVz/tiEsSjzEyIgmrN13jP5t67MwMb3Uz8/fx8L4G8OZFZfi+p52aOBZN3Z3xhWnFd9v0T7mbPzpgrSi/Zb2uVxQjgY+LPzxdIntdQx08t0xO6N2rOSNr94g7raxPHL9PeTbDf4+FoZ2bsiixEM8d2t7z8FbocOJADar5ZLl6NM6hPVJx8tcN236bzILsw+fO/K/iDIDlIhYgX3Ar4A0XE/IvccYs+siy/8OiDDGPOKePmWMqVFWRopURIC6kpOZRcFodLfGniAzbXUyQ5sHMHdPDr4CfgLTO8LOIyd55XgwvhhXRdYkh525hldz69KHbNabOozMO0isXyMwMDg7mUUhHRh5MonYoOauijZzP4sadKZP/iHW+zfmuVrH6ZRziAn2thQiiLub5eGExecqe3dFOzbx62Jpq1gb3t11lNRlKPlW3/OWG02+jx/RzhTXEUxge/wLzzI+/nPmRAyjf/YBFjaJINInj/jCANfRnt1OzE+cq3xztruO2PzanKssG9tdaQfsRDYJJj79JNG9m8CpU8Rsz8HfZmF8v5I/sMgmNYhPP+U6ArRYXEeZ7qOu6BpZUKMGMUd8ie5a1/X+u59KVhBnzxKzPrX0SrdxEPGHTl/4Q+3amLU7D7mClqlPZNOaxKflEt3KB1JSiHE0cR11b/mSOZEj6N+6HguTT7o+o70rmdV1GNhs5a5wyqpMo29sAceOEbPnzFWr6LwprbRKd1Zciut3UXiYRY56rtZc5/7g61vi8/P3sfD+Q65W3oT/bSa/0IHdYUpUfuVNK62SLKpM7+3ZzNW6HhDOtDUHGNo5lHkbUz3zi8pmdzgxgE+xA067w3nJA9DyphXtd+7GVHxtFvxsFtfB6qEcXvlqjyetaH3A876ovKWVo7TtvfrVHvq0qsv65ExGWjOJtYVidxpP2W7r0ojw+kGe/RSt9/ryfViEyz/gLmXdx++82V6QkexTVl1engDVG3jJGHOLe/pZAGPMPy6yfBzwojHmG/d0pQWo4udaJs1LZOq9EeX+YKetTmZorULmHTL0ObKb9Q3bM3L3WtaGuY7+F3Ye5KrI4he6KvZLpEUe3kN8o/ZE718Jfn7ENO9LZHYK8XXCiP5xLfj6EdOkF5HZB4mv04JRu9ewtkUEY7ctZ8YNo8i3+hAtaXA2nxjf1vhbhfH9WzHn+xT613ay8LDTldY3jDkJ6RcedcX9SP/CDBY6QjyBosRR17oD5NudjOrSkLXJWSWOun5uBVX8R1ziB3Hej/1i23s4KuyKK+zLrXDOfR7J5NtNyYDi/oz8fSz84ea2TFtzgIkDwi9Z4VgtckGlUVolCVy1is6b0kqrJItXsNPv786ClTtZeOAUvjYLsx8+F4zyChz4WIXJ7s9+6r0RP6+bu5R64PwD0kdvbFXqgWpR2exO4+kiLxqZduDYua75n3MOt7y9NUX7BS7o/i8611u8HKVtL3pQaxxO2HvkJAsT0/G1WbijWxOGX9+4RN1Z/KC+vAf6lzplUXxdi3/QPmf+6XaUoTyDJJoAqcWm04CepS0oIi2AlsCqYsn+IpIA2IHXjDGLSllvPDAeoHnz5uXI0jmlDY+cOCCcWzqFuiqFQofrZKZxMpgcXv2qkJGWTGKdtQEYfPowry4pZORPm1havx0jD+9iYaveRNozWdhxIKNq5rO29mCiO9VlxrZMYvrcw6g2NVkbEkJ0ZDNmfPejK61DPdaGhDCqXX0WAaMimjDL/xYARnVowKLEorRBF6StDbqZ/s2DiQn8Nf4+FqJvDGdWnA27zYk/roqsV6t6BAfYPEdTPjYLvdo2ILiG37m0Ess5PMsFB5z7NwcH2PCxWXAYw6JtRzyV+XO3ti9RycTuPnpFaQ5z4Xmue3s24/Mth8pc12oRT17tDudl52HKiv2efRX9UIES6aXtz8dmxWEcJbo1ZsWlYHcaBDhw7DRT740gqlWIp0Iq8uiNrTxp4fWDLqg0SjsKLX4u8PyKrvj2SttHdUoD12d/U/v6F1SwnRq7Kqy1GQV0alyT/Rmn2Hkoh2lrDjD9ftc5vi+3HmLKCtfglqhWIZ5zwUUV4+WkbUvLYdbDN3gqyUdvbEWnxq4u/fce6uEZXelw4pl+76EeJf7nRYNqolqFeJYpfi66+PvLTSvab/Hz3bd1aXTB8sXzWrR80TrFR4leanuP3tiKuOTjzFh3wPPZD7++sWfZqfdGXFC2otGvxbdXWjmK5+dS65qzZ05esHIpytOCuhMYYoz5jXv6fqCnMWZSKcv+CWhqjPldsbQmxph0EQnHFbgGGWMuetXW5bag4pKPeyJ+VKsQ/hu7m1djkxl5No2l1gbk2/yIXj8fgJg+9xCZuoP4Zp2JjvvI1c3U69dEZv5IfL2WjKpdwNqCIPq3q3/R/tRLdSkUP3KviKPMs3bnBZV9aee+ykor71FXRYziK21kXHkHj/zcUXylDSm/1FDx0gJK8aNucFWSy3dmeL5fl1LWoJrSRlNW1hD8qnCpSw6KDiaLPte45OOe72lRcCu+/C/x86kq59eZ509XBhHZbIzpUeZyFdnFJyKJwOPGmLiLbGs2sMQY8+nF9lfeAFX8yx+XfJxJszfS//AultUMZ8i+9a7uN2ch4+vmMetUTdeopnb1WbQ9g5FdGxO75yhw6e6h0vpTL9WlcCWVfWlN8eJHmZcaBn25FykCF60wrsUKoLyjNK/lz+hqqcrr5a513vDZV2SAsuEaJDEISMc1SOJeY8zO85ZrDywDWhr3RkWkDnDGGHNWREKA74ERFxtgAZcOUBcEpXmJTIwIwfHlEvamZ7Ow8yD6miy22upSaLFcVh9/af3R5b26u8iV/JO94cuilFKVqbwBqsxzUMYYu4hMApbjGmb+vjFmp4i8DCQYYxa7Fx0DfGRKRrwOwHQRceK6799rlwpOZSnRLRDsZGLuDl79tjV9ciys73wTfcPrsP4ADGxTn9/c6LpQubx9/KX1R5fWn1qkvP2vZSktCF3JdpRS6pem2l2oW9SdN3b9p8zp/Cs6WvP4zq8hoyIa065hTawWPKN+it/qRlsjSinlHSqsBeUNineDRVlPMXbjImIi76RTXV82n3QwqnMoa/cd564ezYhqFeJpDRW1RLQ1opRS1U+1CFCerr3RHSF6MrO6jMHHAruyCnju1vae80XFR6JoUFJKqeqtWgSoqFYhTL03ggnTv6Ww6334+PlzZ49mhNcPYtqaA3RqXMuzTFHLSSmlVPVWLQIUQFQdC11Sd7O+xfWMH9CayTe7LkLW7jyllPplqjZP1I2bvZDd9cOI7hTMnI0/eZ7aGNUqRAdAKKXUL1C1CFBxyceZlBbM1O0LmHx/P6beG8GkeYkXPFpYKaXUL0e1CFDbEpOY+tnfibqtL0CJ801KKaV+mbw2QL27NtnTQnos8Uui0nYS1+923l3ruo2fdu0ppdQvm9cGqKKh5XH7j8H//kfc6EeYtCzFc7NNpZRSv2xeO4qvqBtv0gfxjG3Rjzltb6/Uu+0qpZSqWl7bggJXkBprDhHT5x7G9myuwUkppa4hXh2g4pKPMyevNtH7Ypmz9aiO2lNKqWuI1wYoz62Llv2HyXVydWi5UkpdY7w2QG1Ly2FqvwZE7VwPvXvr0HKllLrGlCtAicgQEdkrIkki8kwp8x8SkWMi8oP79Zti8x4Ukf3u14Plzdhj/VsRdXCra6JXL0CHliul1LWkzFF8ImIF3gZ+BaQBm0RkcSkPHvzYGDPpvHXrAi8CPQADbHavm12u3G3YADVrQseO5VpcKaXUL0d5WlCRQJIx5oAxpgD4CBhRzu3fAnxjjMlyB6VvgCHlzt3330NkJFi8tidSKaXUVVKemr8JkFpsOs2ddr47RGSbiHwqIs0uZ10RGS8iCSKScOzYMVfi6dOwbRv07l2OLCqllPqlqaimyZdAmDGmC65W0geXs7IxZoYxpocxpgf+NV0j9TZtAqcTevUiLvm45xZHSimlrg3lCVDpQLNi003daR7GmExjzFn35Eyge3nXPV+Ar9U1nPzb7QDENWrPpHmJeosjpZS6xpQnQG0C2ohISxHxBcYAi4svICKNik3eDux2v18O3CwidUSkDnCzO+2iavjZXNc85TRiyohoJi1J1lscKaXUNajMUXzGGLuITMIVWKzA+8aYnSLyMpBgjFkMRIvI7YAdyAIecq+bJSJ/wxXkAF42xmSVtc+oViGMPbCemPY3E623OFJKqWuSGGOqOg8l9OjRw8R8vIxJb8Uy1hxiTsh12oJSSqlfEBHZbIzpUdZyXjd++9RZu+sWR1/+H5MDjuotjpRS6hrldQEqr8DB1NEdiDqwBerW1VscKaXUNcrrngdVP9iPqJrubse6dQHXOSnt4lNKqWuL17WgAMhyj6NwByillFLXHq9rQQEaoJSqZgoLC0lLSyM/P7+qs6K8iL+/P02bNsXHx+eK1tcApZT62dLS0ggODiYsLAwRqersKC9gjCEzM5O0tDRatmx5RdvQLj6l1M+Wn59PvXr1NDgpDxGhXr16P6tV7d0Bql69qs2HUqrcNDip8/3c74T3Bih/fwgIqOqcKKWUqiLeGaAyM7V7TylVbpmZmXTt2pWuXbvSsGFDmjRp4pkuKCgo1zYefvhh9u7de8ll3n77bebOnVsRWQYgIyMDm83GzJkzK2ybvyReeaujhObNYf9+2L69qrOjlCqH3bt306FDh6rOBgAvvfQSNWrU4KmnniqRbozBGIPFix6A+tZbb/HJJ5/g6+vLypUrr9p+7HY7NlvVjIkr7btR3lsdee8oPm1BKVU9/f738MMPFbvNrl3hP/+57NWSkpK4/fbbiYiIIDExkW+++Ya//vWvbNmyhby8PO6++27+8pe/ANC3b1+mTp1K586dCQkJ4bHHHmPp0qUEBgbyxRdf0KBBA55//nlCQkL4/e9/T9++fenbty+rVq0iJyeHWbNmERUVxenTp3nggQfYvXs3HTt2JCUlhZkzZ9K1a9cL8jd//nzeeust7rzzTg4fPkyjRq4HQ3z11Ve88MILOBwOQkNDWbFiBSdPnmTSpEkkJiYC8PLLL3PbbbcREhLCiRMnAPjoo4+IjY1l5syZjB07luDgYDZv3syAAQMYPXo0Tz75JPn5+QQGBjJ79mzatGmD3W7n6aef5ptvvsFisfDYY4/RunVrZsyYwaeffgrA0qVLef/991mwYMEV/fuulPcGqFatqjoXSqlfgD179vDhhx/So4frgP21116jbt262O12Bg4cyJ133knHjh1LrJOTk0P//v157bXXmDx5Mu+//z7PPPPMBds2xhAfH8/ixYt5+eWXWbZsGW+99RYNGzbks88+Y+vWrXTr1q3UfKWkpJCVlUX37t256667+OSTT3jiiSc4cuQIEydO5Ntvv6VFixZkuQeNvfTSS9SvX59t27ZhjPEEpUs5fPgwGzZswGKxkJOTw7fffovNZmPZsmU8//zzfPzxx0ybNo1Dhw6xdetWrFYrWVlZ1K5dm/9v7+yDq6quBf5bho9IQiGYSjWhSRq1kIR8QbGEAgYIH+rAoDAmBPkqpULx6aP1UcuMPvuP0qoPqT7AeTYKStKIAo6FoQW1xWefGgifUV6CxJYEQxIxQGB83GS9P87J9SbkiyQk99L1mzlz915n7332OtmTdffe6669fPlyqqurueGGG8jJyWHRokVX+uo7jf8aqB/8oKd7YRhGR+jATOdqEhsb6zVO4MxaXnrpJTweD+Xl5RQVFV1moK6//nqmTZsGwIgRI9i7d2+zbd9zzz3eMqWlpQC8//77rFy5EoCkpCTi4+ObrZuXl8d9990HQGZmJsuWLeOhhx7ib3/7G+np6URFRQEwyF1N2r17N9u2bQMc77iwsDA8Hk+rus+ePY7B/QQAABCsSURBVNu7pPnVV18xb948jh9vfDr57t27efjhhwkKCmr0vOzsbDZv3kx2djb79u0jNze31WddDfzXQNkSn2EYXUBISIg3XVxczHPPPcdHH33EwIEDmTt3brO/0+nTp483HRQU1KIh6Nu3b5tlWiI3N5eqqipeeeUVAMrLy/nss8+uqI3rrrsOXz+Cprr46r5q1SqmTJnCsmXLKCkpYerUqa22vWjRIu69914A7rvvPq8B607atVsoIlNF5JiIlIjIZfNcEVkhIkUickhE9ohIlM+9OhE54F5vNa17GfX1cPGiGSjDMLqcs2fP0r9/f771rW9x6tQpdu1q9YDvDjFmzBjy8/MBOHz4MEVFRZeVKSoqwuPxUFZWRmlpKaWlpTzyyCPk5eWRlpbGu+++y+effw7gXeLLyMjghRdeAJylxTNnznDdddcRFhZGcXEx9fX1bN26tcV+1dTUEBERAcDLL7/slWdkZLB+/Xrq6uoaPW/IkCGEh4fz1FNPsWDBgs69lA7SpoESkSDgBWAaEAdkiUhck2KFwEhVTQS2AL/xuXdRVZPda3qbPXJfkhkowzC6mtTUVOLi4hg6dCjz5s1jzJgxXf6MBx98kLKyMuLi4njiiSeIi4tjwIABjcrk5uYyc+bMRrJ7772X3NxcBg8ezLp165gxYwZJSUlkZ2cD8Pjjj1NRUUFCQgLJycneZcfVq1czZcoU0tLSiIyMbLFfK1eu5JFHHiE1NbXRrOunP/0p3/nOd0hMTCQpKclrXAHmzJlDTEwMt912W6ffS0do081cREYD/66qU9z8owCq+mQL5VOA51V1jJs/r6qh7e3QyPh4LSgqgvx8mD27vdUMw+hB/MnNvKfxeDx4PB6Cg4MpLi5m8uTJFBcX95ibd2d44IEHGD16NPPnz+9wG1fbzTwC+IdP/iRweyvlfwzs9MkHi0gB4AGeUtVtTSuIyBJgCUDs4MGO0GZQhmEEIOfPn2fixIl4PB5UlQ0bNgSkcUpOTiYsLIy1a9f2WB+69K2JyFxgJDDeRxylqmUi8j3gHRE5rKqN3EhU9UXgRYCRt9yiVFSYgTIMIyAZOHAg+/bt6+ludJoDXf1btg7QHieJMmCITz7SlTVCRCYBq4Dpqvp1g1xVy9zPz4D3gJRWn9bgCWMGyjAM45+a9hioj4FbRSRGRPoAmUAjbzx332kDjnE67SMPE5G+bjocGANc7tLiizlJGIZhGLRjiU9VPSKyHNgFBAG/V9WjIvJroEBV3wJ+C4QCr7vh1f/ueuwNAzaISD2OMXxKVVs3UB4P9OoFoe32qzAMwzCuQdq1B6WqO4AdTWSP+aQntVDvA2D4FfXI43FmT3a2jGEYxj81/hPWt4G6Ojuo0DCuYdb/5TgfHK9qJPvgeBXr/3K8hRptk56eftmPbtesWcPSpUtbrRfqrtSUl5cza9asZsvccccdFBQUtNrOmjVruHDhgjd/5513titWXntJTk4mMzOzy9oLFPzPQDXMoAzDuCZJjBzA8s2FXiP1wfEqlm8uJDFyQBs1WyYrK4u8vLxGsry8PLKystpV/+abb/ZG7u4ITQ3Ujh07GDhwYIfb8+WTTz6hrq6OvXv3Ultb2yVtNseVhmrqDsxAGYbRraTFhvP8nBSWby7k2T8dY/nmQp6fk0JabHiH25w1axZ//OMfvYcTlpaWUl5eztixY72/S0pNTWX48OFs3779svqlpaUkJCQAcPHiRTIzMxk2bBgzZ87k4sWL3nJLly5l5MiRxMfH8/jjjwOwdu1aysvLSU9PJz09HYDo6GiqqhwD/Oyzz5KQkEBCQgJr3EC6paWlDBs2jJ/85CfEx8czefLkRs/xJTc3l/vvv5/Jkyc36ntJSQmTJk0iKSmJ1NRUbxDY1atXM3z4cJKSkrwR2H1ngVVVVURHRwNOyKPp06czYcIEJk6c2Oq72rhxozfaxP3338+5c+eIiYnh0qVLgBNGyjffJTQc4uUv14g+fVTnz1fDMAKHoqKiK67zzK5PNWrl2/rMrk+7pA933XWXbtu2TVVVn3zySf35z3+uqqqXLl3SmpoaVVWtrKzU2NhYra+vV1XVkJAQVVU9ceKExsfHO/165hlduHChqqoePHhQg4KC9OOPP1ZV1erqalVV9Xg8On78eD148KCqqkZFRWllZaW3Lw35goICTUhI0PPnz+u5c+c0Li5O9+/frydOnNCgoCAtLCxUVdXZs2frpk2bmtXrtttu088//1x37dqld999t1c+atQoffPNN1VV9eLFi1pbW6s7duzQ0aNHa21tbaP+jh8/3qtDZWWlRkVFqapqTk6ORkREeMu19K6OHDmit956q1fHhvILFizQrVu3qqrqhg0bdMWKFZf1v7mxgeNg16Y9sBmUYRjdzgfHq3j1w7/zLxNu4dUP/37ZnlRH8F3m813eU1V+9atfkZiYyKRJkygrK6OioqLFdv76178yd+5cABITE0lMTPTey8/PJzU1lZSUFI4ePdpsIFhf3n//fWbOnElISAihoaHcc8893hh6MTEx3kMMfY/r8KWgoIDw8HC++93vMnHiRAoLC/nyyy85d+4cZWVl3nh+wcHB9OvXj927d7Nw4UL69esHfHN0RmtkZGR4y7X0rt555x1mz55NeHh4o3YXL15MTk4OADk5OSxcuLDN510J/meg6uvNQBnGNUzDntPzc1JYMfn73uW+zhqpGTNmsGfPHvbv38+FCxcYMWIEAK+99hqVlZXs27ePAwcOMHjw4GaP2GiLEydO8PTTT7Nnzx4OHTrEXXfd1aF2Gmg4qgNaPq4jNzeXTz/9lOjoaGJjYzl79ixvvPHGFT+rV69e1NfXA60fyXGl72rMmDGUlpby3nvvUVdX510m7Sr8z0ABDBrUaa8ewzD8k0MnaxrtOTXsSR06WdOpdkNDQ0lPT2fRokWNnCNqamq48cYb6d27d6NjLFpi3LhxbN68GYAjR45w6NAhwNljCQkJYcCAAVRUVLBz5zchR/v378+5c+cua2vs2LFs27aNCxcuUFtby9atWxk7dmy79Kmvryc/P5/Dhw97j+TYvn07ubm59O/fn8jISO8Bhl9//TUXLlwgIyODnJwcr8NGw9EZ0dHR3vBLrTmDtPSuJkyYwOuvv051dXWjdgHmzZvHnDlzunz2BH5qoD7o/e1Oe/UYhuGfPDA+9jKHiLTYcB4YH9vptrOysjh48GAjA5WdnU1BQQHDhw9n48aNDB06tNU2li5dyvnz5xk2bBiPPfaYdyaWlJRESkoKQ4cOZc6cOY2O6liyZAlTp071Okk0kJqayoIFCxg1ahS33347ixcvJiWl9WhvDezdu5eIiAhuvvlmr2zcuHEUFRVx6tQpNm3axNq1a0lMTCQtLY0vvviCqVOnMn36dEaOHElycjJPP/00AL/4xS9Yt24dKSkpXueN5mjpXcXHx7Nq1SrGjx9PUlISK1asaFTnzJkz7faYvBLaPG6juxnS/wb99sMbeX7B7Z3y6jEMo/uw4zb+edmyZQvbt29n06ZNzd6/2sdtdCunQwfxr7eEmHEyDMPwcx588EF27tzJjh072i7cAfzOQN3YG14t/ZofHq8yI2UYhuHH/O53v7uq7fvdHtTgGwfw/NwRXeLVYxhG9+Fv2wVGz9PZMeF3Bgq6zqvHMIzuITg4mOrqajNShhdVpbq6muDg4A634XdLfA2kxYbbEp9hBAiRkZGcPHmSysrKnu6K4UcEBwcTGRnZ4fp+a6AMwwgcevfuTUxMTE93w7jGaNcSn4hMFZFjIlIiIr9s5n5fEfmDe/9DEYn2ufeoKz8mIlO6ruuGYRjGtUybBkpEgoAXgGlAHJAlInFNiv0YOKOqtwD/Aax268bhHBEfD0wF/tNtzzAMwzBapT0zqFFAiap+pqr/B+QBM5qUmQG84qa3ABPFOft9BpCnql+r6gmgxG3PMAzDMFqlPXtQEcA/fPIngdtbKqOqHhGpAW5w5f/TpG5E0weIyBJgiZs9LyLH2tV7/yQcuBb8400P/8L08C9Mj84R1Z5CfuEkoaovAi/2dD+6AhEpaE8ID3/H9PAvTA//wvToHtqzxFcGDPHJR7qyZsuISC9gAFDdzrqGYRiGcRntMVAfA7eKSIyI9MFxenirSZm3gPluehbwjntq4ltApuvlFwPcCnzUNV03DMMwrmXaXOJz95SWA7uAIOD3qnpURH6Nc2zvW8BLwCYRKQG+xDFiuOXygSLAA/xMVeuuki7+wjWxVInp4W+YHv6F6dEN+N1xG4ZhGIYBfhqLzzAMwzDMQBmGYRh+iRmodiAivxeR0yJyxEc2SET+LCLF7meYKxcRWeuGdzokIqk+dea75YtFZH5zz7qKOgwRkXdFpEhEjorIQwGqR7CIfCQiB109nnDlMW6YrRI37FYfV+7XYbhEJEhECkXk7UDVQ0RKReSwiBwQkQJXFlDjyn3+QBHZIiKfisgnIjI60PQQke+7f4eG66yIPBxoenhRVbvauIBxQCpwxEf2G+CXbvqXwGo3fSewExDgh8CHrnwQ8Jn7Geamw7pRh5uAVDfdH/hfnNBVgaaHAKFuujfwodu/fCDTla8HlrrpZcB6N50J/MFNxwEHgb5ADHAcCOqBsbUC2Ay87eYDTg+gFAhvIguoceX24RVgsZvuAwwMRD189AkCvsD5UWxA6tHtLy1QLyCaxgbqGHCTm74JOOamNwBZTcsBWcAGH3mjcj2gz3YgI5D1APoB+3Eim1QBvVz5aGCXm94FjHbTvdxyAjwKPOrTlrdcN/Y/EtgDTADedvsViHqUcrmBCqhxhfPbzRO4jmOBqkeTvk8G/juQ9bAlvo4zWFVPuekvgMFuurnQUBGtyLsdd3koBWf2EXB6uMtiB4DTwJ9xZg1fqaqnmT41CsMF+Ibh6um/xxrg34B6N38DgamHAn8SkX3ihC2DwBtXMUAlkOMuuf6XiIQQeHr4kgnkuumA1MMMVBegzleMgPDXF5FQ4A3gYVU963svUPRQ1TpVTcaZgYwChvZwl64YEbkbOK2q+3q6L13Aj1Q1FefEg5+JyDjfmwEyrnrhLOOvU9UUoBZnKcxLgOgBgLt3OR14vem9QNLDDFTHqRCRmwDcz9OuvKXwTj0e9klEeuMYp9dU9U1XHHB6NKCqXwHv4iyFDRQnzFbTPvlrGK4xwHQRKcU5IWAC8ByBpweqWuZ+nga24nxpCLRxdRI4qaofuvktOAYr0PRoYBqwX1Ur3HxA6mEGquP4hneaj7On0yCf53rH/BCocafWu4DJIhLmetBMdmXdgogITsSPT1T1WZ9bgabHt0VkoJu+Hmcf7RMcQzWrBT38LgyXqj6qqpGqGo2zFPOOqmYTYHqISIiI9G9I44yHIwTYuFLVL4B/iMj3XdFEnAg4AaWHD1l8s7wHgapHT2zeBdqF84c+BVzC+ab1Y5z1/z1AMbAbGOSWFZwDHo8Dh4GRPu0swjkTqwRY2M06/AhnWn8IOOBedwagHolAoavHEeAxV/49nH/MJTjLGn1debCbL3Hvf8+nrVWufseAaT04vu7gGy++gNLD7e9B9zoKrHLlATWu3OcnAwXu2NqG470WiHqE4MyuB/jIAk4PVbVQR4ZhGIZ/Ykt8hmEYhl9iBsowDMPwS8xAGYZhGH6JGSjDMAzDLzEDZRiGYfglZqAMwzAMv8QMlGEYhuGX/D9DR1Tvii0/RQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation accuracy at 0.7890999913215637\n"
     ]
    }
   ],
   "source": [
    "# Gradient Descent\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate).minimize(loss)    \n",
    "\n",
    "# The accuracy measured against the validation set\n",
    "validation_accuracy = 0.0\n",
    "\n",
    "# Measurements use for graphing loss and accuracy\n",
    "log_batch_step = 50\n",
    "batches = []\n",
    "loss_batch = []\n",
    "train_acc_batch = []\n",
    "valid_acc_batch = []\n",
    "\n",
    "# Object to export the trained model\n",
    "saver = tf.train.Saver()\n",
    "\n",
    "with tf.Session() as session:\n",
    "    session.run(init)\n",
    "    batch_count = int(math.ceil(len(train_features)/batch_size))\n",
    "\n",
    "    for epoch_i in range(epochs):\n",
    "        \n",
    "        # Progress bar\n",
    "        batches_pbar = tqdm(range(batch_count), desc='Epoch {:>2}/{}'.format(epoch_i+1, epochs), unit='batches')\n",
    "        \n",
    "        # The training cycle\n",
    "        for batch_i in batches_pbar:\n",
    "            # Get a batch of training features and labels\n",
    "            batch_start = batch_i*batch_size\n",
    "            batch_features = train_features[batch_start:batch_start + batch_size]\n",
    "            batch_labels = train_labels[batch_start:batch_start + batch_size]\n",
    "\n",
    "            # Run optimizer and get loss\n",
    "            _, l = session.run(\n",
    "                [optimizer, loss],\n",
    "                feed_dict={features: batch_features, labels: batch_labels})\n",
    "\n",
    "            # Log every 50 batches\n",
    "            if not batch_i % log_batch_step:\n",
    "                # Calculate Training and Validation accuracy\n",
    "                training_accuracy = session.run(accuracy, feed_dict=train_feed_dict)\n",
    "                validation_accuracy = session.run(accuracy, feed_dict=valid_feed_dict)\n",
    "\n",
    "                # Log batches\n",
    "                previous_batch = batches[-1] if batches else 0\n",
    "                batches.append(log_batch_step + previous_batch)\n",
    "                loss_batch.append(l)\n",
    "                train_acc_batch.append(training_accuracy)\n",
    "                valid_acc_batch.append(validation_accuracy)\n",
    "\n",
    "        # Export the model\n",
    "        saver.save(session, 'model/edge', global_step=epoch_i)\n",
    "\n",
    "        # Check accuracy against Validation data\n",
    "        validation_accuracy = session.run(accuracy, feed_dict=valid_feed_dict)\n",
    "\n",
    "loss_plot = plt.subplot(211)\n",
    "loss_plot.set_title('Loss')\n",
    "loss_plot.plot(batches, loss_batch, 'g')\n",
    "loss_plot.set_xlim([batches[0], batches[-1]])\n",
    "acc_plot = plt.subplot(212)\n",
    "acc_plot.set_title('Accuracy')\n",
    "acc_plot.plot(batches, train_acc_batch, 'r', label='Training Accuracy')\n",
    "acc_plot.plot(batches, valid_acc_batch, 'x', label='Validation Accuracy')\n",
    "acc_plot.set_ylim([0, 1.0])\n",
    "acc_plot.set_xlim([batches[0], batches[-1]])\n",
    "acc_plot.legend(loc=4)\n",
    "plt.tight_layout()\n",
    "plt.show()\n",
    "\n",
    "print('Validation accuracy at {}'.format(validation_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Test accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch  1/4: 100%|██████████| 1900/1900 [00:02<00:00, 909.90batches/s]\n",
      "Epoch  2/4: 100%|██████████| 1900/1900 [00:02<00:00, 884.38batches/s]\n",
      "Epoch  3/4: 100%|██████████| 1900/1900 [00:02<00:00, 914.58batches/s]\n",
      "Epoch  4/4: 100%|██████████| 1900/1900 [00:02<00:00, 927.90batches/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nice Job! Test Accuracy is 0.8596000075340271\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# The accuracy measured against the test set\n",
    "test_accuracy = 0.0\n",
    "\n",
    "with tf.Session() as session:\n",
    "    \n",
    "    session.run(init)\n",
    "    batch_count = int(math.ceil(len(train_features)/batch_size))\n",
    "\n",
    "    for epoch_i in range(epochs):\n",
    "        \n",
    "        # Progress bar\n",
    "        batches_pbar = tqdm(range(batch_count), desc='Epoch {:>2}/{}'.format(epoch_i+1, epochs), unit='batches')\n",
    "        \n",
    "        # The training cycle\n",
    "        for batch_i in batches_pbar:\n",
    "            # Get a batch of training features and labels\n",
    "            batch_start = batch_i*batch_size\n",
    "            batch_features = train_features[batch_start:batch_start + batch_size]\n",
    "            batch_labels = train_labels[batch_start:batch_start + batch_size]\n",
    "\n",
    "            # Run optimizer\n",
    "            _ = session.run(optimizer, feed_dict={features: batch_features, labels: batch_labels})\n",
    "\n",
    "        # Check accuracy against Test data\n",
    "        test_accuracy = session.run(accuracy, feed_dict=test_feed_dict)\n",
    "\n",
    "\n",
    "assert test_accuracy >= 0.80, 'Test accuracy at {}, should be equal to or greater than 0.80'.format(test_accuracy)\n",
    "print('Nice Job! Test Accuracy is {}'.format(test_accuracy))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Export model\n",
    "Now, let's use the Tensorflow backend to get the TF graphdef and frozen graph\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Source: https://blog.metaflow.fr/tensorflow-how-to-freeze-a-model-and-serve-it-with-a-python-api-d4f3596b3adc\n",
    "def freeze_graph(model_dir, output_node_names):\n",
    "    \"\"\"Extract the sub graph defined by the output nodes and convert \n",
    "    all its variables into constant \n",
    "    Args:\n",
    "        model_dir: the root folder containing the checkpoint state file\n",
    "        output_node_names: a string, containing all the output node's names, \n",
    "                            comma separated\n",
    "    \"\"\"\n",
    "    if not tf.gfile.Exists(model_dir):\n",
    "        raise AssertionError(\n",
    "            \"Export directory doesn't exists. Please specify an export \"\n",
    "            \"directory: %s\" % model_dir)\n",
    "\n",
    "    if not output_node_names:\n",
    "        print(\"You need to supply the name of a node to --output_node_names.\")\n",
    "        return -1\n",
    "\n",
    "    # We retrieve our checkpoint fullpath\n",
    "    checkpoint = tf.train.get_checkpoint_state(model_dir)\n",
    "    input_checkpoint = checkpoint.model_checkpoint_path\n",
    "    \n",
    "    # We precise the file fullname of our freezed graph\n",
    "    absolute_model_dir = \"/\".join(input_checkpoint.split('/')[:-1])\n",
    "    output_graph = absolute_model_dir + \"/frozen_model.pb\"\n",
    "\n",
    "    # We clear devices to allow TensorFlow to control on which device it will load operations\n",
    "    clear_devices = True\n",
    "\n",
    "    # We start a session using a temporary fresh Graph\n",
    "    with tf.Session(graph=tf.Graph()) as sess:\n",
    "        # We import the meta graph in the current default Graph\n",
    "        saver = tf.train.import_meta_graph(input_checkpoint + '.meta', clear_devices=clear_devices)\n",
    "\n",
    "        # We restore the weights\n",
    "        saver.restore(sess, input_checkpoint)\n",
    "\n",
    "        # We use a built-in TF helper to export variables to constants\n",
    "        output_graph_def = tf.graph_util.convert_variables_to_constants(\n",
    "            sess, # The session is used to retrieve the weights\n",
    "            tf.get_default_graph().as_graph_def(), # The graph_def is used to retrieve the nodes \n",
    "            output_node_names.split(\",\") # The output node names are used to select the usefull nodes\n",
    "        ) \n",
    "\n",
    "        # Finally we serialize and dump the output graph to the filesystem\n",
    "        with tf.gfile.GFile(output_graph, \"wb\") as f:\n",
    "            f.write(output_graph_def.SerializeToString())\n",
    "        print(\"%d ops in the final graph.\" % len(output_graph_def.node))\n",
    "\n",
    "    return output_graph_def\n",
    "\n",
    "def load_graph(frozen_graph_filename):\n",
    "    # We load the protobuf file from the disk and parse it to retrieve the \n",
    "    # unserialized graph_def\n",
    "    with tf.gfile.GFile(frozen_graph_filename, \"rb\") as f:\n",
    "        graph_def = tf.GraphDef()\n",
    "        graph_def.ParseFromString(f.read())\n",
    "\n",
    "    # Then, we import the graph_def into a new Graph and returns it \n",
    "    with tf.Graph().as_default() as graph:\n",
    "        # The name var will prefix every op/nodes in your graph\n",
    "        # Since we load everything in a new graph, this is not needed\n",
    "        tf.import_graph_def(graph_def, name=\"prefix\")\n",
    "    return graph"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from model/edge-3\n",
      "INFO:tensorflow:Froze 2 variables.\n",
      "Converted 2 variables to const ops.\n",
      "8 ops in the final graph.\n"
     ]
    }
   ],
   "source": [
    "# saving the model in model/frozen_model.pb\n",
    "g = freeze_graph('model/', 'netout')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import & use model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Result [[9.9945730e-01 5.7841000e-05 6.6928458e-09 5.9107151e-07 2.5566817e-07\n",
      "  1.8387044e-11 1.5552172e-07 5.8738078e-07 4.6154135e-04 2.1867334e-05]]\n"
     ]
    }
   ],
   "source": [
    "# Read the saved model from disk\n",
    "graph = load_graph('model/frozen_model.pb')\n",
    "\n",
    "# Access the input and output nodes \n",
    "netin = graph.get_tensor_by_name('prefix/netin:0')\n",
    "netout = graph.get_tensor_by_name('prefix/netout:0')\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "    # Check accuracy against Test data\n",
    "    res = session.run(netout, feed_dict={netin: [test_features[0]]}) # Feed 'A'\n",
    "    print(\"Result\", res)"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}