RNN_LSTM_text_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "RNN LSTM for text classification\n",
    "=============\n",
    "<span style=\"color: lightsteelblue;\">Deep Learning</span>\n",
    "\n",
    "The goal of this notebook is to train recurrent neural network with LSTM cell for the purpose of text classification."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Data preparation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "import collections\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "import pickle\n",
    "from pathlib import Path\n",
    "import os\n",
    "import time\n",
    "\n",
    "# custom libraries\n",
    "from nlp.preparer import Corpus\n",
    "from nlp.iterator import BatchIterator\n",
    "from nlp.conveyor import Conveyor\n",
    "\n",
    "# View\n",
    "from IPython.display import Markdown, display"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading corpora"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corpus label: good,  length: 6617 articles,  av length: 207 words,  max length: 402 words.\n",
      "Corpus raw article: 3 февраля в большинстве европейских стран закрылось зимнее трансферное окно — период, когда клубы могут заявлять новых футболистов, купленных у других команд. Ценники меняются чуть ли не ежемесячно. Н\n",
      "Corpus data (words): ['num', 'феврал', 'в', 'большинств', 'европейск', 'стран', 'закр', 'зимн', 'трансферн', 'окн', 'период', ',', 'когд', 'клуб', 'могут', 'заявля', 'нов', 'футболист', ',', 'куплен']\n",
      "\n",
      "Corpus label: bad,  length: 7518 articles,  av length: 84 words,  max length: 391 words.\n",
      "Corpus raw article: Неизвестный угрожает взорвать аэропорт Кишинева, если ему не дадут миллион. Неизвестный сообщил о бомбе в аэропорту Международного аэропорта Кишинева и требует миллион рублей, сообщили в пограничной п\n",
      "Corpus data (words): ['неизвестн', 'угрожа', 'взорва', 'аэропорт', 'кишинев', ',', 'есл', 'ем', 'не', 'дадут', 'миллион', '.', 'неизвестн', 'сообщ', 'о', 'бомб', 'в', 'аэропорт', 'международн', 'аэропорт']\n",
      "\n"
     ]
    }
   ],
   "source": [
    "corpora_paths = ['./articles/good.articles', './articles/bad.articles'] \n",
    "corpora = []\n",
    "lengths = []\n",
    "\n",
    "for path in corpora_paths:\n",
    "  corpus = Corpus(path, 321)\n",
    "  corpora.append(corpus)\n",
    "  length = [len(article) for article in corpus.articles]\n",
    "  lengths.append(length)\n",
    "\n",
    "  print(f\"Corpus label: {corpus.label}, \",\n",
    "        f\"length: {len(corpus.articles)} articles, \",\n",
    "        f\"av length: {round(corpus.average_length())} words, \",\n",
    "        f\"max length: {max(length)} words.\")\n",
    "  print(f\"Corpus raw article: {corpus.raw[0][:200]}\")\n",
    "  print(f\"Corpus data (words): {corpus.articles[0][:20]}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Visualizing articles lengths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA34AAAHwCAYAAAD5DL2VAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3XmYZGV5///3h0V2RGAkbDJE0QSNoo6KaxBcQBSMPyQq\nQUEM5htNIBJl+ZovJKKBGDdioqKoxCBIXFEgiggqLuiABFlEQQdZBmYERhAEWe7fH+c01LRd3TXT\nVV3d1e/XdfXVdbbnuevp08vdz3JSVUiSJEmSRtcaww5AkiRJkjRYJn6SJEmSNOJM/CRJkiRpxJn4\nSZIkSdKIM/GTJEmSpBFn4idJkiRJI87ET5K0SpIsSfKCAZR7fpI3tK/3S/K1PpZ9eZJd2tfHJPmv\nPpZ9VJKP9au82SBJJXnMAMvvqc2SfDLJsYOKQ5LmExM/SZojBpVwTVHnUP7wrqpTqupFU53Xa3xV\n9fiqOn+6cSXZJcn148p+V1W9YbpljyrbTJJmBxM/SdLISrLWsGOYz2x/SZo9TPwkaQQkeWmSS5Ks\nSPLdJE/sOLYkyd8nuTTJr5N8Jsm6HcfflmRpkhuTvGFsmF+Sg4H9gLcl+U2SL3dUudNE5SXZPMlX\n2jhuTfLtJBP+rknywiQ/acv4IJCOYwckuaB9nSTvS7Isye1JfpzkCd3ia9/v4UkuBe5MstYEvaXr\ntnHfkeTiJE/qqHulYY5jvYpJNgDOBrZq6/tNkq3GDx1Nslc7tHRFO3z1j3v9WkzxNV4zyXuS/CrJ\nL5K8uY11rfb4VknOaNv96iR/2XHtOkne336Nb2xfr9Nx/K0d98Drp4jjwCRXtm338yRv7Di2S5Lr\n2/a/CTi1xzZ7TnvfrkhyXZIDutQ92X1+eJIb2riuSrJbL+0qSfOFiZ8kzXFJngx8HHgjsBnwEeCM\nzj/sgX2B3YHtgScCB7TX7g68BXgB8Bhgl7ELqupE4BTgX6pqw6p62VTlAYcB1wMLgC2Ao4CaIObN\ngc8Dbwc2B64Bnt3lLb4IeB7wWODhbd23TBHfq4E9gU2q6r4Jytwb+G9gU+DTwBeTrN2lfgCq6k5g\nD+DGtr4Nq+rGce/rsTTJzqFtG5wFfDnJwzpO69Z2tAnNc7qE8Jdt/TsBTwFePu74aTRtvxWwD/Cu\nJLu2x/4vsHN77ZOAp9O0/dg98PfAC4EdaO6FySwDXgpsDBwIvC/JUzqO/wFNu24HvJap22w7muTw\n32jabCfgkvGVTnafJ3kc8GbgaVW1EfBiYMkU70OS5hUTP0ma+w4GPlJVF1bV/VV1MnAPzR/6Y06o\nqhur6lbgyzR/XEOThHyiqi6vqruAY3qss1t59wJbAttV1b1V9e2q+r3ED3gJcHlVfbaq7gXeD9zU\npa57gY2APwJSVVdW1dIe4ruuqn7b5fhFHXW/F1iXldtrdf05cGZVndOW/a/AesCzxsU2UdtRVZtU\n1QVdyt4X+EBVXV9VtwHHjR1Isi1N4nx4Vd1dVZcAH6NJvKDpGf2nqlpWVcuBfwT27yj3E1V1WZvc\nHjPZG6yqM6vqmmp8E/ga8NyOUx4Ajq6qeyZp/06vAb5eVae298wtbfzjTXaf3w+sA+yYZO2qWlJV\n1/RQtyTNGyZ+kjT3bQcc1vYWrUiyAtiWpudnTGdSdRewYft6K+C6jmOdryfTrbx3A1cDX2uHAR7R\n5fqV6m2TwwnrrqpvAB8E/h1YluTEJBtPEd9U76Oz7gd4qKdsurYCrh1X9nXA1h3ndGu7Xsru9rXa\nCri1qu7o2HdtR70rxdW+3qrj2HXjjnWVZI8k32+HlK6gSeI37zhleVXdPdWb6bAtTY/vVLre51V1\nNU0v6zE098hpSfrx9ZSkkWHiJ0lz33XAO9veorGP9avq1B6uXQps07G97bjjE/XWdVVVd1TVYVX1\nh8BewFu6zLVa2llXkkxQd2e5J1TVU4EdaYZ8vnWK+KaKu7PuNWjaYGwI4l3A+h3n/sEqlHsjTYIy\nVvbY+7phiut6MdnX6kZg0yQbdex7VEe9K8XVHht7vyt9LdpjE2qHD3+Opidzi6rahGY4azpOG99G\nU7XZdcCjpzhn7Lyu93lVfbqqnkPzPgs4vocyJWneMPGTpLll7STrdnysBXwU+Kskz0hjgyR7jksC\nujkdODDJHydZH/iHccdvBv6w1+DaxTce0yY8v6YZgvfABKeeCTw+ySva9/C3rJxgdZb5tPa9rQ3c\nCdzdUeYqxdfhqR11H0ozZPD77bFLgNe0i6nsDvxpx3U3A5sleXiXck8H9kyyWxvvYW3Z312NGCcq\n+5AkWyfZBDh87EBVXdfW8c/tffFE4CBgbAGVU4G3J1nQzq/8fx3HTgcOSLJjew8cPUkMD6MZUrkc\nuC/JHjRzMCczVZudArwgyb5pFuLZLMlOE5zX9T5P8rgku7aJ6d3Ab5n4vpOkecvET5LmlrNo/qgd\n+zimqhbTLPzxQeA2mqGWB/RSWFWdDZwAnNdeN5b83NN+Polm3tSKJF/socgdgK8DvwG+B/xHVZ03\nQb2/Al5JM0/tlva673Qpc2OaP/pvoxmGeAvNkNLViW/Ml2jm491GM9ftFe2cPIBDgJcBK2jmxj1Y\nblX9hCaJ+nlb50rDCavqKuAvaBYq+VVbzsuq6ne9BNWuevncLoc/SjOf7lLgRzT3wn00yTU0C9os\npOnJ+wLNPLuvt8eOBRa31/4YuLjdN3YPvB/4Bs098I1u8bVDSf+WJlm8jWZ+3hmTvace2uyXNMNF\nDwNupUm8nzRBOZPd5+vQ3Eu/ohlK+0jgyMnikqT5JhPPuZckzUdpHj1wGbBOl9UwNUu0vW0frqrt\npjxZkjTv2eMnSfNckj9rl8R/BM28qC+b9M0+SdZL8pJ2OOTWNEMyvzDsuCRJc4OJnyTpjTTPZruG\nZtjg/xluOOoiNI9huI1mqOeVNHP1JEmakkM9JUmSJGnE2eMnSZIkSSPOxE+SJEmSRtxaww5gOjbf\nfPNauHDhsMOQJEmSpKG46KKLflVVC6Y6b04nfgsXLmTx4sXDDkOSJEmShiLJtb2c51BPSZIkSRpx\nJn6SJEmSNOJM/CRJkiRpxJn4SZIkSdKIM/GTJEmSpBFn4idJkiRJI25OP85BkiRJ6mbhEWeutL3k\nuD2HFIk0fPb4SZIkSdKIM/GTJEmSpBE3sMQvyceTLEtyWce+dyf5SZJLk3whySYdx45McnWSq5K8\neFBxSZIkSdJ8M8gev08Cu4/bdw7whKp6IvBT4EiAJDsCrwIe317zH0nWHGBskiRJkjRvDCzxq6pv\nAbeO2/e1qrqv3fw+sE37em/gtKq6p6p+AVwNPH1QsUmSJEnSfDLMOX6vB85uX28NXNdx7Pp2nyRJ\nkiRpmoaS+CX5v8B9wCmrce3BSRYnWbx8+fL+BydJkiRJI2bGE78kBwAvBfarqmp33wBs23HaNu2+\n31NVJ1bVoqpatGDBgoHGKkmSJEmjYEYTvyS7A28D9qqquzoOnQG8Ksk6SbYHdgB+MJOxSZIkSdKo\nWmtQBSc5FdgF2DzJ9cDRNKt4rgOckwTg+1X1V1V1eZLTgStohoC+qaruH1RskiRJkjSfDCzxq6pX\nT7D7pEnOfyfwzkHFI0mSJEnz1cASP0mSJGkmLTzizGGHIM1aw3ycgyRJkiRpBpj4SZIkSdKIM/GT\nJEmSpBFn4idJkiRJI87ET5IkSZJGnImfJEmSJI04Ez9JkiRJGnEmfpIkSZI04kz8JEmSJGnEmfhJ\nkiRJ0ogz8ZMkSZKkEWfiJ0mSJEkjzsRPkiRJkkaciZ8kSZIkjTgTP0mSJEkacSZ+kiRJkjTi1hp2\nAJIkSdJMWHjEmV2PLTluzxmMRJp5A+vxS/LxJMuSXNaxb9Mk5yT5Wfv5Ee3+JDkhydVJLk3ylEHF\nJUmSJEnzzSCHen4S2H3cviOAc6tqB+DcdhtgD2CH9uNg4EMDjEuSJEmS5pWBJX5V9S3g1nG79wZO\nbl+fDLy8Y/9/VuP7wCZJthxUbJIkSZI0n6xS4pdkjSQbT6O+Lapqafv6JmCL9vXWwHUd513f7pso\nhoOTLE6yePny5dMIRZIkSZLmhykTvySfTrJxkg2Ay4Arkrx1uhVXVQG1GtedWFWLqmrRggULphuG\nJEmSJI28Xnr8dqyq22mGZZ4NbA/sv5r13Tw2hLP9vKzdfwOwbcd527T7JEmSJEnT1Evit3aStWkS\nvzOq6l5Wo6eudQbwuvb164Avdex/bbu6587ArzuGhEqSJEmSpqGX5/h9BFgC/C/wrSTbAbdPdVGS\nU4FdgM2TXA8cDRwHnJ7kIOBaYN/29LOAlwBXA3cBB67Su5AkSZIkdTVl4ldVJwAndOy6Nsnze7ju\n1V0O7TbBuQW8aaoyJUmSJEmrrpfFXbZIclKSs9vtHXlouKYkSZIkaZbrZY7fJ4GvAlu12z8FDh1U\nQJIkSZKk/upljt/mVXV6kiMBquq+JPcPOC5JUp8sPOLMlbaXHLfnkCKRJEnD0kuP351JNqNdyXNs\n1c2BRiVJkiRJ6pteevzeQvO4hUcn+Q6wANhnoFFJkiRJkvqml1U9L07yp8DjgABXtc/ykyRJkiTN\nAV0TvySv6HLosUmoqs8PKCZJkiRpRjkfWqNush6/l01yrAATP0mSJEmaA7omflV14EwGIklafZ3/\nqfa/1JI0ffYAatT08gD3dyXZpGP7EUmOHWxYkiRJkqR+6eVxDntU1Yqxjaq6DXjJ4EKSJEmSJPVT\nL4nfmknWGdtIsh6wziTnS5IkSZJmkV6e43cKcG6ST7TbBwInDy4kSdJ0jJ+XIkmS1Mtz/I5Pcimw\nW7vrHVX11cGGJUmSJEnql156/Kiqs4GzBxyLJGkIVmXluumscucKeZIkDc9kD3C/oKqek+QOmuf2\nPXgIqKraeODRSZIkSZKmbbLn+D2n/bzRzIUjSZIkSeq3KYd6JvlUVe0/1b5VkeTvgDfQ9CT+mGbB\nmC2B04DNgIuA/avqd6tbhyTNZcMcFunD4CVJGj29zPF7fOdGkrWAp65uhUm2Bv4W2LGqfpvkdOBV\nNM8GfF9VnZbkw8BBwIdWtx5Jmu1MsCbnnEBJkvqn63P8khzZzu97YpLb2487gJuBL02z3rWA9dok\ncn1gKbAr8Nn2+MnAy6dZhyRJkiSJyef4/XOS44GPVdXr+1VhVd2Q5F+BXwK/Bb5GM7RzRVXd1552\nPbB1v+qUJA3HXHymoD2NkqRR1LXHD6CqHgCe1s8KkzwC2BvYHtgK2ADYfRWuPzjJ4iSLly9f3s/Q\nJEmSJGkk9TLH7+IkT6uqH/apzhcAv6iq5QBJPg88G9gkyVptr982wA0TXVxVJwInAixatKgmOkeS\nZoP50HM0V97jXIlTkqRB6SXxewawX5JrgTt56Dl+T1zNOn8J7JxkfZqhnrsBi4HzgH1oVvZ8HdOf\nRyhJI2suDqGcyii+J0mSZoteEr8X97PCqrowyWeBi4H7gB/R9OCdCZyW5Nh230n9rFeSJEmS5qsp\nE7+quhYgySOBdftRaVUdDRw9bvfPgaf3o3xJkiRJ0kMmXdwFIMleSX4G/AL4JrAEOHvAcUmSJEmS\n+qSXoZ7vAHYGvl5VT07yfOAvBhuWJEnduViLJEmrZsoeP+DeqroFWCPJGlV1HrBowHFJkiRJkvqk\nlx6/FUk2BL4FnJJkGc3qnpKkDq5KKUmSZqteEr+9aR678HfAfsDDgX8aZFCSpNnBZFaSpNHQy6qe\nY717DwAnDzYcSZImZhIqaZicW6y5rpc5fpIkSZKkOczET5IkSZJGXNehnknOrardkhxfVYfPZFCS\nNN84jFGS5pbOn9sO+9RcMNkcvy2TPAvYK8lpQDoPVtXFA41MkmYhf9GPHpNuSdJ8MFni9/+AfwC2\nAd477lgBuw4qKEmSJElS/3RN/Krqs8Bnk/xDVb1jBmOSJEmSJPVRL49zeEeSvYDntbvOr6qvDDYs\nSdJctCrDJufKEEuXcJckjYIpE78k/ww8HTil3XVIkmdV1VEDjUySpFnIRFCSNBdNmfgBewI7VdUD\nAElOBn4EmPhJkiRJ0hzQS+IHsAlwa/v64QOKRZLUxVwZFjks9sJJkjS5XhK/fwZ+lOQ8mkc6PA84\nYqBRSZIkSZL6ppfFXU5Ncj7wtHbX4VV100CjkiRJkiT1TU9DPatqKXBGvypNsgnwMeAJNM8EfD1w\nFfAZYCGwBNi3qm7rV52SJEmSNF/1Osev3z4A/E9V7ZPkYcD6NIvFnFtVxyU5gmY46eFDik/SPOVc\nMc0mk83t9N6UJK2KGU/8kjycZp7gAQBV9Tvgd0n2BnZpTzsZOB8TP0mS+qIziTRplPrLfxpqLlhj\nsoNJ1kzykz7XuT2wHPhEkh8l+ViSDYAt2iGlADcBW3SJ6eAki5MsXr58eZ9DkyRJkqTRM2niV1X3\nA1cleVQf61wLeArwoap6MnAn41YJraqimfs3UUwnVtWiqlq0YMGCPoYlSZIkSaOpl6GejwAuT/ID\nmiQNgKraazXrvB64vqoubLc/S5P43Zxky6pammRLYNlqli9JmoTPBJQkaf7pJfH7h35WWFU3Jbku\nyeOq6ipgN+CK9uN1wHHt5y/1s15JApMeNfp5Hzi3R5I0F/TyHL9vJtkO2KGqvp5kfWDNadb7N8Ap\n7YqePwcOpBl2enqSg4BrgX2nWYckSUM1VVJo0ihJmilTJn5J/hI4GNgUeDSwNfBhmp661VJVlwCL\nJji02mVKkiRJkibWy1DPNwFPBy4EqKqfJXnkQKOSJEkDY0+jJM0/vSR+91TV75IAkGQtuqy4KUmz\n0XTmczknUJIkjYJJH+fQ+maSo4D1krwQ+G/gy4MNS5IkSZLUL70kfkfQPHD9x8AbgbOAtw8yKEmS\nJElS//SyqucDSU6mmeNXwFXtA9YlSZIkSXNAL6t67kmziuc1QIDtk7yxqs4edHCSJEnSXOMCSpqN\nelnc5T3A86vqaoAkjwbOBEz8JEkakmH+YdlZt3/QStLc0MscvzvGkr7Wz4E7BhSPJEmSJKnPuvb4\nJXlF+3JxkrOA02nm+L0S+OEMxCZJq8VHMGi2msl70+8DSVKnyYZ6vqzj9c3An7avlwPrDSwiSRpR\n/iGu2WqQw0ad6yRJs0PXxK+qDpzJQCRJ6pdhJtkm+JKk2aiXVT23B/4GWNh5flXtNbiwJEmSJEn9\n0suqnl8ETgK+DDww2HAkadXZwyINj0M5JWlu6CXxu7uqThh4JJIkSZKkgegl8ftAkqOBrwH3jO2s\nqosHFpUkSVol/ez5thddkkZPL4nfnwD7A7vy0FDParclSZIkSbNcL4nfK4E/rKrfDToYSZIkSVL/\n9ZL4XQZsAizrZ8VJ1gQWAzdU1Uvb1UNPAzYDLgL2N9mU5icXi9B85RBLSdKg9JL4bQL8JMkPWXmO\n33Qf53AIcCWwcbt9PPC+qjotyYeBg4APTbMOSZI0g6ZKXv3HjiQNRy+J39H9rjTJNsCewDuBtyQJ\nzZzB17SnnAwcg4mfJEmSJE3blIlfVX1zAPW+H3gbsFG7vRmwoqrua7evB7YeQL2SJGkW6ewBtPdP\nkgZnjalOSHJHktvbj7uT3J/k9tWtMMlLgWVVddFqXn9wksVJFi9fvnx1w5AkSZKkeaOXHr+xXjna\nIZl7AztPo85nA3sleQmwLs0cvw8AmyRZq+312wa4oUs8JwInAixatKimEYckSZIkzQtT9vh1qsYX\ngRevboVVdWRVbVNVC4FXAd+oqv2A84B92tNeB3xpdeuQJEmSJD1kyh6/JK/o2FwDWATcPYBYDgdO\nS3Is8CPgpAHUIUmS5ihXBJWk1dfLqp4v63h9H7CEZrjntFXV+cD57eufA0/vR7mSJGnuMbGTpMHp\nZY7fgTMRiCT1yodcS5IkrZquiV+S/zfJdVVV7xhAPJLmKZM5SZKkwZmsx+/OCfZtABxE89w9Ez9J\nkiRJmgO6Jn5V9Z6x10k2Ag4BDgROA97T7TpJkiRJ0uwy6Ry/JJsCbwH2A04GnlJVt81EYJLmNhdp\nkCRJmj0mm+P3buAVNA9L/5Oq+s2MRSVJkiRJ6pvJHuB+GLAV8HbgxiS3tx93JLl9ZsKTJEmSJE3X\nZHP8JksKJWnGuOKnND/N5Pe+w9MljbpeHuAuSRPq/EPJP5IkzTSTNc0V3quaDUz8JPWFvXKSJEmz\nl8M5JUmSJGnE2eMnSZLmnVUZpeAwPUmjwMRPkiSNvPm4UMxsiUPS7GDiJ0mSRs50Ez3nLUsaNc7x\nkyRJkqQRZ4+fpAc5LEiSRpc/42cPvxYaBhM/SUPhMCpJmpoJgqR+MfGTNCNM9CTp95nYSZopMz7H\nL8m2Sc5LckWSy5Mc0u7fNMk5SX7Wfn7ETMcmSZIkSaNoGD1+9wGHVdXFSTYCLkpyDnAAcG5VHZfk\nCOAI4PAhxCdJkjQQ/VxttN+9g5OVbc/k8Nj26pcZT/yqaimwtH19R5Irga2BvYFd2tNOBs7HxE/q\nu+n80eBwTUmz2Sj8jBqF9yBpdhrqHL8kC4EnAxcCW7RJIcBNwBZdrjkYOBjgUY961OCDlGaBQf6H\nV5I0PXM1WZurcY8if89rJgztOX5JNgQ+BxxaVbd3HquqAmqi66rqxKpaVFWLFixYMAORSpIkSdLc\nNpQevyRr0yR9p1TV59vdNyfZsqqWJtkSWDaM2KT5ZKr/9vrfYEnSdDlHTZodZjzxSxLgJODKqnpv\nx6EzgNcBx7WfvzTTsUmSJE1ltv5TbLbGpVXj11GDMowev2cD+wM/TnJJu+8omoTv9CQHAdcC+w4h\nNmlG9PO/n/4ndXj85SxpmPwZJGlVDGNVzwuAdDm820zGIkmSJEnzwVBX9ZQ0Mf+LK0maLxy5Is0M\nEz+pR1P9YposWfOXmCTNX3Pxn3mzKWYfdSD1h4mfJEmSVsl0/hm6qmX361xpvhvac/wkSZIkSTPD\nHj/NOq54OT3+91OSNNP83SPNfiZ+mtemkxj6S06SJElzhYmfNAuYREqS5gt/503PfBzNpP5wjp8k\nSZIkjTh7/DSvjMJ/GUfhPUiSJGlmmfhJHXxWkCRJs5f//Jyaf8uoGxM/zYi5+IN6LsYsSZIkTcQ5\nfpIkSZI04uzx06xnz5skSZI0PSZ+0gwweZUkSYMwnb8xfDTE/GLiN8f5DStJkqSJTPfvRBeKGS3O\n8ZMkSZKkEWePn1bLVP9BcmijJEmSNHvMusQvye7AB4A1gY9V1XFDDmnOchioJEmSBsG/M+eeWZX4\nJVkT+HfghcD1wA+TnFFVVww3stHkN6wkSZJ64WiuuW+2zfF7OnB1Vf28qn4HnAbsPeSYJEmSJGlO\nm1U9fsDWwHUd29cDzxhSLKutnz1pq/rflUH+N2aysv0vkCRJ0uw21d9rs+XvuUGOSluVskdtdFyq\natgxPCjJPsDuVfWGdnt/4BlV9eaOcw4GDm43HwdcNcNhbg78aobrVMO2Hy7bf7hs/+Gy/YfL9h8u\n2394bPvhmivtv11VLZjqpNnW43cDsG3H9jbtvgdV1YnAiTMZVKcki6tq0bDqn89s++Gy/YfL9h8u\n23+4bP/hsv2Hx7YfrlFr/9k2x++HwA5Jtk/yMOBVwBlDjkmSJEmS5rRZ1eNXVfcleTPwVZrHOXy8\nqi4fcliSJEmSNKfNqsQPoKrOAs4adhyTGNowU9n2Q2b7D5ftP1y2/3DZ/sNl+w+PbT9cI9X+s2px\nF0mSJElS/822OX6SJEmSpD4z8etRkt2TXJXk6iRHDDue+SDJkiQ/TnJJksXtvk2TnJPkZ+3nRww7\nzlGR5ONJliW5rGPfhO2dxgnt98OlSZ4yvMhHQ5f2PybJDe33wCVJXtJx7Mi2/a9K8uLhRD0akmyb\n5LwkVyS5PMkh7X7v/xkwSft7/8+AJOsm+UGS/23b/x/b/dsnubBt58+0i+6RZJ12++r2+MJhxj/X\nTdL+n0zyi477f6d2vz9/+izJmkl+lOQr7fbI3vsmfj1Isibw78AewI7Aq5PsONyo5o3nV9VOHUvp\nHgGcW1U7AOe22+qPTwK7j9vXrb33AHZoPw4GPjRDMY6yT/L77Q/wvvZ7YKd2DjTtz59XAY9vr/mP\n9ueUVs99wGFVtSOwM/Cmto29/2dGt/YH7/+ZcA+wa1U9CdgJ2D3JzsDxNO3/GOA24KD2/IOA29r9\n72vP0+rr1v4Ab+24/y9p9/nzp/8OAa7s2B7Ze9/ErzdPB66uqp9X1e+A04C9hxzTfLU3cHL7+mTg\n5UOMZaRU1beAW8ft7tbeewP/WY3vA5sk2XJmIh1NXdq/m72B06rqnqr6BXA1zc8prYaqWlpVF7ev\n76D5A2BrvP9nxCTt3433fx+19/Fv2s21248CdgU+2+4ff/+PfV98FtgtSWYo3JEzSft348+fPkqy\nDbAn8LF2O4zwvW/i15utges6tq9n8l9K6o8CvpbkoiQHt/u2qKql7eubgC2GE9q80a29/Z6YOW9u\nh/N8PA8Nbbb9B6QduvNk4EK8/2fcuPYH7/8Z0Q51uwRYBpwDXAOsqKr72lM62/jB9m+P/xrYbGYj\nHi3j27+qxu7/d7b3//uSrNPu8/7vr/cDbwMeaLc3Y4TvfRM/zWbPqaqn0AxreFOS53UerGZJWpel\nnSG291B8CHg0zfCfpcB7hhvOaEuyIfA54NCqur3zmPf/4E3Q/t7/M6Sq7q+qnYBtaHpP/2jIIc0r\n49s/yROAI2m+Dk8DNgUOH2KIIynJS4FlVXXRsGOZKSZ+vbkB2LZje5t2nwaoqm5oPy8DvkDzy+jm\nsSEN7edlw4twXujW3n5PzICqurn9g+AB4KM8NJzN9u+zJGvTJB2nVNXn293e/zNkovb3/p95VbUC\nOA94Js0QwrHnPXe28YPt3x5/OHDLDIc6kjraf/d2CHRV1T3AJ/D+H4RnA3slWUIzjWtX4AOM8L1v\n4tebHwI7tKv8PIxmUvkZQ45ppCXZIMlGY6+BFwGX0bT769rTXgd8aTgRzhvd2vsM4LXt6mI7A7/u\nGBKnPhk3b+PPaL4HoGn/V7UrjG1PM8n/BzMd36ho52icBFxZVe/tOOT9PwO6tb/3/8xIsiDJJu3r\n9YAX0sw5P1m2AAAgAElEQVSzPA/Ypz1t/P0/9n2xD/CN8qHQq61L+/+k459OoZlj1nn/+/OnD6rq\nyKrapqoW0vxt/42q2o8RvvfXmvoUVdV9Sd4MfBVYE/h4VV0+5LBG3RbAF9o5s2sBn66q/0nyQ+D0\nJAcB1wL7DjHGkZLkVGAXYPMk1wNHA8cxcXufBbyEZlGFu4ADZzzgEdOl/Xdpl/AuYAnwRoCqujzJ\n6cAVNCsivqmq7h9G3CPi2cD+wI/beTYAR+H9P1O6tf+rvf9nxJbAye3KqGsAp1fVV5JcAZyW5Fjg\nRzTJOe3nTyW5mmZBqlcNI+gR0q39v5FkARDgEuCv2vP9+TN4hzOi937mWKIqSZIkSVpFDvWUJEmS\npBFn4idJkiRJI87ET5IkSZJGnImfJEmSJI04Ez9JkiRJGnEmfpKkGZXk5UkqyR9Ncs4mSf66Y3ur\nJJ+dotzzkyxahTg+mWSfqc9cNUmO6ni9MMllk53fce6hSV7b73jG1bEkyeaTHD8tyQ6DjEGSNBwm\nfpKkmfZq4IL28+9JshawCfBg4ldVN1ZV35O0ATlq6lNW1r7n1wOf7lcQbZmr6kPA2/oVgyRp9jDx\nkyTNmCQbAs8BDqLj4bdJdkny7SRn0DyY+zjg0UkuSfLuzp6zJGsm+dcklyW5NMnfTFDPi5J8L8nF\nSf67rXeyuJ6a5JtJLkry1SRbtvvPT3J8kh8k+WmS57b7109yepIrknwhyYVJFiU5DlivjfuUtvg1\nk3w0yeVJvpZkvQlC2BW4uKruS/LIJBe19Typ7R19VLt9TVv3wvYBz5cmObfj+CeTfDjJhcC/JNms\nrfPyJB+jeRg0STZIcmaS/23b8c/bOL4NvGA1k0ZJ0ixm4idJmkl7A/9TVT8Fbkny1I5jTwEOqarH\nAkcA11TVTlX11nFlHAwsBHaqqicCp3QebIcyvh14QVU9BVgMvKVbQEnWBv4N2Keqngp8HHhnxylr\nVdXTgUOBo9t9fw3cVlU7Av8APBWgqo4AftvGvV977g7Av1fV44EVwP83QRjPBi5qy1gGrJtkY+C5\nbfzPTbIdsKyq7mrjPbnj/Z/QUdY2wLOq6i1tvBe0dX8BeFR7zu7AjVX1pKp6AvA/bd0PAFcDT+rW\nXpKkucn/6EmSZtKrgQ+0r09rty9qt39QVb/ooYwXAB+uqvsAqurWccd3BnYEvpME4GHA9yYp73HA\nE4Bz2vPXBJZ2HP98+/kimoQTml7LD7T1X5bk0knK/0VVXTJBGZ22BK7s2P4uTTL4POBdNIlaaHrk\nAJ4JvKJ9/SngXzqu/e+qur99/byx86rqzCS3tft/DLwnyfHAV6rq2x3XLwO24qGviyRpBJj4SZJm\nRJJNaYY0/kmSokmwKslYj96d/aoKOKeqJpxD2OX8y6vqmV2O39N+vp/V+715T8fr+4GJhnr+Fli3\nY/tbNL192wFfAg4HCjizh/qmbMeq+mmSpwAvAY5Ncm5V/VN7eN02HknSCHGopyRppuwDfKqqtquq\nhVW1LfALmgRnvDuAjbqUcw7wxrF5aG1C2en7wLOTPKY9vkGSx04S11XAgiTPbM9fO8njp3gv3wH2\nbc/fEfiTjmP3tsNHV8WVwGM6tr8N/AXws3b45a00SdoF7fHv8tAcyf14qCdwvG8Br2nj3AN4RPt6\nK+Cuqvov4N00w2zHPBboaSVSSdLcYeInSZopr6aZZ9bpc0ywumdV3UIzVPOyJO8ed/hjwC+BS5P8\nL21i03HtcuAA4NR2COb3gK6Pjqiq39Ekpce35V0CPGuK9/IfNMniFcCxwOXAr9tjJ7axndLt4gmc\nTTMscyymJTQ9kd9qd10ArKiqsaGafwMc2L6//YFDupT7j8DzklxOM+Tzl+3+PwF+kOQSmnmAxwIk\n2YJmjuJNqxC7JGkOSFUNOwZJkuaUJGsCa1fV3UkeDXwdeFybRK5umV8A3lZVP+tXnKsRw98Bt1fV\nScOKQZI0GM7xkyRp1a0PnNcO6Qzw19NJ+lpH0CzyMrTEj2bV0U8NsX5J0oDY4ydJkiRJI845fpIk\nSZI04kz8JEmSJGnEmfhJkiRJ0ogz8ZMkSZKkEWfiJ0mSJEkjzsRPkiRJkkaciZ8kSZIkjTgTP0mS\nJEkacSZ+kiRJkjTiTPwkSZIkacSZ+EmSJEnSiDPxkyRJkqQRZ+InSZIkSSPOxE+SJEmSRpyJnyRJ\nkiSNOBM/SZIkSRpxJn6SJEmSNOJM/CRJkiRpxJn4SZIkSdKIM/GTJEmSpBFn4idJkiRJI87ET5Ik\nSZJGnImfJEmSJI04Ez9JkiRJGnEmfpIkSZI04kz8JEmSJGnEmfhJkiRJ0ogz8ZMkSZKkEWfiJ0mS\nJEkjzsRPkiRJkkaciZ8kSZIkjTgTP0mSJEkacSZ+kiRJkjTiTPwkSZIkacSZ+EmSJEnSiDPxkyRJ\nkqQRZ+InSZIkSSPOxE+SJEmSRpyJnyRJkiSNOBM/SdJqS7IkyQsGUO75Sd7Qvt4vydf6WPblSXZp\nXx+T5L/6WPZRST7Wr/ImqeeAJBf0sbxK8ph+lTdB+T21S5JPJjl2UHFI0nxm4idJc9CgEq4p6hzK\nH+VVdUpVvWiq83qNr6oeX1XnTzeuJLskuX5c2e+qqjdMt+y5zHaRpNnJxE+SNC8kWWvYMYw621iS\nZi8TP0kaMUlemuSSJCuSfDfJEzuOLUny90kuTfLrJJ9Jsm7H8bclWZrkxiRvGBsCmORgYD/gbUl+\nk+TLHVXuNFF5STZP8pU2jluTfDvJhL93krwwyU/aMj4IpOPYg8Ma03hfkmVJbk/y4yRP6BZf+34P\nT3IpcGeStSboLV23jfuOJBcneVJH3SsNgRzrVUyyAXA2sFVb32+SbDV+6GiSvdqhpSva4at/3OvX\nogdJ8sH22p8k2a3jwIFJrmzf08+TvHHchW/t+Dq/fopKupY11rvXtvFNwKk9tstz2ntzRZLrkhzQ\npe7J7uXDk9zQxnVV5/uXJP0+Ez9JGiFJngx8HHgjsBnwEeCMJOt0nLYvsDuwPfBE4ID22t2BtwAv\nAB4D7DJ2QVWdCJwC/EtVbVhVL5uqPOAw4HpgAbAFcBRQE8S8OfB54O3A5sA1wLO7vMUXAc8DHgs8\nvK37liniezWwJ7BJVd03QZl7A/8NbAp8GvhikrW71A9AVd0J7AHc2Na3YVXdOO59PZYmETq0bYOz\ngC8neVjHad3ajjbZec4kYTyDpq02B44GPp9k0/bYMuClwMbAgcD7kjylLXd34O+BFwI70Hy9J9O1\nrNYf0LTddsBre2iX7WiSw39r22Un4JLxlU52Lyd5HPBm4GlVtRHwYmDJFO9DkuY1Ez9JGi0HAx+p\nqgur6v6qOhm4B9i545wTqurGqroV+DLNH97QJCGfqKrLq+ou4Jge6+xW3r3AlsB2VXVvVX27qn4v\n8QNeAlxeVZ+tqnuB9wM3danrXmAj4I+AVNWVVbW0h/iuq6rfdjl+UUfd7wXWZeX2Wl1/DpxZVee0\nZf8rsB7wrHGxTdR2VNUmVTXZAi7LgPe3bfsZ4CqaBJeqOrOqrqnGN4GvAc9trxv7Ol/WJrDHTPYm\npigL4AHg6Kq6Z5I27vQa4OtVdWob+y1V9XuJH5Pfy/cD6wA7Jlm7qpZU1TU91C1J85aJnySNlu2A\nw9reohVJVgDbAlt1nNOZVN0FbNi+3gq4ruNY5+vJdCvv3cDVwNfaIYJHdLl+pXrb5HDCuqvqG8AH\ngX8HliU5McnGU8Q31fvorPsBml7Krbqf3rOtgGvHlX0dsHXHOd3arhc3jEukr23rJMkeSb7fDrFd\nQZNcb94R13XjrutqirIAllfV3asQ97Y0PZVT6XovV9XVND2px9DcB6cl6cfXTJJGlomfJI2W64B3\ntr1FYx/rV9WpPVy7FNimY3vbcccn6q3rqqruqKrDquoPgb2At3SZh7W0s64kmaDuznJPqKqnAjvS\nDPl86xTxTRV3Z91r0LTB2PDEu4D1O879g1Uo90aa5GWs7LH3dcMU1/Vq67bMMY8CbmyH9X6Opodx\ni6rahGaY6di5K7V3e92EeigLfr8dpmqX64BHT3HO2Hld7+Wq+nRVPYemjQs4vocyJWneMvGTpLlr\n7STrdnysBXwU+Kskz0hjgyR7Jtmoh/JOBw5M8sdJ1gf+Ydzxm4E/7DW4dmGOx7TJya9phuc9MMGp\nZwKPT/KK9j38LSsnWJ1lPq19b2sDdwJ3d5S5SvF1eGpH3YfSDCf8fnvsEuA1SdZs58b9acd1NwOb\nJXl4l3JPB/ZMslsb72Ft2d9djRgn8kjgb5OsneSVwB/TJGUPoxkGuRy4L8keNHMjO+M6IMmO7df5\n6EnqmKqsiUzVLqcAL0iyb5rFdjZLstME53W9l5M8LsmubWJ6N/BbJr63JEktEz9JmrvOovmDd+zj\nmKpaDPwlzXDI22iGWh7QS2FVdTZwAnBee91Y8nNP+/kkmjlVK5J8sYcidwC+DvwG+B7wH1V13gT1\n/gp4JXAccEt73Xe6lLkxTUJwG80QxVtohpSuTnxjvkQzH+82YH/gFe2cPIBDgJcBK2hWDX2w3Kr6\nCc3iLT9v61xpqGFVXQX8Bc0iJr9qy3lZVf2ul6DaFTGfO8kpF9K01a+AdwL7tPPl7qBJnk9v39Nr\ngDM64jqbZh7lN2i+zt/oVsFUZXW5Zqp2+SXNcNHDgFtpkusnTVDOZPfyOjT3y69ohss+Ejhysrgk\nab7LxPPsJUnzXZpHD1wGrNNlNUxJkjRH2OMnSXpQkj9rl8t/BM2cqS+b9EmSNPeZ+EmSOr2R5jEB\n19DMyfs/ww1HkiT1g0M9JUmSJGnE2eMnSZIkSSPOxE+SJEmSRtxaww5gOjbffPNauHDhsMOQJEmS\npKG46KKLflVVC6Y6b04nfgsXLmTx4sXDDkOSJEmShiLJtb2c51BPSZIkSRpxJn6SJEmSNOJM/CRJ\nkiRpxJn4SZIkSdKIM/GTJEmSpBE3sMQvybpJfpDkf5NcnuQf2/3bJ7kwydVJPpPkYe3+ddrtq9vj\nCwcVmyRJkiTNJ4N8nMM9wK5V9ZskawMXJDkbeAvwvqo6LcmHgYOAD7Wfb6uqxyR5FXA88OcDjE8z\naOERZ660veS4PYcUiSRJkjT/DKzHrxq/aTfXbj8K2BX4bLv/ZODl7eu9223a47slyaDikyRJkqT5\nYqBz/JKsmeQSYBlwDnANsKKq7mtPuR7Yun29NXAdQHv818BmE5R5cJLFSRYvX758kOFLkiRJ0kgY\naOJXVfdX1U7ANsDTgT/qQ5knVtWiqlq0YMGCaccoSZIkSaNuRlb1rKoVwHnAM4FNkozNLdwGuKF9\nfQOwLUB7/OHALTMRnyRJkiSNsoEt7pJkAXBvVa1Ish7wQpoFW84D9gFOA14HfKm95Ix2+3vt8W9U\nVQ0qPs0dLgwjSZIkTc8gV/XcEjg5yZo0PYunV9VXklwBnJbkWOBHwEnt+ScBn0pyNXAr8KoBxiZJ\nkiRJ88bAEr+quhR48gT7f04z32/8/ruBVw4qHkmSJEmar2Zkjp8kSZIkaXhM/CRJkiRpxJn4SZIk\nSdKIM/GTJEmSpBFn4idJkiRJI26Qj3OQerKqz+nzuX6SJEnSqrHHT5IkSZJGnImfJEmSJI04Ez9J\nkiRJGnEmfpIkSZI04kz8JEmSJGnEuaqnZp3xq3ZKkiRJmh57/CRJkiRpxJn4SZIkSdKIc6inBsLh\nmpIkSdLsYY+fJEmSJI04Ez9JkiRJGnEmfpIkSZI04gaW+CXZNsl5Sa5IcnmSQ9r9xyS5Ickl7cdL\nOq45MsnVSa5K8uJBxSZJkiRJ88kgF3e5Dzisqi5OshFwUZJz2mPvq6p/7Tw5yY7Aq4DHA1sBX0/y\n2Kq6f4AxSpIkSdLIG1iPX1UtraqL29d3AFcCW09yyd7AaVV1T1X9ArgaePqg4pMkSZKk+WJG5vgl\nWQg8Gbiw3fXmJJcm+XiSR7T7tgau67jseiZPFCVJkiRJPRh44pdkQ+BzwKFVdTvwIeDRwE7AUuA9\nq1jewUkWJ1m8fPnyvscrSZIkSaNmoIlfkrVpkr5TqurzAFV1c1XdX1UPAB/loeGcNwDbdly+Tbtv\nJVV1YlUtqqpFCxYsGGT4kiRJkjQSBra4S5IAJwFXVtV7O/ZvWVVL280/Ay5rX58BfDrJe2kWd9kB\n+MGg4tP8tPCIM1faXnLcnkOKRJIkSZo5q5T4JVkD2LAdsjmVZwP7Az9Ockm77yjg1Ul2AgpYArwR\noKouT3I6cAXNiqBvckVPSZIkSZq+KRO/JJ8G/gq4H/ghsHGSD1TVuye7rqouADLBobMmueadwDun\nikmSJEmS1Lte5vjt2PbwvRw4G9iepidPkiRJkjQH9JL4rd0u0vJy4IyqupdmmKYkSZIkaQ7oJfH7\nCM1cvA2AbyXZDuhljp8kSZIkaRaYco5fVZ0AnNCx69okzx9cSNLqc9VOSZIk6ff1srjLFsC7gK2q\nao8kOwLPpHlUgwT8fsI1W8zWuCRJkqSZ1MtQz08CX6V5th7AT4FDBxWQJEmSJKm/ekn8Nq+q04EH\nAKrqPppHO0iSJEmS5oBeEr87k2xGu5Jnkp2BXw80KkmSJElS30w5xw94C3AG8Ogk3wEWAPsMNCpJ\nkiRJUt/0sqrnxUn+FHgcEOCq9ll+msdcNEWSJEmaO7omfkle0eXQY5NQVZ8fUEzSrOSjIiRJkjRX\nTdbj97JJjhVg4idJkiRJc0DXxK+qDpzJQCRJkiRJgzHlqp5J3pVkk47tRyQ5drBhSZIkSZL6pZfH\nOexRVSvGNqrqNuAlgwtJkiRJktRPvTzOYc0k61TVPQBJ1gPWGWxY0vC5cqkkSZJGRS+J3ynAuUk+\n0W4fCJw8uJAkSZIkSf3Uy3P8jk9yKbBbu+sdVfXVwYYlSZIkSeqXXnr8qKqzgbNXpeAk2wL/CWxB\n8/iHE6vqA0k2BT4DLASWAPtW1W1JAnyAZv7gXcABVXXxqtQprSqfzSdJkqT5oOviLkkuaD/fkeT2\njo87ktzeQ9n3AYdV1Y7AzsCbkuwIHAGcW1U7AOe22wB7ADu0HwcDH1rtdyVJkiRJetBkz/F7Tvt5\no9UpuKqWAkvb13ckuRLYGtgb2KU97WTgfODwdv9/VlUB30+ySZIt23IkSZIkSaupl+f4faqXfVOU\nsRB4MnAhsEVHMncTzVBQaJLC6zouu77dJ0mSJEmahl6e4/f4zo0kawFP7bWCJBsCnwMOraqVhoi2\nvXvVa1lteQcnWZxk8fLly1flUkmSJEmal7oO9UxyJHAUsF7HnL4AvwNO7KXwJGvTJH2nVNXn2903\njw3hTLIlsKzdfwOwbcfl27T7VlJVJ47Vv2jRolVKGrVqXPhEkiRJGg1de/yq6p+Bh9PMu9u4/dio\nqjarqiOnKrhdpfMk4Mqqem/HoTOA17WvXwd8qWP/a9PYGfi18/skSZIkafomfZxDVT2Q5GmrWfaz\ngf2BHye5pN13FHAccHqSg4BrgX3bY2fRPMrhaprHORy4mvVKkiRJkjr08hy/i5M8rap+uCoFV9UF\nNENDJ7Lb+B3tfL83rUodkiRJkqSp9ZL4PQPYL8m1wJ00yVxV1RMHGpkkSZIkqS96SfxePPAoJEmS\nJEkDM2XiV1XXAiR5JLDuwCOSJEmSJPVVLw9w3yvJz4BfAN8ElgBnDzguSZIkSVKf9DLU8x3AzsDX\nq+rJSZ4P/MVgw5Jmv/HPORzP5x5KkiRptpiyxw+4t6puAdZIskZVnQcsGnBckiRJkqQ+6aXHb0WS\nDYFvAackWUazuqckSf9/e/ceZWlV3nn8+7MhEYkTxAuruTOIulCGHuwgijKIN4RIJw5hZBxjGGba\nWYILY1YEWRqSqAnEoIOTBAeVAR2kRRKUKCEigmiCXEWkUQSko2BDE0UhGnGAZ/44u+CkUpdT9LnX\n97NWrfO++709Vbt3wVP78kqSpAnQS4/fGjovVP9t4GLgduA1gwxKkiRJktQ/vazqOdO79whw9mDD\nkSRJkiT1Wy89fpIkSZKkCWbiJ0mSJElTbt6hnkkuraqXJTmlqo4fZlAajcVeTyBJkiRpMi00x29l\nkhcBhyVZB6T7YFVdP9DIJEmSJEl9sVDi93vAu4AdgffPOlbAQYMKSpIkSZLUP/MmflV1PnB+kndV\n1buHGJMkSZIkqY96eZ3Du5McBhzQii6vqs8ONixJkiRJUr8smvgl+WNgX+CcVnRckhdV1YkDjUxa\nRmYvrLPh5ENHFIkkSZKm0aKJH3AosKqqHgFIcjbwNcDEb8K5iqckSZK0PPT6Hr9turZ/uZcLkpyZ\nZFOSm7rKfj/JXUluaF+HdB17R5LbktyS5FU9xiVJkiRJWkQvPX5/DHwtyWV0XulwAHBCD9edBfwZ\n8LFZ5R+oqj/tLkiyJ/A64LnA9sAXkjyrqh7u4TmSJEmSpAX0srjLuUkuB36lFR1fVXf3cN0VSXbt\nMY41wLqqehC4I8ltdOYVXtnj9ZIkSZKkefQ01LOqNlbVhe1r0aRvEccmubENBX1KK9sB+F7XOXe2\nMkmSJEnSZup1jl+/nA7sDqwCNgKnLvUGSdYmuTbJtffee2+/45MkSZKkqdPLHL++qap7ZraTfBiY\neR/gXcBOXafu2MrmuscZwBkAq1evrsFEKvWXr2uQJEnSKC2Y+CVZAayvquf042FJVlbVxrb768DM\nip8XAp9I8n46i7vsAVzdj2dqPPkqCUmSJGl4Fkz8qurh9nqFnavqu0u5cZJzgQOBpyW5EzgJODDJ\nKqCADcCb2nPWJzkPuBl4CDjGFT0lSZIkqT96Ger5FGB9kquBn8wUVtVhC11UVUfOUfzRBc5/L/De\nHuKRJEmSJC1BL4nfuwYehSRJkiRpYHp5j9+XkuwC7FFVX0jyJGDF4EOTppdzHCVJkjRMi77OIcl/\nB84H/ncr2gH49CCDkiRJkiT1Ty/v8TsG2B+4H6CqbgWeMcigJEmSJEn908scvwer6udJAEiyBZ1V\nOaWx4LBJSZIkaWG99Ph9KcmJwFZJXgF8CvjrwYYlSZIkSeqXXhK/E4B7gW/Qee/eRcA7BxmUJEmS\nJKl/elnV85EkZwNX0RnieUtVOdRTkiRJkibEoolfkkOBDwG3AwF2S/KmqvqbQQcnSZIkSdp8vSzu\ncirw0qq6DSDJ7sDnABM/SZIkSZoAvczxe2Am6Wu+AzwwoHgkSZIkSX02b49fkte2zWuTXAScR2eO\n328A1wwhNo0ZX5swPLN/1htOPnREkUiSJGkaLDTU8zVd2/cA/6Ft3wtsNbCINDAmbpIkSdLyNG/i\nV1VHDTMQSZIkSdJg9LKq527AW4Bdu8+vqsMGF5YkSZIkqV96WdXz08BHgb8GHhlsOJIkSZKkfusl\n8ftZVX1w4JFIkiRJkgail8TvtCQnAZ8HHpwprKrrBxaVNCIugCNJkqRp1EvitxfwBuAgHhvqWW1f\nkiRJkjTmekn8fgP4t1X186XcOMmZwK8Cm6rqea1sW+CTdBaK2QAcUVX3JQlwGnAI8FPgt+xRlObm\nO/4kSZK0VL0kfjcB2wCblnjvs4A/Az7WVXYCcGlVnZzkhLZ/PPBqYI/29QLg9PYpaREmgpIkSVpM\nL4nfNsC3klzDv5zjt+DrHKrqiiS7zipeAxzYts8GLqeT+K0BPlZVBXw1yTZJVlbVxh7ikyRJkiQt\noJfE76Q+Pm+7rmTubmC7tr0D8L2u8+5sZf8q8UuyFlgLsPPOO/cxNGk62AMoSZKk2RZN/KrqS4N4\ncFVVknoc150BnAGwevXqJV8vSZIkScvNExY7IckDSe5vXz9L8nCS+x/n8+5JsrLddyWPzRu8C9ip\n67wdW5kkSZIkaTP10uP35JnttvrmGmC/x/m8C4E3Aie3z890lR+bZB2dRV1+7Py+x6d7mJ9D/CRJ\nkiRBDz1+3arj08CrFjs3ybnAlcCzk9yZ5Gg6Cd8rktwKvLztA1wEfAe4Dfgw8OalxCVJkiRJmt+i\nPX5JXtu1+wRgNfCzxa6rqiPnOfSyOc4t4JjF7ilJkiRJWrpeVvV8Tdf2Q3RevL5mINFIU2T26pqS\nJEnSqPQyx++oYQQiaTicBypJkrT8zJv4Jfm9Ba6rqnr3AOKRJEmSJPXZQj1+P5mjbGvgaOCpgImf\nJEmSJE2AeRO/qjp1ZjvJk4HjgKOAdcCp810nSZIkSRovC87xS7It8Dbg9cDZwD5Vdd8wApMkSZIk\n9cdCc/zeB7wWOAPYq6r+aWhRSRqJ2SuRuviLJEnSdFjoBe6/A2wPvBP4fpL729cDSe4fTniSJEmS\npM210By/hZJCSZIkSdKEMLmTJEmSpCm36AvcNd5mz8mSlsJ/P5IkScuDPX6SJEmSNOVM/CRJkiRp\nypn4SZIkSdKUc47fFHP+liRJkiQw8ZO0AF/oLkmSNB0c6ilJkiRJU87ET5IkSZKm3EiGeibZADwA\nPAw8VFWrk2wLfBLYFdgAHFFV940iPkmSJEmaJqOc4/fSqvrHrv0TgEur6uQkJ7T940cTmqTN5fxA\nSZKk8TFOi7usAQ5s22cDl2PiJ4217uTOxE6SJGl8jWqOXwGfT3JdkrWtbLuq2ti27wa2G01okiRJ\nkjRdRtXj9+KquivJM4BLknyr+2BVVZKa68KWKK4F2HnnnQcfqaSeLPbeSId+SpIkjc5IEr+quqt9\nbkpyAbAvcE+SlVW1MclKYNM8154BnAGwevXqOZPDaeZL2Zcn612SJEmbY+iJX5KtgSdU1QNt+5XA\nHwIXAm8ETm6fnxl2bNI0MmmUJEnSKHr8tgMuSDLz/E9U1cVJrgHOS3I08A/AESOITdKIuFCMJEnS\n4Aw98auq7wB7z1H+A+Blw45H0vhZrJfSxFCSJGlpRrWqpyRJkiRpSEz8JEmSJGnKmfhJkiRJ0pQb\n1Xv8JOlx852AkiRJS2PiJ2nquWKoJEla7kz8JE2dhVYFtbdQkiQtR87xkyRJkqQpZ+InSZIkSVPO\nxAxX7FoAAAv4SURBVE+SJEmSppxz/CT1bKG5c5IkSRpf9vhJkiRJ0pSzx28MLLTUvD0s0mC5yqck\nSVoOTPzGjImeJEmSpH4z8ZM08Qb5B5OFegTtLZQkSZPCxE/SSExq7/bmvBx+KYmiSaUkSeonEz9J\nGpGF5vdKkiT1k4nfCExqT4ekwenn7wV7CyVJ0mwmfpLUZVL+MOOwUUmStBRjl/glORg4DVgBfKSq\nTh5xSJK0rDgEVZKk6TNWiV+SFcCfA68A7gSuSXJhVd082sgkaXptzoI1m3v+4722nwvpSJK0HIxV\n4gfsC9xWVd8BSLIOWANMVOI3KUPFJA3WMH8XjOvvnYXiGmQSOch7DdMk9r5O6s9akqbduCV+OwDf\n69q/E3jBiGKRpLExLondIBeh2Zzz+3mvpfYejiq53Zy6GGUP6bB6dofZWz0qm/Nvc67z+/nsSTSN\n35M2z7T9m0hVjTqGRyU5HDi4qv5b238D8IKqOrbrnLXA2rb7bOCWoQc6uZ4G/OOog9CirKfJYD1N\nButpMlhPk8F6mgzW02ToZz3tUlVPX+ykcevxuwvYqWt/x1b2qKo6AzhjmEFNiyTXVtXqUcehhVlP\nk8F6mgzW02SwniaD9TQZrKfJMIp6esIwH9aDa4A9kuyW5BeA1wEXjjgmSZIkSZpoY9XjV1UPJTkW\n+Fs6r3M4s6rWjzgsSZIkSZpoY5X4AVTVRcBFo45jSjlEdjJYT5PBepoM1tNksJ4mg/U0GaynyTD0\nehqrxV0kSZIkSf03bnP8JEmSJEl9ZuI3xZJsSPKNJDckubaVbZvkkiS3ts+njDrO5SbJmUk2Jbmp\nq2zOeknHB5PcluTGJPuMLvLlZZ56+v0kd7U2dUOSQ7qOvaPV0y1JXjWaqJefJDsluSzJzUnWJzmu\nldumxsgC9WSbGiNJnpjk6iRfb/X0B618tyRXtfr4ZFuAjyS/2PZva8d3HWX8y8UC9XRWkju62tOq\nVu7vvRFJsiLJ15J8tu2PtC2Z+E2/l1bVqq7lYk8ALq2qPYBL276G6yzg4Fll89XLq4E92tda4PQh\nxai56wngA61NrWpzkkmyJ51ViJ/brvmLJCuGFuny9hDwO1W1J7AfcEyrD9vUeJmvnsA2NU4eBA6q\nqr2BVcDBSfYDTqFTT88E7gOObucfDdzXyj/QztPgzVdPAL/b1Z5uaGX+3hud44Bvdu2PtC2Z+C0/\na4Cz2/bZwK+NMJZlqaquAH44q3i+elkDfKw6vgpsk2TlcCJd3uapp/msAdZV1YNVdQdwG7DvwILT\no6pqY1Vd37YfoPMf2B2wTY2VBeppPrapEWjt4p/a7pbtq4CDgPNb+ez2NNPOzgdeliRDCnfZWqCe\n5uPvvRFIsiNwKPCRth9G3JZM/KZbAZ9Pcl2Sta1su6ra2LbvBrYbTWiaZb562QH4Xtd5d7Lw/yxp\n8I5tQ2XOzGNDpa2nMdCGxvx74CpsU2NrVj2BbWqstKFpNwCbgEuA24EfVdVD7ZTuuni0ntrxHwNP\nHW7Ey9Pseqqqmfb03taePpDkF1uZ7Wk0/ifwduCRtv9URtyWTPym24urah86XfzHJDmg+2B1lnR1\nWdcxY72MtdOB3ekMrdkInDracDQjyS8Bfwm8taru7z5mmxofc9STbWrMVNXDVbUK2JFOL+tzRhyS\n5jC7npI8D3gHnfr6FWBb4PgRhrisJflVYFNVXTfqWLqZ+E2xqrqrfW4CLqDzC/yeme799rlpdBGq\ny3z1chewU9d5O7YyjUBV3dP+Y/sI8GEeG3pmPY1Qki3pJBPnVNVftWLb1JiZq55sU+Orqn4EXAa8\nkM7QwJl3P3fXxaP11I7/MvCDIYe6rHXV08FtSHVV1YPA/8H2NEr7A4cl2QCsozPE8zRG3JZM/KZU\nkq2TPHlmG3glcBNwIfDGdtobgc+MJkLNMl+9XAj8ZluRaz/gx13D1zRks+ZE/DqdNgWdenpdW5Vr\nNzoT6K8ednzLUZsD8VHgm1X1/q5DtqkxMl892abGS5KnJ9mmbW8FvILOfMzLgMPbabPb00w7Oxz4\nYvmC6IGbp56+1fXHrtCZO9bdnvy9N0RV9Y6q2rGqdqWzUNUXq+r1jLgtbbH4KZpQ2wEXtHmhWwCf\nqKqLk1wDnJfkaOAfgCNGGOOylORc4EDgaUnuBE4CTmbuerkIOITOwgY/BY4aesDL1Dz1dGBbHruA\nDcCbAKpqfZLzgJvprF54TFU9PIq4l6H9gTcA32jzXQBOxDY1buarpyNtU2NlJXB2W0H1CcB5VfXZ\nJDcD65K8B/ganSSe9vnxJLfRWQzrdaMIehmar56+mOTpQIAbgP/Rzvf33vg4nhG2pfiHGUmSJEma\nbg71lCRJkqQpZ+InSZIkSVPOxE+SJEmSppyJnyRJkiRNORM/SZIkSZpyJn6SpKFK8mtJKslzFjhn\nmyRv7trfPsn5i9z38iSrlxDHWUkOX/zMpUlyYtf2rkluWuj8rnPfmuQ3+x3PrGdsSPK0BY6vS7LH\nIGOQJI2GiZ8kadiOBL7SPv+VJFsA2wCPJn5V9f2q6nuSNiAnLn7Kv9S+5/8KfKJfQbR7LtXpwNv7\nFYMkaXyY+EmShibJLwEvBo6m6wW1SQ5M8uUkF9J5affJwO5Jbkjyvu6esyQrkvxpkpuS3JjkLXM8\n55VJrkxyfZJPtecuFNfzk3wpyXVJ/jbJylZ+eZJTklyd5NtJXtLKn5TkvCQ3J7kgyVVJVic5Gdiq\nxX1Ou/2KJB9Osj7J55NsNUcIBwHXV9VDSZ6R5Lr2nL1b7+jObf/29uxd28uab0xyadfxs5J8KMlV\nwJ8keWp75vokH6HzYmeSbJ3kc0m+3n6O/6nF8WXg5Y8zaZQkjTETP0nSMK0BLq6qbwM/SPL8rmP7\nAMdV1bOAE4Dbq2pVVf3urHusBXYFVlXVvwPO6T7YhjK+E3h5Ve0DXAu8bb6AkmwJ/C/g8Kp6PnAm\n8N6uU7aoqn2BtwIntbI3A/dV1Z7Au4DnA1TVCcA/t7hf387dA/jzqnou8CPgP84Rxv7Ade0em4An\nJvk3wEta/C9Jsguwqap+2uI9u+v7/2DXvXYEXlRVb2vxfqU9+wJg53bOwcD3q2rvqnoecHF79iPA\nbcDe8/28JEmTyb/oSZKG6UjgtLa9ru1f1/avrqo7erjHy4EPVdVDAFX1w1nH9wP2BP4uCcAvAFcu\ncL9nA88DLmnnrwA2dh3/q/Z5HZ2EEzq9lqe159+U5MYF7n9HVd0wxz26rQS+2bX/93SSwQOAP6KT\nqIVOjxzAC4HXtu2PA3/Sde2nqurhtn3AzHlV9bkk97XybwCnJjkF+GxVfbnr+k3A9jxWL5KkKWDi\nJ0kaiiTb0hnSuFeSopNgVZKZHr2f9OtRwCVVNeccwnnOX19VL5zn+IPt82Ee3383H+zafhiYa6jn\nPwNP7Nq/gk5v3y7AZ4DjgQI+18PzFv05VtW3k+wDHAK8J8mlVfWH7fATWzySpCniUE9J0rAcDny8\nqnapql2raifgDjoJzmwPAE+e5z6XAG+amYfWEspuXwX2T/LMdnzrJM9aIK5bgKcneWE7f8skz13k\ne/k74Ih2/p7AXl3H/l8bProU3wSe2bX/ZeC/ALe24Zc/pJOkfaUd/3semyP5eh7rCZztCuA/tzhf\nDTylbW8P/LSq/i/wPjrDbGc8C+hpJVJJ0uQw8ZMkDcuRdOaZdftL5ljds6p+QGeo5k1J3jfr8EeA\n7wI3Jvk6LbHpuvZe4LeAc9sQzCuBeV8dUVU/p5OUntLudwPwokW+l7+gkyzeDLwHWA/8uB07o8V2\nznwXz+Fv6AzLnIlpA52eyCta0VeAH1XVzFDNtwBHte/vDcBx89z3D4ADkqynM+Tzu618L+DqJDfQ\nmQf4HoAk29GZo3j3EmKXJE2AVNWoY5AkaaIkWQFsWVU/S7I78AXg2S2JfLz3vAB4e1Xd2q84H0cM\nvw3cX1UfHVUMkqTBcI6fJElL9yTgsjakM8CbNyfpa06gs8jLyBI/OquOfnyEz5ckDYg9fpIkSZI0\n5ZzjJ0mSJElTzsRPkiRJkqaciZ8kSZIkTTkTP0mSJEmaciZ+kiRJkjTlTPwkSZIkacr9fxPE13Eq\nZIzPAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11c652898>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# visualization\n",
    "plt.figure().set_size_inches(15, 8)\n",
    "\n",
    "plt.subplot(211)\n",
    "plt.xlabel('Article length (words)')\n",
    "plt.ylabel('Number of articles')\n",
    "plt.title(f'Lengths distribution: {corpora[0].label} articles')\n",
    "plt.hist(lengths[0], 200)\n",
    "\n",
    "plt.subplots_adjust(hspace=.5)\n",
    "\n",
    "plt.subplot(212)\n",
    "plt.xlabel('Article length (words)')\n",
    "plt.ylabel('Number of articles')\n",
    "plt.title(f'Lengths distribution: {corpora[1].label} articles')\n",
    "plt.hist(lengths[1], 200)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Building a dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dictionary from stored file (1194029 bytes)...\n",
      "Top popular words counts: [['UNK', 30414], (',', 128485), ('.', 124529), ('в', 96291), ('num', 61427), ('и', 38544), ('на', 36182), ('с', 18901), ('по', 17567), ('что', 14996), ('эт', 13930), ('год', 12755), ('из', 12583), ('не', 11630), ('котор', 9868)]\n",
      "\n",
      "Example word: 4071 => слон\n",
      "\n",
      "Example data: [1180, 387, 108, 3049, 23, 206, 2, 17, 63, 486, 3780, 223, 16, 15, 1, 82, 6, 120, 206, 2]\n",
      "\n",
      "Validation sizes split train / validation: 6417 / 200\n",
      "Validation dataset size: 400\n"
     ]
    }
   ],
   "source": [
    "STORAGE_PATH = './storage/model' # to restore model's knowledge later\n",
    "validation_size = 200 # articles per each class\n",
    "vocabulary_size = 25000\n",
    "\n",
    "def build_dictionary(words):\n",
    "  count = [['UNK', -1]]\n",
    "  count.extend(collections.Counter(words).most_common(vocabulary_size - 1))\n",
    "  dictionary = dict()\n",
    "  for word, _ in count:\n",
    "    dictionary[word] = len(dictionary)\n",
    "  reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys())) \n",
    "\n",
    "  return dictionary, reverse_dictionary, count\n",
    "\n",
    "def build_dataset(words, dictionary, count):\n",
    "  data = list()\n",
    "  unk_count = 0\n",
    "  for word in words:\n",
    "    if word in dictionary:\n",
    "      index = dictionary[word]\n",
    "    else:\n",
    "      index = 0  # dictionary['UNK']\n",
    "      unk_count = unk_count + 1\n",
    "    data.append(index)\n",
    "\n",
    "  count[0][1] = count[0][1] + unk_count\n",
    "\n",
    "  return data\n",
    "\n",
    "# flatten all words into a single bag\n",
    "all_words = [word for corpus in corpora for words in corpus.articles for word in words]\n",
    "\n",
    "dictionary_file_name = f\"{STORAGE_PATH}.dictionary\"\n",
    "if Path(dictionary_file_name).is_file():\n",
    "  dictionary_file_size = os.path.getsize(dictionary_file_name)\n",
    "  print(f\"Loading dictionary from stored file ({dictionary_file_size} bytes)...\")\n",
    "  with open(dictionary_file_name, 'rb') as fp:\n",
    "    dictionary, reverse_dictionary, count = pickle.load(fp)\n",
    "else:\n",
    "  print('Building dictionary from the corpora...')\n",
    "  dictionary, reverse_dictionary, count = build_dictionary(all_words)\n",
    "  print('Saving dictionary on disk for further access...')\n",
    "  with open(dictionary_file_name, 'wb') as fp:\n",
    "    pickle.dump((dictionary, reverse_dictionary, count), fp)\n",
    "  dictionary_file_size = os.path.getsize(dictionary_file_name)\n",
    "  print(f\"Done. It took {dictionary_file_size} bytes on the disk.\")\n",
    "\n",
    "good_data = [build_dataset(article, dictionary, count) for article in corpora[0].articles]\n",
    "bad_data  = [build_dataset(article, dictionary, count) for article in corpora[1].articles]\n",
    "\n",
    "# shuffling\n",
    "for data in [good_data, bad_data]: np.random.shuffle(data)\n",
    "\n",
    "# Splitting between train and validation datasets\n",
    "good_train_data = good_data[:-validation_size]\n",
    "good_validation_data = good_data[-validation_size:]\n",
    "bad_train_data = bad_data[:-validation_size]\n",
    "bad_validation_data = bad_data[-validation_size:]\n",
    "validation_data = [item for ga, ba in zip(good_validation_data, bad_validation_data) for item in [ga, ba]]\n",
    "\n",
    "print(f\"Top popular words counts: {count[:15]}\\n\")\n",
    "print(f\"Example word: {dictionary['слон']} => {reverse_dictionary[dictionary['слон']]}\\n\")\n",
    "print(f'Example data: {good_data[0][:20]}\\n')\n",
    "print(f'Validation sizes split train / validation: {len(good_train_data)} / {len(good_validation_data)}')\n",
    "print(f'Validation dataset size: {len(validation_data)}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Below is an **example** of two training batches spat out by `BatchIterator`:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "([195, 305, 3, 1892, 244, 1428, 125, 2, 305, 3], [[0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [0.0, 1.0]], [0.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0])\n",
      "['сми', 'ноч', 'в', 'луганск', 'произошел', 'мощн', 'взрыв', '.', 'ноч', 'в']\n",
      "([1892, 244, 1428, 125, 2, 6, 261, 5470, 808, 233], [[0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [0.0, 1.0]], [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0])\n",
      "['луганск', 'произошел', 'мощн', 'взрыв', '.', 'на', 'улиц', 'слышн', 'сир', 'машин']\n"
     ]
    }
   ],
   "source": [
    "batch_size = 64 # Number of articles fed into a model at once\n",
    "num_unrollings = 10 # Number of recurrent steps\n",
    "\n",
    "good = [1., 0.] # Representation of first class\n",
    "bad  = [0., 1.] # Representation of second class\n",
    "\n",
    "min_size = min(len(good_train_data), len(bad_train_data))\n",
    "articles_pool = [(data, good) for data in good_train_data[:min_size]] + \\\n",
    "                [(data, bad)  for data in bad_train_data[:min_size]]\n",
    "\n",
    "conveyors = [Conveyor(articles_pool, num_unrollings) for _ in range(batch_size)]\n",
    "\n",
    "for _ in range(2):\n",
    "  bundle = conveyors[0].next()\n",
    "  print(bundle)\n",
    "  print([reverse_dictionary[index] for index in bundle[0]])\n",
    "conveyors[0].rewind()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. LSTM Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Graph was successfully built\n"
     ]
    }
   ],
   "source": [
    "num_nodes = 64 # Dimension of lstm cell\n",
    "num_classes = 2 # Number of classes among which to predict\n",
    "embedding_size = 128 # Dimension of a word vector\n",
    "hidden_layer_size = 32\n",
    "\n",
    "graph = tf.Graph()\n",
    "with graph.as_default():\n",
    "\n",
    "  # Parameters:\n",
    "  # Input gate: input, previous output, and bias.\n",
    "  ix = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1), name='ix')\n",
    "  im = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1), name='im')\n",
    "  ib = tf.Variable(tf.zeros([1, num_nodes]), name='ib')\n",
    "  # Forget gate: input, previous output, and bias.\n",
    "  fx = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1), name='fx')\n",
    "  fm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1), name='fm')\n",
    "  fb = tf.Variable(tf.zeros([1, num_nodes]), name='fb')\n",
    "  # Output gate: input, previous output, and bias.\n",
    "  ox = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1), name='ox')\n",
    "  om = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1), name='om')\n",
    "  ob = tf.Variable(tf.zeros([1, num_nodes]), name='ob')\n",
    "  # Memory cell: input, state and bias.\n",
    "  cx = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], -0.1, 0.1), name='cx')\n",
    "  cm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], -0.1, 0.1), name='cm')\n",
    "  cb = tf.Variable(tf.zeros([1, num_nodes]), name='cb')\n",
    "  # Variables saving state across unrollings.\n",
    "  saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "  saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "  # Classifier weights and biases.\n",
    "  w = tf.Variable(tf.truncated_normal([num_nodes, hidden_layer_size], -0.1, 0.1), name='w')\n",
    "  b = tf.Variable(tf.zeros([hidden_layer_size]), name='b')\n",
    "  hw = tf.Variable(tf.truncated_normal([hidden_layer_size, num_classes], -0.1, 0.1), name='hw')\n",
    "  hb = tf.Variable(tf.zeros([num_classes]), name='hb')\n",
    "  # Embeddings\n",
    "  embeddings = tf.Variable(tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0), name='embeddings')\n",
    "\n",
    "\n",
    "  # Definition of the cell computation.\n",
    "  def lstm_cell(i, o, state):\n",
    "    # Create an LSTM cell\n",
    "    input_gate =  tf.sigmoid(tf.matmul(i, ix) + tf.matmul(o, im) + ib)\n",
    "    forget_gate = tf.sigmoid(tf.matmul(i, fx) + tf.matmul(o, fm) + fb)\n",
    "    output_gate = tf.sigmoid(tf.matmul(i, ox) + tf.matmul(o, om) + ob)\n",
    "    update = tf.matmul(i, cx) + tf.matmul(o, cm) + cb\n",
    "    state = forget_gate * state + input_gate * tf.tanh(update)\n",
    "\n",
    "    return output_gate * tf.tanh(state), state\n",
    "\n",
    "  # Input data.\n",
    "  train_inputs   = tf.placeholder(tf.int32,   shape=[batch_size, num_unrollings])\n",
    "  train_clearers = tf.placeholder(tf.float32, shape=[batch_size, num_unrollings])\n",
    "  train_labels   = tf.placeholder(tf.float32, shape=[batch_size, num_unrollings, num_classes])\n",
    "\n",
    "  # Unrolled LSTM loop.\n",
    "  lstm_outputs = list()\n",
    "  output = saved_output\n",
    "  state  = saved_state\n",
    "  embedded_batch = tf.nn.embedding_lookup(embeddings, train_inputs)\n",
    "  lstm_inputs = tf.unstack(embedded_batch, axis=1) # list of ~ (batch_size, embedding_size)\n",
    "  lstm_state_clearers = tf.unstack(train_clearers, axis=1) # list of ~ (batch_size, )\n",
    "\n",
    "  for lstm_input_batch, state_clearers_batch in zip(lstm_inputs, lstm_state_clearers):\n",
    "    reshaped_clearers = tf.reshape(state_clearers_batch, (batch_size, 1))\n",
    "    output, state = lstm_cell(lstm_input_batch,\n",
    "                              tf.multiply(reshaped_clearers, output),\n",
    "                              tf.multiply(reshaped_clearers, state))\n",
    "\n",
    "    lstm_outputs.append(output)\n",
    "\n",
    "  # State saving across unrollings.\n",
    "  with tf.control_dependencies([saved_output.assign(output),\n",
    "                                saved_state.assign(state)]):\n",
    "    # Classifier.\n",
    "    # here we use `tf.reshape(tf.stack())` instead of `tf.concat` to preserve 0th and 1th axes order\n",
    "    joined_output = tf.reshape(tf.stack(lstm_outputs, axis=1), (-1, num_nodes))   \n",
    "    joined_labels = tf.reshape(train_labels, (-1, num_classes))\n",
    "\n",
    "    hidden_layer = tf.nn.xw_plus_b(joined_output, w, b)\n",
    "    activation = tf.nn.tanh(hidden_layer)\n",
    "    logits = tf.nn.xw_plus_b(activation, hw, hb)\n",
    "\n",
    "    loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(\n",
    "      labels=joined_labels,\n",
    "      logits=logits\n",
    "    ))\n",
    "\n",
    "  # Optimizer.\n",
    "  global_step = tf.Variable(0, trainable=False, name='global_step')\n",
    "  learning_rate = tf.train.exponential_decay(4.0, global_step, 5000, 0.8, staircase=True)\n",
    "  optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "  gradients, v = zip(*optimizer.compute_gradients(loss))\n",
    "  gradients, _ = tf.clip_by_global_norm(gradients, 1.25)\n",
    "  optimizer = optimizer.apply_gradients(\n",
    "    zip(gradients, v), global_step=global_step)\n",
    "\n",
    "  # Predictions.\n",
    "  train_prediction = tf.nn.softmax(logits)\n",
    "\n",
    "  # Storing knowledge to the disk\n",
    "  saver = tf.train.Saver()\n",
    "\n",
    "  # Sampling and validation evaluation\n",
    "  # ==================================\n",
    "  #\n",
    "  # Sampling eval: one unrolling, no state clearers\n",
    "  sample_input = tf.placeholder(tf.int32, shape=[1])\n",
    "  sample_lstm_input = tf.nn.embedding_lookup(embeddings, sample_input)\n",
    "  saved_sample_output = tf.Variable(tf.zeros([1, num_nodes]), trainable=False)\n",
    "  saved_sample_state = tf.Variable(tf.zeros([1, num_nodes]), trainable=False)\n",
    "  reset_sample_state = tf.group(\n",
    "    saved_sample_output.assign(tf.zeros([1, num_nodes])),\n",
    "    saved_sample_state.assign(tf.zeros([1, num_nodes])))\n",
    "  sample_output, sample_state = lstm_cell(\n",
    "    sample_lstm_input, saved_sample_output, saved_sample_state)\n",
    "  with tf.control_dependencies([saved_sample_output.assign(sample_output),\n",
    "                                saved_sample_state.assign(sample_state)]):\n",
    "    hidden_sample_pred = tf.nn.xw_plus_b(sample_output, w, b)\n",
    "    sample_activation = tf.nn.tanh(hidden_sample_pred)\n",
    "    sample_logits     = tf.nn.xw_plus_b(sample_activation, hw, hb)\n",
    "    sample_prediction = tf.nn.softmax(sample_logits)\n",
    "\n",
    "  valid_labels = tf.placeholder(tf.float32, shape=[None, num_classes])\n",
    "  valid_logits = tf.placeholder(tf.float32, shape=[None, num_classes])  \n",
    "  validation_loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(\n",
    "    labels=valid_labels,\n",
    "    logits=valid_logits\n",
    "  ))\n",
    "\n",
    "print('Graph was successfully built')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Representation helpers\n",
    "\n",
    "Before we dive into the very learning - let us define some useful helpers to better represent training process."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def print_md(string):\n",
    "  display(Markdown(string))\n",
    "\n",
    "def colored_p(text, prediction):\n",
    "  g, r = [round(probability * 255) for probability in prediction]\n",
    "  return f\"<span style='color: rgb({r}, {g}, 0)'>{text}</span>\"\n",
    "\n",
    "def represent(words, predictions, tail=3):\n",
    "  result = []\n",
    "  temp = []\n",
    "  changed_ago = 0\n",
    "  for word, label, i in zip(words, predictions, range(len(words))):\n",
    "    word_to_add = colored_p(word, label)\n",
    "\n",
    "    if i == 0:\n",
    "      temp.append(word_to_add)\n",
    "      continue\n",
    "\n",
    "    if label_to_color(predictions[i - 1]) == label_to_color(label):\n",
    "      temp.append(word_to_add)\n",
    "    else:\n",
    "      if len(temp) > tail * 2:\n",
    "        result += temp[:tail] + ['...'] + temp[-tail:]\n",
    "      else:\n",
    "        result += temp\n",
    "      temp = [word_to_add]\n",
    "\n",
    "  if len(temp) > tail * 2:\n",
    "    result += temp[:tail] + ['...'] + temp[-tail:]\n",
    "  else:\n",
    "    result += temp\n",
    "\n",
    "  overall_label = colored_p(f\"=> {predictions[-1][0]:.2f}\", predictions[-1])\n",
    "  article = \" \".join(result)\n",
    "\n",
    "  return f\"{overall_label} {article}\"\n",
    "\n",
    "def label_to_color(label):\n",
    "  if abs(label[0] - label[1]) < 0.2: color = 'black'\n",
    "  elif label[0] > label[1]:          color = 'green'\n",
    "  else:                              color = 'red'\n",
    "\n",
    "  return color\n",
    "\n",
    "def colored(text, color='black'):\n",
    "  colors = { 'black':'30',\n",
    "             'red':'31',\n",
    "             'green':'32',\n",
    "             'yellow':'33',\n",
    "             'blue':'34',\n",
    "             'purple':'35',\n",
    "             'sea-blue':'36' }\n",
    "\n",
    "  return f\"\\x1b[{colors[color]}m{text}\\x1b[0m\"\n",
    "\n",
    "\n",
    "def logprob(predictions, labels):\n",
    "  \"\"\"Log-probability of the true labels in a predicted batch.\"\"\"\n",
    "  predictions[predictions < 1e-10] = 1e-10\n",
    "  return np.sum(np.multiply(labels, -np.log(predictions))) / labels.shape[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Validation loss: 0.7075, taken: 22.98 seconds\n",
      "Average loss at step 0: 0.6884, learning rate: 4.0000\n",
      "Average loss at step 200: 1.3995, learning rate: 4.0000\n",
      "Validation loss: 0.5853, taken: 23.50 seconds\n",
      "Average loss at step 400: 1.9295, learning rate: 4.0000\n",
      "Average loss at step 600: 0.6441, learning rate: 4.0000\n",
      "Validation loss: 0.2640, taken: 23.52 seconds\n",
      "Average loss at step 800: 0.1415, learning rate: 4.0000\n",
      "Average loss at step 1000: 0.1083, learning rate: 4.0000\n",
      "Validation loss: 0.2188, taken: 23.46 seconds\n",
      "Average loss at step 1200: 0.0993, learning rate: 4.0000\n",
      "Average loss at step 1400: 0.0912, learning rate: 4.0000\n",
      "Validation loss: 0.1917, taken: 22.95 seconds\n",
      "Average loss at step 1600: 0.0836, learning rate: 4.0000\n",
      "Average loss at step 1800: 0.0929, learning rate: 4.0000\n",
      "Validation loss: 0.1471, taken: 22.95 seconds\n",
      "Average loss at step 2000: 0.0670, learning rate: 4.0000\n",
      "Average loss at step 2200: 0.0704, learning rate: 4.0000\n",
      "Validation loss: 0.1188, taken: 23.63 seconds\n",
      "Average loss at step 2400: 0.0475, learning rate: 4.0000\n",
      "Average loss at step 2600: 0.0594, learning rate: 4.0000\n",
      "Validation loss: 0.1767, taken: 23.58 seconds\n",
      "Average loss at step 2800: 0.0499, learning rate: 4.0000\n",
      "Average loss at step 3000: 0.0499, learning rate: 4.0000\n",
      "Validation loss: 0.1457, taken: 23.55 seconds\n",
      "Average loss at step 3200: 0.0534, learning rate: 4.0000\n",
      "Average loss at step 3400: 0.0405, learning rate: 4.0000\n",
      "Validation loss: 0.1937, taken: 23.56 seconds\n",
      "Average loss at step 3600: 0.0585, learning rate: 4.0000\n",
      "Average loss at step 3800: 0.0486, learning rate: 4.0000\n",
      "Validation loss: 0.1421, taken: 23.57 seconds\n",
      "Average loss at step 4000: 0.0423, learning rate: 4.0000\n",
      "Average loss at step 4200: 0.0381, learning rate: 4.0000\n",
      "Validation loss: 0.1399, taken: 23.48 seconds\n",
      "Average loss at step 4400: 0.0585, learning rate: 4.0000\n",
      "Average loss at step 4600: 0.0521, learning rate: 4.0000\n",
      "Validation loss: 0.1027, taken: 23.08 seconds\n",
      "Average loss at step 4800: 0.0449, learning rate: 4.0000\n",
      "Average loss at step 5000: 0.0416, learning rate: 3.2000\n",
      "Validation loss: 0.1571, taken: 22.91 seconds\n",
      "Average loss at step 5200: 0.0638, learning rate: 3.2000\n",
      "Average loss at step 5400: 0.0307, learning rate: 3.2000\n",
      "Validation loss: 0.1878, taken: 22.92 seconds\n",
      "Average loss at step 5600: 0.0451, learning rate: 3.2000\n",
      "Average loss at step 5800: 0.0448, learning rate: 3.2000\n",
      "Validation loss: 0.1477, taken: 22.94 seconds\n",
      "Average loss at step 6000: 0.0276, learning rate: 3.2000\n",
      "Average loss at step 6200: 0.0432, learning rate: 3.2000\n",
      "Validation loss: 0.1810, taken: 23.17 seconds\n",
      "Average loss at step 6400: 0.0313, learning rate: 3.2000\n",
      "Average loss at step 6600: 0.0262, learning rate: 3.2000\n",
      "Validation loss: 0.1323, taken: 22.99 seconds\n",
      "Average loss at step 6800: 0.0338, learning rate: 3.2000\n",
      "Average loss at step 7000: 0.0311, learning rate: 3.2000\n",
      "Learning finished in 8.5 minutes.\n"
     ]
    }
   ],
   "source": [
    "num_steps = 7001\n",
    "summary_frequency = 200\n",
    "# validation_frequencies = [1000, 5000, 30000, 60000, 90000]\n",
    "\n",
    "with tf.Session(graph=graph) as session:\n",
    "  tf.global_variables_initializer().run()\n",
    "\n",
    "  print('Initialized')\n",
    "  mean_loss = 0\n",
    "  losses   = []\n",
    "  v_losses = []\n",
    "  start_at = time.time()\n",
    "  for step in range(num_steps):\n",
    "    feed = ([], [], [])\n",
    "    for conveyor in conveyors:\n",
    "      bundle = conveyor.next()\n",
    "      feed[0].append(bundle[0])\n",
    "      feed[1].append(bundle[1])\n",
    "      feed[2].append(bundle[2])\n",
    "\n",
    "    feed_dict = { train_inputs: feed[0],\n",
    "                  train_labels: feed[1],\n",
    "                  train_clearers: feed[2] }\n",
    "\n",
    "    _, l, lr = session.run(\n",
    "      [optimizer, loss, learning_rate], feed_dict=feed_dict)\n",
    "    mean_loss += l\n",
    "\n",
    "    if step % summary_frequency == 0:\n",
    "      if step > 0:\n",
    "        # The mean loss is an estimate of the loss over the last few batches.\n",
    "        mean_loss = mean_loss / summary_frequency\n",
    "\n",
    "      # ======================================== Validation\n",
    "      if step % (summary_frequency * 2) == 0:\n",
    "        v_start_at = time.time()\n",
    "        logits = []\n",
    "        labels = []\n",
    "\n",
    "        for i, article in enumerate(validation_data):\n",
    "          reset_sample_state.run()\n",
    "          current_label = good if i % 2 == 0 else bad\n",
    "          for word in article:\n",
    "            logits.append(sample_logits.eval({ sample_input: [word] })[0])\n",
    "            labels.append(current_label)\n",
    "\n",
    "        v_loss = validation_loss.eval({ valid_logits: logits, valid_labels: labels })\n",
    "        v_losses.append(v_loss)\n",
    "        v_taken = (time.time() - v_start_at)\n",
    "        print(f\"Validation loss: {v_loss:.4f}, taken: {v_taken:.2f} seconds\")\n",
    "      # ======================================== Validation\n",
    "\n",
    "      print('Average loss at step %d: %.4f, learning rate: %.4f' % (step, mean_loss, lr))\n",
    "\n",
    "      losses.append(mean_loss)\n",
    "      mean_loss = 0\n",
    "\n",
    "  taken = (time.time() - start_at) / 60\n",
    "  print(f\"Learning finished in {taken:.1f} minutes.\")\n",
    "\n",
    "  # save what have been learned\n",
    "  saver.save(session, STORAGE_PATH)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAA3QAAAD8CAYAAAA/m+aTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzs3Xl8XNd93/3Pb1Ysg30Z7itAbba1UZRsKTYoW0ucxFu8\nSG0UN7Utt7WdtGme1kkaO49TP6806dO0qZPHVlLVlZ2IluPElmPZkiwSkh1rISVrFylSFBeAC0gC\nxA4MZuY8f9w7wADESgw4c8Hv+/W6r7nbzJyZA8zM955zzzXnHCIiIiIiIhI8oWIXQERERERERM6P\nAp2IiIiIiEhAKdCJiIiIiIgElAKdiIiIiIhIQCnQiYiIiIiIBJQCnYiIiIiISEAp0ImIiIiIiASU\nAp2IiIiIiEhAzRnozGytme0ys1fN7BUz+61p9jEz+3MzO2BmL5rZNXnbPm5m+/3p44V+ASIiIiIi\nIhcrc87NvoPZSmClc+45M6sCngU+4Jx7NW+f9wKfA94LXA/8D+fc9WZWD+wBtgLOv++1zrme2Z6z\nsbHRbdiw4fxf1RIZHByksrKy2MWQeVJ9BY/qLHhUZ8Gi+goe1VnwqM6Cp1Tr7Nlnnz3tnGuaa7/I\nXDs4544Dx/35fjN7DVgNvJq32/uB+5yXDp8ys1o/CLYBjzrnugHM7FHgduD+2Z5zw4YN7NmzZ66i\nXXDt7e20tbUVuxgyT6qv4FGdBY/qLFhUX8GjOgse1VnwlGqdmdnh+ey3oHPozGwDcDXw9JRNq4Gj\necsd/rqZ1ouIiIiIiMgizdlCl2NmCeA7wL91zvUVuiBmdjdwN0AymaS9vb3QT7FoAwMDJVkumZ7q\nK3hUZ8GjOgsW1VfwqM6CR3UWPEGvs3kFOjOL4oW5v3HO/f00u3QCa/OW1/jrOvG6Xeavb5/uOZxz\n9wD3AGzdutWVYrNnqTbHyvRUX8GjOgse1VmwqL6CR3UWPKqz4Al6nc1nlEsD/hfwmnPuv82w24PA\nr/ujXd4A9Prn3j0M3GpmdWZWB9zqrxMREREREZFFmk8L3Y3AXcBLZva8v+73gHUAzrmvAg/hjXB5\nABgCfsPf1m1mfwTs9u/3pdwAKSIiIiIiIrI48xnl8qeAzbGPAz4zw7Z7gXvPq3QiIiIiIiIyowWN\ncikiIiIiIiKlQ4FuGXrteB//dOB0sYshIiIiIiJLTIFumXHO8dsPvMC/+9bzc+8sIiIiIiKBNu/r\n0EkwvNjRy2vHvcsE9g6NUVMRLXKJRERERERkqaiFbpnZsfvI+PyBU/1FLImIiIiIiCw1BbplZHA0\nzYPPH+P6jfUAHOgaKHKJRERERERkKSnQLSPff+EYg6kMv3PbJZRFQ+w/qUAnIiIiIrKcKdAtI/fv\nPsqWZIKt6+vY1Jhgv1roRERERESWNQW6ZeK14328cPQsd1y3DjOjNZlQl0sRERERkWVOgW6Z2PHM\nEWKREB+6ZjUArc0JOs8OMziaLnLJRERERERkqSjQLQPDqQz/8PNOfvEtK6itiAHQ0lwFwBun1Eon\nIiIiIrJcKdAtAw+9dJy+kTR3XLdufF1LcwLQSJciIiIiIsuZAt0ysGP3ETY2VnLDpvrxdesbKoiG\nTQOjiIiIiIgsYwp0AXegq5/dh3r42HVrMbPx9dFwiI2Nlbp0gYiIiIjIMqZAF3A7njlKJGT86jVr\nztnW0pzgQFd/EUolIiIiIiIXggJdgI2mM3znuQ5uuTxJU1X8nO0tzVUc6R5iZCxThNKJiIiIiMhS\nU6ALsEdeOUnP0Bh3bls37fbW5gRZB2+eHrzAJRMRERERkQthzkBnZveaWZeZvTzD9v/LzJ73p5fN\nLGNm9f62Q2b2kr9tT6ELf7HbsfsIa+rKuamlcdrtuZEuNTCKiIiIiMjyNJ8Wuq8Dt8+00Tn3p865\nq5xzVwG/CzzunOvO22W7v33r4ooq+Q6fGeSfDpzhY1vXEgrZtPtsbKwkZLp0gYiIiIjIcjVnoHPO\nPQF0z7Wf707g/kWVSOblW7uPEjL4yNa1M+5TFg2zvqFSA6OIiIiIiCxTBTuHzswq8FryvpO32gGP\nmNmzZnZ3oZ7rYjeWyfLtZzu4+dJmVtSUzbrv5qaELl0gIiIiIrJMmXNu7p3MNgD/6Jx7yyz7fAz4\nNefcr+StW+2c6zSzZuBR4HN+i990978buBsgmUxeu2PHjoW8jgtiYGCARCJR7GLw7Mk0//Pno/zW\nNXGubo7Muu+396X40aExvnZLBZEZumYuV6VSXzJ/qrPgUZ0Fi+oreFRnwaM6C55SrbPt27c/O5/T\n1mZPAwtzB1O6WzrnOv3bLjP7B2AbMG2gc87dA9wDsHXrVtfW1lbAohVGe3s7pVCur//vZ0hW9/G5\nX72ZSHj2Rtbu6g5+8OYLbHjLdeODpFwsSqW+ZP5UZ8GjOgsW1VfwqM6CR3UWPEGvs4J0uTSzGuBd\nwPfy1lWaWVVuHrgVmHakTJm/zrPDPP76KT66de2cYQ6gtbkKQOfRiYiIiIgsQ3O20JnZ/UAb0Ghm\nHcAXgSiAc+6r/m4fBB5xzuVf8CwJ/IOZ5Z7nb51zPypc0S9OD+w+CsBHZxkMJd/m5kpAI12KiIiI\niCxHcwY659yd89jn63iXN8hfdxC48nwLJufKZB3f3nOUm1oaWVtfMa/7VMQirK4t17XoRERERESW\noYKNcilL74nXT3Gsd4Q7t61b0P1akxrpUkRERERkOVKgC5D7nzlCYyLGey5LLuh+LU0J3jg1QCY7\n94imIiIiIiISHAp0AdHVN8Jje7v41WvXEIssrNpakwlG01k6e4aXqHQiIiIiIlIMCnQB8e1nO8hk\nHXdct7DulgAt/kiX+zXSpYiIiIjIsqJAFwDZrONbu49yw6Z6NjZWLvj+uevPaWAUEREREZHlRYEu\nAJ48eIYj3UMLHgwlp6Y8SnNVXJcuEBERERFZZhToAuBvnzlCbUWU265Ycd6P0ZpMqIVORERERGSZ\nUaArcWcGRnnklRN88OrVlEXD5/04LU0J3ugawDmNdCkiIiIislwo0JW4v3+uk7GMO+/uljktySoG\nRtOc6BspUMlERERERKTYFOhKmHOO+3cf4Zp1tWxJVi3qsVpzA6PoAuMiIiIiIsuGAl0J232oh4On\nBrljka1zoJEuRURERESWIwW6ErbjmSNUxSP88ttWLvqxGipj1FVENdKliIiIiMgyokBXonqHxvjB\nS8d5/9WrqIhFFv14ZkZrcxUHdHFxEREREZFlQ4GuRH33+U5G01nuuG7x3S1zWvxLF2ikSxERERGR\n5UGBrgQ557j/mSO8dXUNb1ldU7DHbWlKcHZojDODqYI9poiIiIiIFI8CXQl6oaOXvSf6uWPb2oI+\nbmtSI12KiIiIiCwnCnQlaMczRyiPhnnflasK+ritzd6lD3QenYiIiIjI8jBnoDOze82sy8xenmF7\nm5n1mtnz/vSFvG23m9k+MztgZp8vZMGXq4HRNA++cIxfuXIlVWXRgj52sjpOIh7RpQtERERERJaJ\n+bTQfR24fY59fuKcu8qfvgRgZmHgL4BfBC4H7jSzyxdT2IvBg88fYyiVKci156YyM1qaE7p0gYiI\niIjIMjFnoHPOPQF0n8djbwMOOOcOOudSwA7g/efxOBeVHbuPcEmyiqvX1i7J47c2J9RCJyIiIiKy\nTBTqHLq3m9kLZvZDM7vCX7caOJq3T4e/TmbwyrFeXuzo5Y5tazGzJXmOluYEp/pH6R0aW5LHFxER\nERGRC2fxV6yG54D1zrkBM3sv8F2gdaEPYmZ3A3cDJJNJ2tvbC1C0whoYGFjSct336ijREDQNHqK9\n/fCSPMdwVxqABx5+gta68JI8R6lY6vqSwlOdBY/qLFhUX8GjOgse1VnwBL3OFh3onHN9efMPmdlf\nmlkj0Ankj7u/xl830+PcA9wDsHXrVtfW1rbYohVce3s7S1Wu4VSGz7X/mF++cjW/fOtVS/IcAJu7\nh/jvz+2ianUrbUtwnl4pWcr6kqWhOgse1VmwqL6CR3UWPKqz4Al6nS26y6WZrTC/f6CZbfMf8wyw\nG2g1s41mFgPuAB5c7PMtVz946Tj9I2nuuK6w156banVtOWXRkM6jExERERFZBuZsoTOz+4E2oNHM\nOoAvAlEA59xXgQ8D/9rM0sAwcIdzzgFpM/ss8DAQBu51zr2yJK9iGdjxzBE2NVWybWP9kj5PKGRs\nbtJIlyIiIiIiy8Gcgc45d+cc278CfGWGbQ8BD51f0S4e+0/2s+dwD7/33kuXbDCUfK3NCXYf6lny\n5xERERERkaVVqFEuZRF27D5KNGz86jVrLsjztSar6Dw7zOBo+oI8n4iIiIiILA0FuiIbGcvwnec6\nuPXyFTQk4hfkOTc3JQB445S6XYqIiIiIBJkCXZE9/MoJzg6Ncce2pR0MJV9r0gt0+08q0ImIiIiI\nBJkCXZHteOYoa+vLuXFz4wV7zvX1FUTDppEuRUREREQCToGuiA6dHuTJg2f42Na1hEJLPxhKTiQc\nYmNjpUa6FBEREREJOAW6Ivr+C8cwgw9fe+G6W+a0NldxoKv/gj+viIiIiIgUjgJdET22t4u3rall\nRU3ZBX/uluYER7qHGBnLXPDnFhERERGRwlCgK5LTA6O80HGWmy9pLsrztzQnyDo4eGqwKM8vIiIi\nIiKLp0BXJO37TuEcvPuy4gS63EiXB3TpAhERERGRwFKgK5Jde7toropzxarqojz/xsZKQgYHTuo8\nOhERERGRoFKgK4KxTJYnXj/F9kuaMbtwo1vmi0fCrG+o1KULREREREQCTIGuCHYf6qZ/NM3NRepu\nmdPSnNClC0REREREAkyBrgh27e0iFg5xU8uFu5j4dFqbE7x5epCxTLao5RARERERkfOjQFcEj+3t\n4vpN9VTGI0UtR2syQTrrOHxGI12KiIiIiASRAt0FdvjMIAdPDXLzpcXtbgnQ0lQFoG6XIiIiIiIB\npUB3ge3c2wVQEoFuc3MlAPtPKtCJiIiIiASRAt0FtnNvF5uaKlnfUFnsolARi7CmrlwjXYqIiIiI\nBNScgc7M7jWzLjN7eYbt/9zMXjSzl8zsZ2Z2Zd62Q/76581sTyELHkSDo2mePtjNu0ugdS5HI12K\niIiIiATXfFrovg7cPsv2N4F3OefeCvwRcM+U7dudc1c557aeXxGXj58eOE0qk2V7CQW61uYEb5wa\nIJN1xS6KiIiIiIgs0JyBzjn3BNA9y/afOed6/MWngDUFKtuys/O1LqriEa7bUF/sooxrba5iNJ2l\no2eo2EUREREREZEFKvQ5dJ8Afpi37IBHzOxZM7u7wM8VKM45du3r4p1bmoiGS+fUxc3NCUAjXYqI\niIiIBJE5N3dXOzPbAPyjc+4ts+yzHfhL4Cbn3Bl/3WrnXKeZNQOPAp/zW/ymu//dwN0AyWTy2h07\ndizwpSy9gYEBEonEed33UG+GP3xyhE++NcZNq6MFLtn5GxxzfOaxIT66Jcp7N8WKXZyCWkx9SXGo\nzoJHdRYsqq/gUZ0Fj+oseEq1zrZv3/7sfE5bK8iVrc3sbcBfA7+YC3MAzrlO/7bLzP4B2AZMG+ic\nc/fgn3+3detW19bWVoiiFVR7ezvnW67/8eP9mL3Ov3r/O2lMxAtbsEX60u4fk0k00dZ25dw7B8hi\n6kuKQ3UWPKqzYFF9BY/qLHhUZ8ET9DpbdN8/M1sH/D1wl3Pu9bz1lWZWlZsHbgWmHSnzYrBzXxdX\nrqktuTAHuZEu+4tdDBERERERWaA5W+jM7H6gDWg0sw7gi0AUwDn3VeALQAPwl2YGkPabBpPAP/jr\nIsDfOud+tASvoeSd6h/lhaNn+e1bthS7KNNqba7i23uO4pzDry8REREREQmAOQOdc+7OObZ/Evjk\nNOsPAsurD995at/XBcDNJXS5gnwtzQkGUxmO946wqra82MUREREREZF5Kp3hFpexXfu6SFbHuWJV\ndbGLMq1Wf6TL/RrpUkREREQkUBTollgqneWJ10+z/ZLmku3O2KJLF4iIiIiIBJIC3RLbc6ibgdF0\nyXa3BGhIxKmvjGlgFBERERGRgFGgW2I793YRC4e4saWx2EWZVUtzgv0n1UInIiIiIhIkCnRLbOfe\nLq7fVE9lvCCX/FsyLc0J9ncNMJ8LzYuIiIiISGlQoFtCh04PcvD0IO8u4e6WOa3NCXqHxzg9kCp2\nUUREREREZJ4U6JbQzr25yxUki1ySubU2VwGwX+fRiYiIiIgEhgLdEtq5t4uW5gTrGiqKXZQ55Ua6\nfEMjXYqIiIiIBIYC3RIZGE3z9JtnSnp0y3zJ6jhV8YiuRSciIiIiEiAKdEvkp/tPMZZxbL8kGIHO\nzGhJaqRLEREREZEgUaBbIjv3dlFVFmHrhrpiF2XeWpoSaqETEREREQkQBbolkM06du07xTu3NBEN\nB+ctbk0mOD0wytkhjXQpIiIiIhIEwUkbAfLysV5O9Y9yc0C6W+bkRro8oFY6EREREZFAUKBbAjv3\ndmEGbZc0FbsoC5Ib6VLdLkVEREREgkGBbgns2tvFVWtraUjEi12UBVldW05ZNKQWOhERERGRgFCg\nK7Cu/hFe6OgNXHdLgFDIaGnWwCgiIiIiIkGhQFdg7ftOAXDzZcELdOCdR3fgZH+xiyEiIiIiIvMw\nr0BnZveaWZeZvTzDdjOzPzezA2b2opldk7ft42a2358+XqiCl6pde7tYUV3G5Suri12U89LSnOBY\n7wgDo+liF0VEREREROYw3xa6rwO3z7L9F4FWf7ob+P8AzKwe+CJwPbAN+KKZBefCbAuUSmf5yf7T\nbL+0CTMrdnHOS25glDfU7VJEREREpOTNK9A5554AumfZ5f3Afc7zFFBrZiuB24BHnXPdzrke4FFm\nD4aBtvtQNwOjaW6+NFnsopy3Vo10KSIiIiISGIU6h241cDRvucNfN9P6Zemx17qIRULc2NJQ7KKc\nt3X1FcTCGulSRERERCQIIsUuQI6Z3Y3XXZNkMkl7e3txCzSNgYGBWcv1g58PcUltiGd+9tMLV6gl\n0FzueOrVQ7SXnyh2URZlrvqS0qM6Cx7VWbCovoJHdRY8qrPgCXqdFSrQdQJr85bX+Os6gbYp69un\newDn3D3APQBbt251bW1t0+1WVO3t7cxUroOnBjj5o8f5N++5lLZ3bLig5Sq0K489x8udvTO+1qCY\nrb6kNKnOgkd1Fiyqr+BRnQWP6ix4gl5nhepy+SDw6/5olzcAvc6548DDwK1mVucPhnKrv27Z2bm3\nC4CbLw3m5QrytTQlONo9xMhYpthFERERERGRWcyrhc7M7sdraWs0sw68kSujAM65rwIPAe8FDgBD\nwG/427rN7I+A3f5Dfck5N9vgKoG1a18Xrc0J1tZXFLsoi9aaTJB1cPDUIJevCublF0RERERELgbz\nCnTOuTvn2O6Az8yw7V7g3oUXLTj6R8Z45s1u/uWNG4tdlIJoba4CYH9XvwKdiIiIiEgJK1SXy4va\nT/efZizj2L4MulsCbGisIGRopEsRERERkRKnQFcAO/d2UV0W4dr1y+Oa6fFImA0NlQp0IiIiIiIl\nToFukbJZx659XbxzSxPR8PJ5O1uaE7q4uIiIiIhIiVs+CaRIXurs5fRAalmMbpmvNZng0OlBxjLZ\nYhdFRERERERmoEC3SDv3dmEGbZcsr0DX0pwgnXUcPjNY7KKIiIiIiMgMFOgWaefeLq5eW0t9ZazY\nRSmo8ZEuT6rbpYiIiIhIqVKgW4SuvhFe6uxddt0tATY3JTBD59GJiIiIiJQwBbpFaN93CoCbL00W\nuSSFVx4Ls7q2XCNdioiIiIiUMAW6RXhs70lW1pRx2cqqYhdlSbRqpEsRERERkZKmQHeeRtMZfrr/\nNG2XNGNmxS7OkmhNVvHGqQEyWVfsooiIiIiIyDQU6M7T7jd7GExlePcyPH8up6UpQSqdpaNnqNhF\nERERERGRaSjQnafH9p4kFgnxjpaGYhdlybQkE4BGuhQRERERKVUKdOfBOcfOvV28Y3MDFbFIsYuz\nZFqa/UCn8+hEREREREqSAt15OHh6kMNnhpbl5QryVZdFSVbHNdKliIiIiEiJUqA7D7v2dgGw/ZLl\nHejAu8D4ga7+YhdDRERERESmoUB3Hnbu7WJLMsHa+opiF2XJtfiXLnBOI12KiIiIiJQaBboF6hsZ\n45k3u9m+zLtb5rQmEwylMhzrHSl2UUREREREZIp5BTozu93M9pnZATP7/DTb/8zMnven183sbN62\nTN62BwtZ+GL46f7TpLOOmy+C7pbgXboA0Hl0IiIiIiIlaM4hGs0sDPwFcAvQAew2swedc6/m9nHO\n/bu8/T8HXJ33EMPOuasKV+Ti2rm3i+qyCNeuryt2US6I1mQVAPtP9vOuLU1FLo2IiIiIiOSbTwvd\nNuCAc+6gcy4F7ADeP8v+dwL3F6JwpSbrHO37unjXJc1EwhdHb9X6yhgNlTG10ImIiIiIlKD5pJLV\nwNG85Q5/3TnMbD2wEdiZt7rMzPaY2VNm9oHzLmkJONSb5fRAipsvvbhaqjY3JxToRERERERKUKGv\nin0H8HfOuUzeuvXOuU4z2wTsNLOXnHNvTL2jmd0N3A2QTCZpb28vcNEWb3fnMIYRObWf9vYDxS7O\nBVOZHuWZE2l27dqFmRW7OPM2MDBQkn9HMjPVWfCozoJF9RU8qrPgUZ0FT9DrbD6BrhNYm7e8xl83\nnTuAz+SvcM51+rcHzawd7/y6cwKdc+4e4B6ArVu3ura2tnkU7cL64s9+yDXra/iVW99R7KJcUIei\nb7Lr+69yxda301xVVuzizFt7ezul+HckM1OdBY/qLFhUX8GjOgse1VnwBL3O5tPlcjfQamYbzSyG\nF9rOGa3SzC4F6oAn89bVmVncn28EbgRenXrfIDjZN8Lhviw3XySXK8jX0uwNjKJulyIiIiIipWXO\nQOecSwOfBR4GXgMecM69YmZfMrP35e16B7DDTb4C9WXAHjN7AdgF/HH+6JiBkU4x9K1P8sHQT7hl\nfbjYpbngWpO6dIGIiIiISCma1zl0zrmHgIemrPvClOU/nOZ+PwPeuojylYaeQ9Sf+Cl/FuvB3fdV\nWLMVWm/1ppVXQoDOKzsfzVVxqsoi7D+pQCciIiIiUkoKPSjKsjRat5l3jP0lH6k/zB9e1Qf7H4Fd\n/w/s+jIkVkDre6D1Nti8HeJVxS5uwZkZLRrpUkRERESk5CjQzcPR7mGqy+PUrmqFtvdA23+EgVNw\n4Mew/2F49fvw829CKArr3+6Fuy23QUPLsmm9a21OsHPvqWIXQ0RERERE8ijQzUNLc4Kfff5mduUP\nZ5pogqvu9KZMGo4+7YW7/Y/CI7/vTXUbvW6ZW26F9TdBNDgjRE7V2lzFA3s6ODuUorYiVuziiIiI\niIgICnTzZmaEZmptC0dgw43edMuX4OwRr1vm64/Ac/fBM1+DaAVsfJcX7lpvhZo1F/YFLFJL88TA\nKFs31Be5NCIiIiIiAgp0S6N2HVz3SW8aG4ZDP4XXH/Za8F7/obdP8xV+uLsN1lznhcISlgt0+xXo\nRERERERKRmmniOUgWg6tt3iT+1M4/bof7h6Bn/1P+OmfQVkttLzbC3ct74HKhmKX+hyra8spj4Z5\n5VhvsYsiIiIiIiI+BboLyQyaLvGmG38TRnrhjV1euNv/KLz8HcD8yyLc5rXgrXhbSQysEgoZN1/a\nzPdfOM7vv/dyymMX3/X4RERERERKjQJdMZXVwBUf8KZsFo4/75979zDs+s/elFjhte5tuQ02tRX1\nsgh3vX09P3jpON9/4RgfvW5t0cohIiIiIiIeBbpSEQrB6mu8qe3zMNDlXRbh9Yfh1Qfh59/wL4vw\nDi/ctd4GjS0XtIjXb6xnSzLBfU8d4iNb12Al0HIoIiIiInIxU6ArVYlmuOqfeVNmzLsswuv+ZREe\n/j1vqt/kjZjZeiusv3HJL4tgZtx1w3r+4Huv8PzRs1y9rm5Jn09ERERERGanQBcE4ShsuMmbbv0j\n6Dnsn3f3CDz7dXj6q95lETa1TQS8mtVLUpQPXrOG//KjfXzjqcMKdCIiIiIiRaZAF0R162Hbp7xp\nbBje/Il/SYRHYN9D3j4b3wkf+GrBg10iHuFD16xmx+6j/Kdfupz6Sl1kXERERESkWELFLoAsUrTc\nGw3zl/5f+Lcvwr95Gt79Beh8Dr72Tm8UzQL7tRvWk0pn+dbuowV/bBERERERmT8FuuXEDJovhV/4\n9/CpXVDZBN/4IDz+J94omgWyJVnFDZvq+ZunD5PJuoI9roiIiIiILIwC3XLVtAU+9Ri89SOw68vw\ntx+Foe6CPfyvv30DHT3DtO/rKthjioiIiIjIwijQLWexSvjQPfBL/w3efNzrgtnxbEEe+pbLkySr\n49z35OGCPJ6IiIiIiCycAt1yZwbXfQL+5cOAwb23wTN/BW5xXSWj4RB3blvH46+f4vCZwcKUVURE\nREREFmRegc7MbjezfWZ2wMw+P832f2Fmp8zseX/6ZN62j5vZfn/6eCELLwuw+hr49OOw+WZ46Hfg\nO5+E0YFFPeSd29YRCRnffEqtdCIiIiIixTBnoDOzMPAXwC8ClwN3mtnl0+z6LefcVf701/5964Ev\nAtcD24AvmpkuXlYsFfVw5w5vFMxX/h7+6mY4te+8Hy5ZXcZtV6zggT0dDKcyBSyoiIiIiIjMx3xa\n6LYBB5xzB51zKWAH8P55Pv5twKPOuW7nXA/wKHD7+RVVCiIU8kbBvOu7MNwN92yHl/7uvB/urrev\np3d4jO+/eKyAhRQRERERkfmYT6BbDeRfcKzDXzfVr5rZi2b2d2a2doH3lQtt07vg00/AyrfBdz4B\nP/gdSI8u+GGu31jPlmSCbzx5GLfI8/JERERERGRhbK4f4Wb2YeB259wn/eW7gOudc5/N26cBGHDO\njZrZp4GPOeduNrPfAcqcc//Z3+8PgGHn3H+d5nnuBu4GSCaT1+7YsaMwr7CABgYGSCQSxS5GQVk2\nzaaD32Btx3fpq2rllSv+A6NlzQt6jMeOjPGNV1P8wQ1lbK4NL1FJF2451tdypzoLHtVZsKi+gkd1\nFjyqs+BfhSjCAAAgAElEQVQp1Trbvn37s865rXPtF5nHY3UCa/OW1/jrxjnnzuQt/jXwJ3n3bZty\n3/bpnsQ5dw9wD8DWrVtdW1vbdLsVVXt7O6VYrkW7+T3w6oNUf+8zvP2F/wAf+mtofc+87751NM3f\nf/nHvJJq4BNtVy1hQRdm2dbXMqY6Cx7VWbCovoJHdRY8qrPgCXqdzafL5W6g1cw2mlkMuAN4MH8H\nM1uZt/g+4DV//mHgVjOr8wdDudVfJ6Xm8vfB3e1QvRr+5sOw88uQnd9AJ4l4hA9ds4Z/fPE43YOp\nJS2miIiIiIhMmDPQOefSwGfxgthrwAPOuVfM7Etm9j5/t980s1fM7AXgN4F/4d+3G/gjvFC4G/iS\nv05KUcNm+OSP4ap/Dk/8CXzzQzB4el53vevt60mls3xr99G5dxYRERERkYKY13XonHMPOee2OOc2\nO+e+7K/7gnPuQX/+d51zVzjnrnTObXfO7c27773OuRZ/+t9L8zKkYKLl8IG/gPd9BY48BV/9BTjy\n9Jx325Ks4oZN9fzN04fJZDU4ioiIiIjIhTCvQCcXoWvugk88CpE4fP298ORfwhwD6Nx1wwY6eoZp\n39d1gQopIiIiInJxU6CTma18m3de3Zbb4eHfhW9/HEb6Ztz91iuSJKvjfOOpwxesiCIiIiIiFzMF\nOpldeS187Jtwyx/Ba/8If7UdTr4y7a7RcIg7t63j8ddPcfjM4AUuqIiIiIjIxUeBTuZmBjf+Jnz8\n+zDaD3/1bnj+/ml3vXPbOsJmfFOtdCIiIiIiS06BTuZvw43w6Z/Amq3w3X8F3/8tGBuZtEuyuozb\nrljBA3s6GBmb32UPRERERETk/CjQycJUJeGu78JNvw3Pfh3uvRV6Dk3a5dduWE/v8BgPvnCsKEUU\nEREREblYKNDJwoUj8J4vwp07vDD3tXfCvh+Ob75hUz1bkgm+8eRh3BwjY4qIiIiIyPlToJPzd8kv\nwqefgLoNcP8d8OM/hEwaM+OuG9bzUmcvL3T0FruUIiIiIiLLlgKdLE7dBviXj8C1vwE//TP4xgeg\n/yQfuHo1lbEw9z15qMgFFBERERFZvhToZPGiZfAr/x0++DXo2ANf+wWqTjzDh65Zwz++eJzuwVSx\nSygiIiIisiwp0EnhXHkHfOoxiFfB//kVfrPsIVLpDA/sOVrskomIiIiILEsKdFJYySvgU7vgsl+m\n6akv8+2ar/DdJ18lk9XgKCIiIiIihaZAJ4VXVg0f+T9w+x9zbWo3Xxv6bfY81V7sUomIiIiILDsK\ndLI0zOCGf032X/yAslCaax75CDx3X7FLJSIiIiKyrCjQyZKKrL+Bf9i2g6eyl8KDn4PvfgZSQ8Uu\nloiIiIjIsqBAJ0vugzddySfSn+cnqz4Bz/8N/K9b4MwbxS6WiIiIiEjgKdDJkktWl3HLFav43Inb\nSd3xLejrhHva4NUHi100EREREZFAm1egM7PbzWyfmR0ws89Ps/23zexVM3vRzB4zs/V52zJm9rw/\n6Rf8RerXbljP2aExvjtwOXz6J9DYCg/cBQ//PmTGil08EREREZFAisy1g5mFgb8AbgE6gN1m9qBz\n7tW83X4ObHXODZnZvwb+BPiYv23YOXdVgcstAXPDpnq2JBN886nDfHTrTfAbP4RH/hM8+RXo2A1X\nfAiqV0L1aqheBZXNEJ7zz1NERERE5KI2n1/M24ADzrmDAGa2A3g/MB7onHO78vZ/Cvi1QhZSgs/M\nuOuG9fzB917h+aNnuWptLbz3T2Ht9fCDfw8/+o9T7hCCxAov3OWCXlVe4KteCVWrIFpWnBckIiIi\nIlIC5hPoVgNH85Y7gOtn2f8TwA/zlsvMbA+QBv7YOffdBZdSloUPXL2aP/7hXu578hBXrfUbbd/6\nYXjLr8JQt3duXd8x6D/m3eamU6/DwcdhtO/cB61o8IJdfvCrXgVVK6kY7ISRPu+6eCIiIiIiy5A5\n52bfwezDwO3OuU/6y3cB1zvnPjvNvr8GfBZ4l3Nu1F+32jnXaWabgJ3Au51z5wxxaGZ3A3cDJJPJ\na3fs2LG4V7YEBgYGSCQSxS5GoN336ihPdKT5s7YKqmK2oPuG00PER89MmmKp3Pxp4qPdxMZ6z7lf\nOlzGaLyRVKye0Xgjo/GGvKmR0Xg9Y9Fqr1VQikr/Y8GjOgsW1VfwqM6CR3UWPKVaZ9u3b3/WObd1\nrv3m00LXCazNW17jr5vEzN4D/D55YQ7AOdfp3x40s3bgauCcQOecuwe4B2Dr1q2ura1tHkW7sNrb\n2ynFcgXJqsv62flnT9AZX8e/etfmwj9BehT6j0PfMV596sdcvqaWSN8xIn2dVPYfh7590HUCXGby\n/cIxqFoxqYVvonunPyVW6Ly+Jab/seBRnQWL6it4VGfBozoLnqDX2Xx+ne4GWs1sI16QuwP4Z/k7\nmNnVwNfwWvK68tbXAUPOuVEzawRuxBswRS5SW5JVXL+xnm8+dZhP/cImwqGFtdLNKRKHug1Qt4Gu\nN1NcfmPbuftkMzDQNaV7Zyf0eUGQzue8UJgemXJHg0QSEk1QVgvltVBW482fs1wzeVnn+omIiIjI\nEpgz0Dnn0mb2WeBhIAzc65x7xcy+BOxxzj0I/CmQAL5tZgBHnHPvAy4DvmZmWbxLJPzxlNEx5SL0\n62/fwGf+9jkef72Lmy9NXvgChML++XYrgWun38c5GO7JC3qdXsjr7YSh0zB8Fk4fgJGzMNILY0Oz\nP2c4PjngzRb+pi7HqyGk7qAiIiIicq559R9zzj0EPDRl3Rfy5t8zw/1+Brx1MQWU5efWK5I0V8W5\n78nDxQl082EGFfXetGIef8LpUW8AllzAGz7rz5/153snbxvogtOv++t7wWVnK4w3sMus4a92+u3l\ntV6rpYgUVv5Bn1AEatZCvPTOvxARkeVPJwTJBRcNh7hz2zr+fOd+Dp8ZZH1DZbGLtHiRuNcVM9G0\n8Ptms5AamBIGe2cOgyO9cHr/xHJ6ePbHL6/zRwJdmXdu4Mq80UFXeftYgbu/igRZasgLa71HvZb5\nc+Y7zm2ZL6+H2nVQuxZq13shb3x5nXegRUQmc877Xxrq9g6SDHf78/7yUI+3vbIJEs3e+e6J5MSk\nUxpEFOikOP7Z9ev4yq4D/M3TR/i9915W7OIUVyjkt8Cd5+UV0qMTLX2TwmCPt9x/fHygGI6/CIOn\ngCmj20bKJgaFqVp5buCrWultD0cX/XLlAslmvADS/Sb0vAnplNfiXF4PFXX+bb3XpfdiC/OZsYku\n1L0d0NcxZb7D+/+ZKrECalZD06XQcos3X70asmk4e8R7v88e8S61sv/H5x5siddMDni16/JC3zod\nWJHgy4z5Iaw7L5j1TDPfkxfYuiEzOvNjRiu90DbUzTnfXeAdKMkPeFUrvOCXWDE5AF4s/1/OeZd5\nGjrjvWeDp/353O0ZGDwzMZ8e8b7ja9b401rvsy03X9FwcbxvAadAJ0WRrC7jtiuSPLDnKL99yxbK\nouFiFym4InH/y6t5fvtnxqD/hB/y/HMExweHOQ6de+C149N8wZr/5bhyctCbNCroSohXFfwlygzG\nRuDsYS+0dR/0gltu/uwRyI7N/RgW9n7ojIe9aULfdLeR2NK/vvORzXo/XHr9YJZrTRuf74SBE+d2\ncy6r8X68VK+GNdd5P2aq10z8sKlatbDX7Jz3Q6r3iFcXZ49OhL6eQ/DmE17LfL5Y4txWvRq/ta92\nrddCoR9WciFkszDamxfCeqa0mk0X2Hog1T/zY4aiE58f5XVQvynvsyf/82bKfO60gUza+98eOAn9\nJ73bgRPeKQz9/m3nHm/bdD1XQlE/8CWnBMDcfC4IJkvr8y2dmghfuVCWH8jGg1peeJvpsz8UhcpG\nL6RVNMDKK70Duv3H4OTL8PqPzh0QLlLmfx6u9sNeLvitnvjMjFUs/fsgs1Kgk6K564YNPPTSCR58\n4Rgf3bp27jtIYYSj/o/FWd5z57wvh/5jUwLfMS8I9hyCwz/zWgKnilVNXOg919WzelVet89V3g9T\nDfQyPyN9flA7ONHa1u1PfZ1MOmIdr/ZGeV3xVrj8fVC30fvRVL8RIuWTuzJNe9sDPYfh2M/nPmoe\nS0B5PddmonBk3czBr9CtgSO9M7Ss5d1mUpPvEynzf4ysgc3bJ+bzf5AU+vw3s4lu2KunGXwpdw5e\nrlUvP/CdPQxHnz73/ytS7v3fTgp9eV07E0n9X8nMMmPe/8jZw97/+WDXuS1luc+CkbOznNtt3vnZ\nueCVaPZarSeFs6kHieq8z4zF/P+HI15rW9UKWDnLfs7BaL8X8AZOTAmA/tRzGI4+44Wh6ZTXTWnh\ny7X45QKgv66sZmGvKT8oD56eR1Dr9lrbZlJWOxHQatfBqqsnwtp4cGv0xwRo8A64zlZe57znznUv\n7+3w5/3P1jce88Lz1JbSioa8g2DTTImkNyCdLBkFOimaGzbV09qc4JtPHVagKzVmUNngTbMNCpMa\nmujOmbsdvxzEcTj9uPfhP/W6f6Go/8U8ubUveeIMvD46MaBLeZ03X0pHSwst15KTH9ryW9um/uCo\nbPJC2oabvKBWv2kiuFXUz/5lXdmwsHKNn9cyTfDzl1MdB7wfHGcPT/wQnMl8WwNjld6Prt7OiS6Q\nuXPXpv64sbDfXWg1rL4GLvuViS5DuSPKc70vxZA/8NLKK6ffZ6QvL/DlTb1H4fjz3g+vfOHYRDep\nXDfO/K6d1auW/nVJ8TjnhZhcYDt7yDv41nPYW9fbee5ncbRyInBV1Hv/N/n/i/n/r7n5sprS/nFu\nNnEaQ2PL7PtmxrzTEHItfJNa/Pzwd+RJLxBOd4ArUnZu987KZja8eQAGvjdNt8cz59ZB/mPlwldl\no/f5PjWQ5Ye08rrCXxvXzHuOykYvHE4nnfK+46cLfD1v+r0PprTUhiLe50/uANqk7p3+wTWdY7wo\nCnRSNGbGXW9fzxe+9wqfum8PlySr2NxcSUtTFZuaKqmM68+z5MUqoGGzN80kd92//G6ducDX1+l1\n89j/KIwNchnA3v9+7mNEK/yQVzcxquek+bzwl7++rKY0LgafzXivfTyoTWltm9T1zrwvuPqNcOkv\nTbSw1W30bi9Ul1YzL1jFKmdtzX1p6sVYsxnv3M1CtQZWNHpf9g2bYeM7J7pA5rpDJlaURh0vhbJq\nKLsCkldMvz016LXs5Vr18lv59j/i/RjNF4pwQ7QGns+1lBhYyJu3kL88dX7KfjPNT3uf/PUz3YdZ\nnj/ktZxWNns/MBPN/rzf8nkxnv850pcX2A5PDmxnj5w7UE8i6bXirr0e3rbBm69b791WrdAoyOHo\nxEHF2Tjn9Q7IhbzpunyeeQMO/xMM97Aeg1P1fhhr8D6/1l0/0XqWW58LbxUN3mdtEERi49f7nVF+\nT4pc4MuFvqNPwSvHvHOP88WrJ4Je9eopgW+NV0c6j39Gy/RbUILiw9eu4cWOXp473MPOvV1kshPN\n+KtqytjcnGBzU4KWZm/a3JSgMRHDLrYv8SDLv+7fdF3PYPzL8uldD3H927ZMXPYhN7BLbsTP4R5v\nvueQ10Ix3DP3NQDj/iUfymvyQl/tzPO5ULjQ6/+lUxPns01tbTt7eHI3wFDU+zKs3wjr3jER2uo3\nea0pQf6RFQpPtO7O19TWwNSg90O0ehVEy5eurEEXq4TmS71pOmMj/g+pifP4evY/x8rmZr87nfPe\n+/H5rLc8vt5NXj/rffDms9l53Geax572Plmv69xMg2GE/fOHKxu9oJdo8sJeZfOU9c1eC1MQuqOm\nUxPnWfqB7fLX98Drf+gFt+HuyfvHq71w1tACm9/t/9D2A1vtOp3bVCiW62ZaC02XzL5vOsXjP/kJ\nbdvffWHKVorKarwpefn027OZiV4YUwNf71HofPbcHgiYdxDi+k/DTf9uyV9C0CjQSVFVxCL81494\n3Y1S6SyHzwxyoGuAN04NcKBrgAOnBvjW7qMMj010Uagpj/rhrnJS0FtTV0E4pKAXSP6X5XDFKliz\ndWH3TadmD39T50/tm5if7RwxC3k/lmZqFYxWeN0Bx89n65h8zkks4bWqNV8Kl7538vls1atLu8vS\nhTbP1kBZoGiZ1+Usr9vZvnA7K/NbVIMgNxjG4CmvNWTw1OT5gS6vy/cJfxTfqUf+wft/rsi18jVN\nDIE/dT7R7O23VN28s1mvVWe6Fraew+eeFxuKkog3wapLYeVVkwNb3YaLZ+TGIInEvK7gMrNQeKJl\ndO110++TGvJ6tkwKfB1eq52cQ4FOSkYsEqI1WUVrcnKXsmzWcbxvxAt6fsh7o2uAnXu7eGBPx6T7\nb2qsZHNzgpamBENdY4y+coKKWJiKWJjyaGRiPhamIhZRAFwOIrGFjfKZb2x4ShCcIxT2Hp3Ynk17\n3WTqNsK6G/K6RfqhTSMSihRG/mAYc8lmvf/P8cDX5Z2/lJsf8MNg9xve/EzX8SyrnSP85XUDndpV\nLtedOBfSxlvb/G6RUw8kVa3yQtqGm7zbug0TXSOrVvLMEz+Z3K1Z5GIQqzjngJTMTIFOSl4oZKyu\nLWd1bTnv2jL5wt1nh1LjrXlvnPJa917q6OWhl47jHPzVS8/O+tixSMgLedGJkFfuh77pQ2CY8miY\nMn//8qi/nDdfHvO3R8NEw6buoaUsWu5NCx0swjnv+n+6oK1IaQmFJgabmatrnHNe997xoJdr8Zsy\nf/JleOOUNzrhdKKVXriLJbwWhJEp+5XVeCGt+TK45HY/rG30AlvNWn2OiMiiKdBJoNVWxLh2fT3X\nrq+ftH5kLMN3H3mct1x1LcNjGYZSGYZTaYZSuXnvdmgsPT7v3Xr7dA+m6OiZvG40PdMQzjMLhywv\nAIbG53OBLz8AlkXzt4fG9yuLhimLhMZDZFlk6nZvPhqe3/khY5ms/z6kx1/34GiaobGJ+dx7NjTq\nv2djE/PDYxmyzrGqppw1dRWsritnTZ0XuFfWlBGZZzkCzUw/wkSCzswbdCWe8FrW55IezWv5O+0H\nwbwuoKMDsP7tE90hc10jy2uX/KWIyMVNgU6WpbJomBWVId6yunDD4Gayzg86aUbHsgz7AWh4zJtG\nUhlG0hmGU962kWm2j68fy3B2eIwTvSPj4Wl0zLv/WGaak//nYSI8hohHvPAXC4f8MmXGQ9xCH7/C\nb7msyGu5dMAT+09xsm9y16FwyFhRXTYe8tbUVbCmdmJ+RU0ZschFEPhEZPmJxCdG3BMRKSEKdCLz\nFA4ZiXiExBJfTiGTdeOhb2Qsw8hY1r+dmB/fls4y6gfHkbS3PbdtdCzLaDpLPBqiIhqmMu53J42G\nqYjnB7TIeJfSyinzZdHQrF1GR9MZjp0dobNnmI6eITrPDtPhzz/1xhmO93WOD4AH3gHxFdVlrM4L\nefnhry/lONk3QjrryGQc6WyWTNZ5y1nHWGbysnebJZ3x5vOX8/errYiyorqMlTXlJGvixCM6YV1E\nRESWBwU6kRITDhmV8UggrsMXj4TZ2FjJxsbpr5+TSmc50TtCx9khP+gNj4e/3Yd6+P6LxyddqgKA\nnY8tebkbKmOsqCljZU2Zf1vuBz5veUVNGRWxpX//05ksg6MZ+kbGGBhN0z+SZiyTpSwapjLuher5\nhmu5sLJZR9/IGKcHUjjnqIhHqPQPkKgVenGcc/SPpjnVP0o644hHQsT9ngexSIh4JEQkpPOT5+Kc\no284zYm+EY73DnOyb4QTvaOc6Buhq2+EynhkfJTozc3e57gOdokEU+n/YhSRwIpFQqxrqGBdw/TX\nQkpnspzo81r4jvYM8/zLr3HZpVuIhIxwKOTfGtHw5OXx23DeurC3PhIKjS+HQ0bIjLNDXvfW473D\n3m3fCCd6R+g8O8Kzh3voGRo7p2w15dG8wFfGiuryScuNiTgj6QwDI2n6RtJ+IBtjwJ/vG0kzMOKv\n88Na/5R9hlKZad6V6ZnhD94ToTLunX9ZmdfSOh7+4hF/m7+vHzKi4cL/+HUOXjiZZuil414LaSY7\nqfU0PbVF1W91nXY5M9HCmnVQGY9QXR6huixKdXmU6rLcvHdb5c+XR8MF+2E/nMpwemCUM4MpzgyM\ncmYgxelB7/aMv/60P989mCI99WCELxo2KnLvvX9wJlcPlfHw5G15t95+ESr8ML/cgkw26+gZStHV\nP+pNfSN09Y9yqn+Urv4Ruvr89f0jjIzNfs5yyLwDSt77M/k9yi1Pv21iff75yWXR8Pi6uH9ecjxv\nW26/eCRUEucJpzNZuvq9cHayd4QT/mda7vZknzc/3fvYUBmjubqM/pExHnzh2Pj6kMHa+gpamhLj\no0Vvbq6kpamKmgpd0FmklCnQiUjRRMIh7zy7ugquBxr7D9B2/fqCP09jIk5Lc2LG7cOpzPhRbC/4\njUzc9g3zcmcvpwdSM95/JrkuulVlERJlEWrKo6ypLfeW4xGqyqIkyiJU5e0TC4cmBqXJG7xmaDTN\n4NR1KS8YdvWNMjSWZmg0w2AqPeeP4YL7+XPz3tUMornQHTLCYS+E5wdzAwZGM/QNj5HKzP5aIiGj\nujxK1TmBb3IYrCqLEgmbF878kHZ6IOUHOG95poBdEQvTkIjRUBlndW0Zb1td4y0n4jRUxoiEbfy9\nH0plvLA+Xl9pBke922Nnh/06m6jDhcoFmfGAEg0RC08ElVg4RNwPHpNCjD9/rDPFi5n9RMJGNBTy\nDoSEQ8Ry9RA2ov6BkmgkNL5PdHx9iGh44mBKzA+ZYxl3TijzQtsop/ongtt0IbgqHqGpOk5zVZyr\n1tbSXBWnuTpOc5V3zu1oeqIL+Wg6Qyqdm/e6nI/P5+2XSmfpGUxNu9/IWGbGMD4fkZCND0iVe9/L\n/NvolANKEwehQlMORk1ZF7ZzDmTlbl97I8VjZ1/2wluf97l0emB0Und2gFg4RLImzorqMt6yuob3\nXJYc73GworqMZHUZzdWTu5wPpzIcPD0xUvQb/nVgf3LgNKm8gcAaEzG/Jc9r0ctdC3ZVTTmh87j8\nj3OOgdE0Z4fG6B2emM4OjXF2OOUtD42Nbx8YnXxtwdwxDZtuZd76/GMfNmlXbykeCVFfGaOhMkZ9\nZZz6RIzGypi3LuGtqy2PntdrXKhcq3/3YIqeoTHODuXfpugfSVMWDVMV974rqsqiJOIRqssmlnPf\nLWVRtbRebBToROSiVx6bvesoeOcLdvWNctxv6Ts9kKI8Gh4PYlX5Aa3Ma2kp1nUO8wfwyQWNc7q2\nFsjPn3uWG7ZtG/8BOvWH6tTlhf4wGhnL0D+Spm9kjL7hMfr8Vs++4Yl1U7ef6h8Y3z5daAqHjIZK\nL5A1JmJsaKjwwlkiRmNlfFJYa0jElqz7bdavp8G8epoIhBkvoPjhZFJYyUwElFR+mPHne4fHJt1n\n6v7ujdeX5PVM1VAZo6kqTnN1Ga3JKi+oVcVpqirzA5sX2spjF/7HZybrGE2fe47y+Lp0Zvw9m3wu\n8+R9RnJB0b/Nnb87ms5Mc76v3yKdyW+1Pve84OnUdBxjRbUXzi5bUU3SD2krauIk/fOD6yqiC269\nLY+FuWJVDVesmjyAWCbr6OgZmrgsUNcgB04N8IMXj9M7PNGjoTwaZlNT5XjXzQ2NlYz5f4Nnh8fo\nHUqNz58d8v5Hz/rhbbbPpFg4RG1FlJryKLUVURoSMUL+a3N+ks2/d364dePrpn/8/NXDYxleOdbH\n6YFR+kemuSA93kGUugov5E0EPS/sNSby1lfGqa+MUVcRJZVxnOgdoccPY2eHxiZuBycHtdy23uEx\nZnpLcufw5/725hILh8a/i8YPKsajeeHPW85tj0VCRMPewZpYOOQdzAl7B4hiEfO35a8L+T1n5t9j\nwDl3zv/TtGMF5MYDyBsbYOJ/NDP+foRs4jslbN5BkbB5ZQr760J5B0ZC5n0fhabsk5vPjFzgA6EF\nNq9vKTO7HfgfQBj4a+fcH0/ZHgfuA64FzgAfc84d8rf9LvAJIAP8pnPu4YKVXkTkAolHwqytr2Bt\n/fTdR0vJpAF8qpb2uboPhLlkxdI9Sa7LW1NV/LzuP5bJ+t1ixxjLOBoTMarLLswR97mE8s+XXeJ6\nytm1axe/8M53kfYHGUpnHGPZLGMZL1yMZSZCR8rfns5kGcvmtk/s493He5xI2GiuKhtvXWtMxOd9\nKZViCIfMHxCq2CWZzDlH1jFpQKin/umn3Pru7Re0HOGQsb6hkvUNldx8aXJS+c4Mpnija4ADpyaC\n3p5DPXzv+WPnPE51WYTaith4MFtTVz4+X1vura/JC265dcU4ZziVztIzlOLMQIruQa8lv3swN5+i\n22/d33uin+5BL4jN6tHpzwcvj4apq4hSWxGjrjLKylovjNdVxLx1/nydHw5rK2JUxSPjn1mpdJZB\nvxt//vnXA6Pewa3clL88MJKmo2cob9/CHOQz45yQl1t2MCm0jaaz57Qqz9f4pZoiYcwgnXVks46M\n87rrZ5x3QCTjrzuf5/n3157fd0ypmDPQmVkY+AvgFqAD2G1mDzrnXs3b7RNAj3OuxczuAP4L8DEz\nuxy4A7gCWAX82My2OOcW3s9ERERkgaLhkPfDqLLEfrkXiZnfXTKMumWVIDMjbBAOTdRNbAnOfz1f\nZkZjwgvs129qmLRtKJXmaPcw8YjXulZVFi1aL4XzEYuESPpdU+cjncnSMzR2bvgbSNFx5BDXvOUS\nP6T5Ac2fX+z/XSwSIhZZ3Geac17vgNz53GP+gZlUxmvFH/MP3qTSbnx+zN+W8vcd8/dLZdzk+/gH\nfQDKo1POQx2/lq53bd7cfDx67nV6y/LOZV1ouB8Pe1lH1rmJAJgX+jJ5y1nnOPDinvN+P0vBfFro\ntgEHnHMHAcxsB/B+ID/QvR/4Q3/+74CvmPfuvx/Y4ZwbBd40swP+4z1ZmOKLiIiISLFVxCJL2lpf\naiLhEE1Vcb/3wOTX3d5+bEnOBy8Us1wLdYTmYhdmCYRCRghjIdm5IxKcgw/TmU9/iNXA0bzlDn/d\ntD2WdHgAAAcoSURBVPs459JAL9Awz/uKiIiIiIjIeSiZQVHM7G7gboBkMkl7e3txCzSNgYGBkiyX\nTE/1FTyqs+BRnQWL6it4VGfBozoLnqDX2XwCXSewNm95jb9uun06zCwC1OANjjKf+wLgnLsHuAdg\n69atrq2tbR5Fu7Da29spxXLJ9FRfwaM6Cx7VWbCovoJHdRY8qrPgCXqdzafL5W6g1cw2mlkMb5CT\nB6fs8yDwcX/+w8BO540X+yBwh5nFzWwj0Ao8U5iii4iIiIiIXNzmbKFzzqXN7LPAw3iXLbjXOfeK\nmX0J2OOcexD4X8A3/EFPuvFCH/5+D+ANoJIGPqMRLkVERERERApjXufQOeceAh6asu4LefMjwEdm\nuO+XgS8voowiIiIiIiIyjdK96qeIiIiIiIjMSoFOREREREQkoMwbu6S0mNkp4HCxyzGNRuB0sQsh\n86b6Ch7VWfCozoJF9RU8qrPgUZ0FT6nW2XrnXNNcO5VkoCtVZrbHObe12OWQ+VF9BY/qLHhUZ8Gi\n+goe1VnwqM6CJ+h1pi6XIiIiIiIiAaVAJyIiIiIiElAKdAtzT7ELIAui+goe1VnwqM6CRfUVPKqz\n4FGdBU+g60zn0ImIiIiIiASUWuhEREREREQCSoFuHszsdjPbZ2YHzOzzxS7PxczM7jWzLjN7OW9d\nvZk9amb7/ds6f72Z2Z/79faimV2Td5+P+/vvN7OPF+O1XAzMbK2Z7TKzV83sFTP7LX+96qxEmVmZ\nmT1jZi/4dfZ/++s3mtnTft18y8xi/vq4v3zA374h77F+11+/z8xuK84rujiYWdjMfm5m/+gvq75K\nmJkdMrOXzOx5M9vjr9PnYgkzs1oz+zsz22tmr5nZ21VnpcvMLvH/v3JTn5n922VbZ845TbNMQBh4\nA9gExIAXgMuLXa6LdQLeCVwDvJy37k+Az/vznwf+iz//XuCHgAE3AE/76+uBg/5tnT9fV+zXthwn\nYCX8/+3cz4uVVRzH8feXHC1GGctCwokaQQoXoRJRJBEOWVZoCxcTQdEPgmoTLYIQ+hOiFpELJQoq\nq+mXtClLoVVWmpUl2liGIzNOZNqPRfbj0+J8Z7wNDENgc89z5/OCwz3nPJfhuffDnHvPfc5zWJX1\nBcAhYLkzq7fkez8/613A7sziVWAg+zcDD2b9IWBz1geAV7K+PMfLeUBfjqPntPv1dWoBHgVeAt7J\ntvOquABHgAsn9XlcrLgAzwP3Z30usNCZNaNQvsuPApd2ama+Qje9q4EhSd9KOg1sAza0+ZxmLUkf\nAicmdW+gDLTk4+0t/S+o+AhYGBEXAzcBOySdkPQTsAO4+f8/+9lH0oikvVn/BTgALMGZVSvf+1+z\n2ZVFwBpgMPsnZzae5SDQHxGR/dsk/S7pO2CIMp7aWRYRvcCtwJZsB86riTwuVioieig/KG8FkHRa\n0kmcWVP0A4clfU+HZuYJ3fSWAEdb2sPZZ/VYLGkk66PA4qxPlZ0zbYNc2rWScsXHmVUsl+/tA8Yo\nH16HgZOS/syntL7/E9nk8VPAIpzZTHoKeAz4O9uLcF61E/BeROyJiAeyz+NivfqAH4Dncmnzlojo\nxpk1xQDwctY7MjNP6KyjqFwf99atlYmI+cDrwCOSfm495szqI+kvSSuAXspVmivafEo2hYi4DRiT\ntKfd52L/yWpJq4B1wMMRcX3rQY+L1ZlDud3jWUkrgd8oy/UmOLM65f3D64HXJh/rpMw8oZveMeCS\nlnZv9lk9judlcfJxLPunys6ZzqCI6KJM5l6U9EZ2O7MGyCVFu4BrKctP5uSh1vd/Ips83gP8iDOb\nKdcB6yPiCOWWgDXA0zivqkk6lo9jwJuUH048LtZrGBiWtDvbg5QJnjOr3zpgr6Tj2e7IzDyhm94n\nwLLcMWwu5bLt9jafk/3bdmB816G7gbdb+u/KnYuuAU7lZfZ3gbURcX7ubrQ2++wsy3tztgIHJD3Z\ncsiZVSoiLoqIhVk/D7iRcu/jLmBjPm1yZuNZbgR25q+e24GBKLsq9gHLgI9n5lXMHpIel9Qr6TLK\n59NOSXfivKoVEd0RsWC8ThnP9uNxsVqSRoGjEXF5dvUDX+PMmuAOziy3hE7NrB07sTStUHa+OUS5\nj2RTu89nNhfKP+UI8AflF7P7KPd/fAB8A7wPXJDPDeCZzO1L4KqWv3Mv5ab/IeCedr+uTi3Aaspy\nhi+AfVlucWb1FuBK4LPMbD/wRPYvpXzBH6IsXZmX/edmeyiPL235W5syy4PAuna/tk4vwA2c2eXS\neVVaMpvPs3w1/r3C42LdBVgBfJpj41uUHQ+dWcUF6KasQOhp6evIzCJP1MzMzMzMzBrGSy7NzMzM\nzMwayhM6MzMzMzOzhvKEzszMzMzMrKE8oTMzMzMzM2soT+jMzMzMzMwayhM6MzMzMzOzhvKEzszM\nzMzMrKE8oTMzMzMzM2uofwAaDUZa5W2+6QAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11c7156a0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.figure().set_size_inches(15, 4)\n",
    "\n",
    "x_axis  = [n * summary_frequency for n in range(0, len(losses))]\n",
    "vx_axis = [n * summary_frequency * 2 for n in range(0, len(v_losses))]\n",
    "\n",
    "plt.plot(x_axis, losses)\n",
    "plt.plot(vx_axis, v_losses)\n",
    "plt.grid()\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 4. Having fun"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let us now see the world with neural network's eyes.\n",
    "\n",
    "First recollect what the model has learned:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def evaluate(articles, session):\n",
    "  for article in articles:\n",
    "    session.run(reset_sample_state)\n",
    "    predicted = []\n",
    "    for word in article:\n",
    "      prediction = session.run(sample_prediction, { sample_input: [dictionary[word]] })\n",
    "      predicted.append(prediction[0])\n",
    "\n",
    "    print_md(represent(article, predicted))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's ask to evaluate some provoking statements:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./storage/model\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "<span style='color: rgb(0.0, 255.0, 0)'>=> 1.00</span> <span style='color: rgb(19.0, 236.0, 0)'>британск</span> <span style='color: rgb(1.0, 254.0, 0)'>совет</span> <span style='color: rgb(5.0, 250.0, 0)'>взорва</span> ... <span style='color: rgb(8.0, 247.0, 0)'>посл</span> <span style='color: rgb(12.0, 243.0, 0)'>испуг</span> <span style='color: rgb(0.0, 255.0, 0)'>вокруг</span>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "<span style='color: rgb(93.0, 162.0, 0)'>=> 0.63</span> <span style='color: rgb(19.0, 236.0, 0)'>британск</span> <span style='color: rgb(136.0, 119.0, 0)'>бомб</span> <span style='color: rgb(248.0, 7.0, 0)'>взорва</span> <span style='color: rgb(251.0, 4.0, 0)'>аэропорт</span> <span style='color: rgb(235.0, 20.0, 0)'>UNK</span> <span style='color: rgb(158.0, 97.0, 0)'>вечер</span> <span style='color: rgb(245.0, 10.0, 0)'>посл</span> <span style='color: rgb(235.0, 20.0, 0)'>испуг</span> <span style='color: rgb(93.0, 162.0, 0)'>вокруг</span>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "<span style='color: rgb(232.0, 23.0, 0)'>=> 0.09</span> <span style='color: rgb(235.0, 20.0, 0)'>бомб</span> <span style='color: rgb(254.0, 1.0, 0)'>взорва</span> <span style='color: rgb(254.0, 1.0, 0)'>аэропорт</span> ... <span style='color: rgb(254.0, 1.0, 0)'>посл</span> <span style='color: rgb(250.0, 5.0, 0)'>испуг</span> <span style='color: rgb(232.0, 23.0, 0)'>вокруг</span>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "<span style='color: rgb(253.0, 2.0, 0)'>=> 0.01</span> <span style='color: rgb(51.0, 204.0, 0)'>секс</span> <span style='color: rgb(196.0, 59.0, 0)'>бомб</span> <span style='color: rgb(253.0, 2.0, 0)'>взорва</span> <span style='color: rgb(254.0, 1.0, 0)'>аэропорт</span> ... <span style='color: rgb(241.0, 14.0, 0)'>вечер</span> <span style='color: rgb(253.0, 2.0, 0)'>посл</span> <span style='color: rgb(253.0, 2.0, 0)'>танц</span>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "<span style='color: rgb(6.0, 249.0, 0)'>=> 0.98</span> <span style='color: rgb(19.0, 236.0, 0)'>британск</span> <span style='color: rgb(3.0, 252.0, 0)'>секс</span> <span style='color: rgb(28.0, 227.0, 0)'>бомб</span> <span style='color: rgb(181.0, 74.0, 0)'>взорва</span> <span style='color: rgb(193.0, 62.0, 0)'>аэропорт</span> <span style='color: rgb(99.0, 156.0, 0)'>UNK</span> <span style='color: rgb(38.0, 217.0, 0)'>вечер</span> <span style='color: rgb(159.0, 96.0, 0)'>посл</span> <span style='color: rgb(144.0, 111.0, 0)'>испуг</span> <span style='color: rgb(6.0, 249.0, 0)'>вокруг</span>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with tf.Session(graph=graph) as session:\n",
    "  saver.restore(session, STORAGE_PATH)\n",
    "\n",
    "  articles = [['британск', 'совет', 'взорва', 'аэропорт', 'UNK', 'вечер', 'посл', 'испуг', 'вокруг'],\n",
    "              ['британск', 'бомб', 'взорва', 'аэропорт', 'UNK', 'вечер', 'посл', 'испуг', 'вокруг'],\n",
    "              ['бомб', 'взорва', 'аэропорт', 'UNK', 'вечер', 'посл', 'испуг', 'вокруг'],\n",
    "              ['секс', 'бомб', 'взорва', 'аэропорт', 'UNK', 'вечер', 'посл', 'танц'],\n",
    "              ['британск', 'секс', 'бомб', 'взорва', 'аэропорт', 'UNK', 'вечер', 'посл', 'испуг', 'вокруг']]\n",
    "\n",
    "  evaluate(articles, session)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As you can see, the results are quite impressive. The model does recognize the words by their context. Neural network here is at its best.\n",
    "\n",
    "Moreover, prepending a statement with strongly provoking word `'неизвестный'` does not move the model off to a downward spiral of misunderstanding :)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Restoring parameters from ./storage/model\n"
     ]
    },
    {
     "data": {
      "text/markdown": [
       "<span style='color: rgb(254.0, 1.0, 0)'>=> 0.00</span> <span style='color: rgb(253.0, 2.0, 0)'>неизвестн</span> <span style='color: rgb(243.0, 12.0, 0)'>секс</span> <span style='color: rgb(251.0, 4.0, 0)'>бомб</span> ... <span style='color: rgb(253.0, 2.0, 0)'>все</span> <span style='color: rgb(254.0, 1.0, 0)'>снима</span> <span style='color: rgb(254.0, 1.0, 0)'>вокруг</span>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "<span style='color: rgb(255.0, 0.0, 0)'>=> 0.00</span> <span style='color: rgb(253.0, 2.0, 0)'>неизвестн</span> <span style='color: rgb(254.0, 1.0, 0)'>бомб</span> <span style='color: rgb(247.0, 8.0, 0)'>раздел</span> ... <span style='color: rgb(255.0, 0.0, 0)'>все</span> <span style='color: rgb(255.0, 0.0, 0)'>снима</span> <span style='color: rgb(255.0, 0.0, 0)'>вокруг</span>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "<span style='color: rgb(253.0, 2.0, 0)'>=> 0.01</span> <span style='color: rgb(253.0, 2.0, 0)'>неизвестн</span> <span style='color: rgb(243.0, 12.0, 0)'>секс</span> <span style='color: rgb(251.0, 4.0, 0)'>бомб</span> <span style='color: rgb(210.0, 45.0, 0)'>раздел</span> <span style='color: rgb(247.0, 8.0, 0)'>в</span> <span style='color: rgb(253.0, 2.0, 0)'>аэропорт</span>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "<span style='color: rgb(236.0, 19.0, 0)'>=> 0.07</span> <span style='color: rgb(253.0, 2.0, 0)'>неизвестн</span> <span style='color: rgb(243.0, 12.0, 0)'>секс</span> <span style='color: rgb(251.0, 4.0, 0)'>бомб</span> ... <span style='color: rgb(233.0, 22.0, 0)'>для</span> <span style='color: rgb(243.0, 12.0, 0)'>фотосесс</span> <span style='color: rgb(236.0, 19.0, 0)'>вокруг</span>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "<span style='color: rgb(255.0, 0.0, 0)'>=> 0.00</span> <span style='color: rgb(253.0, 2.0, 0)'>неизвестн</span> <span style='color: rgb(255.0, 0.0, 0)'>сообщ</span> <span style='color: rgb(255.0, 0.0, 0)'>о</span> ... <span style='color: rgb(255.0, 0.0, 0)'>все</span> <span style='color: rgb(255.0, 0.0, 0)'>испуга</span> <span style='color: rgb(255.0, 0.0, 0)'>вокруг</span>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/markdown": [
       "<span style='color: rgb(255.0, 0.0, 0)'>=> 0.00</span> <span style='color: rgb(253.0, 2.0, 0)'>неизвестн</span> <span style='color: rgb(255.0, 0.0, 0)'>сообщ</span> <span style='color: rgb(255.0, 0.0, 0)'>о</span> ... <span style='color: rgb(255.0, 0.0, 0)'>все</span> <span style='color: rgb(255.0, 0.0, 0)'>испуга</span> <span style='color: rgb(255.0, 0.0, 0)'>вокруг</span>"
      ],
      "text/plain": [
       "<IPython.core.display.Markdown object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "with tf.Session(graph=graph) as session:\n",
    "  saver.restore(session, STORAGE_PATH)\n",
    "\n",
    "  articles = [['неизвестн', 'секс', 'бомб', 'раздел', 'в', 'аэропорт', 'UNK', '.', 'все', 'снима', 'вокруг'],\n",
    "              ['неизвестн', 'бомб', 'раздел', 'в', 'аэропорт', 'UNK', '.', 'все', 'снима', 'вокруг'],\n",
    "              ['неизвестн', 'секс', 'бомб', 'раздел', 'в', 'аэропорт'],\n",
    "              ['неизвестн', 'секс', 'бомб', 'раздел', 'полност', 'в', 'аэропорт', 'для', 'фотосесс', 'вокруг'],\n",
    "              ['неизвестн', 'сообщ', 'о', 'бомб', 'в', 'аэропорт', 'UNK', '.', 'все', 'испуга', 'вокруг'],\n",
    "              ['неизвестн', 'сообщ', 'о', 'секс', 'бомб', 'в', 'аэропорт', 'UNK', '.', 'все', 'испуга', 'вокруг']]\n",
    "\n",
    "  evaluate(articles, session)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "However, having someone notified of a sex-bomb in an airport is still considered a bad news. On the other hand - given the source of our _bad_ news - perhaps that is what we had taught our model to."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}