emotion_train.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import tensorflow as tf \n",
    "from collections import Counter\n",
    "import itertools\n",
    "\n",
    "import utils.load_data as load_data\n",
    "from speech_models.emotion_recognition import ClassifyEmotion\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder, OneHotEncoder\n",
    "from sklearn.metrics import confusion_matrix\n",
    "\n",
    "import matplotlib.pyplot as plt \n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load IEMOCAP Data \n",
    "\n",
    "The IEMOCAP data is saved into 5 sessions. Here scripted readings are mixed with improvised conversations. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished data session Session1\n",
      "Finished data session Session2\n",
      "Finished data session Session3\n",
      "Finished data session Session4\n",
      "Finished data session Session5\n"
     ]
    }
   ],
   "source": [
    "data, orig_lengths, labels, metadata = load_data.load_sentences(max_sequence_length=1000, \n",
    "                                                                win_len=.04, win_step=.02)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Counter({'ang': 1103,\n",
       "         'exc': 1041,\n",
       "         'fru': 1849,\n",
       "         'hap': 595,\n",
       "         'neu': 1708,\n",
       "         'sad': 1084})"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnt = Counter()\n",
    "for emotion in labels:\n",
    "    cnt[emotion] += 1\n",
    "emotions = list(cnt.keys())\n",
    "sizes = list(cnt.values())\n",
    "cnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([<matplotlib.patches.Wedge at 0x7fb805858e80>,\n",
       "  <matplotlib.patches.Wedge at 0x7fb805882908>,\n",
       "  <matplotlib.patches.Wedge at 0x7fb805891358>,\n",
       "  <matplotlib.patches.Wedge at 0x7fb805891dd8>,\n",
       "  <matplotlib.patches.Wedge at 0x7fb8058fa898>,\n",
       "  <matplotlib.patches.Wedge at 0x7fb8058fc358>],\n",
       " [Text(-0.49772,0.980956,'ang'),\n",
       "  Text(-1.0805,0.206211,'exc'),\n",
       "  Text(-0.829873,-0.722018,'sad'),\n",
       "  Text(-0.153554,-1.08923,'hap'),\n",
       "  Text(0.819337,-0.733953,'neu'),\n",
       "  Text(0.779141,0.776492,'fru')],\n",
       " [Text(-0.271484,0.535067,'14.9%'),\n",
       "  Text(-0.589363,0.112479,'14.1%'),\n",
       "  Text(-0.452658,-0.393828,'14.7%'),\n",
       "  Text(-0.0837568,-0.594125,'8.1%'),\n",
       "  Text(0.446911,-0.400338,'23.1%'),\n",
       "  Text(0.424986,0.423541,'25.1%')])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWQAAADuCAYAAAAOR30qAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd8G+X9B/DPc6fTsoZHPJM4znCckJ0QEhIIJGwwAQr8DIWyodAwCoGWUkppoS2ldYFCIJQWaMsyqyWYslcgAbIniTO995Rkbd3z++PkxNkeku9O/r5fL79sSyfdV4n98aPnnsE45yCEEKI+Qe0CCCGEKCiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIyiQCSFEIwxqFzBYMcb+C2A4ADOAJzjnf2OMeQA8AaAQgA/ABZzzBsbYaAAvA0gC8A6An3LObSqVPiDy7n3PACANwJDoZweUn1cx+tH9a/HGiufDZjkgAJABRA76cANoB9DW9XlxSWl4QF8QIT3AOOdq1zAoMcZSOeetjDELgNUATgHQDGAh5/xdxtijAFyc84cZY6UAXuacv8oYuxnAn/UWyHn3vpcGIAv7A3bI0b7mnDsYY6ynz79ozzNNAkN6L0rqxIEhfUBgA6gGsDf6Ubm4pDTUi+cmpE8okFXCGHsQwEXRb/MAnAXgSwBmzjlnjBUBOINzfgNjrAVAJuc8zBhzAKjVYiBHW7WjAIzr9lHAOS9gjKXF89w37VziMxkES5yePoIDA7r7xx4A9YtLSukXifQbdVmogDF2KoDTAZzIOfcyxr6A0nUR4vv/Qkag0f+fvHvfcwIYD6AA0eDlnI8DMJoxJh18fC8aun3GGOQ4Pr0IYET049TD3O8vLircC2ATgHUA1gJYt7iktC2ONZEEpMlf+EHACaAtGsbjAMw+xvHfArgYQAmAy+JdXHd5977nADAXwMmc89kAJjDGMg4+biBCV8PMUP5AjQdQ1HVjcVFhOaLhHP28dnFJabMaBRJ9oEBWxwcAbmaMbQNQBiVwj+anAF5ijP0y+tiOeBWWd+97WQBO5lyeB1meD0EczxgTgEEfun2RF/24uOuG4qLCKuwP6FUAvl5cUtqpRnFEe6gPWQcYY1YAvmjf8mUALuecXxCL5867970xAE7mcuQUAKcyQRwRi+cdaD/etaTTKApJatfRByEowfxZ9OObxSWlAXVLImqhQNYBxtjJAJ4CwKCMAriOc76rL8+Vd+97EwGcyuXIqQDmMUHszcgEzdJxIB/MB2AFgA8BvL+4pHSryvWQAUSBnODy7n1P4LI8l0dCRUwQL2CiYZjaNcVDAgXywaoQDWcAnywuKXWpXA+JIwrkBJR373tmLkfO5OHgD5konc1Eg1PtmuItgQO5uyCUawivAli2uKTUq3I9JMYokBNE3r3vSTwcOptHQtczyXQmE8R4jcnVpEESyN11AlgGJZw/oIkriYECWcfy7n1P4OHgfDkUuEkwWs5lokFzk0UGyiAM5O5aAbwFJZy/XFxSGs8x2SSOKJB1KPeuN8fwSOheQbL8gBmkFLXr0YJBHsjd1QJ4HcCri0tKV6ldDOkdCmSdsObPYsnzri4Src67BKvz+N6s8zAYUCAf1lYoi1W9tLik1Kd2MeTYKJA1Lvvqx2yC0fozISn5JtFsy1S7Hq2iQD6qFgB/A7BkcUlpjdrFkCOjQNaonGufLGAm64MGW+qFzGA0q12P1lEg90gYwJsAnlhcUnqs2aFEBRTIGpNz/dMXCmbbL0RbykzGBOqW6CEK5F77Dkp3xpsDOUKDMXY7gFsArOOcXzFQ59ULCmQNyL76MYtgtNwpWJN/IlrsQ9WuR48okPusBsDTAJ5eXFLaHu+TMca2Azidc17d7TYD55w2DAAFsqqcsy+xWcfNe0BKG/oTQTJTmPQDBXK/tQF4BMCT8boAyBhbCuA6KAtq5UIZRz0KQCWU2YjHc85vjR5bCmUjhi/iUYtW0Z56KrDmz7JnXPyr3zlOuLjSlDX6HgpjogEpAP4IYGdxUeGNxUWFYqxPwDm/GcqwvPkAHgNwHJTW8uWxPpdeUSAPIGv+LOOQwrt+nHr6zdus+bPvE60OGkNMtGYolBEZW4uLCi8+1sH9tIxzTsPxuqH1kAeANX+WYB45/ZzkeVc/Kg3JPY6GEBMdKADwZnFR4SoA9y4uKf08Dufovg50GAc2EAflyCIK5Diy5s9ixqz8ac4Ti54wZo05kQlizN8GEhJnJwD4rLio8CMowbw+TucpB/CT6GYIQ6PnHXQokOPEPqNwuGPmhY8ZswsKBclkUrseQvrpTABnFBcV/gvA4sUlpS0xfv4VUDaN/R7ANii7qgw6NMoixqz5s5y2yWf8yjR80o2i2eZQu57BgkZZDKhmKKH8L7ULSTQUyDFizZ9lFh3p59qnnPUHY8aosWrXM9hQIKviEwA3Ly4p3a12IYmCAjkGrPmzxphzJ//WNvmM8wVT0qBdAlNNFMiq8QH4FYDHaNnP/qNhb/1gzZ8lJR13yiWOmRe9bT/+gssojMkgZAHwZwDLi4sK89UuRu+ohdxH1vxZOcbM0ffZp517uZiUkqp2PYMdtZA1wQvgPgB/XVxSSsHSB9RC7iVr/izBOnb2fNvkM99yzrnsZgpjQvaxAngcwCfFRYUJsZv5QKNA7gVr/qxUQ0rOAykLbnzZmj97No0rJuSwFgBYW1xUeLzahegNBXIPWPNnMWv+rOnWsXNeS5l31b1Scla22jURonHDAXxVXFR4rdqF6An1IR+DNX9WEkTDD50nXHy3KaeAhrNpFPUha9ozAO6gnbGPjVrIR2HNnzVSMNseSTn1ul9TGBPSZ7cA+Ly4qDBL7UK0jgL5CKz5s2YYnJmPpCy48YdSchYtGk9I/8yF0q98otqFaBkF8kGi/cXnGrMLHko+5ZpC0WKnURSExEYOgC+KiwpvUrsQraJA7saaP0sCcM1NY9vueXbajvFZrJVmHhESW0YAzxYXFT6gdiFaRBf1ohYWSLat8ojfXT0hMuv+Ka0nCAwsLCPysXtUzSPCdY4KaVSy2jXGWvP/Hodv92qIVidyrn/6gPtcq95G2+fPY9htL0O0Og95bNsXL8C3ezUAwDnnMiSNnwcAaHr3Twg1VcAyeiZSTrkaANC+8jUYh4yAdWz83q3SRT1d+t3iktL71S5CS6iFDGBhgZQM4OcPTW047/4prbMEBgYABgHiOc49uZ/b7nf+K3RP1fjg1lgvOagq26TTkXHpbw65Pexqgm/veoiOw4/t9+5ejWD9bmRf+ySyfvQXuFb9B3LAi2DjXggGE3KuewrBup2QA50Ie1oRrC2LaxgT3fplcVHho2oXoSWDPpAXFkgZAO67e46x8IcThNHCYTbzEBjYPHvN8Pcdv0t7K3x7zczAdw0DXmgcmIdPhGixH3J726fPIWX+tQAOv7NJqLkSpuETwAQRgtEMKT0Pvj1rwQQD5HAAnMvgchhgAjq+egnOk2i3d3JE9xQXFT6udhFaMagDeWGBNBzA/T+fazxj3gjD1J48Zoateegbzicy35dvaVjg/6w2ziUOOO/ObyHa02DMGHXEY4wZI+Hfuw5yyI+ItwOByk2IuJsgDRkO0eJE3Yt3wDrmBITb6sA5hylrzAC+AqJDdxQXFS4pLioc9HubDdodQxYWSCMB/OymGdJxc3N7Fsbdjbd2ZD5v/TsqfK80/yVwgf8d07lDwURd/0DJIT86vnkdmUUPHfU4y8jpCNbtRP1L90C0OGEcOg5gyizy1NP3X0BvfPM3SD3rVnSsLEGwcS/MeVNhn3p2XF8D0a2fADAWFxX+eDAv4zkoW8gLC6QsAHdfNM4w/Nx8w8n9ea4RFu+QJ5JfHfatcEP7lf5XK0Ue1O0PU7i9HuGOBtQ+fxuqn7kOEXcz6l78KSKetkOOdc4pQs61TyLzsocBDkipOQfc7935LYxZY8BDfoTa65B+4b3wlq2AHPIP1Msh+nMDgOeLiwoHZS4BgzCQFxZITgB3nZQrZv5oinSWEKMtoLNMgZSHk9/NXS3e0HmL/x8VJtkXjsXzDiRjeh6G3/Yyht3yPIbd8jxE+xBkX/M4RFvKAcdxOYKIzwUACDbuRahpL8wjp++/PxKGa807cMy6GDwcwL6+aC4DEd39s5CBdTWAYrWLUMugCuSFBZIFwO3jhwgj7phlPNcgsJh32aQaw/afJ386Yo3xpuDP/E9W2CIdgVifI1aalj2K+n/fjVBrDaqXXA33xo+OeGygbida3v+r8o0cQcPLP0ft329BywdPYUjh3WDC/oXv3Oveg23iaRAkM6T0keDhAGr/sQjGrDEQzLSGPzmmnw7WRYkGzTjkhQWSAcBPsm3sxD+faT7bbmIDMq7YH2GBN9yT6x6TrstoFdOtA3HOwYjGISecIIBTF5eUfqN2IQNpULSQFxZIDMDlNiNm/e400ykDFcYAYBa56UfJG/O+tdxhfDzwYEVOqMo1UOcmRMeMAN4uLiocpnYhA2lQtJAXFkjnGQRc9vjZ5hNzncJoNWuJyJA/84yoeQTX2nYbx6Yc+xGkJ/TSQm73+vDqdxvgDgTBAMwelYuTx47Eh1t24Lu9lbCZTACAcyYVYHx2xiGPL1m1Ed/XNcJmMuKes0/Zd3vpxm0oq29CTrIDl89SBg2trahGZyCEeWNHDshri5O1AE5eXFLqU7uQgZDww94WFkgnAij69Smm49QOYwAQBQhnOCqGn8Yf5N95sqr/IF9l3mSaOkTtusjAEBjD+VOPw7AUJ/yhMB7/+GvkZyr//fPyR+LUcUf/ET1+5DDMzc/Dq99t2HebLxhCTbsLi8+ah9dXb0JduwtDbElYvbcaN847Ia6vZwDMAPA8gMvVLmQgJHSXxcICaTyAmy4aZ0iakiVOP+YDBpDAwE601w9b5nx0yLLIrXVzAisSYvYfOTqHxYxhKcraIGbJgEyHDS5fz4cCjk5Pg9UoHXAbYwwRWQbnHKFIBKIg4IuyPTgpPw+ikBC/4pcVFxX+Qu0iBkJC/G8dzsICKRXAbdk25r1sonSW2vUczeSk1uxXnEsyP5Zvajzb/0ENkPjdSARo7fSipr0DuWnKJY0VuypQ/OFylKzaCG+w55trmCUDxmdn4LGPv4bdbIJZMqCytR0ThybUevAPFxcVnqN2EfGWkH3ICwskAcDtAI577CzzSaNThfFq19QbVX5LyxP+87xvmS4YxnU++2+g6KUPuUsgFMbTX3yD08ePwaRh2XD7A0gyGgEGfLilDC5fAEUnTDnsY1s7vfjHV6sP6EPu7vXVmzBnzAjUtHWgrL4ZOcl2nH5cfjxfzkCpAzBhcUnpoTOVEkSitpDnAph21RRpiN7CGACGm31pf05+c/gq4TrXdf5/VxrkYETtmkjsRGQZ/1y5FtNzh2LSMGW/XLvZBEFgEBjDrFG5qGxt79Nz17R1AOBItydhY1UdrpozHc0eL5rcnTF8BarJBpDQCxElXCAvLJAyAVw1Mpl5FhYYzlW7nv5IN4WcDyS/n7tWut53h29phVn20CaROsc5x+urNyHTYcMpBfsXcOrej7yluh7ZzkNX4euJD7bswFkTCyDLHF3vfgUGhCIJ8zf9quKiwvPULiJeEqrLYmGBJAL4GQNyl5xnPnOYQzjykmU61BkW/K94ZtQ/Zbgms8OQYlG7Hi3RS5fF3qZWLPn8G2Q77eiatX/OpAKsr6xFbbsLDEBKkgWXzJgEh8WMDp8fb6zehBuioyVe+mY9dje1oDMQhN1swpkT8jFrVC4AYEtNPWraXDhrorIf77sbvkdZQzOynXZcMXuaKq83TmoAHLe4pDThxvQnWiCfCeCKa6dKzovGSxeoXU+8BCIs+B/3hLpiw3VDmgxZmg+hgaCXQCYx8/jiktI71S4i1hKmy2JhgTQUQJHThMazxhhOU7ueeDKJ3HhZ8pYR31jvMi8J/KpyeGhvh9o1ETLAbi0uKpykdhGxlhCB3DU1GkDwJzONs6wSGxQr2BgEiOc5d+d+aful48XQz6oKgt8n1BZThByFAcDTxzxKZxIikAFMAjCpIE0IzBwqDrrN2wQGdqq9evgH9ofT3gjfUTMjsKZR7ZoIGQAnFRcVJtQMPt0H8sICyQjgSgCtN86QTovHkpp6wRgw09Y09C3nXzL+F7ml/lT/F3Vq10RInN2XSFs/6T6QAZwMIH1qliDlpwoT1S5GK45L6sh6Mflv2Z/zG5vO95fWcK7bjUwIOZqJAHQ9vLU7XQfywgLJCuASAA2XTZROitHmHwllpKUz/cnkV4Z+w65vu9xfUinwECUzSTQ/U7uAWNF1IAOYA8A0Nk0wjRsiJNwV11jKMQdS/pD8Tu4a8XrPj30vVBh1uMUUIUcwr7iocJbaRcSCbgN5YYFkAnABgKYrJklzBMZ0+1oGUqox7PhFyscj1ko3Be/2L6lIirg0u8UUIb3wc7ULiAU9h9gJAGxpFiZPyhQ0tbSmHtiliPXW5BUj1phvwW/8fypPjTR71a6JkH64oLiocKzaRfSXLgM5uj/ehQCaLznOMGkwj6zoL4vITVcnr8/71nKHVBz4bUV2qMatdk2E9IEA4B61i+gvXQYygHEAUgF0njBUTKhJ+moxCly62Ll9xNdJ9yQ9G7yvcmRoV8IucUgS1o+KiwoP3fdKR2ISyIyxKxljqxhjGxhjzzLGRjDGdjLGhjDGBMbYV4yxM6PHXsUY28QY28gY+3cfT3kqAN/MHCEjPUnIicVrIApRgHCWozz3U9sDyS+HFldPDG5qVrsmQnrIBOWds271O5AZY+MBFAGYyzmfCiAC4BQAfwTwDIDFAL7nnH/EGJsA4H4ACzjnUwDc0dvzLSyQkgFMB9B8br5EreM4ERjYXHvdsFLHI0P+E76tdnZgJW0xRfTgfLUL6I9YtJBPg7IR4WrG2Ibo96M4538H4ABwM4C7o8cuAPAG57wZADjnrX0433Qok9Lk8enChH5XT45pmq0l5zXnU5kfyjc3nOH/uEbtegg5itOKiwp1uzRtLAKZAfgn53xq9KOAc/4gY8wKYFj0mJgs9hNdROh0AK0n5Yo5Von1bRVv0icFVlfmc8kvDF3Or2/5ge8/VeCRxFm7lSQKC5SM0KVYBPKnAC5hjGUAAGMslTE2AkqXxcsAHgDwXPTYzwBcyhhL6zq2l+fKBJAFwD1nuFgQg9pJH+RafGl/SXlj+Crh+o6rfS/TFlNEaxaqXUBf9TuQOeffQ+kX/ogxtgnAxwDyAMwE8EfO+csAgoyxaznnWwH8DsCXjLGNAP7Sy9Pt2x9vbJowpr+1k/7JMAWTf5PyXu5q6Qbfrf6/lZtkL20xRbSgUK8LDsVk/C7nvARAyUE3z+52/w+6ff1PAP/s46lmAXBnJjFLmpVl9/E5SIylSGHb3clf2G4OL/e97Dm+Zonh6iyXIcWsdl1k0MoCcDyA1WoX0lu6GYccXUgoH0DHrGHiUEGjKwld944PGX9yY+LTnkPuK14ZAPuNC83ew6/vc/ZLnUh+xIXCVw6cNHfF215MfsaD+z7dvxHmw8sD+O92bTVIbQbZ8uPkVXmrLLcKvw/8oWJIuCEhtjomuqTL0Ra6CWQAI6Kf5bFp2h17fM1UCR9caT3k9qoOGR/tCSPXeeS/I/fMMeHfFx14gXhTQwQWA8OmW2xYXRtBh5+jzi3ju5oILhwnxbz+WDCL3PhD5+YR31jvNP018OuKYaGKhNuMkmjeSWoX0Bd6CuRcKCM6kOsUhqpcyxHNG2FAquXQ0L3zQz8ePd2MozXrTxtlgN104BGSAPjCHDLnCEUAUQAe+DyA35xqinHlsScJMCx07hzxpe0XtueDP68aG9zel2GOhPSFLte10FMgjwfgAYCMJKbZFvLhvLM9hKF2AVOyxF4/dny6iHSrgOnPduL8sQbsapUhc2B6du+fSy0ig7DAUTX8Q/tvU0vCP62ZFlxLW0yReMspLirU3S7kuliUJzr+OB9Ae66T2fS0iak3xPH7rwP46Mq+/2w8fvb+62Pnv+rFs4Vm/G55ABsbIjhjlAE3zjDGotS4YwyYZWsc+h8UY0tncv2j4cv4ctM8ujhL4oEBGANgo9qF9IZeWsipUAZ8h0alCMlqF9Mbu1tl7G3jmLLUg7zH3ah2cUx/thP1nt5v3PHO9hBmZAvwBDl2t8l4/VIr3twWgjekv/kZE5Pas/7lXJr9Gb+x6Tz//2iLKRIPuuu20EsgZwCQAWCoXV+BPClTROM9dpT/VPkY5mBY9+MkZNl6908finA8/l0QP5trgi+EfX3RERnQ87SMUZbO9CXJLw39ht3QepnvDdpiisRSvtoF9JZeAtmJaK3pScypci1HdflbXpz4j06UtcgY9hc3/rEueMRj19RGcMMy377vT36hE5e+4cOne8MY9hc3Pty1f5elJauDuHqKBKvEMDlTgDfMMekZD2Zki0g2a3IEYK/kmP2pj6T8J3eVeIPnRv+LFZIc0PGfGaIRumshM861/3Z3YYF0PpRl9ap+O9907tQscabaNZH4coXEzhc7T2xeKv0o2yvaj9lJ/uNdSzqNoqC7izgkrlYuLimdq3YRvaGXFnIWAD8A2I36uaBH+s4hRZJuT/56xBrTLfID/uLy5HCL79iPIuQAo9UuoLf0EsiZiAayJEKbsyFIXFgNsvm65LV531luN/wp8HBFVpi2mCI9prt3THoJZAeAMAAYBArkwcgkculS5/cjvrbeY30m+MvKEaHdHWrXRDRPd1mhl0CWoOxEAoOgj7HTJD4MAsRzHHtzP7P9yvGs+07fpMDGFrVrIpqlu0DWS7gZAXQCFMhEITKw+WkNliG82P8vf26ZNzzBEIjwfT8bjHEceL2aA+g+c53zru/ZviOUB7Buj2AAAzjf/3XXzYe7GH7AbdFj+VGGwPBuxx1Q2wF1HPicnB/+rp7fxw65r/v5efd7GDg42P6HsAMed7TzHVZ0RbDD/Tt1e4xy3n23sCMdpzzXwecSGCAbDQYXAFZcVCgsLinVzVBKvYSbEdFxyDoYFEJiLMw5L+MG12pm7FwHybRbMlobbEbmt4smZmApV77q9YopC0wWZ94QtWslmtE1PFZXiaH5QI5Om5YQDeRgBAF1KyLxInOOnTC41zDJvckgBXeYTGKdRbJ0Wg3JEJkT+3/JAChNpZzdkcA5e1nOh2JFJ5x5qtRNNCuwaOkCCuQ4CCP6liUYwZFnWhDd2C0L7rVMcm+QjKEyo5HVWSSLx2pI5gbBDqBneyXKMu76L9wSY0MyfS2t7hjt3UgShu4ab5oP5GVlIb6wQPJDqTUUjHDd/SMPZpWy0LmGSe4NBmNgu0lCjdlocVsNyVzqRfAewbnLZFduEEMAIN1TbqbxcOQg/mMfoi2aD+QoHwARQChAXRaaVMeZdzUk1waDMbDNaES1RTK7LFKybBSSEIfxoKlNkc7Lv+dJXZeJstxVKbs5h0Y3kiHqqFW7gN7SUyBbAKDdz6khpKJGmfnXQOpYbzAGthklXmUxmtotBqdsEq0ADt0qJU7ueiXUbmKGfRsVmHlQCoU6O4xGm6bXOiEDaofaBfSWXgK5E9G3tw0e3qZyLYNCK0dgLZc61hmMvm1GIyrMRmOb1eCMKMGr6gamp3wZrB7rNQw75A5vkwcUyGS/nWoX0Ft6CeQGAHkAOqpcMm0DFEMdMkLrIbWvNRh9WyWjXGGRjK0WyRk2i0lQlj3VFKs7ErhuBVION9LV7KkJyckjB74oolXUQo6TGkRbZbtaZWoh94GHI7wehvZ1gtG3xWiMlJuNUovF4AiZRTsYS1e7vp669ZVQg4UZcg93X4q7QmzR596WJD6ohRwnzYiOQ67s4J5QhAclkelj36IB5pd5ZAOT2tcJknezZIrsMUtSs0WyBy2iHYzpeuLE5A2h+hktYu6R5oFluMttNI+adEMt5DhpQ7cZN01eXptjZ3mqVaMBfpnLWyC1rxUl72aDMbzHbDQ0WiRbwCI6IbA0AGlq1xhLhoAcue0DbmbsyMuvpHvrnVvlSEQQRP3sAEvipXXR0gW6+/usl0BugTLsDQBQ2SFX59iFPPXKGTghzvlWLnWsFSTPZkkK7zKbDA0WQ5LfYnBCYKlQ9htMeNe/HqxyckPe0Y4xQBZCAVeLyZKSUH+MSJ/ornUM6CSQl5WF3AsLpGYoQ99825vlqtmHXmPXta71GtYKkmejZAzvNBmFBouU5LUYnBBZMgBd7SUYSyN3hVvnVx25q6I7obPBCwpkAnypdgF9oYtAjvoewAkAfN9WR6qvmap2OX1z6HoNRqHOYkzqtIhOGIRD1msY7FhE5ne9HYkITOzRUrFJnqpIcMi4eJdFtO9/ahfQF3oK5G0ATgaAWjf3tvt5c7JZ2xeputZr2CgZg2Umo1Br7sN6DYPcpctClZkRcURPj09zlRvr4lkQ0YN2ACvVLqIv9BTI1eh2YW9HS6TshKEGTQRyPNdrGMwyasOuC7YJ2T3pquiS6am0UyAPeh8tWrogfOzDtEdPgVwHZfUmI4DgispI2QlDDQO6o+xAr9cw2C1+LeKRmOjozWNSg632cCQUMIiSKV51Ec3TZXcFoKNAXlYWiiwskFYDmA2g7suKSPXNx/NOi8RiHoRaWa9hMDvjk2DVyIA4vC+Pjfha2w22zMxY10R0gQN4X+0i+ko3gRy1BtF+ZJmD726TyyZmiNP7+mRaXq9hMLO3R3w/WoW03nRVdCd11vlhozwepNYuWrqgUe0i+kpvgbwTymanIoDI15WRrT0J5K71GtYp6zXwcoskaXm9hsHu9ldCTeYjTI/uCbu7Ep2ZOh2GQ/rrHbUL6A9dBfKyspB/YYG0CcA4AE3v7wzvuXKy1GEzMicAdHIeXs+l9rWivtdrGMxmrgrWTenoexgDwBB3ubkzVgURPQkC+LvCLFypAAAc70lEQVTaRfSHrgI5agWAaYDSWfTbZuM3G8clT0qU9RoGM6NPDt/8af/76LM9lc6KWBRE9KZk0dIF9WoX0R96DOStQUB2C8IUH2P2nVsCKeKJ5qFMoJ0i9O6m14I1dhh6POb4SKwRnzkU8nkkyUJ77A0uj6tdQH/1aPaTliwrCwV2GaUtO41SdpVk8DZ28s3BhqAu562T/fK3h5tPqhP71VXRHfc2u2L1XEQXvlq0dME6tYvoL90FMgA0GgwlfkHYFhCE9SFBKHetc+ly3jpRCGGZ3/lOBEIMN8QzeWpod/LBRfetY0Cngexa76qDMpU6DQC8O721weag7hajJoofvh2qGCKLMe37d7ordPmzTfpkL4D/ql1ELOj5h3YZuk1L7ljV8YV6pZC+yq4Od5y7S4j52n0ZngqaOTl4PLVo6QJZ7SJiQc+BXAZlzdMhAODdQa1kPVpcEvEaGIv5xeVMT00y53JC/JKSo6oF8De1i4gV3Qaya72LA3gLwL4r6dRK1pfzPghU5gbF7Hg8t4SwGAp4OuLx3ERT7lm0dIFH7SJiRbeBHFUGZfae0pe8w1sbqA1sUrck0hPOloj3snVCXGdJMm8DzQ9JbF8tWrrgFbWLiCU9jkPex7XexR3THG8DuBdAKwDe8mnLx9mXZxcwA9P9al/V/6iGe4MbBocB+b/LP+C+5vebUV9Sj3FPjoPBfuB/o2ebB/Wv7B8fH6gLYPgtw+GY4UDV0ir4q/2wT7Uj65IsAEDjskaYh5rhmNGrhdX65c5XQq0mZojrvi8WT3U4nJp/7ANV0uZpxL8+fwRubxvAGOaOPw/zJ12M0tUvYFP5CjAmwG5JxpWn/gzJSYde81zy3r0ob/weo7Im4pZzfr/v9hc//T1qW/dgYu5sLJx1AwDgg3UvITslD1NGJsau3JzzCGPsVrXriDW9t5ABYDuA1QCyACDUEvJ4tnu+ULWiGEk5KQV5i/MOuT3YEoRnqwdSmnTYx9nG2zDmoTEY89AY5P08D4JJgG2iDf4qPwSjgPyH8+Hb60PEG0GoPQTfbt+AhvGcFcGa4zzxDWMASHVXHv4fSCMEJuIHs2/G/UUv4O4Ln8Lyre+grq0cp035P9x36d/xi0v+hom5s/H+2n8f9vGnT/k/XDX/3gNuq2nZDclgxH2X/h0VTWXwBTzo6GxBecO2hAljAGCMLV20dEHCvRvWfSBH+5JLoCw4ZASA1s9bV0U6I7pd8alLUkESxKRDN1Cuf7Uemf/Xs9XMXGtcsE2yQTAJgAjIQRlc5uBhDghA49uNyLho4NZXMnsiwRuXY0DSP9NdrumZes6kNAxPHwsAMButyEoegfbOZliM+weIBMJ+HGl4dsGw6TAZD5xpLgoGhMJByFxGRI5AEES8t+ZFnHf81fF7IQOMc94M4Fdq1xEPug9kAHCtdzUB+A8A5QJRBHLb122lnHN+1AfqkGudC1KKBEuupUfHd3zXAedsZZs+c44ZBrsBu3+9G46pDgQbguCcw5LXs+eKhZ+8FqpLgjAgu6ik+RqdETkcGohz9VeLux7VLbuQlzEeALBs1T9w/0uXYc3OT3He8df0+HmyUkbAZnbij2/djEkjZqOpowacy/uCPxEwxu5btHRBm9p1xIOu+5AP8gmA+VA2Ce3o3NZZZR1jXWEdbU2Y92lyQEZTaRPy7s7r0fGh9pDSXzxxf/5lX7F/UEPFYxXIuSYHjcsa4a/ywzbBhtRTU2Nd9j7HbQk1zmrs2e7RsSAwIOxvbxetQzS90l8g5MPfP3oQF5/4k32t44UnXI+FJ1yPD9e/guVb/ovzZl7T4+e7ZO6ifV8vff+XuGzenfhg3cuoadmNccNmYO7482L9EgYM53wFY+wfatcRLwnRQgYA13pXAMALAFKhdF+g+f3mz8PucMJssRZsDCLYFMSuX+1C2eIyhNpC2P3r3Qi1H74R2LGqA47pDjDDoQnoWueCOc8MOSAj2BRE7qJcuNa4IAfiM3RXDMryHaWygcVwenSPzttZ7xvI8/VWJBLGcx89iOPzT8PUUScfcv/MMadhw96v+vTcm8pXYHj6WARCPjS7anH9GQ9g/Z7lCIb8/S1bFZzzFsbYZYkyCeRwEiaQAcC13rUVwIcAhgIAD3O55eOWt3mE63LDw4OZh5sx/snxKCguQEFxAaQUCaN/MxpS8uGvXXV8u7+7ojse5mj5qAXp56ZDDu7/2d7XtxwHV70ZrEzhYvya30dgd1dqttuKc46Xv/wzspJzcdrkS/fd3thRve/rTRUrkZnc+52sIpEwPt/8Fs6YUoRQJLjvTQnnMsKy/n4dOOecMXbloqULqo99tH4lUpdFl7cBTIIyNrnFX+lv9mz1fGSfbD9X5bp6reqZKnRu70TYE8b2O7cj48IMpJ5y+Ezz7fWh9fNWDL1uKAAg2BREqDWEpIJDZxC3fNqC5LnJEEwCzMPN4EGOnffvhH2y/bAXEftreHm4/cxycfhAdVV0l+auMGr1N3hP/Ras2vkxclJH4g9v3gRA6apYuf19NLZXgTGGVFsmLpv3UwBARVMZvv7+XVxxyt0AgMfeuQMN7VUIhHy4/6Ui/PCUu3Hc8JkAgOVb38GssWfCKJkxNHUUguEAfvfGDZgw/ARYTZq+1nlYHPyRW5ee9oHadcQbS8DrXnBMc4wA8CCUaZUhAMi6LOtiU5Zpopp1DUqyjMeLQw05YVGVTe5cBrt3zUmP0Ma0OibLka8EQZy/aOmCiNq1xFtCdVl0ca13VQB4HcAwQGmXNf6n8Z2wO6zr3QT06KL3QpVqhTEAOMJuazgc8Kp1ftI/siw3C4L4f4MhjIEEDeSoj6DsUj0MAOSAHG4qbXpNDsr0yzlAUpsinZdsFlTf/jnia6E1LXSIcy4LglCk922ZeiNhA9m13hUB8DyARgDpABBsCHa0fdn2Opd5wl6l1ZK7Xgm1S0z9KexGTy0tVq9L/P5FSxd8pnYVAylhAxkAXOtdnQCegDIMzgYAnq2eCvcGd6mqhQ0Cp3wZrB7rNQxVuw4AcLoracNFnQlHQn+99dnT/6B2HQMtoQMZAFzrXfUAnoLSSpYAoG1523rPNs8nqhaWwKyuSOC6FUhRu44uQzzlZrVrID3nD3pfu+O5s+5Quw41JHwgA4BrvWsLgJcADEd00kjLhy0rvHu8K1QtLEHd+mqowcIEzezYkeWuSknE0USJyBfsfN9stF6hdh1qGRSBHPUJlPUuRiD6upuWNX3ir/brfqdaLZm8IVQ/oyV2u0fHgpkHpVCwky7saZw/6P3aYky6IJFn4h3LoAnk6Kpw/4Uyk28EosPhGt5qKA3UB7aoWVuiMATkyG0fcPMAz47uGW9TwuwqkYj8Ie96s9F65qKlC3SxGFS8DJpABvaF8msAVkAJZYCD179e/za1lPvv+pJglZMLyUc75p+trTh/7x4s3LsHd9fWIHDQtndrvF5cXL4Xk8q240O3a9/te4MBXFK+Fxfu3YsNPmV5ijDnuK6qEr4ebJ1n9lQP6l90LQuEfGVmyXrKoqULNL3uyEAYVIEM7BsO9wKA9QDyADDI4A1vNrzr3eNdqWpxOjZyV7h1fvXRuyoaQiG81N6GN0bkYdnIUYgA+F+30AWAbMmA32dl4zzHgUsmv97ejl9kZGLpsGF4obUFAPBaexvOdzhgEY79Y5zirkzEZQJ0zxfs3M6YMGfR0gVutWvRgkEXyADgWu8KAXgGwLdQQrmrT/ljzzbPpyqWpkssIvO73o5EBMaO+fMU4Rx+zhHmHH5ZRobhwIWRhkpGFJjNh/xgGhiDP/pYA2NwRSL4wuPBBY5DF086nAxPuWYuMhJFh7flu4gcnnbn389pVbsWrRi0rQbXelfQMc3xHIBOAGcAqAAQafmw5WvZL/vsU+3nDfRSkXp16TuhysyIOOJYx2VKEq5NTcVpu3fBLAiYY03C3KSe5eTlySn4RV0tgpzjwcwsPNPSjJvS0iD08L8o3Vvv3CpHIoIgxn71JNJrje3V71U277jghU8eHhRTontqULaQu0S7L14G8A6UPmVlnPKXbWs7vut4i2b0HVtGbdh1wXYh+9hHAh2RCD7zePDxqNH4YvQY+LiMZR09G/yQI0n4Z+4IvDoiD2ZBQEM4jFFGE35eV4u7amtQHjz6ZDwDZCEU6Gjv0clI3HDOeU3Lnqfe/nbp+RTGhxrUgQwArvUuGcqSnS9DGadsAoCObzu2ti1ve4VHOF0MOorFr0U8EmPGnhz7jbcTQyUJqQYDJMZwhs2ODf7eX8d5orkJtw9Jx0ttbbjEmYzF6RlY0tx8zMeJnQ2dvT4ZiZmIHI5UNu1Y/Ps3brhtc/lKGhh+GIM+kAFl9IVrvetDAM8ByAFgAQD3Bvfuxncbn4t4Iy2qFqhRZ3wSrBoZEHN6eny2QcJGnw8+WQbnHN96OzHK2KMs32e114sMgwF5RiP8XAaD8kPs78GbmSRPNYWASoLhgK+icfulj759y2Nq16JlCbkecn84pjlmAFgEwAWgHQAEq2DMuDDjQlOGabyqxWmIvT3iW/J0hJuZ0Ku1hp9sbsIHbjdEAOPNZjyUmYVnW1swwWzGApsdm30+3F5bA1ckAiNjGGIw4N2RowAoO2zcUF2F4pyhSBZF7A4E8LO6WkQ48EBmJqZbj17KttSptXWTb+zxHxASG96Au6W6efe5T7x71yq1a9E6CuTDcExzjAJwG4AkKIvcAwDSzkibkzQ+6TQmHHs0QaL75dP+yikdBk3NyDuWVmOqe8OchwZkx2uiaGivWr+7fsu5L3/xp0GzhGZ/DPpgORzXetceAL8GUAZgJKKjUVo+blnZ+mnrv+SgPKhnfc1cFazTWxgDQGqw1R6OhAJq1zEYRORw+PvKVf/6z7fPnkhh3HMUyEfgWu9yAXgcyvoXw6G0luHZ6qmoe6XumWBTcIea9anF6JPDN38K3Y7pjfhaaaRFnLl97W3flH140/Lvl127uXwl/QHshUHfZcEYywNQyjk/4n57jmmOyQBugbL+xb6/9qnzU2faJtrOZCIbNOO5b33BXzGv3nDMMcda9cn4GyuEzKm6rV/LOOeoaCpbu3bX51d8tumNMrXr0SNqIfeAa71rE4BfASiH0oUhAUDr562rG99t/FvYE25UsbwBM3Z7uPmkOm2t5NZbdnel2iUkJH/Q6/1ux0d//WDdS6dSGPddwrSQGWNJ2L+xqQjgIQAFAM6HMoxtJYAfc845Y2wGlO2dAGXvvXOO1kLu4pjmEAGcBeBSKKMw2gCAGZlhyNlDTrXkWU5M1At+QljmTxWHWobI4hC1a+mP3c6Choppt6u+z18iqW3du2vNrs9urW3d+xGNL+6fRArkiwGczTm/Mfq9E4DIOW+Nfv9vAK9zzt9ljG0CcCvnfDlj7E/oYSB3iY7C+AmAFAA1AGQAMA83p6WemnqOlCaNjumL04ArXw+UL9wt5qldR395RYv/25P/TDuIxIDH39GxfveXb26tWnXf5vKVfX6XGO02fB/A1wDmQPmdugDKnIAlUHb78QK4kXO+nTH2IpRuxjejj/dwzm39eS1akUituc0AzmCM/ZExdjLnvAPAfMbYd4yxzQAWAJjAGEsGkMw5Xx593L97e6LoKIwHAHwFIBfAEADwV/lbav9d+1LbiraSiD+SMBePsqvDHefuEoapXUcsWCM+cyjkpZXF+iESCYe3VHy7+vWvn7xqa9Wqm/sTxt3kA1jCOZ8AZfz/xQD+BuA2zvkMAHcDeDoG59G0hLkYxTnfwRibDuBcAA8zxj6FMsHjeM55FWPsQQAxaxm51ru8AF50THN8BeBHUPqWGwB4Xatd2z2bPLtST0s9yTraOlfvF/3uLon4DEzs2bJqOsC9zW44c2k8ch/UtZZXfFP2wT8bO6qf2Vy+MpbD2fZyzjdEv14LZRXGOQDe6LbGl+o7mMebroOiO8ZYDoBWzvlLjLF2ADdE72pmjNkAXALgTc55O2OsnTF2Euf8awD92r/Ltd612zHN8RCAEwFcDiAVQJ0ckMPN/2v+wpRt2pA6P/UsY4ZxXH/Oo5bz3g9UDg/q+0LewUye2iB3JtRLirtOv8u1Ztfnn2yrXv0QgI1x6CvuPjwuAiATQDvnfOphjg0j+u6eKUu+9m7+vYYlTCADmATgT4wxGUAIyjC1CwFsgTJUbXW3Y68F8DxjjEO5qNcv0VXjvnZMc2wEUAjgTAA+AI2BukB73St1Jfap9tHOE5zniFYxrb/nGyjOloj3svVCBhJsEVKnu0Jox2y1y9CFiBwOb69et3HVzo8fDYR8724uXzlQu3q4AOxljF3KOX8juhTuZM75RiijnWZAuYi/ENFRT4kgYS7qaYljmiMXSst7HIBGKGsug0lMTJ2fOjupIGkeE3u2QpqaHnzSX3WcxzBc7TpirTopt2XHzJ/r5g+jGiJyJFzeuG3nut1fvtnirlu6uXxl7bEf1TcHzwVgjN0NwAbgn1A2ksiGErqvcc5/yxjLhLJkrgXABwAWJcpFPQrkOHFMcwgAjgdwJZQfrloob7UgpUm25LnJJ1pyLcczgzaDec6KYM1PlwtD1a4jHkIQI8tPeZwx1oO9nwYZWY6Eyxu371i185OV7Z1NSwGso6FsA4cCOc4c0xxWAOcAOA9KINcjOkxOtIvmlLkpMy2jLLMEo6CZ6chmTyT4zJORQBKEhL3w9cHs37cZzc4UtevQiv1B/PHq9s7mVwF8tbl8pVftugYbCuQB4pjmyIYyAmQulIsWDYi2mJmRGVLmpkyzFljniGbxqLs2D4S7/u6vmN2k3+nRPfHh5J9WS6n5CTGUrz9kORIubyrbsWrHx2vaO5teAQWxqiiQB5hjmiMdwGnRDxFKMCtXmAUw5yznBNsE20kGm0GV2WTHbQk1/noZ0hN9P8Hloy4pD+fOz1O7DrVE5HCosmnHru92fLy6vbPpVQDLKYjVR4GsEsc0hxPAyVC6MswAmqDMRgIA2KfZx9in2E+SkqUBa6mKQVl++i+h9hQupg7UOdWyZcjMmsaJ1yRkH/nReAOe1p11G3du3Pv1Vm/A/QYoiDWFAlll0T7m2VCmijqgrI/h6ro/aVzSMMd0x1wpXRoX70brda/4K86uSOyuii5N5vSOzbMfTJjJLkfDucybOmp3b678ZvfO2o17ASyDEsS0x6DGUCBrhGOawwhgOoCLoAyKdwFo7brfmGVMdkx1TDLnmqfEYyzz8PJw+6OvcLvImBjr59YimQOfnvJEUBQMmhzlEgvBsN+zt2Fb2fo9y3e3dzbtAPA/ABsGcCwx6SUKZI2Jrig3Ecqkljwok1yaop8BANax1hzbcbbJphzTJMHYuz3tDkuW8XhxqCEnLA6qVdDen/lgoykpPUPtOmKtzdNUta169a4tFd/tlXlkBYAvAOyl4WvaR4GsUY5pDgZgFJQp2XOh9DP7ADQjOmwOIgTHFMdo61jrFGO6saCva2Zc9G6g4vIt4qDoquju4wm3VIrpExNiDrXH11FX2VRW8X3V6upmd10llNbw6s3lK13HeizRDgpkHYh2Z4yHchFwGpR5/B2I7ooNAGKSaHJMd4y3jLJMMSQb8nra35zaFOl88jnZIDGW8Au3HOyb3PMqfKPO1e0fIo+/o76qedeu7yu/q29y1XoAbATwMYDtm8tXRlQuj/QBBbLOOKY57AAmQ1lOdGT05mZ0G6FhzDA67NPskyy5lsliknjUt+QPP+GvGes1DLrRBgCwI3liXfXUW7LVrqM3PH5XfXXzrl1bq76rb+qo8QCohtIlsWlz+comdasj/UWBrGOOaY5MKBcCT4OyylwESn9zsOsYU7YpOWlc0hhTjilfSpVGMpHtW4jllC+D1YtWJsY6x33hNti8q0/6Y//74ONI5rLs8bXX1LWVV26tXFXf2FHthrKA++dQ1gBvor7hxEGBnACi62bkATgBwDwo68ZyKEPo9g1tYhITnaOsE1OSjSebR1jCz7zIbRamnSnbavjwpGKvZDBrKpS9AXdTs6t+T1XzjrqdtRvd/pA3AmUtlC8AbALQSCGcmCiQE4xjmkOCcjHwOCgB3TVywgvO24c148zkTlSbwtiUYzDYZliseaNNppE5BkOeTUz8CSEHe3/GffUm+9AsNWsIhHyuVk/jnpqWPVU7aze0d3hbQlCuE9QC+BJKCDdQCCc+CuQEFh2pkQZgDIDjzUF+Yl4DCqwBbGLKOGcXukZsABguSY7pFkveaKNpZJZkGGETxIRffOfTgmsrWPbxA3Zhj3MOf8jb6va21TZ0VFfvrtvcWt9e0bU4eweU3TK2AtizuXxlx0DVRbSBAnkQyTjOLhXUIM8gYySUvucxwL7l510A3OgW0CmiaB5nMmXkSsbMLMmQmSYaMpyimCkxbS4Z2hdrck6rdI39QVyGvnHOuS/Y2eL2tdW1eRrr6tsqGyqayry+oKfr3y8ApfW7HsAeUH/woEeBPIjNt9tNUDZpHQMloEdB6XsWoKxE1xn9CHc9hgHIMxqT842mzKGSlJFhMGSmiGKmTRDSBB0uSFRuH920Z8Zd6f19HlmORPwhb6vL21bX6mmorW+rbKxsKuv0h7xG7P+jFwKwE8A6ALsB1NDwNNIdBTLZZ77dboayO0MmgBEARkc/S1CCmgHwQwlpb/Q2AICJMXG8yZSeZzRl5kRb0w5RTDMzpunp2AEmhb6e95ihJ6vbhSJBnz/obfUFPG0ef0eb29fe1t7Z1NrkqnW3uOo4xwG7VgShhG4ZgEoo/cEtm8tXyod7bkIACmRyDPPtdgFACoAMAFlQQnpU9GsZSmuaQwlpD7oNueuSLhqs2ZLBnm4wOFJE0Z4sig67INqTBMFuYsxqEpjVxASrWpNT3p/9+1aIJikUCXSGwoHOQNjfGQz5vb6gx+3ytrW3ehpaG9ur3Z0BlwBl26CuOrveTfgB7ML+8K2DEr70y0V6hQKZ9Ml8u90IIB1KUA+D0u0xEsp2VV2tQAYlsEJQ+ku7PsLo1rruIjEmpIuiNdVgsCYLotUhClaJMYMAxhjABAaBAYzt/54d9P2B9wOMMbAg5yG/zP1eWfZ3ynKglTNzuTFtQthg+V8LF/weCGEo7wIkKDsYm6OfZRwYuvVQxgDXQJmM0wplaGE7hS+JBQpkEjPz7XYGpQXpAGCPfnZACe10AEOgjPqwQQm67m/fu8JbhjLBhWN/aB/t85HuY1B2VTdA2QigqyUvc0Bw2UdMD0n2NVC2kedQWvcdUAK2DkoXQxv2h66PQpfEGwUyGXDRbhALAGv0o/vXDiiB3RXQQvRr8aDvD/4sHnRb10VJd/SjE0orNwDA7zUPkb1J2e3R2ylsiSZQIBNCiEbQNuikVxhjeYyxLWrXQUgiokAmhBCNoEAmfSEyxp5jjG1ljH3EGLMwxm5kjK1mjG1kjL3FGLMCAGPsRcbYUsbYGsbYDsZYodrFE6JVFMikL/IBLOGcT4CySP7FAN7mnM/knE8BsA3A9d2Oz4Oy0NF5AJYyxswDXC8hukCBTPpiL+d8Q/TrtVACdyJj7CvG2GYAVwCY0O341znnMud8J5Q1G8YNaLWE6AQFMumLQLevI1DG+r4I4FbO+SQAv4EyuaLLwUN5aGgPIYdBgUxixQ6gjjEmQWkhd3cpY0xgjHVNuy4b8OoI0YE+7VJMyGH8CsB3ULaQ+g5KQHepBLAKyqSPmznn/oEvjxDto4khJK4YYy8CKOWcv6l2LYRoHXVZEEKIRlALmRBCNIJayIQQohEUyIQQohEUyIQQohEUyIQQohEUyIQQohEUyIQQohEUyIQQohEUyIQQohEUyIQQohEUyIQQohEUyIQQohEUyIQQohEUyIQQohEUyIQQohEUyIQQohEUyIQQohEUyIQQohEUyIQQohH/DxiK1uVHaGzuAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.pie(sizes, labels=emotions, shadow=True, startangle=90, autopct='%1.1f%%')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convert Y to one hot\n",
    "int_labels = LabelEncoder().fit_transform(labels)\n",
    "int_labels = int_labels.reshape(len(int_labels), 1)\n",
    "onehot_labels = OneHotEncoder().fit_transform(int_labels).toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Subset limited amount of data for hyperparameter search \n",
    "lim_data = data[::10]\n",
    "lim_lengths = orig_lengths[::10] \n",
    "lim_onehot_labels = onehot_labels[::10]\n",
    "\n",
    "# Split into train and test\n",
    "lim_train_x, lim_val_x, lim_train_y, lim_val_y, lim_train_lengths, lim_val_lengths = train_test_split(lim_data,\n",
    "                                                                              lim_onehot_labels,\n",
    "                                                                              lim_lengths,\n",
    "                                                                              test_size=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperparameter search using limited data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'dense_hidden': 64, 'num_classes': 6, 'learning_rate': 0.0001, 'num_features': 26, 'num_hidden': 256}\n",
      "Created 21 batches of size 32\n",
      "Created Validation 3 batches of size 32\n",
      "Data Batch Shape: (32, 1000, 26) Labels Shape: (32, 6) \n",
      "BLSTM-0 (?, 512)\n",
      "BLSTM-1 (?, 512)\n",
      "(?, 64)\n",
      "(?, 6)\n",
      "Epoch 0 mean batch val loss 1.7716026306152344\n",
      "Epoch 5 mean batch val loss 1.672549843788147\n",
      "Epoch 10 mean batch val loss 1.6704819202423096\n",
      "Epoch 15 mean batch val loss 1.6707208156585693\n",
      "Epoch 20 mean batch val loss 1.6707878112792969\n",
      "Epoch 25 mean batch val loss 1.6708301305770874\n",
      "Epoch 30 mean batch val loss 1.6708418130874634\n",
      "Epoch 35 mean batch val loss 1.671813726425171\n",
      "Epoch 40 mean batch val loss 1.6795858144760132\n",
      "Epoch 45 mean batch val loss 1.6695420742034912\n",
      "Epoch 49 mean batch val loss 1.6695420742034912\n",
      "Accuracy: 0.20270270270270271\n",
      "{'dense_hidden': 64, 'num_classes': 6, 'learning_rate': 0.0005, 'num_features': 26, 'num_hidden': 256}\n",
      "Created 21 batches of size 32\n",
      "Created Validation 3 batches of size 32\n",
      "Data Batch Shape: (32, 1000, 26) Labels Shape: (32, 6) \n",
      "BLSTM-0 (?, 512)\n",
      "BLSTM-1 (?, 512)\n",
      "(?, 64)\n",
      "(?, 6)\n",
      "Epoch 0 mean batch val loss 1.6961473226547241\n",
      "Epoch 5 mean batch val loss 1.6712663173675537\n",
      "Epoch 10 mean batch val loss 1.7001367807388306\n",
      "Epoch 15 mean batch val loss 1.6667330265045166\n",
      "Epoch 20 mean batch val loss 1.6670869588851929\n",
      "Epoch 25 mean batch val loss 1.6742384433746338\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-11-4d00964723c8>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     11\u001b[0m                           \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m                           \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m50\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mload_model\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m                           save_path='final_model/lr_test.ckpt')\n\u001b[0m\u001b[1;32m     14\u001b[0m     \u001b[0macc\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_pred\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0margmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlim_val_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m/\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mval_pred\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'Accuracy: {}'\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0macc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/Emotional_Dialogue/models/emotion_recognition.py\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(self, train_x, train_y, train_lengths, val_x, val_y, val_lengths, batch_size, epochs, verbose, save_path, load_model)\u001b[0m\n\u001b[1;32m    135\u001b[0m                         \u001b[0mseq_len\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mlen_batch\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    136\u001b[0m                     }\n\u001b[0;32m--> 137\u001b[0;31m                     \u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    138\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    139\u001b[0m                         \u001b[0merr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mfeed_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    898\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    899\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 900\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    901\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    902\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1133\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1134\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1135\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1136\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1137\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1314\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1315\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[0;32m-> 1316\u001b[0;31m                            run_metadata)\n\u001b[0m\u001b[1;32m   1317\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1318\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1320\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1321\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1322\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1323\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1324\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1305\u001b[0m       \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1306\u001b[0m       return self._call_tf_sessionrun(\n\u001b[0;32m-> 1307\u001b[0;31m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[1;32m   1308\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1309\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/.local/lib/python3.5/site-packages/tensorflow/python/client/session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[0;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[1;32m   1407\u001b[0m       return tf_session.TF_SessionRun_wrapper(\n\u001b[1;32m   1408\u001b[0m           \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1409\u001b[0;31m           run_metadata)\n\u001b[0m\u001b[1;32m   1410\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1411\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mraise_exception_on_not_ok_status\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mstatus\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "learning_rates = [.0001, .0005, .0007, .001]\n",
    "truth = np.argmax(lim_val_y, 1) \n",
    "\n",
    "for lr in learning_rates:\n",
    "    lstm = ClassifyEmotion(lr=lr, num_hidden=256)\n",
    "    print(vars(lstm))\n",
    "    val_pred = lstm.train(train_x=lim_train_x, val_x=lim_val_x, \n",
    "                          train_y=lim_train_y, val_y=lim_val_y, \n",
    "                          train_lengths=lim_train_lengths, \n",
    "                          val_lengths=lim_val_lengths, \n",
    "                          verbose=False, \n",
    "                          batch_size=32, epochs=50, load_model=False, \n",
    "                          save_path='final_model/lr_test.ckpt')\n",
    "    acc = sum(val_pred == np.argmax(lim_val_y, 1))/len(val_pred)\n",
    "    print('Accuracy: {}'.format(acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'num_features': 26, 'num_classes': 6, 'learning_rate': 0.0005, 'num_hidden': 512}\n",
      "Created 42 batches of size 32\n",
      "Created Validation 5 batches of size 32\n",
      "Data Batch Shape: (32, 750, 26) Labels Shape: (32, 6) \n",
      "(?, 1024)\n",
      "(?, 1024)\n",
      "(?, 256)\n",
      "(?, 6)\n",
      "Epoch 0 mean batch val loss 1.6801049709320068\n",
      "Epoch 5 mean batch val loss 1.6791181564331055\n",
      "Epoch 10 mean batch val loss 1.6758193969726562\n",
      "Epoch 15 mean batch val loss 1.6723060607910156\n",
      "Epoch 20 mean batch val loss 1.7721226215362549\n",
      "Epoch 25 mean batch val loss 1.736720323562622\n",
      "Epoch 30 mean batch val loss 1.733214020729065\n",
      "Epoch 35 mean batch val loss 1.728166937828064\n",
      "Epoch 40 mean batch val loss 1.7222931385040283\n",
      "Epoch 45 mean batch val loss 1.715497612953186\n",
      "Accuracy: 0.25\n",
      "{'num_features': 26, 'num_classes': 6, 'learning_rate': 0.0005, 'num_hidden': 128}\n",
      "Created 42 batches of size 32\n",
      "Created Validation 5 batches of size 32\n",
      "Data Batch Shape: (32, 750, 26) Labels Shape: (32, 6) \n",
      "(?, 256)\n",
      "(?, 256)\n",
      "(?, 64)\n",
      "(?, 6)\n",
      "Epoch 0 mean batch val loss 1.6830295324325562\n",
      "Epoch 5 mean batch val loss 1.6776962280273438\n",
      "Epoch 10 mean batch val loss 1.671523094177246\n",
      "Epoch 15 mean batch val loss 1.6703250408172607\n",
      "Epoch 20 mean batch val loss 1.6647762060165405\n",
      "Epoch 25 mean batch val loss 1.6592369079589844\n",
      "Epoch 30 mean batch val loss 1.6718149185180664\n",
      "Epoch 35 mean batch val loss 1.6593239307403564\n",
      "Epoch 40 mean batch val loss 1.6648643016815186\n",
      "Epoch 45 mean batch val loss 1.664939522743225\n",
      "Accuracy: 0.20270270270270271\n",
      "{'num_features': 26, 'num_classes': 6, 'learning_rate': 0.0005, 'num_hidden': 64}\n",
      "Created 42 batches of size 32\n",
      "Created Validation 5 batches of size 32\n",
      "Data Batch Shape: (32, 750, 26) Labels Shape: (32, 6) \n",
      "(?, 128)\n",
      "(?, 128)\n",
      "(?, 32)\n",
      "(?, 6)\n",
      "Epoch 0 mean batch val loss 1.700989007949829\n",
      "Epoch 5 mean batch val loss 1.6753206253051758\n",
      "Epoch 10 mean batch val loss 1.6685558557510376\n",
      "Epoch 15 mean batch val loss 1.6681592464447021\n",
      "Epoch 20 mean batch val loss 1.6567184925079346\n",
      "Epoch 25 mean batch val loss 1.651473045349121\n",
      "Epoch 30 mean batch val loss 1.6535756587982178\n",
      "Epoch 35 mean batch val loss 1.663648247718811\n",
      "Epoch 40 mean batch val loss 1.6639769077301025\n",
      "Epoch 45 mean batch val loss 1.6430658102035522\n",
      "Accuracy: 0.19594594594594594\n"
     ]
    }
   ],
   "source": [
    "num_hidden = [512, 128, 64]\n",
    "\n",
    "for h in num_hidden: \n",
    "    lstm = ClassifyEmotion(lr=.0005, num_hidden=h)\n",
    "    print(vars(lstm))\n",
    "    val_pred = lstm.train(train_x=lim_train_x, val_x=lim_val_x, \n",
    "                          train_y=lim_train_y, val_y=lim_val_y, \n",
    "                          train_lengths=lim_train_lengths, \n",
    "                          val_lengths=lim_val_lengths, \n",
    "                          verbose=False, \n",
    "                          batch_size=32, epochs=50, load_model=False, \n",
    "                          save_path='final_model/num_hidden_test.ckpt')\n",
    "    acc = sum(val_pred == np.argmax(lim_val_y, 1))/len(val_pred)\n",
    "    print('Accuracy: {}'.format(acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'num_features': 26, 'num_classes': 6, 'learning_rate': 0.0005, 'num_hidden': 64, 'dense_hidden': 512}\n",
      "Created 42 batches of size 32\n",
      "Created Validation 5 batches of size 32\n",
      "Data Batch Shape: (32, 750, 26) Labels Shape: (32, 6) \n",
      "(?, 128)\n",
      "(?, 128)\n",
      "(?, 512)\n",
      "(?, 6)\n",
      "Epoch 0 mean batch val loss 1.6846727132797241\n",
      "Epoch 5 mean batch val loss 1.6806259155273438\n",
      "Epoch 10 mean batch val loss 1.6762855052947998\n",
      "Epoch 15 mean batch val loss 1.6878869533538818\n",
      "Epoch 20 mean batch val loss 1.6715717315673828\n",
      "Epoch 25 mean batch val loss 1.666272521018982\n",
      "Epoch 30 mean batch val loss 1.6656242609024048\n",
      "Epoch 35 mean batch val loss 1.6539623737335205\n",
      "Epoch 40 mean batch val loss 1.6508935689926147\n",
      "Epoch 45 mean batch val loss 1.6635900735855103\n",
      "Epoch 49 mean batch val loss 1.6635900735855103\n"
     ]
    }
   ],
   "source": [
    "# Dense layer hidden units = 512 \n",
    "lstm = ClassifyEmotion(lr=.0005, num_hidden=64, dense_hidden=512)\n",
    "print(vars(lstm))\n",
    "val_pred = lstm.train(train_x=lim_train_x, val_x=lim_val_x, \n",
    "                      train_y=lim_train_y, val_y=lim_val_y, \n",
    "                      train_lengths=lim_train_lengths, \n",
    "                      val_lengths=lim_val_lengths, \n",
    "                      verbose=False, \n",
    "                      batch_size=32, epochs=50, load_model=False, \n",
    "                      save_path='final_model/num_hidden_test.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.20270270270270271\n"
     ]
    }
   ],
   "source": [
    "acc = sum(val_pred == np.argmax(lim_val_y, 1))/len(val_pred)\n",
    "print('Accuracy: {}'.format(acc))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train using all data "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Split into train and test\n",
    "train_x, val_x, train_y, val_y, train_lengths, val_lengths = train_test_split(data,\n",
    "                                                                              onehot_labels,\n",
    "                                                                              orig_lengths,\n",
    "                                                                              test_size=0.2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'dense_hidden': 512, 'num_classes': 6, 'learning_rate': 0.0002, 'num_features': 26, 'num_hidden': 128}\n",
      "Created 185 batches of size 32\n",
      "Created Validation 47 batches of size 32\n",
      "Data Batch Shape: (32, 1000, 26) Labels Shape: (32, 6) \n",
      "BLSTM-0 (?, 256)\n",
      "BLSTM-1 (?, 256)\n",
      "(?, 512)\n",
      "(?, 6)\n",
      "Epoch 0 mean batch val loss 1.5604650974273682\n",
      "Epoch 5 mean batch val loss 1.5549639463424683\n",
      "Epoch 10 mean batch val loss 1.5510139465332031\n",
      "Epoch 15 mean batch val loss 1.5486640930175781\n",
      "Epoch 19 mean batch val loss 1.5486640930175781\n"
     ]
    }
   ],
   "source": [
    "lstm = ClassifyEmotion(lr=.0002, num_hidden=128, dense_hidden=512)\n",
    "print(vars(lstm))\n",
    "val_predictions = lstm.train(train_x=train_x, val_x=val_x, train_y=train_y, val_y=val_y, \n",
    "                             train_lengths=train_lengths, val_lengths=val_lengths, \n",
    "                             batch_size=32, epochs=20, load_model=False, verbose=False,\n",
    "                             save_path='final_model/emotion_recognition_1000.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.24661246612466126"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "truth = np.argmax(val_y, 1)\n",
    "correct = sum(val_predictions == truth)\n",
    "\n",
    "correct/len(truth)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Using Even Class Sizes "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3595, 1000, 26) 3595 3595\n"
     ]
    }
   ],
   "source": [
    "ang_idx = [i for i, x in enumerate(labels) if x=='ang'][:600]\n",
    "fru_idx = [i for i, x in enumerate(labels) if x=='fru'][:600]\n",
    "neu_idx = [i for i, x in enumerate(labels) if x=='neu'][:600]\n",
    "exc_idx = [i for i, x in enumerate(labels) if x=='exc'][:600]\n",
    "hap_idx = [i for i, x in enumerate(labels) if x=='hap']\n",
    "sad_idx = [i for i, x in enumerate(labels) if x=='sad'][:600]\n",
    "\n",
    "all_idx = ang_idx + fru_idx + neu_idx + exc_idx + hap_idx + sad_idx\n",
    "\n",
    "balanced_data = data[all_idx, :]\n",
    "balanced_labels = [label for i, label in enumerate(labels) if i in all_idx]\n",
    "balanced_lengths = [l for i, l in enumerate(orig_lengths) if i in all_idx]\n",
    "\n",
    "print(balanced_data.shape, len(balanced_labels), len(balanced_lengths))\n",
    "\n",
    "# Convert Y to one hot\n",
    "balanced_int_labels = LabelEncoder().fit_transform(balanced_labels)\n",
    "balanced_int_labels = balanced_int_labels.reshape(len(balanced_int_labels), 1)\n",
    "balanced_onehot_labels = OneHotEncoder().fit_transform(balanced_int_labels).toarray()\n",
    "\n",
    "# Split into train and test\n",
    "train_x, val_x, train_y, val_y, train_lengths, val_lengths = train_test_split(balanced_data,\n",
    "                                                                              balanced_onehot_labels,\n",
    "                                                                              balanced_lengths,\n",
    "                                                                              test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'dense_hidden': 256, 'num_classes': 6, 'learning_rate': 0.0005, 'num_features': 26, 'num_hidden': 128}\n",
      "Created 203 batches of size 16\n",
      "Created Validation 23 batches of size 16\n",
      "Data Batch Shape: (16, 1000, 26) Labels Shape: (16, 6) \n",
      "BLSTM-0 (?, 256)\n",
      "BLSTM-1 (?, 256)\n",
      "(?, 256)\n",
      "(?, 6)\n",
      "Epoch 0 mean loss 1.7924994230270386\n",
      "Epoch 0 mean batch val loss 1.7941268682479858\n",
      "Epoch 1 mean loss 1.7924065589904785\n",
      "Epoch 1 mean batch val loss 1.7898458242416382\n",
      "Epoch 2 mean loss 1.7921665906906128\n",
      "Epoch 2 mean batch val loss 1.7896699905395508\n",
      "Epoch 3 mean loss 1.7918757200241089\n",
      "Epoch 3 mean batch val loss 1.7889657020568848\n",
      "Epoch 4 mean loss 1.7912245988845825\n",
      "Epoch 4 mean batch val loss 1.7889125347137451\n",
      "Epoch 4 mean batch val loss 1.7889125347137451\n"
     ]
    }
   ],
   "source": [
    "# Test to see if balanced dataset has decreasing error rates \n",
    "lstm = ClassifyEmotion(lr=.0005, num_hidden=128, dense_hidden=256)\n",
    "print(vars(lstm))\n",
    "val_pred_balanced = lstm.train(train_x=train_x, val_x=val_x, train_y=train_y, val_y=val_y, \n",
    "                             train_lengths=train_lengths, val_lengths=val_lengths, \n",
    "                             batch_size=16, epochs=5, load_model=False, verbose=True,\n",
    "                             save_path='final_model/emotion_recognition_balanced.ckpt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "val_pred_truth = np.argmax(val_y, 1)\n",
    "accuracy = sum(val_pred_truth == val_pred_balanced) / len(val_pred_truth)\n",
    "accuracy"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}