Decision trees and random forests.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['pixels', 'overfeat', 'labels', 'names', 'allow_pickle'])\n"
     ]
    }
   ],
   "source": [
    "with np.load('cifar4-train.npz', allow_pickle=False) as npz_file:\n",
    "    data = dict(npz_file.items())\n",
    "    print (data.keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Decision Trees"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Size of X: (5000, 4096) float32\n",
      "Size of y: (5000,) int64\n",
      "Size of X_tr, y_tr: (4000, 4096) (4000,)\n",
      "Size of X_te ,y_te: (1000,) (1000,)\n",
      "Size of X_train, y_train: (3200, 4096) (3200,)\n",
      "Size of X_valid, y_valid: (800, 4096) (800,)\n"
     ]
    }
   ],
   "source": [
    "#create X/y arrays\n",
    "X = data['overfeat']\n",
    "y = data['labels']\n",
    "names = data['names']\n",
    "\n",
    "print ('Size of X:',X.shape,X.dtype)\n",
    "print ('Size of y:',y.shape,y.dtype)\n",
    "\n",
    "X_tr, X_te, y_tr, y_te = train_test_split(X, y, test_size=1000, random_state=0,stratify=y)\n",
    "print ('Size of X_tr, y_tr:',X_tr.shape,y_tr.shape)\n",
    "print ('Size of X_te ,y_te:',y_te.shape,y_te.shape)\n",
    "\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(X_tr, y_tr, test_size=800, random_state=0,stratify=y_tr)\n",
    "\n",
    "print ('Size of X_train, y_train:', X_train.shape, y_train.shape)\n",
    "print ('Size of X_valid, y_valid:', X_valid.shape, y_valid.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.decomposition import PCA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "#define parameters and values\n",
    "depth = np.arange(1,11)\n",
    "\n",
    "pipe = Pipeline([('pca',PCA(n_components=150)),('tree',DecisionTreeClassifier())])\n",
    "\n",
    "#save accuracy for train/vallid sets\n",
    "train_scores = []\n",
    "valid_scores = []\n",
    "\n",
    "for d in depth:\n",
    "    pipe.set_params(tree__max_depth=d)\n",
    "    pipe.fit(X_train,y_train)\n",
    "    \n",
    "    train_accuracy = pipe.score(X_train,y_train)\n",
    "    train_scores.append({'tree__max_depth': d, 'accuracy': train_accuracy})\n",
    "\n",
    "    valid_accuracy = pipe.score(X_valid,y_valid)\n",
    "    valid_scores.append({'tree__max_depth': d, 'accuracy': valid_accuracy})  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accuracy</th>\n",
       "      <th>tree__max_depth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.924687</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.898438</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.847812</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.797813</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.747812</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   accuracy  tree__max_depth\n",
       "9  0.924687               10\n",
       "8  0.898438                9\n",
       "7  0.847812                8\n",
       "6  0.797813                7\n",
       "5  0.747812                6"
      ]
     },
     "execution_count": 124,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_df = pd.DataFrame(train_scores)\n",
    "train_df.sort_values(by='accuracy',ascending=False).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accuracy</th>\n",
       "      <th>tree__max_depth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.65375</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.65250</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.64875</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.64250</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.63625</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   accuracy  tree__max_depth\n",
       "4   0.65375                5\n",
       "5   0.65250                6\n",
       "3   0.64875                4\n",
       "6   0.64250                7\n",
       "8   0.63625                9"
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_df = pd.DataFrame(valid_scores)\n",
    "valid_df.sort_values(by='accuracy',ascending=False).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "sns.set()\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEFCAYAAADqujDUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VOW9+PHPbNl3Mgn7Th4QAmGTXcV9Q8EN7LVebatd7F5vb9tfe7W92sXWW3tt7WJrrdYFKqCAKMoVBUGULZCwPBACYQmQyUL2ZZbz++MMIQECA2Q4ycz3/XrllcycMzNfHpLne85zvud5bIZhIIQQIvrYrQ5ACCGENSQBCCFElJIEIIQQUUoSgBBCRClJAEIIEaWcVgcQKo+n9oLLldLTE6iqaujMcLo1aY/2pD1OkrZoLxLaw+1OtnW0LSrOAJxOh9UhdCnSHu1Je5wkbdFepLdHVCQAIYQQp5MEIIQQUUoSgBBCRClJAEIIEaUkAQghRJSSBCCEEFFKEoAQQkQpSQBCCNEFNfoa+ezoZl7d9QaH646E5TMkAVyERx55iE2bNrR77plnfsPSpW+ecf8jR0p5+OEHAHjssR/i9XrbbV+/fh1PPvl4h5/X3Nzc+t7Lly/l448/uvDggXfeWcY3v/kVvv3tr/Gtb32Nzz5bf9b933prET6f76I+UwjRsTpvPetKN/Dc1hf4wZqf8Y8dr7O29LOwJYBuMxVEV3TbbXN49923GT9+IgBer5e1a9fw5S8/cs7X/vSnvzjvz6usrGDp0jeZNWs2N98867xf31ZdXR0vvvhX/vnPf+FyuSgv9/DQQ//OwoXLsNvPfFzw8st/58Ybb8HplF8bITpLdXMNWz2FbPEUUnS8mIARAKBPUi/y3KPIc+fSO6lnWD47Yv6SF3xQxIZdZWfc5nDY8PvPfyqhicOzuOfqoR1uv+qqa/jLX56jqamJuLg41qz5iMsvn0R8fDxbtmzi739/HoCmpiZ+/OOf4nK5Wl97112zeOWVNzhypJRf/OJnxMXFEx8fR3JyCgALF87no49W4fP5SEpK4sknf81LL73A/v37+PvfnycQCNCjRw9mz76LZ5/9Ldu25QNw3XU3cs899/Lkk4/jcrk4evQIFRXl/OhHj6PU8NbPj4+Px+/3s3jxG0ybNoM+ffoyf/6b2O12jh07ylNP/ZyWlmZiYmL5/vd/xIYN66msrODxx3/EL37x9Hm3pRDipIrGSvI9heR7CtlXXYKB2T8NSOnHWHcuY9yjyErIDHscEZMArBAbG8uMGVeyevUqrr/+JpYvX8JDD30NgH37ivmv//pvMjPdvPTSC6xatZLrr7/ptPf461//yJe+9GUmTpzMP//5IiUl+wkEAlRXV/PMM89ht9v57ne/zs6d27n//i+wd28RDz74EH/7258BWLt2DUeOlPKXv7yI3+/nq1/9YusZSc+evfj+9/8fS5YsZsmSRfzHf/yo9XMdDgfPPPMcCxa8yve+9w28Xi/33fcAc+bcxR/+8DvuumsuU6ZMY+PGz/jTn37PY489wYsv/o3HH//5JWhZISLPsfqyYKdfwIHawwDYsDEkbSB57lzy3KNIj0u7pDFFTAK45+qhHR6tu93JeDy1YfncWbPm8Ic//I5x4yZQW1vbepTtdrt55plfEx+fgMdTRm7umDO+ft++YkaMGAVAbm4eJSX7sdvtuFwuHn/8/xEfH09ZWVmHY+8lJfsYMyYPm82G0+lk5Mhc9u8vBmDYMAVAVlY2BQVb272uvNxDc3Mz3/3ufwJw4EAJ3/veNxk9Oo/i4iJefvnvvPLKPwBkyEeIC2AYBqX1R9lSVkC+p4Aj9ccAsNvsjMjIIc89itHukaTEJFsWo/xlX6QhQ4bS2FjPggWvccstt7U+/6tfPcGCBW+RkJDIE0881uHr+/cfSGHhNiZPnsquXdsBKCraw+rVH/L88/+gqamJL37xPgBsNjtGcHzwhAEDBrF8+RLmzv03fD4fhYXbuOmmW4F12GwdzgJLRUUFP//54zz77F9ISUmhZ89epKWl4nI56d9/IPfeex+5uWMoKdnPli2b2nz+Bc/KLUTEMwyDktqD5JeZR/qexgoAnHYnuZmXMdadS27mCBJcCRZHapIE0AluueU2/vCH/2XhwmWtz91ww808/PADJCcnk57eg/Jyzxlf+73v/YDHHvshr732MmlpacTExNK3bz/i4+P54hc/T0yMix49Mikv9zByZC5er4/nnvtfYmNjAZg2bQZbtmziy19+EK/Xy9VXX9turL8jSg3nrrvm8a1vfYW4uDh8Pj+33jqb/v0H8sgj3+Lpp39JS0sLzc1NfOtbjwIwZkwejz76TZ599s9nTS5CRJOAEWDv8f1sDY7pVzUfByDGEcO4rNHkuUcxssdw4pxxFkd6Olt3OaK7mAVhwjkE1B1Je7Qn7XGStEV7HbWHP+Bn9/G95JcVsLV8O7UtdQDEO+MZnXkZY9yjGJGRQ4zDddprL7WzLQgTtjMApZQdeA4YAzQDX9JaF7XZ/p/AvUAN8JTWetkZ30gIIboAr9/Lrqo9bCkroKB8Bw2+RgCSXIlM6z2JPPcoctKH4LR3n4GVcEY6G4jTWk9RSk0GngZuB1BK5QKfAyYF912nlPpAa929114TQkSUJm8Tm8u2kV9WQGHFTpr9LQCkxaYysec4xrpHMSRtEHZb97ynNpwJYDrwLoDWer1SakKbbSOAD7XWTQBKqT3AaKDDW1HT0xMuank2t9u6K+1dkbRHe9IeJ0lbQHl9Ja9sW8xnh7fi9Zt37GcnZjKp31gm9R3LkIwB3bbTbyucCSAFqG7z2K+UcmqtfUAB8EOlVDIQA0wF/nK2N7uYhZllXLM9aY/2pD1Oiva28Af8rDr0MW/ve58Wfwt9UnoyOmMkee5c+iT1MosfAlBRXm91qCE7W0IPZwKoAdp+sj3Y+aO13qmU+j3wDlAEfAqUhzEWIYQ4q/01B3ht1yIO1ZWS5EpkXs4cbsm9kvLyOqtDC5twJoC1wCxgQfAaQMGJDUopN5CptZ6ulEoF3gMKwxiLEEKcUaOvkSV7V7Dm8CcYGEzpNZHZQ28myZUY8eXO4UwAi4HrlFLrABvwoFLqu5hH/EuBwUqpDUAL8B9aa38YYwmLZ5/9LVrvpLKygqamJnr37kNaWjpPPPGrc752zx7Nxx+v5sEHHzrnvs3NTfzmN7+kvNyDzWYjMTGJRx/9AampZ75tvKammvXrP+H6628873+TENHCMAy2eAp4Y/dbVLfUkp2Qxb3qDoalD7Y6tEtG7gPoBMuXL6WkZD9f/eo3wvL+ixb9i2PHjra+/4IFr1JaWsq3v/3oGfffvHkjb721sMMZR6N9nPdU0h4nRUtbVDRWMn/3m2yv2IXT7uTGAddw7YArcZ1SwhkJ7WHJfQCX2qKiZWwpKzjjNofdhj9w/vljbFYudwy99bxft3nzRv74x2dxuVzcdtscYmNjWbToX63TKDzxxFMUFxe1dtLz5s0hN3cMBw6UkJGRwRNPPIXDcbLiqWfPXixb9ia5uWMYO3Ycd945t/W9PvhgJfPnv4Ldbmf06Dy++tVv8NJLL1BUtIe33lrE7bffcd7xCxGp/AE/Hxxcw9v73scb8DI8fRhz1ZxLMvNmVxQxCaCraWlp4fnnzcnUXnrpBX79698RFxfHU089yWeffUJmprt139LSw/zud38kO7snX/3qF9i5cwejRuW2bp86dTpebwtvv/0WP//5Txk8eAjf+c73cbvdvPDCn/nrX18mLi6O//7vn7Bhw3ruv/8LvPXWQun8hWijuLqE13YtpLT+KMmuJD43/E4mZo+N+HH+s4mYBHDH0Fs7PFq34jSuf/8BrT+np2fwxBOPkZCQQEnJfkaNGt1u39TUNLKzzQUfsrKyaWlpbre9sHAb48dfzpVXXo3f72fFiuU8+eTjPProDzh+vIpHH/0mAA0NDRw+fLjdZwsR7Rq8Dby19x3Wln6GgcG03pdz+5CbSewiE7JZKWISQFdjt5tHFXV1dfztb39unSjuO9955LQZNc91BLJy5QoSEhJ5+OGv4XA4GDJkGDExMfTq1YesrGyeeeY5nE4ny5cvZdiwHOrr6wlcwJCXEJHEMAw2HcvnjaKl1LbU0Ssxm3vVnQxJG2h1aF2GJIAwS0xMJDd3DF/4wn3Ex8eTnJxMebmHXr16h/weDz/8Nf7nf57igQc+R3x8HHFx8fzgBz8hPT2duXP/ja9//WH8fj+9evXm6quvo7a2huLiIhYseJV77vlcGP91QnRNnoYK5u9ezM7K3bjsTm4bfCPX9L+iW83TcylIFVAUkvZoT9rjpO7eFr6Aj5UHVvPu/pV4Az5GZOQwT80hM77HBb1fd28PiJIqICFEdCs6vo/X9CKO1h8jOSaJzw+7jXFZY6L6Iu+5SAIQQnRr9d4G3ixazrojn2HDxvQ+k7l98E0kuOKtDq3LkwQghOiWDMNgw7EtLNyzlDpvPX2SenGvuoNBqVIFFypJAEKIbqeswcPrejG6qgiX3cXsITdzdb8ZOOwXPmV8NJIEIIToNrwBHytLPuTdkg/wBXyM7DGcuTmz6RGfYXVo3ZIkACFEt7Cnai+v6UUca/CQGpPMXTm3M9adKxd5L4IkACFEl1bXUs/ivW+z/shGbNi4su9UZg2+gXinXOS9WJIAhBBdkmEYfHp0E4uKllHvbaBvUm/uHX4HA1P6Wx1axJAEIIToco7Wl/G6XsSe48XE2F3cMfRWruo7TS7ydjJJAEKILsPr97KiZBXvl6zCZ/jJzRzBPTmzyYhLtzq0iCQJQAjRJeys2M383YvxNFaQFpvK3Tm3MyZzpFzkDSNJAEIISx1vrmbRnmVsKtuK3Wbn6n4zuGXQdcQ546wOLeJJAhBCWCJgBFh9+BOW7l1Bk7+JgSn9mafuoF9y6DPliosjCUAIccmV1Bzkdb2IA7WHiXfGM0/dwbTel2O32a0OLapIAhBCXDKNvkaW7F3BmsOfYGBwec9xzBl6CykxyVaHFpUkAQghwu7E6lwLi5ZR01JLdoKbeWoOOelDrQ4tqkkCEEKEVVmDh/n6TXZV7cFldzJr8A1c0/9KXLI6l+Xkf0AIERZev5f3SlbxXrCm/7IMxVw1+4JX5xKdTxKAEKLT7azczXxt1vSnxqRwV85tMnFbFyQJQAjRaaqba1i4ZymbyrZiw8bMftO5ZdD1xEtNf5ckCUAIcdFOrekfkNKPe9Ud9EvuY3Vo4iwkAQghLkr7mv445qk5TOs9SWr6uwFJAEKIC9Loa2Rp8QpWHzJr+idmj+OOYVLT351IAhBCnBfDMNhctpU39iylpqWWrIRM5uXcgcqQmv7uRhKAECJkbWv6nXYntw66gWsHSE1/dyX/a0KIc/L6vbx34EOzpj/gY0RGDnNz5uBOkJr+7kwSgBDirHZV7mG+XkxZY7ksxh5hJAEIIc6ourmWRUVL2Xgs36zp7zudWwZLTX8kCVsCUErZgeeAMUAz8CWtdVGb7Y8C9wIB4Oda68XhikUIEbqAEWDN4fUsLX6XRl8TA5L7MW/4HPon97U6NNHJwnkGMBuI01pPUUpNBp4GbgdQSqUB3wSGAolAPiAJQAiLFVeW8NzGf3Kg9hDxzjjm5sxheh+p6Y9U4UwA04F3AbTW65VSE9psqwdKMDv/RMyzgLNKT0/A6XRccDBut9QmtyXt0V60t0dDSyOvFy5hRdFHGIbB9AGXc/+YO0iLT7U6NMtF8u9GOBNAClDd5rFfKeXUWvuCjw8COwAH8ItzvVlVVcMFB+J2J+Px1F7w6yONtEd70dweJ2r6F+5ZSnVLLb2Ssrhr6O0MzxiGtw48ddHZLidEwu/G2RJYOBNADdD2k+1tOv+bgF7AoODjFUqptVrrz8IYjxCijbKGchbsfpOdlbuDNf3Xc+/4Wzle2WR1aOISCWcCWAvMAhYErwEUtNlWBTQCzVprQyl1HEgLYyxCiCBvwMf7JatY0aam/56c2WQlZOJyuABJANEinAlgMXCdUmodYAMeVEp9FyjSWi9RSl0LrFdKBYCPgffDGIsQgmBN/+7FlDWYNf13DruNcVmjpaY/SoUtAWitA8BXTnl6V5vtjwGPhevzhRAnnVrTf2XfacwafD3xznirQxMWkhvBhIhgASPAx4fXsyRY098/uS/3qjvonyI1/UISgBAR60DtIV7ftZiS2oPEOeKYmzOb6X0mS02/aCUJQIgI0+hrYlnxCj46tA4DgwnZedwxdBapsZFbzy4ujCQAISKEWdO/jYV7llDdUktWfCZz1RyGZwyzOjTRRUkCECICeBoqmL97cWtN/82DruP6/lcFyzqFODNJAEJ0Y96Aj5UlH/JuyQf4Aj6Gpw9jrppNVoLb6tBENyAJQIhuSlcWMX/3Yo41eEiJSeauYbMYlzVGavpFyCQBCNHN1LTUsmjPMjYc2yI1/eKiSAIQopswa/o/ZUnxO8Ga/j7MU3cwIKWf1aGJbkoSgBDdwMHaw7ymF1FSY9b035MzmxlS0y8ukiQAIbqwRl8Tbxe/x4eH1mJgMD5rDHcOm0VqbIrVoYkIIAlAiC7IMAy2eAp4Y/cSqltqcMf3YK6aw4iMHKtDExFEEoAQXYynoYIFu99kR6XGaXNITb8IG0kAQnQRZk3/R6wo+T+8UtMvLgFJAEJ0Aburinhdn6zpv2/YLMZLTb8IM0kAQljIrOl/mw3HNgdr+qcya/ANUtMvLglJAEJY4PR5+qWmX1x6kgCEuMRKag7yul7MgdpDUtMvLCUJQIhLpMHbyNLiFaw5/AkGBhOzxzJn6K0yT7+wjCQAIcLMMAw2HNvCoqJl1LbUkZ3gZm7OHFTGUKtDE1HunAlAKdVTa330UgQjRKQ5Wl/GfL2Y3cf34rI7mTX4Rq7pfwUuuxx7CeuF8lu4Wim1B3gReEtr3RLekITo/lr8Lby7/wNWHvgIv+FnVI/h3J0zm8z4DKtDE6LVOROA1jpHKTUD+HfgV0qp5cCLWuuNYY9OiG6ooHwH/9r9FhVNVaTHpnF3zm2MzhwpNf2iywnpPFRrvUYptRG4G3gSuE0p5QEe0VqvD2eAQnQXlU1VvLF7CVvLt2O32bmu/1XcOPAa4pyxVocmxBmFcg3gGuB+4FpgOTBXa71OKZULvAP0DW+IQnRt/oCfDw6uYfm+92kJeBmSOoh5ag69k3paHZoQZxXKGcBjwN+Ar2qtG048qbUuUEr9JmyRCdEN7KkqZv7uxRypP0aSK5G5ag6Teo6X4R7RLYSSAG4B7tdaNyil+gBfBn6ptW7QWj8T3vCE6JpqW+pYXPQ2nx7dhA0b03tP4rYhN5HoSrA6NCFCFkoCeAUoCP5cC9iBl4E7wxWUEF1VwAiwtvQzlux9hwZfI32TejNPzWFQ6gCrQxPivIWSAAZorW8D0FrXAD9WSuWHNywhup6DtYd5XS9mf80B4hyx3DXsNq7oMwWH3WF1aEJckFASgKGUytVaFwAopYYD3vCGJUTX0ehrYlnxCj46tK51WcY7ht1KWmyq1aEJcVFCSQCPAu8rpQ4FH7uBz4cvJCG6BsMw2Fy2lYV7llLdUktWfCb3qNmyLKOIGKHcCLZSKdUfyMU88tda6+awRyaEhY41eFig32RX1R6cdie3Drqea/tfKcsyiogSyn0Aw4CvA0mADXAopQZpra8Id3BCXGotfi/vlXzA+yUf4jP8XJahuCdnNu6EHlaHJkSnC2UI6DXgbWAG5nxAc4DCc71IKWUHngPGAM3Al7TWRcFteUDbEtLJwGyt9bvnE7wQnWl7hWbB7jcpb6wgLTaVu4bdRp57lNT0i4gVSgKI0Vo/ppRyAZuB54FQ5gGaDcRpracopSYDTwO3A2it84GrAJRSdwOl0vkLq1Q0VPF8wWvkewqw2+xc0+8Kbh50LXHOOKtDEyKsQkkADUqpWGA3MF5r/bFSKpT3ng68C6C1Xq+UmnDqDkqpROCngAwniUuuxe9l9eF1vLN/JU2+ZganDmCeuoM+Sb2sDk2ISyKUBPBPYCnwb8AnSqkbgcMhvC4FqG7z2K+UcmqtfW2e+yLwL611+bneLD09Aafzwuut3W5ZdamtaG6PqsZq3itazXt7V1PbXEdyTCIPTLyHqwbJsowQ3b8bZxLJ7RHSegDAP7TWtUqpq4CJwHshvK4GaNty9lM6fzCTyl2hBFpV1XDunTrgdifj8dRe8OsjTbS2x4HaQ6w6+DGbjm3Fb/hJcMZz/YCZ3DP2ZpprDCrK660O0XLR+rvRkUhoj7MlsFASwHyt9QgArfUh4NA59j9hLTALWBC8BlDQdqNSKhWI1VofDPH9hDhvASPAtvIdrDq4hqLj+wDITshiZr/pTOo5jhhHDCmxSXjo3n/kQlyIUBLADqXUfwGfAo0nntRarz7H6xYD1yml1mGWjz6olPouUKS1XgLkAPsvKGohzqHR18QnRzbw4cG1VDRVAjAiI4eZ/WYwImOYDPUIQWgJIAOYGfw6wQCuPtuLtNYB4CunPL2rzfYNmJVCQnSa8sYKPjy4lk+ObKDJ34zL7mRa70nM7DedXonZVocnRJcSyp3AM8+1jxBWMgyDouPFrDr4MdvKd2BgkBqTwvUDZjKtzySSXIlWhyhElxTKncCrMI/429Fan/UMQIhw8wZ8bD62lVUH13CwrhSA/sl9ubrfDMZljZZZOoU4h1CGgB5v87ML82auqrBEI0QIalvqWHP4E1Yf/oTaljps2BjrzuXq/jMYlDJA7twVIkShDAF9dMpTK5VSnwL/FZ6QhDizw3VHWHXwYzYc24Iv4CPeGcc1/a/gyj7T6BGfbnV4QnQ7oQwB9W/z0AaMBGRmLHFJBIwA2yt28cHBj9ldVQSAO74HV/WbzuSeE4hzxlocoRDdVyhDQG3PAAzAA3wjPOEIYWryNbP+6EY+OriWskbzRvGc9KFc3W86I3sMlzJOITpBKENAg5RSLq21NzghXIzWWm6ZFGFR0VjFR4fXsq70Mxp9TTjtTqb0msjMftNljh4hOlkoQ0B3Y4735wL9gQ+VUl/XWr8V7uBEdDAMg301JXxw8GO2egoJGAGSY5K4ZdB1zOgzheSYJKtDFCIihTIE9BPgWgCt9V6l1HjMuYAkAYiL4g/42Vy2jVUHP6ak1pwRpG9Sb2b2m8747Dxc9lB+PYUQFyrU9QCOnXigtS5TSkmdnbhgZQ3lbDq2lTWHP6G6pQYbNsZkjmRmv+kMTRssZZxCXCKhJICPlVKvAa9gXgSeB3wS1qhERDEMgyP1x9jiKSC/rIDS+qMAxDpimNl3Olf2nSZLLgphgVASwCOYVT9fxlwU/iPgj+EMSnR/hmFwoPYQ+Z5C8j0FlDWYlTxOm4NRPUaQ5x5FXtYo4p3xFkcqRPQKJQG4gEat9SylVB/MROAEWsIameh2AkaA4uoStnoKyfcUUtlk3jAeY3eR585lrHsUIzNHEC9LLQrRJYSSAF7l5Fz+tYAdeBm4M1xBie7DH/Cz53gx+Z5CtnoKqWkx59WPc8QxMXsseVm5XJaRQ4wjJqxxGIZBU4ufxmYfDU0+GprNr8bTfvbS0OQz92v20djsx+m0YxgGDpsNu92Gw97+u91ua93WbrvtDPu1/W4783ucaX+Xw0FGSiyZqXEkxbvkOoi4JEJJAAO01rcBaK1rgB8rpfLDG5boyrwBH7pyD1s8BRR4dlDvM1drS3QlMLXXRPKycslJH3peVTyBgEFjS7DzbjrRObfpzJu8NDb7T+vA2/5snDZl4dm5nHbiYxzY7Xa8Pj8Bw8AfMAgEzO/n+36dJcZlJzM1nh4pcWSmxtEjNfg9+DglMUYShOgUofyFGkqpXK11AYBSajjmtQARRZr9Leyo0OR7Cigs30mTvxmA1JhkrugzlbFZoxiSOuicM3Aer2tm824PW4sqqKptau3Em1r85x1TbIyDhFgnacmx9MpMJCHWSUKck4RYJ/Ftfk6IcxEf6yAh1tVuu8tp3k3c0bJ/AcNMBicSwokEYQTaJAqjzfYzfT9le9v3avt8s9dPRU0TFdXBr5omSjtYotLpsJtJISWWHqnxpyWItKRY7HZJEOLcQkkAjwLvK6UOYVYBZQH3hTUq0SU0+hopKN/JVk8h2ys03oCZ93vEpTOt9yTyskYxMKX/OadlKD/eyKbdHjbt9rD3UHXr3OInOmJ3WnyHnXf8iU78REfeuo8Dhz2800HYbTbsDhtYNKt0Y7OPiuomyoMJoby6sd3j7ZUNnGliXofdFhxOOvNZRHpKbNjbTnQPoUwFsTI4IdwY4Kbg1zuA3J4Zgepa6tlWvoN8TwG6cg8+wzwyz05wk+fOJS9rFP2S+pxzCOJIRT2bd3vYqD2UHDWPrm02yOmXxnjlZlyOm4wUuRh8NvGxTvpmJdE368x/as0t/mBiaJ8gTiSJnSVnnrXdbrORnhxjnj2ckiCG2WzgC7SeHYnIFspUEIOAh4EvAGnAk5iLvYsIUd1cw1ZPIVs8hRQdLyZgBADok9SLse5c8rJyz7mcomEYHPLUs0mXsUl7OBwcvnDYbYwalME45WbcMDcpieG9GBxNYmMc9M5MpHfmmVc88/r8VNQ0BxNC48lkEUwQew4eZ/cZXmcDUpNizDOIU4aXegR/jnHJYjuRoMMEoJSag1nyOR5zgff7gOe11j+7RLGJMKporGyt0d9XfQAjODAzMKW/WaPvzj3nzVmGYbDvSK3Z6e/2UFbVCJhj1GOHZTIux03esEwS41xh//eI07mcDnpmJNAzI+GM233+AJW1bRJEdRN1zX5Ky2opr26iuLSGosPVZ3xtSoLrjNcfTiSI+FiZxqM7ONv/0kJgATBFa10EoJQKXJKoRFhUNFay6dhWCrZsp7jqAAA2bAxJG2gO77hHkR6Xdtb3CAQM9hw6ziZtjulX1ZoXg2NdDiYOz2K8cjN6SA/iYqQD6OqcDjtZafFkpcUD5oI6bS+I+wMBjte2nPH6Q3l1EwfLatl3pOaM750Y5+zwDCIzNY4ECw4KAoaB1xugyeunucUsPGj2Br9a/Ccft3l+UJ80BmQlkp1+5iTa3Z3tr3Q08CDmVBD7gdfOsb/ogmpb6thcto3xiP5pAAAVEElEQVSNx7ZQXF0CgMNmZ0RGDnnuUYx2jyQlJvms7+HzB9h1oIrN2sPm3R5qGsyLwQmxTqaO6sl45WbkwAwZFogwDrtZbdQjNY6cfqcfGAQMg+q6ltOvPwSrmUor6ik5dnp1FUB8rIMeKfHtzhraJoi4GEe7DrnpRMccfK7t46Y2nXi7fU99rff8K83AnKTQnRbHyEE9GDUogxED0iPmDMdmnKPYWSnlBG4FHsC8ALwS+IPWennYo2vD46m94Krsjsr8IlWjr4mtnkI2HstHVxURMALYsDEsfQgTs/O4ZvhkGmvOfjLn9fnZvq+KTbqM/KJy6pt8gHnqPzbHzXjlZnj/dJyO7n+xMNp+P86mM9vCMAxqG7wdVjGVVzfRfAHlv6Gw2SAuxkmsy05sjJM4l4PYGAdxMQ5iXI52j2ODP8e6Tj6Oi3HgdNqpavCxflspO0sqaWw2Y3XYbQzpncLIwWZCGNAzGXsXvi/D7U7uMLhzJoC2lFJu4H7gfq31mE6ILWSSAM7O6/eyvWIXG4/lU1ixE2/A7LAHpPRjQnYe47JGkxabCnTcHk0tPgqKK9mky9i6t6L1jzM9OZbxwU5/WN+0iKsxj4bfj1BdyrYwDIP6pjalrtWNrWcPLV4/sTHOk51yjNlpx5zyODamTefdplN3OuydcrPcifbw+QMUl9ZQuK+S7fsq2H+ktrWcOSnexchBGYwalMHIQRmkJXWtZUo7LQFYSRLA6fwBP7uP72XjsXzyywpp8jcBkJ2QxcTsPMZn55GVkHna69q2R0OTl/yicjZpD4X7KvH6zDMDd1oc45U5pj+oV0qXPsK5WJH6+3EhpC3a66g9ahta2LG/iu37KincV8HxupNTo/V1JzFqsJkQhvVNs7yk9mwJIDIGsqKIYRjsrznAxmP5bCrbSm1LHQDpsWlM7zOJCdlj6ZvU66xHP9V1zXyUf5hNuz3s3F+FP2Dm1t6Zia1H+v2ykmS6ASE6kJwQw6TLspl0WTaGYXC4vJ7CYvPsQB+s5pCnjnc/PUCM047qn86oQRmMGpxBz4yELvV3JQmgmyitO8qmY/lsPJZPeVMlYM69M73PZCZmj2Vw6oAz3pFrGAYVNWZJX3FpDXtLq9lXWkOwz2dAz+TWTr9XjzPXkwshOmaz2ejrTqKvO4kbJ/Wn2etn98HjFBabZwcFxeYX/wc9UuJah4suG5huSTVUW5IAurCKxio2lZmd/uG6IwDEOGKYmD2WCdl5jMjIOW3uncZmH/uO1LR2+MVHaqipP3l6arfZUAMyGD04g/E5bjLTZD5+ITpTrMtB7uAe5A7uAQyjsqaJwn2VFBZXsGN/Fau3lrJ6ayl2m43BvVPMaweDMxjUM+WSX1+TawBdTG1LHVvKtrHhWD7F1fsBcNgcXNZDMTE7j9zMy1qnVvYHAhz21FMc7PD3ldZQWl5P24bKSIllcK8UBvdOZXDvFAb0TKZv77Ru0x6XQnf6/Qg3aYv2Ors9/IEA+4/UmglhXwXFpTWts84mxjm5bODJi8mdNVWKXAPo4pp8TWz1bGfjsXx2Ve1pLdvMSR/KhOwxjHXnkuBKoKq2mYKi4+wNHt3vP1pDi/dkOWesy4Hqn8ag3ikM7mV2+OnJXasiQYho5rDbGdInlSF9Url9+iDqm7zs3F9F4b4KCvdVsmFXGRt2lQHQJzPRHC4anMHIgRlhuXYgCcAiXr+X7ZXaLNss33GybDO5HxOyxzAqPZeqKig+UMPf1++l+EhN6123YM7X0tudGDy6T2FI71R6ZyZGXImmEJEsMc7FhOFZTBieZa6dXdHQenaw+8Bx3ttwkPc2HOTz1+cwc1zfTv98SQCXUMAIsLsqWLbpKaDRZ5ZtZiW4UUmXkdIyEE+Zg48KanjVk0+gzfBcamIMY4dlMri3OZwzsGdyxNyNKIQwLyafmNzv+on98Pr87D5YTfGRGkYPOb2cuzNID3KJfHpkE2/uXd66ZGKiI5n+thy85b0ozXdQ0hIAjgHmSlWD+6QwJNjZD+6VQkZKbJcqHxNChJfL6WBk8HpAuEgCuASqm2t4dddCAgEbzuMDqT+aRWNtOuWYHXrPjLhgZ292+H3ciRExxYIQomsLWwJQStmB5zAXkmkGvnRiVtHg9puAx4IPNwOPaK27R0nSeXq7eCU+w0fL/pHE1w1mVO8UBueZHf6gXikyXbIQwhLhPAOYDcRpracopSYDTwO3AyilkoFfA1dprcuVUt8HMgFPGOOxREVjFeuOfEagKZ6bhk1lzoyhMpQjhOgSwpkApgPvAmit1yulJrTZNhUoAJ5WSg0G/qq1Pmvnn56egNN54dMNu91nn/I4XF5btwiDAK6K4dw/b3SXuXBrVXt0VdIeJ0lbtBfJ7RHO3igFaLuckF8p5dRa+zCP9mcCeUAdsEYp9YnW+kwr1AFQVdVwwYFYdXNLWYOHjw9+SqAxkdvVVOpqGqm75FGcTm72aU/a4yRpi/YioT3OlsDCeaWxBmj7yfZg5w9QAWzQWh/VWtcBqzGTQURZvHsFYBBXeVlYaniFEOJihDMBrAVuBgheAyhos20TMEoplRlccGYysCOMsVxypXVH2VaxjUB9MrNzp+K6iOErIYQIh3AOAS0GrlNKrcO8cfVBpdR3gSKt9RKl1A+BFcF9F2itC8MYyyW3UC8HGyRWj2L6Lb2sDkcIIU4TtgSgtQ4AXznl6V1ttr8OvB6uz7fSgZpD7KreRaAulbvHTcFhl5p+IUTXIz1TGPxr19sApNeNYeKIbIujEUKIM5ME0MmKju+juG4v/poM5k2aHNFLKQohujdJAJ3IMAz+tdM8+s9uHhtcEEIIIbomSQCdaFfVHg41HsB/3M3nplwud/wKIbo0SQCdxDAMFgSP/gcY41H90y2OSAghzk4SQCfZVr6DsuYj+CuzuXfahHO/QAghLCYJoBMEjABv7FqOYUCO63IG9kyxOiQhhDgnSQCdYNOxbVR6PQQqejNv2jirwxFCiJBIArhI/oCfRfodjICN0UmT6Z2ZaHVIQggREkkAF2n9kU3U+KsIVPTlnqljrA5HCCFCJgngIvgCPt7c8x5GwMbl6dPITIu3OiQhhAiZJICLsObgpzQEajDKB3Dn1JFWhyOEEOdFEsAFavF7WVa8EsPvYEb2DFKTYq0OSQghzoskgAv0Qclamox6bOUDuX3KcKvDEUKI8yYJ4AI0+Zp5d/8HGD4n1/S/ksQ4l9UhCSHEeZMEcAFW7PsIL004KoZw88ShVocjhBAXRBLAeWrwNvB/B1djeF3cNPQq4mLCuaiaEEKEjySA87SsaBV+WnBVDeO6sYOsDkcIIS6YHL6eh9qWOtaUrsXwxjJ7+ExcTsmfQojuS3qw8/Cmfp+AzUd89XCuGN3P6nCEEOKiyBlAiI43V/Np2acEWuK4K/cqWehdCNHtSS8Wojd2rsCwBUipG8mkEb2tDkcIIS6anAGEoLyxki0Vmwg0J3Dv2KtloXchRESQM4AQLNixHGwGmY2jGTMk0+pwhBCiU8gZwDkcrS9j+/ECAo1J3Ddxpiz0LoSIGHIGcA6vFb4NNoNevrGy0LsQIqJIAjiLg7WHKarfSaA+hc9PusLqcIQQolNJAjiLVwveBmAgExjUK9XiaIQQonNJAuhA8fESDjQVEahN5/NTplsdjhBCdDpJAB14tXAZAMNjJtM7M8niaIQQovNJAjiDXRVFHGkpIVDdg3+bOtnqcIQQIiwkAZzCMAxe224e/Y9JmkZmqiz0LoSITJIATrGtbCflvlKM41ncO2Wi1eEIIUTYhO1GMKWUHXgOGAM0A1/SWhe12f6/wDSgNvjU7Vrr6nDFEwrDMJi/06z8uTzjClnoXQgR0cJ5J/BsIE5rPUUpNRl4Gri9zfZxwA1a6/IwxnBeNhzZRnXAA1W9ufuWsVaHI4QQYRXOIaDpwLsAWuv1wIQTG4JnB8OAvyil1iqlvhDGOEISMAK8sWs5hgEzsmWhdyFE5AvnGUAK0HZIx6+UcmqtfUAi8CzwP4ADWKWU2qi13tbRm6WnJ+B0Oi44GLc7+azb39PrqKcKR3U/HnpgKnGxkT1N0rnaI9pIe5wkbdFeJLdHOHu5GqBty9mDnT9AA/A7rXUDgFLqA8xrBR0mgKqqhgsOxO1OxuOp7XC7P+Dnn/lvYQRsXNt3JrU1jXS8d/d3rvaINtIeJ0lbtBcJ7XG2BBbOIaC1wM0AwWsABW225QAfK6UcSikX5nDR5jDGclYf7FtPEzW4qgdw87jLrApDCCEuqXCeASwGrlNKrQNswINKqe8CRVrrJUqpV4D1gBd4SWu9PYyxdMgb8LF8/0oMw84tg6+Vhd6FEFEjbAlAax0AvnLK07vabH8KeCpcnx+qd4vW0GKrJ/b4UK65eqjV4QghxCUT2Vc6z6HF38LKAx9iGA7uGH69LPQuhIgqUd3jLdn1IT57I4l1w5h22QCrwxFCiEsqas8AGn1NfHRkNUbAyT2jbpCF3oUQUSdqzwAWbl9JwN5CWsMIJgztbXU4QghxyUXlGUC9t4H1nk8w/C4+N/YGWehdCBGVovIM4PWCdzHsXjKbcxk1IMvqcIQQwhJRdwZQ3VTDlsoNGL5Y7p9wvdXhCCGEZaLuDOCVbe9g2P309ucxtHeG1eEIIYRlouoMoKKhiu21Wwi0xPPA5ddZHY4QQlgqqs4AXspfBrYAg2zj6etOsTocIYSwVNQkgKN1HooaCzGaEvn3yVdbHY4QQlguahLAP/KXgs1AxVxOdlqS1eEIIYTloiIBFFcc4kDzLozGZO6fPNPqcIQQokuIigTw+zULwAajE6eSnhRndThCCNElRHwC2FNRwqHmPVCfxn2TZlgdjhBCdBkRnwDe3bkRgMvTZ5AUH2NxNEII0XVE/H0AV/aZSsy+nsy7fLLVoQghRJcS8Qlg9KBeXHN5Trdf2FkIITpbxA8BCSGEODNJAEIIEaUkAQghRJSSBCCEEFFKEoAQQkQpSQBCCBGlJAEIIUSUkgQghBBRymYYhtUxCCGEsICcAQghRJSSBCCEEFFKEoAQQkQpSQBCCBGlJAEIIUSUkgQghBBRShKAEEJEqYheEEYpZQeeA8YAzcCXtNZF1kZlDaWUC3gBGAjEAk9orZdYGlQXoJTKAjYB12mtd1kdj5WUUj8EbgNigOe01n+zOCRLBP9W/oH5t+IHHorU341IPwOYDcRpracAPwCetjgeK90HVGitZwA3Ab+3OB7LBf/Q/ww0Wh2L1ZRSVwFTgWnAlUA/SwOy1s2AU2s9FfgZ8KTF8YRNpCeA6cC7AFrr9cAEa8Ox1L+An7R57LMqkC7kN8CfgFKrA+kCbgAKgMXAUmCZteFYajfgDI4gpABei+MJm0hPAClAdZvHfqVURA97dURrXae1rlVKJQNvAD+2OiYrKaUeADxa6xVWx9JFZGIeIN0NfAV4RSllszYky9RhDv/sAp4H/tfSaMIo0hNADZDc5rFdax21R75KqX7AKuBlrfWrVsdjsS8A1ymlPgTygJeUUj2tDclSFcAKrXWL1loDTYDb4pis8h3MtsjBvH74D6VUnMUxhUWkHw2vBWYBC5RSkzFPcaOSUiobeA/4utb6/6yOx2pa6ytO/BxMAl/RWh+1LiLLfQx8Syn1P0AvIBEzKUSjKk4O+1QCLsBhXTjhE+kJYDHmUd46wAY8aHE8VvoRkA78RCl14lrATVrrqL8AKkBrvUwpdQXwGebIwCNaa7/FYVnlt8ALSqk1mBVRP9Ja11scU1jIdNBCCBGlIv0agBBCiA5IAhBCiCglCUAIIaKUJAAhhIhSkgCEECJKRXoZqOhmlFIDgX3AX7TWX27zfB6wBXhQa/3iRbz/VZjTHBRhlgbHAcuBn2it6y7wPVdprWcGfza01hd0B61SahzwJrD/xH0KSqkewIn7Nk7cqHbifoVrtNbRWqsvOoEkANEVVQA3KqUcbWrR5wKeTnr/jVrrq6DdLKl/BD5/ge93VeeExa3AP7XWPzrxRLCDzwNQSj0efO7xTvo8EeUkAYiuqA7IB67AnLoC4Hpg5YkdlFJfx+ywE4EW4F6gAXNq5yuBvcBG4Ida67c7+iCttVcp9R/AQaXUN4Ba4NeYnboDeFFr/dvgmcNPMO8QHYR5w9SXgvuilPpUaz0p+POfgCnBj7jz1CnIg3el/w7z7KMc+DKQA3wtuL1Ja/2zUBpKKbUf+BQzScwAbgS+jTm8uwnzhq4mpdSNmDNbujDPsB6Sswch1wBEV7UAuAtAKTUR2IbZ0aOUSsGc6vsqrfUozCGdr2utDwL/iXk0/xiw7myd/wnBKSCqMDvhh4LPjQMuB25XSs0I7joV+BYwHLPzfkRr/c3g/pPavOVKrfUY4H3Mzr2VUioGeD0Y7xjM2Uhf01ovD/78p1A7/zbe0VorzLl7HgKmaq3zgDLgUaWUG/glcIPWeiywAvjVeX6GiECSAERXtQS4KTgl71xg/okNWusa4HPAPKXULzDne0oKbvs75vz+nwO+dx6fZwRfdy1wm1IqH/PIui+QG9xntTYZwMvA1R2815vB79sxZ9lsKweo0lpvCMb7L2CoUir1PGI91afB7zOBYcD6YPy3YyarSUB/YFXw+a8H9xNRThKA6JKCF2S3Yq7pcDXth3/6AZ8AacA7wIuYF3QJztrYD3N4s28onxWcKC8Vc9jIAXxfa50XPIqejHmNANqvoWCngzUV2sw4a5yI65TXncrGxU02dmI+JwewoE3sl2N29g7g4zbPTwTuvIjPExFCEoDoyhZgDl1sPGUa74lAkdb6t8AGYA4nO9D/Bj7AnNL3RaXUWTtWpVQs5jj+i1rrhuBrH1JKuZRSSZizZE4O7j5dKdUneFZyP2bygfNbZ0IDPYLDWiil7gFKtNaVIb7+bD4E5iilsoJz+f8R83rAp8AUpVROcL+fYC6GI6KcJADRlS3FvLg5/5Tn3wPsSqkdwGbMhTsGBS+u3g38P631G5jVRGcaBpqglMoPDodsAo5jju2DOQ6/B7PkdCPwd631h8FtpcBLwA7gMPDX4PNvAVtDmTNea92MOaT1e6VUIeYR+txzvS4UWuutwE8xk9h2zKT4y+A1ji9gToteAIzj/IbHRISS2UCFCEGwCujxE+WjQkQCOQMQQogoJWcAQggRpeQMQAghopQkACGEiFKSAIQQIkpJAhBCiCglCUAIIaLU/wc5hBACdBiYrwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x254a59fd390>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(valid_df['accuracy'],label='Validation Set')\n",
    "plt.plot(train_df['accuracy'],label='Train Set')\n",
    "\n",
    "plt.legend()\n",
    "plt.xlabel('Max Depth of Tree')\n",
    "plt.ylabel('Accuracy')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Looking at the graph above, accuracy seems to increase at 1. For the train set, accuracy continued to increase with the depth of the tree. However, I'd say it's a sign of overfitting and high variance as the accuracy of the validation set doesn't increase, rather it decreased at 3. \n",
    "\n",
    "I chose PCA with 150 components, as shown in our earlier exercise, 150 components will cover 90% of the variance explained. PCA will also help with overfitting and increase prediction accuracy and speed. This is essential, given that there are more than 4000 components in the set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 170,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid_df['accuracy'].idxmax()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top DECISION TREE accuracy on validation set: 0.654 with max_depth of 5\n"
     ]
    }
   ],
   "source": [
    "print ('Top DECISION TREE accuracy on validation set: {:.3f} with max_depth of {}'.format(valid_df.loc[4,'accuracy'],\n",
    "                                                                                      valid_df.loc[4,'tree__max_depth']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Random Forest Classifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestClassifier"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "estimators = [100,200,300,400,500]\n",
    "\n",
    "pipe = Pipeline([('pca',PCA(n_components=150)),('forest', RandomForestClassifier())])\n",
    "\n",
    "valid_scores_forest = []\n",
    "\n",
    "for estimator in estimators:\n",
    "    pipe.set_params(forest__n_estimators=estimator)\n",
    "    pipe.fit(X_train,y_train)\n",
    "    accuracy = pipe.score(X_valid,y_valid)\n",
    "    valid_scores_forest.append({'accuracy':accuracy,'estimator':estimator})\n",
    "\n",
    "forest_df = pd.DataFrame(valid_scores_forest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>accuracy</th>\n",
       "      <th>estimator</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.7775</td>\n",
       "      <td>500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.7725</td>\n",
       "      <td>400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.7675</td>\n",
       "      <td>200</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.7625</td>\n",
       "      <td>100</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.7575</td>\n",
       "      <td>300</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   accuracy  estimator\n",
       "4    0.7775        500\n",
       "3    0.7725        400\n",
       "1    0.7675        200\n",
       "0    0.7625        100\n",
       "2    0.7575        300"
      ]
     },
     "execution_count": 159,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest_df.sort_values(by='accuracy',ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "forest_df['accuracy'].idxmax()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Top RANDOM FOREST accuracy on validation set: 0.777 with max_depth of 500\n"
     ]
    }
   ],
   "source": [
    "print ('Top RANDOM FOREST accuracy on validation set: {:.3f} with max_depth of {}'.format(forest_df.loc[4,'accuracy'],\n",
    "                                                                                      forest_df.loc[4,'estimator']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tuned models on entire train set (4000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "#tree with max_depth of 5\n",
    "tree = DecisionTreeClassifier(max_depth=5)\n",
    "pca = PCA(n_components=150)\n",
    "pipe = Pipeline([('pca',pca),('tree',tree)])\n",
    "pipe.fit(X_tr,y_tr)\n",
    "acc_tree_tuned = pipe.score(X_te,y_te)\n",
    "\n",
    "#forest with n_estimators of 500\n",
    "forest = RandomForestClassifier(n_estimators=500)\n",
    "pipe = Pipeline([('pca',pca),('forest',forest)])\n",
    "pipe.fit(X_tr,y_tr)\n",
    "acc_forest_tuned = pipe.score(X_te,y_te)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Decision tree accuracy (test set): 0.642\n",
      "Random forest accuracy (test set): 0.776\n"
     ]
    }
   ],
   "source": [
    "print ('Decision tree accuracy (test set): {:.3f}'.format(acc_tree_tuned))\n",
    "print ('Random forest accuracy (test set): {:.3f}'.format(acc_forest_tuned))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}