MachineLearning_HW1.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.7"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uMMASUUdm4E7"
      },
      "source": [
        "# Homework 1 (Coding)\n",
        "\n",
        "**Due Tuesday 20th September, 11:59pm**\n",
        "\n",
        "If you are working in pairs make sure to add your team member’s name on Gradescope when submitting"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mqy1LtZCIQuG"
      },
      "source": [
        "\"\"\"\n",
        "Import libraries that you might require\n",
        "\"\"\"\n",
        "\n",
        "import numpy as np\n",
        "import math\n",
        "import matplotlib.pyplot as plt\n",
        "import operator\n",
        "import pandas as pd\n",
        "import io\n",
        "import requests\n",
        "from sklearn.metrics import accuracy_score"
      ],
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RAfePG6PyjNp"
      },
      "source": [
        "# Load Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "y2lGPqwTymkU",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 248
        },
        "outputId": "fe08f06e-95f4-4c65-d984-e92d36120012"
      },
      "source": [
        "url=\"https://raw.githubusercontent.com/susanli2016/Machine-Learning-with-Python/master/diabetes.csv\"\n",
        "response = requests.get(url).content\n",
        "diabetes = pd.read_csv(io.StringIO(response.decode('utf-8')))\n",
        "diabetes.head()"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  \\\n",
              "0            6      148             72             35        0  33.6   \n",
              "1            1       85             66             29        0  26.6   \n",
              "2            8      183             64              0        0  23.3   \n",
              "3            1       89             66             23       94  28.1   \n",
              "4            0      137             40             35      168  43.1   \n",
              "\n",
              "   DiabetesPedigreeFunction  Age  Outcome  \n",
              "0                     0.627   50        1  \n",
              "1                     0.351   31        0  \n",
              "2                     0.672   32        1  \n",
              "3                     0.167   21        0  \n",
              "4                     2.288   33        1  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e942b3f6-6ce1-4dd9-9acc-af7c0c329a50\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Pregnancies</th>\n",
              "      <th>Glucose</th>\n",
              "      <th>BloodPressure</th>\n",
              "      <th>SkinThickness</th>\n",
              "      <th>Insulin</th>\n",
              "      <th>BMI</th>\n",
              "      <th>DiabetesPedigreeFunction</th>\n",
              "      <th>Age</th>\n",
              "      <th>Outcome</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>6</td>\n",
              "      <td>148</td>\n",
              "      <td>72</td>\n",
              "      <td>35</td>\n",
              "      <td>0</td>\n",
              "      <td>33.6</td>\n",
              "      <td>0.627</td>\n",
              "      <td>50</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>85</td>\n",
              "      <td>66</td>\n",
              "      <td>29</td>\n",
              "      <td>0</td>\n",
              "      <td>26.6</td>\n",
              "      <td>0.351</td>\n",
              "      <td>31</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>8</td>\n",
              "      <td>183</td>\n",
              "      <td>64</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>23.3</td>\n",
              "      <td>0.672</td>\n",
              "      <td>32</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1</td>\n",
              "      <td>89</td>\n",
              "      <td>66</td>\n",
              "      <td>23</td>\n",
              "      <td>94</td>\n",
              "      <td>28.1</td>\n",
              "      <td>0.167</td>\n",
              "      <td>21</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>137</td>\n",
              "      <td>40</td>\n",
              "      <td>35</td>\n",
              "      <td>168</td>\n",
              "      <td>43.1</td>\n",
              "      <td>2.288</td>\n",
              "      <td>33</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e942b3f6-6ce1-4dd9-9acc-af7c0c329a50')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e942b3f6-6ce1-4dd9-9acc-af7c0c329a50 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e942b3f6-6ce1-4dd9-9acc-af7c0c329a50');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PGU3nr9PzMq3"
      },
      "source": [
        "# Split the data into training, validation and testing sets\n",
        "\n",
        "from sklearn.model_selection import train_test_split\n",
        "\n",
        "data = diabetes.to_numpy()\n",
        "X, y = data[:, :-1], data[:, -1]\n",
        "\n",
        "X_train, X_val_test, y_train, y_val_test = train_test_split(X, y, stratify=y, test_size = 0.25, random_state=66)\n",
        "X_val, X_test, y_val, y_test = train_test_split(X_val_test, y_val_test, test_size = 0.4, random_state=66)"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "len(X_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NCQ0Zn9WVDRJ",
        "outputId": "67a0efe4-4557-4844-b30c-98b3fb33a746"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "576"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HMQDN_YzIQuK"
      },
      "source": [
        "# Question 3: KNN Classification"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "WgH67-bV4AJ9"
      },
      "source": [
        "We will implement the KNN algorithm for the diabetes dataset. Refer to the pdf and the following functions for the instructions. Complete all the functions as indicated below. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5qmuIim2IQuK"
      },
      "source": [
        "\"\"\"\n",
        "Task 1: Classification\n",
        "\n",
        "Please implement KNN for K: 3, 5, and 7 \n",
        "with the following norms:\n",
        "L1\n",
        "L2\n",
        "L-inf\n",
        "\n",
        "\"\"\"\n",
        "\n",
        "def distanceFunc(metric_type, vec1, vec2):\n",
        "    \"\"\"\n",
        "    Computes the distance between two d-dim vectors\n",
        "    Args:\n",
        "        metric_type: String\n",
        "        vec1 (numpy vector): Vector\n",
        "        vec2 (numpy vector): Vector\n",
        "    Returns:\n",
        "        distance (float): distance between the two vectors\n",
        "    \"\"\"\n",
        "\n",
        "    diff = vec1 - vec2\n",
        "    if metric_type == \"L1\":\n",
        "        #complete\n",
        "        distance = sum(abs(diff))\n",
        "\n",
        "    if metric_type == \"L2\":\n",
        "        #complete\n",
        "        distance = np.sqrt(sum(diff**2))\n",
        "        \n",
        "    if metric_type == \"L-inf\":\n",
        "        #complete\n",
        "        distance = np.max(abs(diff))\n",
        "        \n",
        "    return distance\n",
        "\n",
        "\n",
        "def computeDistancesNeighbors(K, metric_type, X_train, y_train, sample):\n",
        "    \"\"\"\n",
        "    Compute the distances between every datapoint in the train_data and the \n",
        "    given sample. Then, find the k-nearest neighbors.\n",
        "    Return a numpy array of the label of the k-nearest neighbors.\n",
        "    \n",
        "    Args:\n",
        "        K (int): K-value\n",
        "        metric_type (String): metric type\n",
        "        X_train (numpy array): Training data\n",
        "        y_train : Training labels\n",
        "        sample (numpy vector): Sample whose distance is to computed with every entry in the dataset\n",
        "        \n",
        "    Returns:\n",
        "        neighbors (list): K-nearest neighbors' labels\n",
        "    \"\"\"\n",
        "\n",
        "    # You will also call the function \"distanceFunc\" here\n",
        "    # Complete this function\n",
        "    output = []\n",
        "    for i in range(len(X_train)):\n",
        "      output.append(distanceFunc(metric_type, X_train[i], sample))\n",
        "    neighbors = y_train[np.where(np.argsort(np.argsort(output))<K)]\n",
        "    return neighbors\n",
        "\n",
        "\n",
        "def Majority(neighbors):\n",
        "    \"\"\"\n",
        "    Performs majority voting and returns the predicted value for the test sample\n",
        "    Args:\n",
        "        neighbors (list): K-nearest neighbors' labels\n",
        "    Returns:\n",
        "        predicted_value (int or float): predicted label for the given sample\n",
        "    \"\"\"\n",
        "    \n",
        "    # Performs majority voting\n",
        "    # Complete this function\n",
        "    dict_ans = {}\n",
        "    for i in neighbors:\n",
        "      if not dict_ans.get(i,''):\n",
        "        dict_ans[i] = 1\n",
        "      else:\n",
        "        dict_ans[i] += 1\n",
        "    predicted_value = max(dict_ans, key=dict_ans.get)\n",
        "    \n",
        "    return predicted_value\n",
        "\n",
        "\n",
        "def KNN(K, metric_type, X_train, y_train, X_val):\n",
        "    \"\"\"\n",
        "    Returns the predicted values for the entire validation or test set\n",
        "    Args:\n",
        "        K (int): K-value\n",
        "        metric_type (String): metric type\n",
        "        X_train (numpy array): Training data\n",
        "        y_train : Training labels\n",
        "        X_val or X_test (numpy array): Validation or test data\n",
        "    Returns:\n",
        "        predicted_values (list): output for every entry in validation/test dataset \n",
        "    \"\"\"\n",
        "    \n",
        "    # Complete this function\n",
        "    # Loop through the val_data or the test_data (as required)\n",
        "    # and compute the output for every entry in that dataset  \n",
        "    # You will also call the function \"Majority\" here\n",
        "    predictions = []\n",
        "    for i in range(len(X_val)):\n",
        "      neighbors = computeDistancesNeighbors(K, metric_type, X_train, y_train, X_val[i])\n",
        "      predictions.append(Majority(neighbors))\n",
        "    \n",
        "        \n",
        "    return predictions\n",
        "\n",
        "\n",
        "def evaluation(predicted_values, actual_values):\n",
        "    \"\"\"\n",
        "    Computes the accuracy of the given datapoints.\n",
        "    \n",
        "    Args:\n",
        "        predicted_values: vector\n",
        "        actual_values: numpy vector\n",
        "    \n",
        "    Returns:\n",
        "        accuracy (float): accuracy\n",
        "    \"\"\"\n",
        "    \n",
        "    return accuracy_score(predicted_values, actual_values)\n",
        "\n",
        "\n",
        "def main():\n",
        "    \"\"\"\n",
        "    Calls the above functions in order to implement the KNN algorithm.\n",
        "    \n",
        "    Test over the following range K = 3,5,7 and all three metrics. \n",
        "    In total you will have nine combinations to try.\n",
        "    \n",
        "    PRINTS out the accuracies for the nine combinations on the validation set,\n",
        "    and the accuracy on the test set for the selected K value and appropriate norm.\n",
        "    \"\"\"\n",
        "    \n",
        "    ## Complete this function\n",
        "    \n",
        "    K = [3,5,7]\n",
        "    norm = [\"L1\", \"L2\", \"L-inf\"]\n",
        "    \n",
        "    print(\"<<<<VALIDATION DATA PREDICTIONS>>>>\")\n",
        "\n",
        "    for k in K:\n",
        "        for metric_type in norm:      \n",
        "          print('K = ',k, 'Norm = ',metric_type,': ',evaluation(KNN(k, metric_type, X_train, y_train, X_val), y_val))\n",
        "\n",
        "    ## Complete\n",
        "\n",
        "    print(\"<<<<TEST DATA PREDICTIONS>>>>\")\n",
        "    for k in K:\n",
        "        for metric_type in norm:      \n",
        "          print('K = ',k, 'Norm = ',metric_type,': ',evaluation(KNN(k, metric_type, X_train, y_train, X_test), y_test))\n",
        "\n",
        "    ## Complete\n",
        "    "
      ],
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DW4D4AbBjhK5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e99bac9a-bde0-4367-d3fa-03caed890989"
      },
      "source": [
        "# Finally, call the main function\n",
        "main()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<<<<VALIDATION DATA PREDICTIONS>>>>\n",
            "K =  3 Norm =  L1 :  0.7217391304347827\n",
            "K =  3 Norm =  L2 :  0.6956521739130435\n",
            "K =  3 Norm =  L-inf :  0.7304347826086957\n",
            "K =  5 Norm =  L1 :  0.7565217391304347\n",
            "K =  5 Norm =  L2 :  0.7652173913043478\n",
            "K =  5 Norm =  L-inf :  0.7304347826086957\n",
            "K =  7 Norm =  L1 :  0.7304347826086957\n",
            "K =  7 Norm =  L2 :  0.7739130434782608\n",
            "K =  7 Norm =  L-inf :  0.7304347826086957\n",
            "<<<<TEST DATA PREDICTIONS>>>>\n",
            "K =  3 Norm =  L1 :  0.7402597402597403\n",
            "K =  3 Norm =  L2 :  0.7012987012987013\n",
            "K =  3 Norm =  L-inf :  0.7012987012987013\n",
            "K =  5 Norm =  L1 :  0.7272727272727273\n",
            "K =  5 Norm =  L2 :  0.7012987012987013\n",
            "K =  5 Norm =  L-inf :  0.7012987012987013\n",
            "K =  7 Norm =  L1 :  0.7532467532467533\n",
            "K =  7 Norm =  L2 :  0.7142857142857143\n",
            "K =  7 Norm =  L-inf :  0.7142857142857143\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(X_val[0])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "um6zLQktOHfr",
        "outputId": "f98ae61f-9a30-468e-dee8-4dff95468534"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "8"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XmURULzoXCvq"
      },
      "source": [
        "# Question 4: Decision Tree Classification"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S-_QAM3jXCvq"
      },
      "source": [
        "### Helper functions\n",
        "The block below contains helper functions for this task."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9KC2aNmoXCvr"
      },
      "source": [
        "import numpy as np\n",
        "from sklearn.tree import DecisionTreeClassifier\n",
        "import matplotlib.pyplot as plt\n",
        "\n",
        "# Below are a list of helper functions to use to help you on this task\n",
        "\n",
        "def train_decision_tree(X, y, depth=None, leaf_count=None):\n",
        "  \"\"\"\n",
        "  Trains a decision tree classifier on the given X, y data with the specified \n",
        "  tree depth d and max leaf node count max_leaf_num.\n",
        "  \n",
        "  Args:\n",
        "    X ((n,p) np.ndarray): The input Xs, which are in an n (number of samples) by\n",
        "                          p (number of features) matrix\n",
        "    y ((n,) np.ndarray): The input ys, which are in an n length array\n",
        "    depth (int): The maximum depth of the tree. A value of None means no restrictions\n",
        "             on the depth of the tree.\n",
        "    leaf_count (int): The maximum leaf count of the tree's leaf nodes. A value of None means \n",
        "    no restrictions on the leaf count of the tree.\n",
        "  \n",
        "  Returns:\n",
        "    clf(DecisionTreeClassifier): the trained decision tree classifier\n",
        "  \"\"\"\n",
        "  clf = DecisionTreeClassifier(max_depth=depth, max_leaf_nodes=leaf_count, criterion=\"entropy\", random_state=1)\n",
        "  clf.fit(X,y)\n",
        "  return clf\n",
        "\n",
        "def predict(clf, X_test):\n",
        "  \"\"\"\n",
        "  Uses a trained decision tree classifier to predict on a given test set.\n",
        "  \n",
        "  Args:\n",
        "    clf (DecisionTreeClassifier): Trained Decision Tree Classifer\n",
        "    X_test ((n,p) np.ndarray): The input Xs, which are in an n (number of samples) by\n",
        "                               p (number of features) matrix\n",
        "  \n",
        "  Returns:\n",
        "    y_pred ((n,) np.ndarray): The output predictions, which are in an n length array\n",
        "  \"\"\"\n",
        "  y_pred = clf.predict(X_test)\n",
        "  return y_pred\n",
        "\n",
        "def evaluate(predicted_values, actual_values):\n",
        "    \"\"\"\n",
        "    Computes the accuracy of the given datapoints.\n",
        "    \n",
        "    Args:\n",
        "        predicted_values: numpy array\n",
        "        actual_values: numpy array\n",
        "    \n",
        "    Returns:\n",
        "        a floating point number representing the accuracy\n",
        "    \"\"\"\n",
        "    from sklearn.metrics import accuracy_score\n",
        "    return accuracy_score(predicted_values, actual_values)\n",
        "  \n",
        "def plot_line_graph(x_vals, y_vals_1, y_vals_2, y_vals_1_label, y_vals_2_label, x_axis_label, y_axis_label, title):\n",
        "  \"\"\"\n",
        "  Plots a line graph of two lines of different values with common x-values\n",
        "  \n",
        "  Args:\n",
        "    x_vals ((j,) list): Values to be displayed on horizontal axis, where j is number of values\n",
        "    y_vals_1 ((j,) list): First set of values to be graphed on a line in respect to x_vals, where j is number of values\n",
        "    y_vals_2 ((j,) list): Second set of values to be graphed on a line in respect to x_vals, where j is number of values\n",
        "    y_vals_1_label (string): Label for first set of y values\n",
        "    y_vals_2_label (string): Label for second set of y values\n",
        "    x_axis_label (string): Label for x axis\n",
        "    y_axis_label (string): Label for y axis\n",
        "    title (string): Plot title\n",
        "  \"\"\"\n",
        "  \n",
        "  plt.plot(x_vals, y_vals_1, color='g', label=y_vals_1_label)\n",
        "  plt.plot(x_vals, y_vals_2, color='orange', label=y_vals_2_label)\n",
        "  plt.xlabel(x_axis_label)\n",
        "  plt.ylabel(y_axis_label)\n",
        "  plt.title(title)\n",
        "  plt.legend(loc='upper right')\n",
        "  plt.show()"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eP9j8K21XCvs"
      },
      "source": [
        "### Compare Accuracy for full classification dataset as well as smaller classification dataset\n",
        "We will be using the diabetes classification dataset. You are also given a smaller training dataset with the same data as the full dataset but with only half of the sample number. We will observe the performance changes when less data is available.\n",
        "\n",
        "To start, uncomment the code below and run to create small dataset. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LRothkZeXCvt"
      },
      "source": [
        "# We will also use the same diabetes classification dataset in Task 1.\n",
        "# Let's create a smaller version of the training dataset using only half of the data available\n",
        "\n",
        "train_sample_num_small = int(X_train.shape[0] / 2)\n",
        "X_train_small, y_train_small = X_train[:train_sample_num_small], y_train[:train_sample_num_small]"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PEsSQwRMXCvv"
      },
      "source": [
        "### Base Metrics on Full and Partial Data\n",
        "To start, you will be comparing the training and testing accuracies of both datasets given a vanilla decision tree.\n",
        "\n",
        "Note: Make sure to create two separate classifiers for each dataset."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xfRXN92DXCvv"
      },
      "source": [
        "def base_metrics(X_train, y_train, X_train_small, y_train_small, X_val, y_val, X_test, y_test):\n",
        "  \"\"\"\n",
        "  Create a decision tree classifer on the full dataset and the partial dataset (only half of n).\n",
        "    \n",
        "  Args: (Note that n is not the same among train and test sets, but merely refers to sample size)\n",
        "    X_train ((n,p) np.ndarray): Input feature matrix of full dataset for training/fitting\n",
        "    y_train ((n,) np.ndarray): Input label array of full dataset for training/fitting\n",
        "    X_train_small ((n,p) np.ndarray): Input feature matrix of partial/small dataset for training/fitting\n",
        "    y_train_small ((n,) np.ndarray): Input label array of partial/small dataset for training/fitting\n",
        "    X_val ((n,p) np.ndarray): Input feature matrix of full dataset for validation\n",
        "    y_val ((n,) np.ndarray): Input label array of full dataset for validation\n",
        "    X_test ((n,p) np.ndarray): Input feature matrix of full dataset for testing\n",
        "    y_test ((n,) np.ndarray): Input label array of full dataset for testing    \n",
        "\n",
        "  To observe:\n",
        "    train_acc_full_set (float): Training accuracy using a model trained on the full dataset\n",
        "    val_acc_full_set (float): Validation accuracy using a model trained on the full dataset\n",
        "    test_acc_full_set (float): Test accuracy using a model trained on the full dataset\n",
        "    train_acc_small_set (float): Training accuracy using a model trained on the small dataset\n",
        "    val_acc_small_set (float): Validation accuracy using a model trained on the small dataset\n",
        "    test_acc_small_set (float): Test accuracy using a model trained on the small dataset\n",
        "  \"\"\"\n",
        "  \n",
        "  # <---- Your code here ----->\n",
        "  train_acc_full_set = evaluate(predict(train_decision_tree(X_train, y_train, depth=None, leaf_count=None), X_train), y_train)\n",
        "  val_acc_full_set = evaluate(predict(train_decision_tree(X_train, y_train, depth=None, leaf_count=None), X_val), y_val)\n",
        "  test_acc_full_set = evaluate(predict(train_decision_tree(X_train, y_train, depth=None, leaf_count=None), X_test), y_test)\n",
        "  train_acc_small_set = evaluate(predict(train_decision_tree(X_train_small, y_train_small, depth=None, leaf_count=None), X_train_small), y_train_small)\n",
        "  val_acc_small_set = evaluate(predict(train_decision_tree(X_train_small, y_train_small, depth=None, leaf_count=None), X_val), y_val)\n",
        "  test_acc_small_set = evaluate(predict(train_decision_tree(X_train_small, y_train_small, depth=None, leaf_count=None), X_test), y_test)\n",
        "\n",
        "  # <---- Your code ends here ----->\n",
        "  \n",
        "  print(\"Train Accuracy on Full Dataset: \", train_acc_full_set)\n",
        "  print(\"Validation Accuracy on Full Dataset: \", val_acc_full_set)\n",
        "  print(\"Test Accuracy on Full Dataset: \", test_acc_full_set)\n",
        "  print(\"Train Accuracy on Small (Half) Dataset: \", train_acc_small_set)\n",
        "  print(\"Validation Accuracy on Small (Half) Dataset: \", val_acc_small_set)\n",
        "  print(\"Test Accuracy on Small (Half) Dataset: \", test_acc_small_set)\n",
        "  \n",
        "  return (train_acc_full_set, \n",
        "          val_acc_full_set, \n",
        "          test_acc_full_set,\n",
        "          train_acc_small_set, \n",
        "          val_acc_small_set, \n",
        "          test_acc_small_set)"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "QTH2bUBY3SCP"
      },
      "source": [
        "Uncomment the code below and run the code. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VM9BV2i3XCvx",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b291f460-30b3-476d-afcf-83c4e4e95575"
      },
      "source": [
        "base_metrics(X_train, y_train, X_train_small, y_train_small, X_val, y_val, X_test, y_test)"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Train Accuracy on Full Dataset:  1.0\n",
            "Validation Accuracy on Full Dataset:  0.7043478260869566\n",
            "Test Accuracy on Full Dataset:  0.7532467532467533\n",
            "Train Accuracy on Small (Half) Dataset:  1.0\n",
            "Validation Accuracy on Small (Half) Dataset:  0.7130434782608696\n",
            "Test Accuracy on Small (Half) Dataset:  0.6753246753246753\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1.0,\n",
              " 0.7043478260869566,\n",
              " 0.7532467532467533,\n",
              " 1.0,\n",
              " 0.7130434782608696,\n",
              " 0.6753246753246753)"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7gUUoJSO3SCR"
      },
      "source": [
        "### Question 4.1 Report on LaTeX\n",
        "Answer the following questions on LaTeX in the respective section.\n",
        "1. Report the results of the accuracies on LateX \n",
        "2. Which dataset had a higher difference between training and test accuracy? Briefly explain why."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6xiF5-LvXCvz"
      },
      "source": [
        "### Improving Decision Tree for Smaller Dataset by Tuning Hyperparameters\n",
        "Classifiers often overfit on smaller datasets, so now, we will optimize hyperparameters on tree depth and max leaf count to improve the performance of our model. \n",
        "\n",
        "Fill out the helper functions below which will take an array of hyperparameter values for tree depth and an array of hyperparameter values for max leaf count. The helper function will return a training and validation accuracy scores for each pair of hyperparameter values. This is referred to as grid search for hyperparameter tuning.\n",
        "\n",
        "At the end, the function identifies the best value of the tree depth and tree node count hyperparameters for a dataset, as well as the final training and testing scores.\n",
        "\n",
        "Note: Use the highest validation score to choose the optimal hyperparameter combination. If there is a tie, use the lower hyperparameter value."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k1nPIPFmXCvz"
      },
      "source": [
        "def grid_search_depth_and_leaf_count(depth_search_space, leaf_count_search_space, X_train, y_train, X_val, y_val):\n",
        "  \"\"\"\n",
        "  Perform a decision tree hyperparameter grid search on tree depth and leaf count given training and validation data.\n",
        "    \n",
        "  Args:\n",
        "    depth_search_space ((d,) list): Tree depth values to search over, i.e. [1, 3, 6, 10, 30]\n",
        "    leaf_count_search_space ((l,) list): Max leaf count values to search over, i.e. [2, 3, 4, 5, 6]\n",
        "    X_train ((n, p) np.ndarray): The input feature matrix for training\n",
        "    y_train ((n, p) np.ndarray): The input ys for training\n",
        "    X_val ((n, p) np.ndarray): The input feature matrix that will be used to validate accuracy scores\n",
        "    y_val ((n, p) np.ndarray): The input ys that will be used to validate accuracy scores\n",
        "    \n",
        "  To return:\n",
        "    best_depth (int): The depth count in the hyperparameter combination with the largest validation score\n",
        "    best_leaf_count (int): The leaf count in the hyperparameter combination with the largest validation score\n",
        "  \"\"\"\n",
        "  \n",
        "  \n",
        "  \n",
        "  # <---- Your code here ----->\n",
        "  acc = 0\n",
        "  for i in depth_search_space:\n",
        "    for j in leaf_count_search_space:\n",
        "      acc1 = evaluate(predict(train_decision_tree(X_train, y_train, depth=i, leaf_count=j), X_val), y_val)\n",
        "      if acc < acc1:\n",
        "        acc = acc1\n",
        "        best_depth = i\n",
        "        best_leaf_count = j\n",
        "\n",
        "  # <---- Your code ends here ----->\n",
        "  print(\"Chosen Depth: \", best_depth)\n",
        "  print(\"Chosen Leaf: \", best_leaf_count)\n",
        "  \n",
        "  return best_depth, best_leaf_count"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RvCBkKLJ3SCV"
      },
      "source": [
        "Uncomment and run the code below to and record the best depth and best leaf count hyperparameters on LaTeX in the respective section."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RyZZ2CHb3SCW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "60ad2c9c-16b8-4be7-b61a-2243cc30e234"
      },
      "source": [
        "# Search spaces for grid search to tune tree depth and leaf count hyperparameters\n",
        "depth_search_space = [1, 3, 6, 10, 30]\n",
        "leaf_count_search_space = [2, 3, 4, 5, 6]\n",
        "\n",
        "print(\"FULL DATASET\")\n",
        "grid_search_depth_and_leaf_count(depth_search_space, \n",
        "                                 leaf_count_search_space, \n",
        "                                 X_train, \n",
        "                                 y_train, \n",
        "                                 X_val, \n",
        "                                 y_val)\n",
        "\n",
        "print(\"\\nSMALL DATASET\")\n",
        "grid_search_depth_and_leaf_count(depth_search_space, \n",
        "                                 leaf_count_search_space, \n",
        "                                 X_train_small, \n",
        "                                 y_train_small, \n",
        "                                 X_val,\n",
        "                                 y_val)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "FULL DATASET\n",
            "Chosen Depth:  3\n",
            "Chosen Leaf:  4\n",
            "\n",
            "SMALL DATASET\n",
            "Chosen Depth:  1\n",
            "Chosen Leaf:  2\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1, 2)"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "T_oY8swF3SCX"
      },
      "source": [
        "### Question 4.2 Report on LaTeX\n",
        "Answer the following questions on LaTeX in the respective section.\n",
        "1. Report the chosen hyperparameters for both the complete set and the partial set.\n",
        "2. Did the small dataset have higher or lower chosen hyperparameter values than the full dataset? Briefly explain why."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e85OLhHfXCv_"
      },
      "source": [
        "### Retrain Decision Tree and Plot Hyperparameter Search\n",
        "Now retrain your decision tree with the optimal hyperparameters. Report training, validation, and testing error for the small dataset.\n",
        "\n",
        "Also for the small dataset, create a graph plotting the training and validation scores for each leaf node hyperparameter value, holding the tree depth hyperparameter consistent at the chosen value."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JrWrji_iXCwA"
      },
      "source": [
        "def retrain_decision_tree(X_train, y_train, X_val, y_val, X_test, y_test):\n",
        "  \n",
        "  \"\"\"\n",
        "  Perform a decision tree hyperparameter grid search given training and validation data and search values for\n",
        "  tree depth and leaf node count.\n",
        "    \n",
        "  Args: (Note that n is not the same among train and test sets, but merely refers to sample size)\n",
        "    X_train ((n,p) np.ndarray)\n",
        "    y_train ((n,) np.ndarray)\n",
        "    X_val ((n,p) np.ndarray)\n",
        "    y_val ((n,) np.ndarray)\n",
        "    X_test ((n,p) np.ndarray)\n",
        "    y_test ((n,) np.ndarray)\n",
        "\n",
        "  To return:\n",
        "    train_acc (float): Optimal Hyperparameters Train Accuracy\n",
        "    val_acc (float): Optimal Hyperparameters Train Accuracy\n",
        "    test_acc (float): Optimal Hyperparameters Train Accuracy\n",
        "    \n",
        "    leaf_count_train_scores (list): Report training scores for max leaf count search space\n",
        "    leaf_count_val_scores (list): Report validation scores for max leaf count search space\n",
        "  \"\"\"\n",
        "  # Select best hyperparameters\n",
        "  depth_search_space = [2, 4, 6, 8, 10, 16, 20]\n",
        "  leaf_count_search_space = [2, 3, 4, 5, 6, 7, 8, 9, 10] \n",
        "\n",
        "  chosen_depth, chosen_leaf_count = grid_search_depth_and_leaf_count(depth_search_space, \n",
        "                                                                     leaf_count_search_space, \n",
        "                                                                     X_train, \n",
        "                                                                     y_train, \n",
        "                                                                     X_val, \n",
        "                                                                     y_val)\n",
        "  \n",
        "  # <---- Your code here ----->          \n",
        "  train_acc = evaluate(predict(train_decision_tree(X_train, y_train, depth=chosen_depth, leaf_count=chosen_leaf_count), X_train), y_train)\n",
        "  val_acc = evaluate(predict(train_decision_tree(X_train, y_train, depth=chosen_depth, leaf_count=chosen_leaf_count), X_val), y_val)\n",
        "  test_acc = evaluate(predict(train_decision_tree(X_train, y_train, depth=chosen_depth, leaf_count=chosen_leaf_count), X_test), y_test)\n",
        "  leaf_count_train_scores = []\n",
        "  leaf_count_val_scores = []\n",
        "  for i in leaf_count_search_space:\n",
        "    leaf_count_train_scores.append(evaluate(predict(train_decision_tree(X_train_small, y_train_small, depth=chosen_depth, leaf_count=i), X_train_small), y_train_small))\n",
        "    leaf_count_val_scores.append(evaluate(predict(train_decision_tree(X_train_small, y_train_small, depth=chosen_depth, leaf_count=i), X_val), y_val))\n",
        "\n",
        "  fig = plt.figure(figsize=(8,6))\n",
        "  ax = fig.add_subplot(111)\n",
        "  ax.plot(range(2,len(leaf_count_train_scores)+2),leaf_count_train_scores,label='Train')\n",
        "  ax.plot(range(2,len(leaf_count_val_scores)+2),leaf_count_val_scores,label='Validation')\n",
        "  ax.set_title('Training and Validation Scores For Each Leaf Node Hyperparameter Value')\n",
        "  ax.set_xlabel('leaf counts')\n",
        "  ax.set_ylabel('leaf scores')\n",
        "  plt.legend()\n",
        "  plt.show()\n",
        "\n",
        "  # <---- Your code ends here ----->\n",
        "  \n",
        "  print(\"Optimal Hyperparameters Train Accuracy: \", train_acc)\n",
        "  print(\"Optimal Hyperparameters Validation Accuracy: \", val_acc)\n",
        "  print(\"Optimal Hyperparameters Test Accuracy: \", test_acc)\n",
        "  \n",
        "  print(\"Training Scores per Max Leaf Node Count:\", leaf_count_train_scores)\n",
        "  print(\"Validation Scores per Max Leaf Node Count:\", leaf_count_val_scores)\n",
        "  \n",
        "  return (train_acc, val_acc, test_acc, leaf_count_train_scores, leaf_count_val_scores)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F0BXjFYN3SCa"
      },
      "source": [
        "Run the code above for the small dataset and by uncommenting code below. Report all necessary values and both graphs on Latex."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jLJCMcNWXCwD",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 900
        },
        "outputId": "a3e088c8-9bd8-4922-c17b-36e28d0444bd"
      },
      "source": [
        "retrain_decision_tree(X_train_small, y_train_small, X_val, y_val, X_test, y_test)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Chosen Depth:  6\n",
            "Chosen Leaf:  9\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 576x432 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfgAAAGDCAYAAADHzQJ9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3yV9fn/8ddFAoQZ9gpTARVEhpGhqOBoXZXW1gpalbqq31pHba1aq9S2v9pWW79trd+6iqsidVWtI6ggLoSgKNsAIoQNypaR5Pr9cd+hh5CEjHNyn3Pyfj4eeeTc49z3+8zr3J97fMzdERERkfTSIOoAIiIiEn8q8CIiImlIBV5ERCQNqcCLiIikIRV4ERGRNKQCLyIikoZU4CthZq+Y2cXxnjdKZrbczE5JwHKnmdll4e0LzCyvKvPWYD3dzWy7mWXUNKvUPTMbb2bvRJyhiZm9aGZbzOxfdbC+nmbmZpaZ6HVJ6qjN9191pV2BD7/8S/9KzOyrmOELqrMsdz/d3R+J97zJyMxuMrPp5YxvZ2Z7zOzIqi7L3Z9w96/FKdd+P0jcfYW7N3f34ngsv8y6xpjZHDPbamYbzexNM+sV7/XEQ1gwi8u83/8ap2UvL/O5iduyq5kh3j9EvwN0BNq6+7nlrHNCWJC/GzMuMxzXM85ZKhVmebyc8W5mvesyS6ozs4lm9us4Li9u35WJlnYFPvzyb+7uzYEVwDdixj1ROp9+VR/gceDYcgraWGCuu8+LIFOdCb80HwVuALKBXsC9QNx+SFggnp+592Pf7+5+dRzzfKM2y05SPYBP3b2oknm+AH6pFqL/ivd3Zap995aTN2W+K9OuwFfEzEaZWaGZ/czM1gL/MLPWZvaSmW0wsy/D211j7hPb7DzezN4xs7vCeT8zs9NrOG8vM5tuZtvM7HUzu7e8X+vhvFXJ+CszezdcXp6ZtYuZfqGZfW5mm8zs5xU9P+5eCLwJXFhm0kXAowfLUSbzfs2xZnaqmS2yoGn0r4DFTDs03FLeFG41P2FmrcJpjwHdgRfDrcgbrUyzp5l1MbMXzOwLM1tiZpfHLHuCmU02s0fD52a+meVW8BQMAj5z9zc8sM3dn3H3FeGyMszsFjNbGi5rtpl1C6cda2azwsc3y8yOLfP6/MbM3gV2AoeY2eFmNiXMvNj232I8w8wWhOtYZWY/qeg1q0h181Rz2RW+XuH0bmb2bPg+2WRltvwr+kxUY/0NLNiCWhouf7KZtYmZ/i8zWxs+9ulm1j8c/0vgNuC88L10aQWreBXYA3yvgvVnh++nDeHn6lYLfySF75G7wudlGXBmOfd9yMzWhK/tr62GPyTMrJOZ7TSztjHjhoS5GoafwXfN7K/hc7HIzE6uSpaY+/7JzDYBE6qwvO+b2cLwfbvMzH4QM62m372/NrP3wtfrRTNrG77ftobv654x85f7mTKzK4ALgBtLlxOO72Jmz4Tr/8zMrolZ1gQze9rMHjezrcD42Oc+zt+V+7XU2IHfb7V7z7h72v4By4FTwtujgCLgd0BjoAnQFvg20BRoAfwLeD7m/tOAy8Lb44G9wOVABnAVsBqwGsz7PnAX0AgYCWwFHq/gMVQl41Kgb/iYpgF3htP6AduBE8LH/MfwOTilgnVdABTEDB9G8GXXvgbP1Tvh7XbANoLm0YbA9WGG0nl7A6eG+doD04F7ynsNw+GegAOZ4fB04G9AFkGR3gCcFE6bAOwCzghfh98CMyp47IeE8/4JGA00LzP9p8Dc8DkxYGD4nLQBviT4sGcC48LhtjHPywqgfzg9G1gJfD8cHgxsBPqF868Bjg9vtwaGVJB333NcZnx18zSs7HNTzrQKX6/wOf44fA6bha/JyKp8JqqaAbgWmAF0DTP8HXgyZvolBO/PxsA9wJyYaROo4HMWOx04G1hG8H7NJHi/9QzneRT4d7iOnsCnwKXhtCuBRUC38HWYyv7v1efCvM2ADsBM4AeVZSlnvAO9w9svA1fFTPsT8JeY57uI4PPWEDgP2AK0OViWmPv+KHz8TaqwvDOBQwk+GycS/HgcUsvv3iXhMrOBBeFzfUqY6VHgH+G8zaj8MzUR+HXMshsAswl+8DUi+OwvA74e89zvBb4Zztskgd+V+73OHPj9VuX3TLnvo6rOmIp/HFjg9wBZlcw/CPiyghdiPLAkZlrT8IXoVJ15CbZIi4CmMdMfp5IvnipkvDVm+H+AV8PbtwGTYqY1C5+Dir68mxL82Dg2HP4N8O8aPlelBf4iYooqwRdAYem85Sz3m8BH5b2GZT8ABF+kxUCLmOm/BSaGtycAr8dM6wd8VclzOxyYTPAjYRfBF0PzcNpiYEw597kQmFlm3PvA+Jjn5Y6YaecBb5eZ/+/A7eHtFcAPgJYHeR+MD99Hm2P+hlc3TyWfm+1lln35wV4vYET43GVWkLfCz09ln90y4xcCJ8cMdyb4Mi5vna3CdWTHvB8OWuDD2x8Q/AjZV+AJfpjsISwc4Xw/AKaFt98EroyZ9rWY92pHYDcxxYLgx9fUSrLsKfMabGb/An8e8G54OwNYCwyNeb73+wFFUBwuPFiW8L4rynn9yl1eBfmfB64Nb4+iZt+9P48Zvht4JWb4G4Q/3jj4Z2oi+xf4YeU8vpv57w+GCcD0g3xG4vVdue89Fw73rOl7pry/lNoXEgcb3H1X6YCZNSX41XsawdYSQAszy/DyD+JaW3rD3XeaGUDzCtZV0bztgC/cfWfMvCsJitUBqphxbcxddsZk6hIuuzTHjrDJrVxhzn8BF5nZ+wS/Um+oRo7ylM3gZrZv2Mw6Av8LHE/wa7cBwRZnVXQheC63xYz7HIhthi/73GSZWaaXsx/W3WcApU17xwBPAT8n+PB3I2gpKS/D52XGfQ7kxAyvjLndAxhmZptjxmUCj4W3vw3cCtxpZp8AN7n7++WsF4IfTiNjR5jZidXMU5FvuvvrZUce5PXqBnxe3nMbqs7npyI9gOfMrCRmXDHQMWz+/Q1wLsGWVOk87Qi2NqvjVuAf/Pd1KV1OQ/Z/fmOf2/3e62Xm6xHed034uCF47ip7LSa7+367CszMYwb/DfyfBfuCDwO2uPvMmOmrPKwKMXm6VDFLebkqWh4W7G65naAlsQFBAZwbM29NvnvXxdz/q3KGS987B/tMldUD6FJm/gzg7ZjhSj8jCfquLC9ndd8z+6k3++BDXmb4BoIPxjB3b0nQlA0x+4gTYA3QJnwTlCq3uIdqk3FN7LLDdbateHYAHiEocqcSfIG/WMscZTMY+z/e/0fwugwIl/u9Msss+5rFWk3wXLaIGdcdWHWQTAfl7rOAZ4HSI2JXEjQXlpehR5lxZTPEPoaVwFvu3irmr7m7X1W6XncfQ9Ac9zxBi0J1VDdPdVX2eq0EultiD6JaCZxe5vnLcvdVwPnAGIJm3GyCrSGowefZ3acQNBH/T8zojQStBbHPb+xzu997PZwWm3s30C4md0t371/dbDEZdxG8P75HsGVetqDlWExlCPOsrmKW8t4j5S7PzBoDzxDsduzo7q0Idh9U9jmO53dvpZ+pcta9kuB4m9j5W7j7GZXkLU88vit3EPwYKtWpTM5avWfqW4EvqwXBL8HNFhyoc3uiV+junwP5BAeuNDKzEQTNTYnI+DRwlpmNNLNGwB0c/DV/m6Ap8H6C5v09tczxH6C/mZ0TfvFfw/5v4hYEzcFbzCyHYF93rHVUcCCYu68E3gN+a2ZZZnYUcCnBLo9qCZ+jy82sQzh8OMG+2BnhLA8CvzKzPhY4yoIDnF4G+prZ+RacUnUewa6AlypY1Uvh/BdacDBUQzM7xsyOCN8PF5hZtrvvJWgCLKlgORWpbp7qquz1mklQ5O40s2bha3JcLdbVMFxG6V8m8H/Ab8ysB4CZtTezMTHZdgObCL40/18t1g1B682NpQPh1tfkcP0twgw/5r/vt8nANWbW1cxaAzfF3HcNkAfcbWYtLThY8NCwxaU2HiVoPj+bAwt8hzBPQzM7FzgCeLkWWcpdHsF+7MYEu2eKwq35g50mG8/v3go/U+H0st8hM4FtFhz018SCgyOPDFvtqiMe35VzgBMsuL5HNkFrIRCf90x9L/D3EBzwsZHgi/zVOlrvBQT7KzcBvyZoCt5dwbw1zuju84EfAv8k+OL9kmD/d2X3cYIvjR7h/1rlcPeNBE2mdxI83j7AuzGz/BIYQtCE+h+CreZYvwVuNbPNVv4R5eMIttRWExyQcnt5TctVsJngS3KumW0neHzPAb8Pp/+R4As8j6DwPkSwb2wTcBbBr/ZNBAXhrPBxHyDcnfA1glNqVhM0W5cefATBlthyC47evZLgvVJl1c1TidIzF0r/ngvHV/h6hQXwGwQH4q0geK+dV831xnqZ4Iuy9G8Cwe6BF4A8M9tG8F4cFs7/KEGz8SqCg7JmUAvu/i5BMYj1I4KtrmXAOwSfrYfDaQ8ArxEcaPghB76XLyIohgsIPotPExxDUNuMJcCH4cZDrA8IPm8bCXZdfCd8f9Q0S7nLC9/T1xB8Pr4kaEl54SDLitt3bxU+Uw8B/cLvkOfD9+lZhGfOhBkeJGj1qc56a/1dGbYUPQV8QnDgX9kf4rV6z5Qe1S0RMrOngEXunvAWBBFJL2b2JvBPd38wZtx4ggO5RlZ4x+qtI67Lk7pR37fgIxE2Hx0aNrmcRrDf8Pmoc4lIagmblYcQbAWK7Ke+HUWfLDoRNN+1JWjGvMrdP4o2koikEjN7hOA0xWvLnEkiAqiJXkREJC2piV5ERCQNJbTAm9lpFlwXeImZ3VTO9O5mNtXMPjKzT8zsjHD8qRZc63tu+P+kROYUERFJNwlrorfggvifElwEoBCYBYxz9wUx89xPcJnL+8ysH8E5mj3NbDCwzt1XW9D13mvunlPOavZp166d9+zZMyGPRUREJBnNnj17o7u3L29aIg+yG0pw7ellAGY2ieBo8QUx8zjQMrydTXAOI2UOOJsPNDGzxu5e0bni9OzZk/z8/DjGFxERSW5mVvb6B/skssDnsP81cwv57wUpSk0guGDFjwg6QjmlnOV8m+AiDhUWdxEREdlf1AfZjSPo+asrQZeej1nYtzKABX05/46gx6YDmNkVZpZvZvkbNmyok8AiIiKpIJEFfhX7d7zQlQM7AbmUsDONsMesLIIemzCzrgSXCr3I3cvrxQt3v9/dc909t337cndBiIiI1EuJbKKfBfSxoCvDVQTXCT6/zDwrgJOBiWHHAFnABjNrRXCd65vCay3XyN69eyksLGTXrl0Hn1mqJCsri65du9KwYcOoo4iISCUSVuDdvcjMribofCEDeNjd55vZHUC+u79A0CHGA2Z2PcEBd+PD/sKvJuiw4jYzuy1c5NfcfX11MhQWFtKiRQt69uyJWSJ7gK0f3J1NmzZRWFhIr169oo4jIiKVSOilat39ZYIeoWLH3RZzewFwQHeS7v5rgl7WamXXrl0q7nFkZrRt2xYd7yAikvyiPsgu4VTc40vPp4hIakj7Ah+lTZs2MWjQIAYNGkSnTp3IycnZN7xnz55K75ufn88111xTR0lFRCTdqDe5BGrbti1z5swBYMKECTRv3pyf/OQn+6YXFRWRmVn+S5Cbm0tubm6d5BQRkfSjLfg6Nn78eK688kqGDRvGjTfeyMyZMxkxYgSDBw/m2GOPZfHixQBMmzaNs846Cwh+HFxyySWMGjWKQw45hD//+c9RPgQREUkB9WYL/pcvzmfB6q1xXWa/Li25/Rv9q32/wsJC3nvvPTIyMti6dStvv/02mZmZvP7669xyyy0888wzB9xn0aJFTJ06lW3btnHYYYdx1VVX6VQ1ERGpUL0p8Mnk3HPPJSMjA4AtW7Zw8cUXU1BQgJmxd+/ecu9z5pln0rhxYxo3bkyHDh1Yt24dXbt2rcvYIiJ17vNNO1i2cUfUMeLmyC7ZtG/RuE7WVW8KfE22tBOlWbNm+27/4he/YPTo0Tz33HMsX76cUaNGlXufxo3/+4bIyMigqKgo0TFFRCL14YovGXv/DPYUlUQdJW7uv/Bovta/U52sq94U+GS1ZcsWcnKCnnAnTpwYbRgRkSRR+OVOrng0n04ts7j7uwPJbJAep+ge0q55na1LBT5iN954IxdffDG//vWvOfPMM6OOIyISuW279nLpxHx2F5Uw6Ypj6N2h7opiOjF3jzpDXOTm5nrZ/uAXLlzIEUccEVGi9KXnVUQSpai4hMsezeftgo088v2hjOzTLupISc3MZrt7uedU6zQ5ERFJGr/+z0KmLd7AHWP6q7jXkgq8iIgkhcfeX87E95Zz6cheXDCsR9RxUp4KvIiIRG76pxuY8OICTjq8A7ecoV2A8aACLyIikSpYt40fPvEhfTo058/jBpORJkfMR00FXkREIrNp+24ueWQWjRtm8ND4Y2jeWCd3xYsKvIiIRGJ3UTE/eGw267fu5oGLjianVZOoI6UVFfgEGz16NK+99tp+4+655x6uuuqqcucfNWoUpaf7nXHGGWzevPmAeSZMmMBdd91V6Xqff/55FixYsG/4tttu4/XXX69ufBGRhHB3bnpmLvmff8nd3x3I4O6to46UdlTgE2zcuHFMmjRpv3GTJk1i3LhxB73vyy+/TKtWrWq03rIF/o477uCUU06p0bJEROLt3qlLeO6jVdxwal/OOqpL1HHSkgp8gn3nO9/hP//5D3v27AFg+fLlrF69mieffJLc3Fz69+/P7bffXu59e/bsycaNGwH4zW9+Q9++fRk5cuS+LmUBHnjgAY455hgGDhzIt7/9bXbu3Ml7773HCy+8wE9/+lMGDRrE0qVLGT9+PE8//TQAb7zxBoMHD2bAgAFccskl7N69e9/6br/9doYMGcKAAQNYtGhRIp8aEamnXvpkNXflfcq3Budw9Um9o46TturP0Qyv3ARr58Z3mZ0GwOl3VjpLmzZtGDp0KK+88gpjxoxh0qRJfPe73+WWW26hTZs2FBcXc/LJJ/PJJ59w1FFHlbuM2bNnM2nSJObMmUNRURFDhgzh6KOPBuCcc87h8ssvB+DWW2/loYce4kc/+hFnn302Z511Ft/5znf2W9auXbsYP348b7zxBn379uWiiy7ivvvu47rrrgOgXbt2fPjhh/ztb3/jrrvu4sEHH6ztsyQiss+clZu5YfLH5PZozZ3fHoCZjphPFG3B14HYZvrS5vnJkyczZMgQBg8ezPz58/drTi/r7bff5lvf+hZNmzalZcuWnH322fumzZs3j+OPP54BAwbwxBNPMH/+/EqzLF68mF69etG3b18ALr74YqZPn75v+jnnnAPA0UcfzfLly2v6kEVEDrBq81dc9kg+HVo25u8XHk3jzIyoI6W1+rMFf5At7UQaM2YM119/PR9++CE7d+6kTZs23HXXXcyaNYvWrVszfvx4du3aVaNljx8/nueff56BAwcyceJEpk2bVquspd3SqktaEYmn7buLuHTiLHbvLebJy4fRtnnd9Ilen2kLvg40b96c0aNHc8kllzBu3Di2bt1Ks2bNyM7OZt26dbzyyiuV3v+EE07g+eef56uvvmLbtm28+OKL+6Zt27aNzp07s3fvXp544ol941u0aMG2bdsOWNZhhx3G8uXLWbJkCQCPPfYYJ554YpweqYjIgYpLnGuf/IiC9dv56wVD6NOxRdSR6gUV+Doybtw4Pv74Y8aNG8fAgQMZPHgwhx9+OOeffz7HHXdcpfcdMmQI5513HgMHDuT000/nmGOO2TftV7/6FcOGDeO4447j8MMP3zd+7Nix/OEPf2Dw4MEsXbp03/isrCz+8Y9/cO655zJgwAAaNGjAlVdeGf8HLCIS+u3LC3lj0XomfKMfJ/ZtH3WcekPdxUq16XkVkar65wcruOW5uYw/ticTzu4fdZy0o+5iRUSkzr1TsJFf/Hseow5rz61naqOgrqnAi4hI3C1Zv52rnphN7/bN+cu4wWRmqNzUNT3jIiISV1/s2MOlj8yicWYDHrw4lxZZDaOOVC+lfYFPl2MMkoWeTxGpzO6iYq58bDZrtuzi7xfm0q1N06gj1VtpXeCzsrLYtGmTilKcuDubNm0iKysr6igikoTcnVuencfM5V/wh+8cxdE91IFMlNL6Qjddu3alsLCQDRs2RB0lbWRlZdG1a9eoY4hIErrvraU882Eh153ShzGDcqKOU++ldYFv2LAhvXr1ijqGiEjae3XeGn7/6mLOHtiFa0/uE3UcIc2b6EVEJPE+KdzMdU/NYUj3Vvz+O0epA5kkoQIvIiI1tmZL0IFM22aN+fuFuWQ1VAcyySKtm+hFRCRxduwu4tKJ+ezcU8wzVw2jfQt1IJNMtAUvIiLVVlziXDtpDovWbuUv5w/msE7qQCbZqMCLiEi1/f7VRby+cB23ndWP0Yd1iDqOlEMFXkREqmXSzBX8ffoyLhrRg/HH6UylZKUCLyIiVfbe0o3c+vw8ju/TjtvO6hd1HKmECryIiFTJsg3buerxD+nVrhn3XjBEHcgkOb06IiJyUJt37uHSR/LJaGA8PP4YWqoDmaSnAi8iIpXaU1TClY/PZtWXX3H/hUerA5kUofPgRUSkQu7Orc/PZcayL7jnvEHk9mwTdSSpIm3Bi4hIhe6fvozJ+YVcc1JvvjlYHcikEhV4EREp12vz13Lnq4s486jOXHdK36jjSDWpwIuIyAHmrdrCdZPmcFTXVtx97kAaNFAHMqlGBV5ERPazdssuLn1kFq2bNuSBi45WBzIpSgfZiYjIPjv3FHHZo7PYvquIp686lg4tsqKOJDWkAi8iIgCUlDjXPzWHBau38uDFuRzRuWXUkaQWEtpEb2anmdliM1tiZjeVM727mU01s4/M7BMzOyNm2s3h/Rab2dcTmVNEROAPeYt5bf46fn5mP046vGPUcaSWErYFb2YZwL3AqUAhMMvMXnD3BTGz3QpMdvf7zKwf8DLQM7w9FugPdAFeN7O+7l6cqLwiIvXZ5PyV3DdtKRcM684lx/WMOo7EQSK34IcCS9x9mbvvASYBY8rM40BpG1A2sDq8PQaY5O673f0zYEm4PBERibMZyzbx8+fmMrJ3Oyac3R8zHTGfDhJZ4HOAlTHDheG4WBOA75lZIcHW+4+qcV/M7Aozyzez/A0bNsQrt4hIvbF84w6ufHw23ds05d4LhtBQHcikjahfyXHARHfvCpwBPGZmVc7k7ve7e66757Zv3z5hIUVE0tGWnXu5ZOIsDHh4/DFkN1EHMukkkUfRrwK6xQx3DcfFuhQ4DcDd3zezLKBdFe8rIiI1tLe4hKuemM3KL3fyxGXD6dG2WdSRJM4SuQU/C+hjZr3MrBHBQXMvlJlnBXAygJkdAWQBG8L5xppZYzPrBfQBZiYwq4hIveHu3Pbveby3dBN3nnMUQ3upA5l0lLAteHcvMrOrgdeADOBhd59vZncA+e7+AnAD8ICZXU9wwN14d3dgvplNBhYARcAPdQS9iEh8PPTOZzw5cyU/HH0o3z66a9RxJEEsqKepLzc31/Pz86OOISKS1F5fsI7LH8vn9CM78ddxQ3SN+RRnZrPdPbe8aVEfZCciInVk/uotXDPpIwbkZHP3uYNU3NOcCryISD2wfusuLnskn+wmDXnwolyaNFIHMulO16IXEUlzX+0p5vJH89ny1V7+deUIOrRUBzL1gQq8iEgaKylxbvjXHD5ZtYX7L8ylf5fsqCNJHVETvYhIGvvjlE95ee5abjn9CE7tpw5k6hMVeBGRNPXM7EL+OnUJ44Z247Lje0UdR+qYCryISBqa+dkX3PTsJxx7aFvuGHOkOpCph1TgRUTSzIpNO/nBY/l0a92U+y44Wh3I1FN61UVE0kjhlzu54KEZOPDQ+GPIbqoOZOorHUUvIpImVn6xk7H3z2Dbrr08dukwerVTBzL1mQq8iEga+HzTDsbdP4Mde4p54rLhDOiq0+HqOxV4EZEU99nGoLjvKirmicuGcWSOiruowIuIpLSlG7Zz/gMz2Fvs/POy4fTr0jLqSJIkVOBFRFLUkvXbGPfAB5SUOE9ePpzDOrWIOpIkERV4EZEU9Om6bZz/wAzAmHTFcPp0VHGX/ek0ORGRFLNo7VbG3T+DBqbiLhVTgRcRSSELVgfFPTMjKO69OzSPOpIkKTXRi4ikiHmrtvC9hz6gScMMnrx8OD11nrtUQlvwIiIp4JPCzZz/wAyaNcrkqStGqLjLQanAi4gkuTkrN3PBgx/QsklDJl0xnO5tm0YdSVKAmuhFRJLYhyu+5OKHZtK6WSP+efkwurZWcZeqUYEXEUlS+cu/YPw/ZtG2eSOevHw4XVo1iTqSpBA10YuIJKGZn33BRQ/PpH2Lxjx1xQgVd6k2FXgRkSTz/tJNXPzwTDpnZ/HUFcPplJ0VdSRJQSrwIiJJ5N0lG/n+xJl0bd2EJ68YToeWKu5SM9oHLyKSJN4u2MBlj+TTs20znrh8GO2aN446kqQwbcGLiCSBaYvXc+kj+fRq14x/qrhLHGgLXkQkYlMXrecHj82md4fmPHHZMFo3axR1JEkD2oIXEYnQ6wvWccVj+fTt1Jx/Xq7iLvGjAi8iEpHX5q/lqidm069zS564dDitmqq4S/yoiV5EJAKvzF3Dj578iCNzsnn00qG0zGoYdSRJM9qCFxGpYy99spqrn/yIgd1a8ZiKuySItuBFROrQv+es4seTP2ZI91b84/tDad5YX8OSGNqCFxGpI899VMj1T83h6B6tmajiLgmmAi8iUgeenl3Ijyd/zLBebZn4/WNopuIuCaZ3mIhIgk2etZKfPfsJxx3ajgcuyqVJo4yoI0k9oC14EZEE+ucHK7jxmU8Y2bsdD16s4i51RwVeRCRBHpvxObc8N5dRh7XngYtyyWqo4i51R030IiIJ8Mh7y7n9hfmcfHgH/va9ITTOVHGXuqUCLyISZw+/8xl3vLSAU/t15N7zh9AoU42lUvdU4EVE4ujBt5fx6/8s5LT+nfjzuMEq7hIZFXgRkTj5v7eWcucrizhzQGfuGTuIhhkq7hIdFXgRkTi4d+oS/vDaYr4xsAt/+u5AMlXcJWIq8CIitfTnNwr445RP+eagLtx1roq7JAcVeBGRGnJ3/vR6AX9+oz5rUbQAACAASURBVIBzhuTwh+8MJKOBRR1LBFCBFxGpEXfn7rxP+evUJXw3tyu/PecoFXdJKirwIiLV5O78/rXF3DdtKeOGduM33xxAAxV3STIq8CIi1eDu/PaVRdw/fRkXDOvOr8YcqeIuSUkFXkSkitydX720kIff/YyLR/Rgwtn9MVNxl+SU0EM9zew0M1tsZkvM7KZypv/JzOaEf5+a2eaYab83s/lmttDM/mz6FIlIhNydX764gIff/YzvH9dTxV2SXsK24M0sA7gXOBUoBGaZ2QvuvqB0Hne/Pmb+HwGDw9vHAscBR4WT3wFOBKYlKq+ISEVKSpzbXpjH4zNWcNnIXvz8zCNU3CXpJXILfiiwxN2XufseYBIwppL5xwFPhrcdyAIaAY2BhsC6BGYVESlXSYnz8+eD4v6DEw9RcZeUkcgCnwOsjBkuDMcdwMx6AL2ANwHc/X1gKrAm/HvN3ReWc78rzCzfzPI3bNgQ5/giUt+VlDg3PzuXJ2eu4IejD+Wm0w5XcZeUkSyXWxoLPO3uxQBm1hs4AuhK8KPgJDM7vuyd3P1+d89199z27dvXaWARSW/FJc6Nz3zCU/krueak3vzka4epuEtKSWSBXwV0ixnuGo4rz1j+2zwP8C1ghrtvd/ftwCvAiISkFBEpo7jE+em/Pubp2YVcf0pffqziLikokQV+FtDHzHqZWSOCIv5C2ZnM7HCgNfB+zOgVwIlmlmlmDQkOsDugiV5EJN6Kikv48eQ5PPvRKn7ytb5ce0qfqCOJ1EjCCry7FwFXA68RFOfJ7j7fzO4ws7NjZh0LTHJ3jxn3NLAUmAt8DHzs7i8mKquICMDe4hKufWoO/56zmhtPO4yrT1Jxl9Rl+9fV1JWbm+v5+flRxxCRFLVu6y5++eJ8Xp67llvOOJwrTjg06kgiB2Vms909t7xpupKdiNQr7s7qLbuYt2rLf/9Wb2XDtt0A3HrmEVx2/CERpxSpPRV4EUlb7k7hl18xNyzkc1dtYf7qrXyxYw8ADQz6dGjB8X3aMSAnm9webRjQNTvi1CLxoQIvImmhpMT5/IudMVvlW5i3aitbvtoLQGYDo0/HFpxyRAeOzMnmyJxsjujUkiaNMiJOLpIYKvAiknKKS5zPNu7YV8znrtrCgtVb2ba7CIBGGQ04rFMLzhjQmSNzWnJkl2wO69SCrIYq5lJ/qMCLSFIrKi5h6YYdMU3sQTP7zj3FADTObMARnVsyZnAXBuRk079LNn07tqBRZrJcx0skGirwIpI09haXULBu+74m9rmrtrBwzVZ27S0BoEnDDPp1acl3c7vRv0tLBnTNpnf75mRmqJiLlKUCLyKR2F1UzKdrtwcHwK0OmtoXrd3GnqKgmDdvnEm/Li25YFiPfc3sh7RvTkYDXVFOpCpU4EUk4XbtLWbhmq3MW72VeYVBQf903Tb2FgfX4WiZlcmROdmMP7ZncABcl5b0bNuMBirmIjWmAi8icbVzTxEL12xlbmFwfvm8VVsoWL+d4pKgmLdu2pAjc7K57PhDOLJLNgNysunWpomu9S4SZyrw5XB33l+2KeoYIimhqNgpWL993xHtSzdsJ6zltGveiCNzsjm1X0f6d8lmQNdsumRnqZiL1AEV+HK4w/kPfBB1DJGU0rFlYwbkZHPGgM4MCM8z79iysYq51M7u7bAzjTa4mrWHRk3rZFUq8OUwg0lXDI86hkhKaGBGz3ZN6dAiK+ookm5KSuC+EbB5RdRJ4mfsP+HwM+tkVSrw5TAzhh/SNuoYIiL125o5QXEfdiV0OirqNPFRh4/joAXezJoBX7l7iZn1BQ4HXnH3vQlPJyIi9VfBFMDghBuhmTa6qqsqV4eYDmSZWQ6QB1wITExkKBEREQryoGuuinsNVaXAm7vvBM4B/ubu5wL9ExtLRETqtR0bYdVs6PO1qJOkrCoVeDMbAVwA/Cccpx4bREQkcZa8ATj0OTXqJCmrKgX+OuBm4Dl3n29mhwBTExtLRETqtYI8aNYBOg2MOknKOuhBdu7+FvCWmTUNh5cB1yQ6mIiI1FMlxbDk9eB0sgbqSKimDvrMmdkIM1sALAqHB5rZ3xKeTERE6qfCfNi1Wc3ztVSVn0b3AF8HNgG4+8fACYkMJSIi9VhBHlgGHDI66iQprUptH+6+ssyo4gRkERERCQp89+HQpFXUSVJaVQr8SjM7FnAza2hmPwEWJjiXiIjUR1vXwNpP1DwfB1Up8FcCPwRygFXAoHBYREQkvpa8HvzX+e+1VulR9GaWAfyvu19QR3lERKQ+K8iDljnQoV/USVJepVvw7l4M9DCzRnWUR0RE6qvivbB0atA8r26Ga60qvcktA941sxeAHaUj3f2PCUslIiL1z4r3Yc826PP1qJOkhaoU+KXhXwOgRWLjiIhIvVWQBxmNoJfOxI6HqlzJ7pcAZtY8HN6e6FAiIlIPFUyBHsdB4+ZRJ0kLVbmS3ZFm9hEwH5hvZrPNTL3JiYhI/Hz5OWxYpKPn46gqp8ndD/zY3Xu4ew/gBuCBxMYSEZF6ZcmU4L8KfNxUpcA3c/d9vce5+zSgWcISiYhI/VMwBVr3graHRp0kbVSlwC8zs1+YWc/w71aCI+tFRERqb+8uWPZWsPWu0+PipioF/hKgPfAs8AzQLhwnIiJSe5+/A0VfqXk+zqpyFP2XqP93ERFJlIIpkNkEeh4XdZK0UpWj6KeYWauY4dZm9lpiY4mISL1RkBec+96wSdRJ0kpVmujbufvm0oFwi75D4iKJiEi9sWkpfLFMvcclQFUKfImZdS8dMLMegCcukoiI1BsFecF/Ffi4q8qlan8OvGNmbwEGHA9ckdBUIiJSPxTkQbvDoHXPqJOknaocZPeqmQ0BhoejrnP3jYmNJSIiaW/PDlj+DgzVNmMiVOUgu+OAr9z9JaAVcEvYTC8iIlJzn02H4j06PS5BqrIP/j5gp5kNBH5M0LPcowlNJSIi6a8gDxo1h+4jok6SlqpS4Ivc3YExwL3ufi/qNlZERGrDPTj//ZBRkNko6jRpqSoFfpuZ3Qx8D/iPmTUAGiY2loiIpLX1C2HLSuj79aiTpK2qFPjzgN3Ape6+FugK/CGhqUREJL2Vnh7XW6fHJUpVjqJfC/wxZngF2gcvIiK1UTAFOg2Alp2jTpK2qrIFLyIiEj+7tsCK93X0fIKpwIuISN1aOhW8WAU+wSos8Gb2Rvj/dzVduJmdZmaLzWyJmd1UzvQ/mdmc8O9TM9scM627meWZ2UIzW2BmPWuaQ0REkkjBFMhqBTm5USdJa5Xtg+9sZscCZ5vZJILL1O7j7h9WtmAzywDuBU4FCoFZZvaCuy+IWcb1MfP/CBgcs4hHgd+4+xQzaw6UVPExiYhIsiopgSVToPfJkFGVq6VLTVX27N4G/ILgqPm72b/AO3DSQZY9FFji7ssAwh8JY4AFFcw/Drg9nLcfkOnuUwDcfftB1iUiIqlg7SewfZ2a5+tAhQXe3Z8GnjazX7j7r2qw7BxgZcxwITCsvBnDS9/2At4MR/UFNpvZs+H414Gb3L24BjlERCRZFEwBDA49Oeokaa8qp8n9ysxaA32ArJjx0+OYYyzwdEwBzyTotW4wsAJ4ChgPPBR7JzO7grBnu+7duyMiIkmuIA9yhkDz9lEnSXtV6WzmMmA68Brwy/D/hCosexXQLWa4aziuPGOBJ2OGC4E57r7M3YuA54EhZe/k7ve7e66757ZvrzeLiEhS27EJCmepeb6OVOU0uWuBY4DP3X00wVb15srvAsAsoI+Z9TKzRgRF/IWyM5nZ4UBr4P0y921lZqVV+yQq3ncvIiKpYOmbgEMfXb2uLlSlwO9y910AZtbY3RcBhx3sTuGW99UEW/wLgcnuPt/M7jCzs2NmHQtMCju0Kb1vMfAT4A0zm0twgN8DVX1QIiKShAryoGk76Dz44PNKrVXlHIVCM2tF0Ew+xcy+BD6vysLd/WXg5TLjbiszPKGC+04BjqrKekREJMmVFMOS14POZRroGmt1oSoH2X0rvDnBzKYC2cCrCU0lIiLpZdWH8NUXap6vQ1W6yoCZjQT6uPs/wv3iOcBnCU0mIiLpoyAPrAEcerBLqEi8VOUo+tuBnwE3h6MaAo8nMpSIiKSZgteg2zBo0jrqJPVGVXaEfAs4G9gB4O6rgRaJDCUiImlk21pY87FOj6tjVSnwe8Ij3B3AzJolNpKIiKSVJa8H/1Xg61RVCvxkM/s7wXnplxNcNlanrImISNUU5EGLLtCxf9RJ6pWqHEV/l5mdCmwlOP/9ttJOYERERCpVvDfo/73/t8Ds4PNL3FTpKPqwoKuoi4hI9az8AHZvVfN8BCos8Ga2jXC/e9lJgLt7y4SlEhGR9FCQBw0awiEnRp2k3qmsu1gdKS8iIrVTMAV6HAuNVVLqmq4XKCIiibF5JaxfoOb5iKjAi4hIYiwJD91SgY+ECryIiCRGwRRo1QPa9Yk6Sb2kAi8iIvFXtBuWTQu23nV6XCRU4EVEJP4+fxf27lTzfIRU4EVEJP4KpkBmFvQcGXWSeksFXkRE4q8gD3oeD42aRp2k3lKBFxGR+Nq0FDYtUfN8xFTgRUQkvvb1HndqtDnqORV4ERGJr09fg7Z9oE2vqJPUayrwIiISP3t2wPJ3oO/Xo05S76nAi4hI/Hz2NhTvVvN8ElCBFxGR+CnIg0bNofuIqJPUeyrwIiISH+7B+e+HjILMxlGnqfdU4EVEJD42LIYtK9Q8nyRU4EVEJD4K8oL/vVXgk4EKvIiIxEdBHnQ8ErJzok4iqMCLiEg87NoKK95X83wSUYEXEZHaWzYNSop0edokkhl1gKRUUgK/6Rh1ChGpa03awOVvqom5JgryoHE2dB0adRIJqcBXZMQPo04gInXJS+D9v8E7f4Qz7446TWopPT2u90mQobKSLPRKlKdBAzhlQtQpRKSu7doKsx+B466DVt2iTpM61s6F7WvVPJ9ktA9eRKTU8TeAGbx9V9RJUsu+0+NOiTaH7EcFXkSkVKtuMORi+Ohx+PLzqNOkjoIp0GUwNO8QdRKJoQIvIhLr+B+DZcD0P0SdJDXs/AIKZ6p5PgmpwIuIxGrZBXK/D3P+CV8sizpN8lv6ZnCAogp80lGBFxEpa+T1kNEQpmtf/EEV5EHTtkETvSQVFXgRkbJadILcS+HjJ2HT0qjTJK+SYljyenBwXYOMqNNIGSrwIiLlGXkdZDSGt34fdZLktfoj2LlJzfNJSgVeRKQ8zTvA0Mtg7mTYWBB1muRUkAfWAA49KeokUg4VeBGRihx3HWQ2gbd+F3WS5FSQF1yatmmbqJNIOVTgRUQq0qwdDL0c5j4N6xdFnSa5bF8fNNGr97ikpQIvIlKZY6+BRs3grTujTpJclrwe/Nf+96SlAi8iUplmbWHYlTD/eVg3P+o0yaMgD5p3gk4Dok4iFVCBFxE5mBE/hMYtYJq24gEoLoIlbwbN82ZRp5EKqMCLiBxM0zYw/CpY+ELQc1p9VzgTdm9R83ySU4EXEamK4f8DjbO1FQ9B83yDTDhkVNRJpBIq8CIiVdGkVdBUv+glWD0n6jTRKpgC3UdAVsuok0glVOBFRKpq+JWQ1ap+b8VvWQXr5ql5PgUktMCb2WlmttjMlpjZTeVM/5OZzQn/PjWzzWWmtzSzQjP7ayJziohUSVY2HHs1fPoKrJoddZpoLJkS/FeBT3oJK/BmlgHcC5wO9APGmVm/2Hnc/Xp3H+Tug4C/AM+WWcyvgOmJyigiUm3DroQmrevvVnzBFMjuDu0PizqJHEQit+CHAkvcfZm77wEmAWMqmX8c8GTpgJkdDXQE8hKYUUSkehq3CC5+U5AHK2dFnaZuFe2GpVN1elyKSGSBzwFWxgwXhuMOYGY9gF7Am+FwA+Bu4CeVrcDMrjCzfDPL37BhQ1xCi4gc1NArgj7Qp/2/qJPUrc/fg7071DyfIpLlILuxwNPuXhwO/w/wsrsXVnYnd7/f3XPdPbd9+/YJDykiAkDj5kFHNEvfhBUzok5TdwqmBF3o9jo+6iRSBYks8KuAbjHDXcNx5RlLTPM8MAK42syWA3cBF5lZPd3hJSJJ6ZhLoVl7mFqPtuIL8qDnyODa/JL0ElngZwF9zKyXmTUiKOIvlJ3JzA4HWgPvl45z9wvcvbu79yRopn/U3Q84Cl9EJDKNmsHI6+Gzt2D5u1GnSbwvlsGmAuj79aiTSBUlrMC7exFwNfAasBCY7O7zzewOMzs7ZtaxwCR390RlERFJiNxLoHlHmPbbqJMkXkHYe1zvU6LNIVWWmciFu/vLwMtlxt1WZnjCQZYxEZgY52giIrXXsAmM/DG8+jP4bDr0OiHqRIlTkAdte0PbQ6NOIlWULAfZiYikpqPHQ4vOMPW3kK4NkXt2wvK3dfR8ilGBFxGpjYZZcPwNsOI9WDYt6jSJsfwdKNoVnP8uKUMFXkSktoZcBC1zgiPq03ErviAPGjaFHsdFnUSqQQVeRKS2MhsHW/GFM2HpG1GniS93KHgt6Bo2s3HUaaQaVOBFROJh8IWQ3S39tuI3FsDmFWqeT0Eq8CIi8ZDZCE74adDLXMGUqNPET0HYHUhvFfhUowIvIhIvg86HVj1g6m/SZyu+IA869INW3Q4+ryQVFXgRkXjJaAgn3ghr5sDiV6JOU3u7twUdzKh5PiWpwIuIxNNRY6F1r6CnuVTfil/2FpTs1fnvKUoFXkQknjIy4cSfwdq5sOilqNPUTkEeNG4J3YZFnURqQAVeRCTeBpwLbQ6FaXdCSUnUaWrGPThY8NDRwa4HSTkq8CIi8ZaRCaNugnXzYOEBnWimhnXzYNtqNc+nMBV4EZFEOPLb0K5v6m7F7zs9Tr3HpSoVeBGRRGiQEeyL37AQFjwXdZrqK5gCnQdCi05RJ5EaUoEXEUmU/t+C9keEW/HFUaepuq++hJUfQJ+vR51EakEFXkQkURpkwKifwcZPYd6zUaepuqVvgpdo/3uKU4EXEUmkI8ZAh/7w1p1QXBR1mqopmAJN2kDOkKiTSC2owIuIJFKDBjD6Zti0BOb+K+o0B1dSEhT43qcELRCSslTgRUQS7fCzoNMAeOt3yb8Vv+Yj2LlRzfNpQAVeRCTRzGDULfDlZ/DJpKjTVK5gCmDQ++Sok0gtqcCLiNSFw06HzoPgrd9D8d6o01SsIA+6HgNN20SdRGpJBV5EpC6YwehbYPPnMOefUacp3/YNsOpDNc+nCRV4EZG60udrkHM0TP8DFO2JOs2Blr4BuLqHTRMq8CIidaV0X/yWlTDn8ajTHKggD5p3hE5HRZ1E4kAFXkSkLvU+GboOhel3QdHuqNP8V3ERLHkDep8anNonKU+voohIXTILzovfugo+fDTqNP+1Kh92bVbzfBpRgRcRqWuHjIbuI+Dtu2HvrqjTBD59DSwj6P9d0oIKvIhIXSs9on7bGpg9Meo0gYIpwY+OrOyok0icqMCLiESh1wnQYyS880fY+1W0WbauhnVz1TyfZlTgRUSiMvpm2L4O8h+ONkfBlOC/zn9PKyrwIiJR6Tky2JJ/50+wZ0d0OQryoGVX6HBEdBkk7lTgRUSiNOoW2LEBZj0UzfqL9sCyadD3a8GxAZI2VOBFRKLUYwQcehK8ew/s3l7361/xPuzZrub5NKQCLyIStVG3wM5NMOuBul93QR5kNAp2FUhaUYEXEYlat2OCK8i9+7+we1vdrrtgSnAsQKNmdbteSTgVeBGRZDDqZvjqS/jg73W3zi+Xw8bFap5PUyrwIiLJoOvR0Pc0eO8vsGtL3axTp8elNRV4EZFkMerm4HrwM/6vbtZXMAXaHAJtD62b9UmdUoEXEUkWXQbBYWfC+/fCV5sTu669X8Fn07X1nsZU4EVEksmom2D3Fpjxt8SuZ/m7UPSVLk+bxlTgRUSSSeej4IhvwIz7YOcXiVtPQR5kNgmuhy9pSQVeRCTZjLoZdm8NmuoTwR0KXoNDToSGWYlZh0ROBV5EJNl07A/9vwUf/B/s2BT/5W9aGpwip+b5tKYCLyKSjE68KeiA5v2/xH/ZBa8F/3urwKczFXgRkWTU4XA48tvwwf2wY2N8l12QB+0Ph9Y94rtcSSoq8CIiyerEnwVHur/7v/Fb5u7twRH0ap5PeyrwIiLJqn1fGHAuzHwAtq+PzzI/ewtK9ur893pABV5EJJmd+DMo3gPv3BOf5RXkQaMW0G14fJYnSSuhBd7MTjOzxWa2xMxuKmf6n8xsTvj3qZltDscPMrP3zWy+mX1iZuclMqeISNJqeygcdR7kPwTb1tZuWe7B5WkPHQ2ZjeKTT5JWwgq8mWUA9wKnA/2AcWbWL3Yed7/e3Qe5+yDgL8Cz4aSdwEXu3h84DbjHzFolKquISFI78adQvBfe+VPtlrN+AWxdpeb5eiKRW/BDgSXuvszd9wCTgDGVzD8OeBLA3T9194Lw9mpgPdA+gVlFRJJXm0Ng0DjI/wdsXV3z5RTkBf97nxKfXJLUElngc4CVMcOF4bgDmFkPoBfwZjnThgKNgKUJyCgikhpO+Cl4Mbz9x5ovo2AKdDoKWnaOXy5JWslykN1Y4Gl3L44daWadgceA77t7Sdk7mdkVZpZvZvkbNmyoo6giIhFo3RMGfw8+fAS2FFb//l9thhUz1DxfjySywK8CusUMdw3HlWcsYfN8KTNrCfwH+Lm7zyjvTu5+v7vnuntu+/ZqwReRNHf8T4ID5d6+u/r3XTY1aAFQga83ElngZwF9zKyXmTUiKOIvlJ3JzA4HWgPvx4xrBDwHPOruTycwo4hI6mjVDYZcBB8+Bl9+Xr37FkyBJq2ha25isknSSViBd/ci4GrgNWAhMNnd55vZHWZ2dsysY4FJ7u4x474LnACMjzmNblCisoqIpIzjbwAzePuuqt+npCQ8Pe5kaJCRuGySVDITuXB3fxl4ucy428oMTyjnfo8Djycym4hISsrOgaPHQ/7DMPLH0KbXwe+z9mPYsV7N8/VMshxkJyIiVTXyx2AZML2KW/EFUwCD3icnNJYkFxV4EZFU07Iz5F4CHz8Z9O1+MAV5kHM0NGuX+GySNFTgRURS0cjrIaMRTP9D5fPt2AiF+Wqer4dU4EVEUlGLjnDMpfDJU7CxoOL5lrwBuLqHrYdU4EVEUtVx10JmFrz1+4rnKciDZu2hs05Eqm9U4EVEUlXzDjD0cpj7L9iw+MDpJcWw5HXofSo00Nd9faNXXEQklR17LTRsCm/97sBphfmwa7Oa5+spFXgRkVTWrC0M+wHMexbWLdh/WkFecDrdoaOjySaRUoEXEUl1x/4IGjWHt+7cf3xBHnQbFlyiVuodFXgRkVTXtA0MvxIW/BvWzgvGbV0Daz+Bvjo9rr5SgRcRSQcjfgiNW8K03wbDS14P/uv893pLBV5EJB00aQ3D/wcWvQRrPg6a51vmQId+USeTiKjAi4iki+FXQVY2vPErWDo1OHreLOpUEhEVeBGRdNGkFYy4GpZMgT3b1Dxfz6nAi4ikk2FXQlYraNAQep0YdRqJUEL7gxcRkTqW1RK+cQ9sXgGNm0edRiKkAi8ikm76fyvqBJIE1EQvIiKShlTgRURE0pAKvIiISBpSgRcREUlDKvAiIiJpSAVeREQkDanAi4iIpCEVeBERkTSkAi8iIpKGVOBFRETSkAq8iIhIGlKBFxERSUMq8CIiImnI3D3qDHFhZhuAz+O82HbAxjgvMwrp8jhAjyVZpctjSZfHAXosySrej6WHu7cvb0LaFPhEMLN8d8+NOkdtpcvjAD2WZJUujyVdHgfosSSrunwsaqIXERFJQyrwIiIiaUgFvnL3Rx0gTtLlcYAeS7JKl8eSLo8D9FiSVZ09Fu2DFxERSUPaghcREUlDKvBlmFk3M5tqZgvMbL6ZXRt1ppoysywzm2lmH4eP5ZdRZ6otM8sws4/M7KWos9SGmS03s7lmNsfM8qPOU1Nm1srMnjazRWa20MxGRJ2pJszssPC1KP3bambXRZ2rpszs+vAzP8/MnjSzrKgz1YSZXRs+hvmp9nqY2cNmtt7M5sWMa2NmU8ysIPzfOpEZVOAPVATc4O79gOHAD82sX8SZamo3cJK7DwQGAaeZ2fCIM9XWtcDCqEPEyWh3H5Tip//8L/Cqux8ODCRFXxt3Xxy+FoOAo4GdwHMRx6oRM8sBrgFy3f1IIAMYG22q6jOzI4HLgaEE762zzKx3tKmqZSJwWplxNwFvuHsf4I1wOGFU4Mtw9zXu/mF4exvBF1ZOtKlqxgPbw8GG4V/KHnRhZl2BM4EHo84iYGbZwAnAQwDuvsfdN0ebKi5OBpa6e7wvnFWXMoEmZpYJNAVWR5ynJo4APnD3ne5eBLwFnBNxpipz9+nAF2VGjwEeCW8/AnwzkRlU4CthZj2BwcAH0SapubBJew6wHpji7in7WIB7gBuBkqiDxIEDeWY228yuiDpMDfUCNgD/CHebPGhmzaIOFQdjgSejDlFT7r4KuAtYAawBtrh7XrSpamQecLyZtTWzpsAZQLeIM9VWR3dfE95eC3RM5MpU4CtgZs2BZ4Dr3H1r1Hlqyt2Lw2bHrsDQsNkr5ZjZWcB6d58ddZY4GenuQ4DTCXYDnRB1oBrIBIYA97n7YGAHCW5yTDQzawScDfwr6iw1Fe7XHUPwA6wL0MzMvhdtqupz94XA74A84FVgDlAcaag48uAUtoS2qKrAl8PMGhIU9yfc/dmo88RD2HQ6lQP3CaWK44CzzWw5MAk4ycwejzZSzYVbWbj7eoJ9vUOjTVQjhUBhTKvQ0wQFP5WdDnzo7uuiDlILpwCfufsGd98LPAscG3GmGnH3h9z9aHc/AfgS+DTqTLW0zsw6A4T/1ydyZSrwZZiZEexTXOjuf4w6T22YWXszaxXebgKcCiyKNlXNuPvN7t7V3XsSNKG+6e4pt1UCYGbNzKxF6W3gawTNkSnF3dcCK83s/7d3PyFWlWEcx78/RLbbugAAA51JREFUdCEiTKgUFtRKst1QQUYZoilES7MIEtPAkjJcFMUsEgehqWnXpoIQJRUkswTlVkgKgYbi3/5gCwkRl4kwDJnl4+J9hplkYi537ni57/w+MMyZM+c+78Nd3Oe859zzPg/mruXArx1MqR1epIsvz6dLwGJJs/PzbDld+uVHSXfn7/sp9993dzajSTsArM3ttcA3UznYzKkM3qWeANYA5/PeNUBfRBzqYE6tWgDskDSDcjK3NyK6+vGyStwD7C+fvcwEdkdEo7MptWwTsCsvbV8E1nU4n5blydYK4NVO5zIZEfGTpC+BU5Sngk7TvSvB7ZM0D7gBvN5NX+KUtAdYCsyXdBnYAgwAeyW9Qul++vyU5uCV7MzMzOrjS/RmZmYVcoE3MzOrkAu8mZlZhVzgzczMKuQCb2ZmViEXeLNKSRqa+Kj/fe3q7A73QztzanLsvjs9plmN/JicWaUkDUXEnBZf2wC2RcSPbU6rmbFbztvMRnkGbzYNSHpb0glJ5yRtHbP/62x488tI0xtJ7wFPAp9LGhwn1jvZy/6spIHc1yvpeMbfP9LnWtIRSY/m9vxcahhJL0v6SlIje2N/mPsHKF3Qzkjalav+Hcyxfpb0wtS+U2b18Ep2ZpWTtBJYSFnvXsABSU9lO8v1EfFnLmV8QtK+iOiXtAx4KyJO3hbrGUojk8ciYljS3PzXTmBTRByV1E9ZtWvzBKn1Uro1XgcuSPo4It6V9EY2SELSKuBKRDybf/dM/h0xmx48gzer38r8OU1ZvnQRpeADvCnpLHCc0opz4bgRRj0NbI+IYYA8OegB7oqIo3nMDkqf+IkcjohrEfEXZQ37B8Y55jywQtIHkpZExLUm4poZnsGbTQcC3o+IT/+zU1pKKdiP52z8CDCrzWP/w+hE4vbY18ds/8s4n0cR8bukhym9wLdJOhwR/W3O0axKnsGb1e9bYL2kOQCS7ssuXT3A1Szui4DFTcT6HlgnaXbGmpuz6quSluQxa4CR2fwfwCO5/VyT+d7Ils1IuhcYjogvgEG6vx2t2R3jGbxZ5SLiO0kPAceyg90Q8BLQAF6T9BtwgXKZfqJYDUm9wElJfwOHgD5K68tPsvCP7Sr3EaV71gbgYJMpfwack3SKcm9/UNJNSkexjU3GMJv2/JicmZlZhXyJ3szMrEIu8GZmZhVygTczM6uQC7yZmVmFXODNzMwq5AJvZmZWIRd4MzOzCrnAm5mZVegWLaltWczPeokAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Optimal Hyperparameters Train Accuracy:  0.8159722222222222\n",
            "Optimal Hyperparameters Validation Accuracy:  0.7739130434782608\n",
            "Optimal Hyperparameters Test Accuracy:  0.7142857142857143\n",
            "Training Scores per Max Leaf Node Count: [0.7743055555555556, 0.7743055555555556, 0.7743055555555556, 0.7743055555555556, 0.7743055555555556, 0.7777777777777778, 0.7951388888888888, 0.8159722222222222, 0.8159722222222222]\n",
            "Validation Scores per Max Leaf Node Count: [0.7478260869565218, 0.7478260869565218, 0.7478260869565218, 0.7478260869565218, 0.7478260869565218, 0.7478260869565218, 0.7130434782608696, 0.7739130434782608, 0.7739130434782608]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0.8159722222222222,\n",
              " 0.7739130434782608,\n",
              " 0.7142857142857143,\n",
              " [0.7743055555555556,\n",
              "  0.7743055555555556,\n",
              "  0.7743055555555556,\n",
              "  0.7743055555555556,\n",
              "  0.7743055555555556,\n",
              "  0.7777777777777778,\n",
              "  0.7951388888888888,\n",
              "  0.8159722222222222,\n",
              "  0.8159722222222222],\n",
              " [0.7478260869565218,\n",
              "  0.7478260869565218,\n",
              "  0.7478260869565218,\n",
              "  0.7478260869565218,\n",
              "  0.7478260869565218,\n",
              "  0.7478260869565218,\n",
              "  0.7130434782608696,\n",
              "  0.7739130434782608,\n",
              "  0.7739130434782608])"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z0-UXUer3SCd"
      },
      "source": [
        "### Question 4.3 Report on LaTeX\n",
        "Answer the following question on LaTeX in the respective section.\n",
        "1. Report the values on LaTeX.\n",
        "2. How did the training accuracy and testing accuracy change after tuning compared to before? Briefly explain why.\n",
        "3. Paste the plot and explain any trends or patterns with the plot within validation and training scores and briefly explain why."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zbNi5JkX0HnG"
      },
      "source": [
        "## Additional Exercise (Ungraded)\n",
        "\n",
        "This section is ungraded. You can try it out for fun. sklearn's [DecisionTreeClassifier](https://scikit-learn.org/stable/modules/generated/sklearn.tree.DecisionTreeClassifier.html) used in this homework has a parameter called **min_samples_split**. Try playing around with it's values to see if you can get better accuracy for the X_train_small dataset. \n",
        "\n",
        "You don't need to add results of this section in your submission. It is for you to try out. \n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G2xmO04DXCwF"
      },
      "source": [
        "# Question 5: Feature Scaling Effects on KNNs and DTs"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NG7pZLBoXCwG"
      },
      "source": [
        "### Observing effects of standardizing features\n",
        "\n",
        "Up until now, we have not been using standardized features. Let's observe the effects of standardized features with decision trees and KNNs.\n",
        "\n",
        "Standardization, or feature scaling / data normalization, is a common preprocessing step for data within machine learning. We will see why it's important.\n",
        "\n",
        "Here is a definition taken from SK-Learn's website on Standardization:\n",
        "\n",
        "*Standardization of datasets is a common requirement for many machine learning estimators implemented in scikit-learn; they might behave badly if the individual features do not more or less look like standard normally distributed data: Gaussian with zero mean and unit variance.*\n",
        "\n",
        "*In practice we often ignore the shape of the distribution and just transform the data to center it by removing the mean value of each feature, then scale it by dividing non-constant features by their standard deviation.*\n",
        "\n",
        "Learn More: https://scikit-learn.org/stable/modules/preprocessing.html\n",
        "\n",
        "To start, uncomment the code below and run to retrieve the data. (Recomment before submission.)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T5RokryGXCwH"
      },
      "source": [
        "# We will use the same data as the previous tasks\n",
        "# Normalize data\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "\n",
        "scaler = StandardScaler()\n",
        "X_train_scaled = scaler.fit_transform(X_train)\n",
        "X_val_scaled = scaler.transform(X_val)\n",
        "X_test_scaled = scaler.transform(X_test)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t9COC_Pa3SCg"
      },
      "source": [
        "### Helper Functions\n",
        "  \n",
        "We implemented above the KNN algorithm. Sci-kit learn also has their own version of the KNN algorithm which we will use in this following task. Use the two helper functions below in this next task."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LJvoFpqh3SCg"
      },
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "\n",
        "def train_KNN(X, y, norm=2, K=5):\n",
        "  \"\"\"\n",
        "  Trains a KNN classifier on the given X, y data with the specified \n",
        "  norm and K.\n",
        "  \n",
        "  Args:\n",
        "    X ((n,p) np.ndarray): The input Xs, which are in an n (number of samples) by\n",
        "                          p (number of features) matrix\n",
        "    y ((n,) np.ndarray): The input ys, which are in an n length array\n",
        "    norm (int): The number form of the norm. Note that sklearn only allows L1 and L2 norms,\n",
        "                (norm would be 1 and 2 respectively). Default is 2.\n",
        "    K (int): The value of K for the KNN algorithm. Default is 5.\n",
        "  \n",
        "  Returns:\n",
        "    clf(KNeighborsClassifier): the trained KNN model\n",
        "  \"\"\"\n",
        "  \n",
        "  clf = KNeighborsClassifier(n_neighbors=K, p=norm)\n",
        "  clf.fit(X,y)\n",
        "  return clf\n",
        "\n",
        "def train_decision_tree(X, y, depth=None, leaf_count=None):\n",
        "  \"\"\"\n",
        "  This helper function is defined again from a previous section. \n",
        "  \n",
        "  Trains a decision tree classifier on the given X, y data with the specified \n",
        "  tree depth d and max leaf node count max_leaf_num.\n",
        "  \n",
        "  Args:\n",
        "    X ((n,p) np.ndarray): The input Xs, which are in an n (number of samples) by\n",
        "                          p (number of features) matrix\n",
        "    y ((n,) np.ndarray): The input ys, which are in an n length array\n",
        "    depth (int): The maximum depth of the tree. A value of None means no restrictions\n",
        "             on the depth of the tree.\n",
        "    leaf_count (int): The maximum leaf count of the tree's leaf nodes. A value of None means \n",
        "    no restrictions on the leaf count of the tree.\n",
        "  \n",
        "  Returns:\n",
        "    clf(DecisionTreeClassifier): the trained decision tree\n",
        "  \"\"\"\n",
        "  clf = DecisionTreeClassifier(max_depth=depth, max_leaf_nodes=leaf_count, criterion=\"entropy\", random_state=1)\n",
        "  clf.fit(X,y)\n",
        "  return clf\n",
        "\n",
        "def predict(clf, X_test):\n",
        "  \"\"\"\n",
        "  This helper function is defined again from a previous section. \n",
        "  \n",
        "  Uses a trained model to predict on a given test set.\n",
        "  \n",
        "  Args:\n",
        "    clf (Classifier): Trained classifier such as KNN or Decision Tree\n",
        "    X_test ((n,p) np.ndarray): The input Xs, which are in an n (number of samples) by\n",
        "                               p (number of features) matrix\n",
        "  \n",
        "  Returns:\n",
        "    y_pred ((n,) np.ndarray): The output predictions, which are in an n length array\n",
        "  \"\"\"\n",
        "  y_pred = clf.predict(X_test)\n",
        "  return y_pred\n",
        "\n",
        "def evaluate(predicted_values, actual_values):\n",
        "    \"\"\"\n",
        "    This helper function is defined again from a previous section. \n",
        "    \n",
        "    Computes the accuracy of the given datapoints.\n",
        "    \n",
        "    Args:\n",
        "        predicted_values: numpy array\n",
        "        actual_values: numpy array\n",
        "    \n",
        "    Returns:\n",
        "        a floating point number representing the accuracy\n",
        "    \"\"\"\n",
        "    return accuracy_score(predicted_values, actual_values)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iVfyIhUiXCwK"
      },
      "source": [
        "### Retrieving Metrics for Unstandardized Data\n",
        "Fill out this function to retrieve training and test accuracies for both KNN and decision tree models. Use default hyperparameters."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K0XxTvHMXCwM"
      },
      "source": [
        "def get_classifier_metrics(X_train, y_train, X_test, y_test):\n",
        "  \"\"\"\n",
        "  Create a decision tree and KNN classifer on the normal dataset.\n",
        "    \n",
        "  Args: (Note that n is not the same among train and test sets, \n",
        "         but merely refers to sample size)\n",
        "    X_train ((n,p) np.ndarray)\n",
        "    y_train ((n,) np.ndarray)\n",
        "    X_test ((n,p) np.ndarray)\n",
        "    y_test ((n,) np.ndarray)\n",
        "\n",
        "  To return:\n",
        "    knn_train_accuracy (float): Accuracy of KNN for train set\n",
        "    knn_test_accuracy (float): Accuracy of KNN for test set\n",
        "    dt_train_accuracy (float): Accuracy of DT for train set\n",
        "    dt_test_accuracy (float): Accuracy of DT for test set    \n",
        "  \"\"\"\n",
        "  \n",
        "  # <---- Your code here ----->\n",
        "  knn_train_accuracy = evaluate(predict(train_KNN(X_train, y_train, norm=2, K=5), X_train), y_train)\n",
        "  knn_test_accuracy = evaluate(predict(train_KNN(X_train, y_train, norm=2, K=5), X_test), y_test)\n",
        "  dt_train_accuracy = evaluate(predict(train_decision_tree(X_train, y_train, depth=None, leaf_count=None), X_train), y_train)\n",
        "  dt_test_accuracy = evaluate(predict(train_decision_tree(X_train, y_train, depth=None, leaf_count=None), X_test), y_test)\n",
        "\n",
        "  # <---- Your code ends here ----->\n",
        "  \n",
        "  print(\"knn_train_accuracy: \", knn_train_accuracy)\n",
        "  print(\"knn_test_accuracy: \", knn_test_accuracy)\n",
        "  print(\"dt_train_accuracy: \", dt_train_accuracy)\n",
        "  print(\"dt_test_accuracy: \", dt_test_accuracy)\n",
        "  \n",
        "  return knn_train_accuracy, knn_test_accuracy, dt_train_accuracy, dt_test_accuracy"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rF3Jmuy53SCk"
      },
      "source": [
        "Uncomment the code below and run the code."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kWS33o6OXCwO",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5b70e2bf-f0d3-4ffd-fa96-cf8e523bf367"
      },
      "source": [
        "print(\"FOR UNSTANDARDIZED DATA\")\n",
        "get_classifier_metrics(X_train, y_train, X_test, y_test)\n",
        "\n",
        "print(\"\\nFOR STANDARDIZED DATA\")\n",
        "get_classifier_metrics(X_train_scaled, y_train, X_test_scaled, y_test)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "FOR UNSTANDARDIZED DATA\n",
            "knn_train_accuracy:  0.7899305555555556\n",
            "knn_test_accuracy:  0.7012987012987013\n",
            "dt_train_accuracy:  1.0\n",
            "dt_test_accuracy:  0.7532467532467533\n",
            "\n",
            "FOR STANDARDIZED DATA\n",
            "knn_train_accuracy:  0.8177083333333334\n",
            "knn_test_accuracy:  0.8181818181818182\n",
            "dt_train_accuracy:  1.0\n",
            "dt_test_accuracy:  0.7532467532467533\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(0.8177083333333334, 0.8181818181818182, 1.0, 0.7532467532467533)"
            ]
          },
          "metadata": {},
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-ZCmPf0G3SCn"
      },
      "source": [
        "### Question 5 Report on LaTex\n",
        "Answer the following question on LaTeX in the respective section.\n",
        "1. Report the values on LaTeX.\n",
        "2. What happens to performance when we use standardization for data with decision trees? What about KNN? Briefly explain why each happened."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "G-7QVbownVl4"
      },
      "source": [
        "# Submission\n",
        "\n",
        "**Due Tuesday 20th September, 11:59pm**\n",
        "\n",
        "There are 3 deliverables you need to submit to Gradescope.\n",
        "\n",
        "1. A PDF compilation of hw1.tex with your name(s) in agreement.\n",
        "\n",
        "2. Your .ipynb file. Named, hw1.ipynb\n",
        "\n",
        "3. Your .py file. Named, hw1.py\n",
        "\n",
        "If you are working in pairs make sure to add your team member’s name on Gradescope when submitting\n"
      ]
    }
  ]
}