model_assessment.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "model_assessment.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyPOhMfjy88vdfKwiwBMKS6e",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/wolframalexa/FrequentistML/blob/master/model_assessment.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eiXQPh9VywyN",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# Assignment\n",
        "\n",
        "# Re-implement the example in section 7.10.2 using any simple, out of the box classifier (like K nearest neighbors from sci-kit). \n",
        "# Reproduce the results for the incorrect and correct way of doing cross-validation."
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nQyOytqX0Bpn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import random\n",
        "from sklearn.neighbors import KNeighborsClassifier\n",
        "from sklearn.model_selection import cross_validate # uses Kfold to cross validate\n",
        "from sklearn.model_selection import KFold\n",
        "from sklearn.feature_selection import SelectKBest, f_classif"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZnA97MfNv_G-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#----------GENERATE DATASET\n",
        "np.random.seed(100)\n",
        "\n",
        "data = np.random.normal(size=(50,500))\n",
        "dataframe = pd.DataFrame(data)\n",
        "\n",
        "# split data into inputs and outputs\n",
        "X = dataframe\n",
        "y = np.ravel(np.random.randint(0, 2, (50,1)))\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e_c9i51n9C-P",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#----------THE WRONG WAY\n",
        "\n",
        "# prescreen features by computing their correlation with output class\n",
        "num_features = 10\n",
        "X_new = SelectKBest(f_classif, k=num_features).fit_transform(X,y) #f_classif works with negative data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OU4dR7EL9GkL",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "0e96635b-1c7e-42cd-ca77-13e88b619422"
      },
      "source": [
        "# construct classifier using k nearest neighbours using the two selected features\n",
        "neigh = KNeighborsClassifier(n_neighbors=3)\n",
        "neigh.fit(X_new, y)\n",
        "\n",
        "# perform cross-validation using kfold\n",
        "cv_results = cross_validate(neigh, X_new,y,cv=2)\n",
        "print(\"Average scores from incorrect cross validation:\",np.mean(np.abs(cv_results['test_score'])))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Average scores from incorrect cross validation: 0.78\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2JE4lV_BHX78",
        "colab_type": "text"
      },
      "source": [
        "After pre-screening for predictors and choosing the ones having the biggest bearing on the outcome classification, then constructing a model around those traits and performing cross-validation, we obtain a score of .78."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Dn3UiglAebm",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#-------THE RIGHT WAY\n",
        "\n",
        "# prepare data to work with Kfolds function\n",
        "kf = KFold(n_splits=2, shuffle=True, random_state=0) # draw samples randomly for kfolds\n",
        "\n",
        "X_trains = []\n",
        "X_tests = []\n",
        "y_trains = []\n",
        "y_tests = []\n",
        "X = X.to_numpy()\n",
        "y = np.array(y)\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cWmC2RFePrdQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "dbb83ff1-a2b9-4579-b11b-75ee8d4d75cf"
      },
      "source": [
        "# split into folds \n",
        "for train_index, test_index in kf.split(X): # there might have been a better way to do this with indexing\n",
        "  X_trains.append(X[train_index])\n",
        "  X_tests.append(X[test_index])\n",
        "  y_trains.append(y[train_index])\n",
        "  y_tests.append(y[test_index])\n",
        "\n",
        "\n",
        "# for each fold, find \"good\" predictors using all samples except fold k\n",
        "scores = 0\n",
        "\n",
        "for fold in range(0,len(X_trains)):\n",
        "  # create arrays dropping the current fold\n",
        "  temp_X_trains = np.delete(X_trains, fold, axis=0)\n",
        "  temp_X_trains = temp_X_trains[0]\n",
        "  temp_X_tests = np.delete(X_tests, fold, axis=0)\n",
        "  temp_X_tests = temp_X_tests[0]\n",
        "  temp_y_trains = np.delete(y_trains, fold, axis=0)\n",
        "  temp_y_trains = np.ravel(np.transpose(temp_y_trains))\n",
        "  temp_y_tests = np.delete(y_tests, fold, axis=0)\n",
        "\n",
        "  # select good features\n",
        "  kbest = SelectKBest(f_classif, k=num_features)\n",
        "  X_new = kbest.fit_transform(temp_X_trains,temp_y_trains)\n",
        "  selected_features = kbest.get_support()\n",
        "\n",
        "  # build a multivariate classifier using all samples except fold k\n",
        "  X_selected = np.concatenate((np.zeros((temp_X_trains.shape[0],len(selected_features)-num_features)) ,temp_X_trains[:, selected_features]), axis=1) # fit to selected features\n",
        "\n",
        "  # use regressor to predict for samples in fold k\n",
        "  neigh = KNeighborsClassifier(n_neighbors=3)\n",
        "  neigh.fit(X_selected, temp_y_trains)\n",
        "\n",
        "\n",
        "  score = neigh.score(X_tests[fold],y_tests[fold])\n",
        "  scores += abs(score)\n",
        "\n",
        "# compute average score\n",
        "avg_score = scores/len(X_trains)\n",
        "print(\"Average score using correct method:\",scores)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Average score using correct method: 1.2000000000000002\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "S0vMIWOeb5OV",
        "colab_type": "text"
      },
      "source": [
        "Done correctly, cross-validation should yield a higher average error rate. This is because it has not seen the test samples, so it cannot use them in its inference. Here, we can see that the \"correct\" error value is much higher than the \"incorrect\" error rate: 0.78 vs 1.2."
      ]
    }
  ]
}