Default_Cross_val.ipynb

{
 "cells": [
  {
   "cell_type": "heading",
   "metadata": {},
   "level": 1,
   "source": [
    "Analyse the Default dataset and use k-fold Cross validation to split the dataset into 5 sets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.linear_model import Lasso\n",
    "from sklearn.metrics import mean_squared_error\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import train_test_split, LeaveOneOut, KFold, cross_val_score\n",
    "from sklearn.preprocessing import PolynomialFeatures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 10000 entries, 0 to 9999\n",
      "Data columns (total 5 columns):\n",
      "Unnamed: 0    10000 non-null int64\n",
      "default       10000 non-null object\n",
      "student       10000 non-null object\n",
      "balance       10000 non-null float64\n",
      "income        10000 non-null float64\n",
      "dtypes: float64(2), int64(1), object(2)\n",
      "memory usage: 468.8+ KB\n"
     ]
    }
   ],
   "source": [
    "dataset = pd.read_csv('/Users/arsheenkhatib/Desktop/ML dataset+screenshots/Default.csv', na_values='?').dropna()\n",
    "dataset.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[1 1 1]\n",
      " [2 2 2]\n",
      " [3 3 3]]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX4AAAD8CAYAAABw1c+bAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4wLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvpW3flQAAFhNJREFUeJzt3X+QXWd93/H3p4sXGmDARgtxbQuZ\nqSeDoWDDHYHiTFlGBQtIUNJmWrmUcQiMptRuQ9vJjCEzeGom47T5IxmmEKOmKpAJNgRwqjIYUAVb\np2Ht+IoaGxsMQpBYFVMriPAjMOxI+faPexSuV/vj7O69dyWf92vmju59znPO+e6ZR58995x790lV\nIUnqjr+z2QVIkibL4JekjjH4JaljDH5J6hiDX5I6xuCXpI4x+CWpYwx+SeoYg1+SOuZJm13AUrZs\n2VLbtm3b7DIk6bxx+PDhv6yqmTZ9z8ng37ZtG/1+f7PLkKTzRpI/b9vXSz2S1DEGvyR1jMEvSR1j\n8EtSxxj8ktQxqwZ/ksuSfC7Jl5M8lOTXluiTJO9OciTJA0leMrTs+iRfax7Xj/oHkCStTZsz/lPA\nv6+q5wMvB25IcuWiPq8Brmgee4HfA0hyEXAz8DJgO3BzkgtHVPtZ5ufnufXWW5mfnx/XLtRhji+N\n0yTH16qf46+qbwHfap5/P8mXgUuAh4e67QY+WIN5HO9J8swkFwOzwMGqOgmQ5CCwC7h9pD8Fg4O2\nc+dOFhYWmJ6e5tChQ+zYsWPUu1FHOb40TpMeX2u6xp9kG3A1cO+iRZcAjw69Pta0Lde+1Lb3Jukn\n6Z84cWItZQEwNzfHwsICp0+fZmFhgbm5uTVvQ1qO40vjNOnx1Tr4kzwN+Bjwtqr63uLFS6xSK7Sf\n3Vi1r6p6VdWbmWn1rePHmZ2dZXp6mqmpKaanp5mdnV3zNqTlOL40TpMeX63+ZEOSCxiE/h9W1ceX\n6HIMuGzo9aXA8aZ9dlH73HoKXc2OHTs4dOgQc3NzzM7O+jZcI+X40jhNenxlcFl+hQ5JgA8AJ6vq\nbcv0eR1wI/BaBjdy311V25ubu4eBM5/y+QLw0jPX/JfT6/XKv9UjSe0lOVxVvTZ925zxXwO8EXgw\nyf1N2zuArQBVdRvwSQahfwT4IfCmZtnJJO8C7mvWu2W10JckjVebT/X8b5a+Vj/cp4Abllm2H9i/\nruokSSPnN3clqWMMfknqGINfkjrG4JekjjH4JaljDH5J6hiDX5I6xuCXpI4x+CWpYwx+SeoYg1+S\nOsbgl6SOMfglqWMMfknqGINfkjpm1b/Hn2Q/8PPAY1X1wiWW/zrwhqHtPR+YaSZh+SbwfeA0cKrt\n7DCSpPFpc8b/fmDXcgur6rer6qqqugp4O/C/Fs2y9cpmuaEvSeeAVYO/qu4G2k6XeB1w+4YqkiSN\n1ciu8Sf5KQbvDD421FzAZ5IcTrJ3VPuSJK1fm8nW2/oF4E8XXea5pqqOJ3k2cDDJV5p3EGdpfjHs\nBdi6desIy5IkDRvlp3r2sOgyT1Udb/59DLgT2L7cylW1r6p6VdWbmZkZYVmSpGEjCf4kzwBeAfz3\nobanJnn6mefAq4EvjWJ/kqT1a/NxztuBWWBLkmPAzcAFAFV1W9Ptl4DPVNVfD636HODOJGf286Gq\n+tToSpckrceqwV9V17Xo834GH/scbjsKvHi9hUmSxsNv7kpSxxj8ktQxBr8kdYzBL0kdY/BLUscY\n/JLUMQa/JHWMwS9JHWPwS1LHGPyS1DEGvyR1jMEvSR1j8EtSxxj8ktQxBr8kdYzBL0kds2rwJ9mf\n5LEkS06bmGQ2yXeT3N883jm0bFeSR5IcSXLTKAuXJK1PmzP+9wO7VunzJ1V1VfO4BSDJFPAe4DXA\nlcB1Sa7cSLGSpI1bNfir6m7g5Dq2vR04UlVHq2oBuAPYvY7tSJJGaFTX+Hck+WKSu5K8oGm7BHh0\nqM+xpk2StIlWnWy9hS8Az62qHyR5LfDHwBVAluhby20kyV5gL8DWrVtHUJYkaSkbPuOvqu9V1Q+a\n558ELkiyhcEZ/mVDXS8Fjq+wnX1V1auq3szMzEbLkiQtY8PBn+Snk6R5vr3Z5reB+4ArklyeZBrY\nAxzY6P4kSRuz6qWeJLcDs8CWJMeAm4ELAKrqNuCXgbcmOQX8CNhTVQWcSnIj8GlgCthfVQ+N5aeQ\nJLWWQUafW3q9XvX7/c0uQ5LOG0kOV1WvTV+/uStJHWPwS1LHGPyS1DEGvyR1jMEvSR1j8EtSxxj8\nktQxBr8kdYzBL0kdY/BLUscY/JLUMQa/JHWMwS9JHWPwS1LHGPyS1DEGvyR1zKrBn2R/kseSfGmZ\n5W9I8kDz+HySFw8t+2aSB5Pcn8SZVSTpHNDmjP/9wK4Vln8DeEVVvQh4F7Bv0fJXVtVVbWeGkSSN\n16pz7lbV3Um2rbD880Mv7wEu3XhZkqRxGfU1/jcDdw29LuAzSQ4n2bvSikn2Jukn6Z84cWLEZUmS\nzlj1jL+tJK9kEPw/N9R8TVUdT/Js4GCSr1TV3UutX1X7aC4T9Xq9c28GeEl6ghjJGX+SFwG/D+yu\nqm+faa+q482/jwF3AttHsT9J0vptOPiTbAU+Dryxqr461P7UJE8/8xx4NbDkJ4MkSZOz6qWeJLcD\ns8CWJMeAm4ELAKrqNuCdwLOA9yYBONV8guc5wJ1N25OAD1XVp8bwM0iS1qDNp3quW2X5W4C3LNF+\nFHjx2WtIkjaT39yVpI4x+CWpYwx+SeoYg1+SOsbgl6SOMfglqWMMfknqGINfkjrG4JekjjH4Jalj\nDH5J6hiDX5I6xuCXpI4x+CWpYwx+SeqYVsGfZH+Sx5IsOYNWBt6d5EiSB5K8ZGjZ9Um+1jyuH1Xh\nkqT1aXvG/35g1wrLXwNc0Tz2Ar8HkOQiBjN2vYzBfLs3J7lwvcVKkjauVfBX1d3AyRW67AY+WAP3\nAM9McjFwLXCwqk5W1XeAg6z8C2RD5ufnufXWW5mfnx/XLtRhji+N0yTH16pTL7Z0CfDo0OtjTdty\n7SM3Pz/Pzp07WVhYYHp6mkOHDrFjx45x7Eod5PjSOE16fI3q5m6WaKsV2s/eQLI3ST9J/8SJE2su\nYG5ujoWFBU6fPs3CwgJzc3Nr3oa0HMeXxmnS42tUwX8MuGzo9aXA8RXaz1JV+6qqV1W9mZmZNRcw\nOzvL9PQ0U1NTTE9PMzs7u+ZtSMtxfGmcJj2+UrXkCfjZHZNtwCeq6oVLLHsdcCPwWgY3ct9dVdub\nm7uHgTOf8vkC8NKqWul+Ab1er/r9ftuf4W/Nz88zNzfH7Oysb8M1co4vjdNGx1eSw1XVa9W3TfAn\nuR2YBbYA/4/BJ3UuAKiq25IE+M8Mbtz+EHhTVfWbdX8VeEezqd+sqv+22v7WG/yS1FVrCf5WN3er\n6rpVlhdwwzLL9gP72+xHkjR+fnNXkjrG4JekjjH4JaljDH5J6hiDX5I6xuCXpI4x+CWpYwx+SeoY\ng1+SOsbgl6SOMfglqWMMfknqGINfkjrG4JekjjH4JaljDH5J6phWwZ9kV5JHkhxJctMSy38nyf3N\n46tJ/mpo2emhZQdGWbwkae1WnYEryRTwHuBVDCZPvy/Jgap6+Eyfqvq3Q/3/NXD10CZ+VFVXja5k\nSdJGtDnj3w4cqaqjVbUA3AHsXqH/dcDtoyhOkjR6bYL/EuDRodfHmrazJHkucDnw2aHmpyTpJ7kn\nyS+uu1JJ0ki0mWw9S7TVMn33AB+tqtNDbVur6niS5wGfTfJgVX39rJ0ke4G9AFu3bm1RliRpPdqc\n8R8DLht6fSlwfJm+e1h0maeqjjf/HgXmePz1/+F++6qqV1W9mZmZFmVJktajTfDfB1yR5PIk0wzC\n/axP5yT5GeBCYH6o7cIkT26ebwGuAR5evK4kaXJWvdRTVaeS3Ah8GpgC9lfVQ0luAfpVdeaXwHXA\nHVU1fBno+cD7kvwNg18yvzX8aSBJ0uTl8Tl9buj1etXv9ze7DEk6byQ5XFW9Nn395q4kdYzBL0kd\nY/BLUscY/JLUMQa/JHWMwS9JHWPwS1LHGPyS1DEGvyR1jMEvSR1j8EtSxxj8ktQxBr8kdYzBL0kd\nY/BLUscY/JLUMa2CP8muJI8kOZLkpiWW/0qSE0nubx5vGVp2fZKvNY/rR1m8JGntVp16MckU8B7g\nVQwmXr8vyYElplD8cFXduGjdi4CbgR5QwOFm3e+MpHpJ0pq1OePfDhypqqNVtQDcAexuuf1rgYNV\ndbIJ+4PArvWVKkkahTbBfwnw6NDrY03bYv8kyQNJPprksjWuS5K9SfpJ+idOnGhRliRpPdoEf5Zo\nWzxD+/8AtlXVi4D/CXxgDesOGqv2VVWvqnozMzMtypIkrUeb4D8GXDb0+lLg+HCHqvp2Vf24eflf\ngJe2XVeSNFltgv8+4IoklyeZBvYAB4Y7JLl46OXrgS83zz8NvDrJhUkuBF7dtEmSNsmqn+qpqlNJ\nbmQQ2FPA/qp6KMktQL+qDgD/JsnrgVPASeBXmnVPJnkXg18eALdU1ckx/BySpJZSteQl903V6/Wq\n3+9vdhmSdN5Icriqem36+s1dSeoYg1+SOsbgl6SOMfglqWMMfknqGINfkjrG4JekjjH4JaljDH5J\n6hiDX5I6xuCXpI4x+CWpYwx+SeoYg1+SOsbgl6SOMfglqWNaBX+SXUkeSXIkyU1LLP93SR5O8kCS\nQ0meO7TsdJL7m8eBxetKkiZr1akXk0wB7wFexWDy9PuSHKiqh4e6/R+gV1U/TPJW4D8B/6xZ9qOq\numrEdUuS1qnNGf924EhVHa2qBeAOYPdwh6r6XFX9sHl5D3DpaMuUJI1Km+C/BHh06PWxpm05bwbu\nGnr9lCT9JPck+cXlVkqyt+nXP3HiRIuyJEnrseqlHiBLtC05Q3uSfwH0gFcMNW+tquNJngd8NsmD\nVfX1szZYtQ/YB4PJ1lvUJUlahzZn/MeAy4ZeXwocX9wpyT8CfgN4fVX9+Ex7VR1v/j0KzAFXb6Be\nSdIGtQn++4ArklyeZBrYAzzu0zlJrgbexyD0HxtqvzDJk5vnW4BrgOGbwpKkCVv1Uk9VnUpyI/Bp\nYArYX1UPJbkF6FfVAeC3gacBf5QE4C+q6vXA84H3JfkbBr9kfmvRp4EkSROWqnPvcnqv16t+v7/Z\nZUjSeSPJ4arqtenrN3clqWMMfknqGINfkjrG4JekjjH4JaljDH5J6hiDX5I6xuCXpI4x+CWpYwx+\nSeoYg1+SOsbgl6SOMfglqWMMfknqGINfkjqmVfAn2ZXkkSRHkty0xPInJ/lws/zeJNuGlr29aX8k\nybWjK12StB6rBn+SKeA9wGuAK4Hrkly5qNubge9U1d8Hfgf4j826VzKYqvEFwC7gvc32JEmbpM0Z\n/3bgSFUdraoF4A5g96I+u4EPNM8/CuzMYA7G3cAdVfXjqvoGcKTZ3ljMz89z6623Mj8/P65dqMMc\nXxqnSY6vVefcBS4BHh16fQx42XJ9mjl6vws8q2m/Z9G6l6y72hXMz8+zc+dOFhYWmJ6e5tChQ+zY\nsWMcu1IHOb40TpMeX23O+LNE2+KJepfr02bdwQaSvUn6SfonTpxoUdbjzc3NsbCwwOnTp1lYWGBu\nbm7N25CW4/jSOE16fLUJ/mPAZUOvLwWOL9cnyZOAZwAnW64LQFXtq6peVfVmZmbaVT9kdnaW6elp\npqammJ6eZnZ2ds3bkJbj+NI4TXp8pWrJE/CfdBgE+VeBncD/Be4D/nlVPTTU5wbgH1TVv0yyB/jH\nVfVPk7wA+BCD6/p/DzgEXFFVp1faZ6/Xq36/v+YfZn5+nrm5OWZnZ30brpFzfGmcNjq+khyuql6r\nvqsFf7PB1wK/C0wB+6vqN5PcAvSr6kCSpwB/AFzN4Ex/T1Udbdb9DeBXgVPA26rqrtX2t97gl6Su\nGnnwT5rBL0lrs5bg95u7ktQxBr8kdYzBL0kdY/BLUscY/JLUMefkp3qSnAD+fJ2rbwH+coTljIp1\nrY11rY11rc0Tsa7nVlWrb7+ek8G/EUn6bT/SNEnWtTbWtTbWtTZdr8tLPZLUMQa/JHXMEzH49212\nAcuwrrWxrrWxrrXpdF1PuGv8kqSVPRHP+CVJKzhvgj/J/iSPJfnSMsuT5N3NxO4PJHnJ0LLrk3yt\neVw/4bre0NTzQJLPJ3nx0LJvJnkwyf1JRvpX6VrUNZvku82+70/yzqFlu5I80hzLmyZc168P1fSl\nJKeTXNQsG+fxuizJ55J8OclDSX5tiT4TH2Mt65r4GGtZ18THWMu6Jj7GkjwlyZ8l+WJT139Yos+T\nk3y4OSb3Jtk2tOztTfsjSa7dcEFVdV48gH8IvAT40jLLXwvcxWDWr5cD9zbtFwFHm38vbJ5fOMG6\nfvbM/hhMWH/v0LJvAls26XjNAp9Yon0K+DrwPGAa+CJw5aTqWtT3F4DPTuh4XQy8pHn+dAZzUFy5\nqM/Ex1jLuiY+xlrWNfEx1qauzRhjzZh5WvP8AuBe4OWL+vwr4Lbm+R7gw83zK5tj9GTg8ubYTW2k\nnvPmjL+q7mbwt/6Xsxv4YA3cAzwzycXAtcDBqjpZVd8BDgK7JlVXVX2+2S8M5h++dFT73khdK9gO\nHKmqo1W1ANzB4NhuRl3XAbePat8rqapvVdUXmuffB77M2fNDT3yMtalrM8ZYy+O1nLGNsXXUNZEx\n1oyZHzQvL2gei2+w7gY+0Dz/KLAzSZr2O6rqx1X1DeAIg2O4budN8Lew1KTwl6zQvhnezOCM8YwC\nPpPkcJK9m1DPjuat510ZzJYG58jxSvJTDMLzY0PNEzlezVvsqxmclQ3b1DG2Ql3DJj7GVqlr08bY\nasdr0mMsyVSS+4HHGJwoLDu+quoU8F3gWYzheD1pIyufYzY84fs4JXklg/+UPzfUfE1VHU/ybOBg\nkq80Z8ST8AUGX/H+QQYzrP0xcAXnyPFi8Bb8T6tq+N3B2I9XkqcxCIK3VdX3Fi9eYpWJjLFV6jrT\nZ+JjbJW6Nm2MtTleTHiM1WDK2auSPBO4M8kLq2r4XtfExtcT6Yx/uYndW0/4Pi5JXgT8PrC7qr59\npr2qjjf/PgbcyQbfvq1FVX3vzFvPqvokcEGSLZwDx6uxh0Vvwcd9vJJcwCAs/rCqPr5El00ZYy3q\n2pQxtlpdmzXG2hyvxsTHWLPtvwLmOPty4N8elwzmOn8Gg8uioz9eo7yBMe4HsI3lb1a+jsffePuz\npv0i4BsMbrpd2Dy/aIJ1bWVwTe5nF7U/FXj60PPPA7smWNdP85PvcWwH/qI5dk9icHPycn5y4+0F\nk6qrWX5mwD91User+dk/CPzuCn0mPsZa1jXxMdayromPsTZ1bcYYA2aAZzbP/y7wJ8DPL+pzA4+/\nufuR5vkLePzN3aNs8ObueXOpJ8ntDD4lsCXJMeBmBjdIqKrbgE8y+NTFEeCHwJuaZSeTvAu4r9nU\nLfX4t3bjruudDK7TvXdwn4ZTNfgjTM9h8HYPBv8RPlRVn5pgXb8MvDXJKeBHwJ4ajLJTSW4EPs3g\n0xf7q+qhCdYF8EvAZ6rqr4dWHevxAq4B3gg82FyHBXgHg1DdzDHWpq7NGGNt6tqMMdamLpj8GLsY\n+ECSKQZXWj5SVZ9IcgvQr6oDwH8F/iDJEQa/lPY0NT+U5CPAw8Ap4IYaXDZaN7+5K0kd80S6xi9J\nasHgl6SOMfglqWMMfknqGINfkjrG4JekjjH4JaljDH5J6pj/D6/9TT2CUzdKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x1a1547f748>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 0.  0.  0.]\n",
      " [ 0.  0.  0.]\n",
      " [ 0.  0.  0.]]\n",
      "          balance\n",
      "0      729.526495\n",
      "1      817.180407\n",
      "2     1073.549164\n",
      "3      529.250605\n",
      "4      785.655883\n",
      "5      919.588530\n",
      "6      825.513331\n",
      "7      808.667504\n",
      "8     1161.057854\n",
      "9        0.000000\n",
      "10       0.000000\n",
      "11    1220.583753\n",
      "12     237.045114\n",
      "13     606.742343\n",
      "14    1112.968401\n",
      "15     286.232560\n",
      "16       0.000000\n",
      "17     527.540184\n",
      "18     485.936864\n",
      "19    1095.072735\n",
      "20     228.952550\n",
      "21     954.261793\n",
      "22    1055.956605\n",
      "23     641.984389\n",
      "24     773.211725\n",
      "25     855.008523\n",
      "26     642.999739\n",
      "27    1454.863272\n",
      "28     615.704277\n",
      "29    1119.569353\n",
      "...           ...\n",
      "9970  1294.500408\n",
      "9971   180.620128\n",
      "9972   755.432801\n",
      "9973   876.119027\n",
      "9974   933.332025\n",
      "9975   908.315934\n",
      "9976   218.417559\n",
      "9977   915.439827\n",
      "9978  2202.462395\n",
      "9979   173.249172\n",
      "9980   770.015741\n",
      "9981   739.418018\n",
      "9982   623.526119\n",
      "9983   506.625454\n",
      "9984   875.241640\n",
      "9985   842.949429\n",
      "9986   401.332674\n",
      "9987  1092.906583\n",
      "9988     0.000000\n",
      "9989   999.281112\n",
      "9990   372.379239\n",
      "9991   658.799558\n",
      "9992  1111.647317\n",
      "9993   938.836241\n",
      "9994   172.412987\n",
      "9995   711.555020\n",
      "9996   757.962918\n",
      "9997   845.411989\n",
      "9998  1569.009053\n",
      "9999   200.922183\n",
      "\n",
      "[10000 rows x 1 columns]\n",
      "          balance\n",
      "0      729.526495\n",
      "1      817.180407\n",
      "2     1073.549164\n",
      "3      529.250605\n",
      "4      785.655883\n",
      "5      919.588530\n",
      "6      825.513331\n",
      "7      808.667504\n",
      "8     1161.057854\n",
      "9        0.000000\n",
      "10       0.000000\n",
      "11    1220.583753\n",
      "12     237.045114\n",
      "13     606.742343\n",
      "14    1112.968401\n",
      "15     286.232560\n",
      "16       0.000000\n",
      "17     527.540184\n",
      "18     485.936864\n",
      "19    1095.072735\n",
      "20     228.952550\n",
      "21     954.261793\n",
      "22    1055.956605\n",
      "23     641.984389\n",
      "24     773.211725\n",
      "25     855.008523\n",
      "26     642.999739\n",
      "27    1454.863272\n",
      "28     615.704277\n",
      "29    1119.569353\n",
      "...           ...\n",
      "9970  1294.500408\n",
      "9971   180.620128\n",
      "9972   755.432801\n",
      "9973   876.119027\n",
      "9974   933.332025\n",
      "9975   908.315934\n",
      "9976   218.417559\n",
      "9977   915.439827\n",
      "9978  2202.462395\n",
      "9979   173.249172\n",
      "9980   770.015741\n",
      "9981   739.418018\n",
      "9982   623.526119\n",
      "9983   506.625454\n",
      "9984   875.241640\n",
      "9985   842.949429\n",
      "9986   401.332674\n",
      "9987  1092.906583\n",
      "9988     0.000000\n",
      "9989   999.281112\n",
      "9990   372.379239\n",
      "9991   658.799558\n",
      "9992  1111.647317\n",
      "9993   938.836241\n",
      "9994   172.412987\n",
      "9995   711.555020\n",
      "9996   757.962918\n",
      "9997   845.411989\n",
      "9998  1569.009053\n",
      "9999   200.922183\n",
      "\n",
      "[10000 rows x 1 columns]\n",
      "          balance\n",
      "0      729.526495\n",
      "1      817.180407\n",
      "2     1073.549164\n",
      "3      529.250605\n",
      "4      785.655883\n",
      "5      919.588530\n",
      "6      825.513331\n",
      "7      808.667504\n",
      "8     1161.057854\n",
      "9        0.000000\n",
      "10       0.000000\n",
      "11    1220.583753\n",
      "12     237.045114\n",
      "13     606.742343\n",
      "14    1112.968401\n",
      "15     286.232560\n",
      "16       0.000000\n",
      "17     527.540184\n",
      "18     485.936864\n",
      "19    1095.072735\n",
      "20     228.952550\n",
      "21     954.261793\n",
      "22    1055.956605\n",
      "23     641.984389\n",
      "24     773.211725\n",
      "25     855.008523\n",
      "26     642.999739\n",
      "27    1454.863272\n",
      "28     615.704277\n",
      "29    1119.569353\n",
      "...           ...\n",
      "9970  1294.500408\n",
      "9971   180.620128\n",
      "9972   755.432801\n",
      "9973   876.119027\n",
      "9974   933.332025\n",
      "9975   908.315934\n",
      "9976   218.417559\n",
      "9977   915.439827\n",
      "9978  2202.462395\n",
      "9979   173.249172\n",
      "9980   770.015741\n",
      "9981   739.418018\n",
      "9982   623.526119\n",
      "9983   506.625454\n",
      "9984   875.241640\n",
      "9985   842.949429\n",
      "9986   401.332674\n",
      "9987  1092.906583\n",
      "9988     0.000000\n",
      "9989   999.281112\n",
      "9990   372.379239\n",
      "9991   658.799558\n",
      "9992  1111.647317\n",
      "9993   938.836241\n",
      "9994   172.412987\n",
      "9995   711.555020\n",
      "9996   757.962918\n",
      "9997   845.411989\n",
      "9998  1569.009053\n",
      "9999   200.922183\n",
      "\n",
      "[10000 rows x 1 columns]\n",
      "          balance\n",
      "0      729.526495\n",
      "1      817.180407\n",
      "2     1073.549164\n",
      "3      529.250605\n",
      "4      785.655883\n",
      "5      919.588530\n",
      "6      825.513331\n",
      "7      808.667504\n",
      "8     1161.057854\n",
      "9        0.000000\n",
      "10       0.000000\n",
      "11    1220.583753\n",
      "12     237.045114\n",
      "13     606.742343\n",
      "14    1112.968401\n",
      "15     286.232560\n",
      "16       0.000000\n",
      "17     527.540184\n",
      "18     485.936864\n",
      "19    1095.072735\n",
      "20     228.952550\n",
      "21     954.261793\n",
      "22    1055.956605\n",
      "23     641.984389\n",
      "24     773.211725\n",
      "25     855.008523\n",
      "26     642.999739\n",
      "27    1454.863272\n",
      "28     615.704277\n",
      "29    1119.569353\n",
      "...           ...\n",
      "9970  1294.500408\n",
      "9971   180.620128\n",
      "9972   755.432801\n",
      "9973   876.119027\n",
      "9974   933.332025\n",
      "9975   908.315934\n",
      "9976   218.417559\n",
      "9977   915.439827\n",
      "9978  2202.462395\n",
      "9979   173.249172\n",
      "9980   770.015741\n",
      "9981   739.418018\n",
      "9982   623.526119\n",
      "9983   506.625454\n",
      "9984   875.241640\n",
      "9985   842.949429\n",
      "9986   401.332674\n",
      "9987  1092.906583\n",
      "9988     0.000000\n",
      "9989   999.281112\n",
      "9990   372.379239\n",
      "9991   658.799558\n",
      "9992  1111.647317\n",
      "9993   938.836241\n",
      "9994   172.412987\n",
      "9995   711.555020\n",
      "9996   757.962918\n",
      "9997   845.411989\n",
      "9998  1569.009053\n",
      "9999   200.922183\n",
      "\n",
      "[10000 rows x 1 columns]\n",
      "          balance\n",
      "0      729.526495\n",
      "1      817.180407\n",
      "2     1073.549164\n",
      "3      529.250605\n",
      "4      785.655883\n",
      "5      919.588530\n",
      "6      825.513331\n",
      "7      808.667504\n",
      "8     1161.057854\n",
      "9        0.000000\n",
      "10       0.000000\n",
      "11    1220.583753\n",
      "12     237.045114\n",
      "13     606.742343\n",
      "14    1112.968401\n",
      "15     286.232560\n",
      "16       0.000000\n",
      "17     527.540184\n",
      "18     485.936864\n",
      "19    1095.072735\n",
      "20     228.952550\n",
      "21     954.261793\n",
      "22    1055.956605\n",
      "23     641.984389\n",
      "24     773.211725\n",
      "25     855.008523\n",
      "26     642.999739\n",
      "27    1454.863272\n",
      "28     615.704277\n",
      "29    1119.569353\n",
      "...           ...\n",
      "9970  1294.500408\n",
      "9971   180.620128\n",
      "9972   755.432801\n",
      "9973   876.119027\n",
      "9974   933.332025\n",
      "9975   908.315934\n",
      "9976   218.417559\n",
      "9977   915.439827\n",
      "9978  2202.462395\n",
      "9979   173.249172\n",
      "9980   770.015741\n",
      "9981   739.418018\n",
      "9982   623.526119\n",
      "9983   506.625454\n",
      "9984   875.241640\n",
      "9985   842.949429\n",
      "9986   401.332674\n",
      "9987  1092.906583\n",
      "9988     0.000000\n",
      "9989   999.281112\n",
      "9990   372.379239\n",
      "9991   658.799558\n",
      "9992  1111.647317\n",
      "9993   938.836241\n",
      "9994   172.412987\n",
      "9995   711.555020\n",
      "9996   757.962918\n",
      "9997   845.411989\n",
      "9998  1569.009053\n",
      "9999   200.922183\n",
      "\n",
      "[10000 rows x 1 columns]\n",
      "          balance\n",
      "0      729.526495\n",
      "1      817.180407\n",
      "2     1073.549164\n",
      "3      529.250605\n",
      "4      785.655883\n",
      "5      919.588530\n",
      "6      825.513331\n",
      "7      808.667504\n",
      "8     1161.057854\n",
      "9        0.000000\n",
      "10       0.000000\n",
      "11    1220.583753\n",
      "12     237.045114\n",
      "13     606.742343\n",
      "14    1112.968401\n",
      "15     286.232560\n",
      "16       0.000000\n",
      "17     527.540184\n",
      "18     485.936864\n",
      "19    1095.072735\n",
      "20     228.952550\n",
      "21     954.261793\n",
      "22    1055.956605\n",
      "23     641.984389\n",
      "24     773.211725\n",
      "25     855.008523\n",
      "26     642.999739\n",
      "27    1454.863272\n",
      "28     615.704277\n",
      "29    1119.569353\n",
      "...           ...\n",
      "9970  1294.500408\n",
      "9971   180.620128\n",
      "9972   755.432801\n",
      "9973   876.119027\n",
      "9974   933.332025\n",
      "9975   908.315934\n",
      "9976   218.417559\n",
      "9977   915.439827\n",
      "9978  2202.462395\n",
      "9979   173.249172\n",
      "9980   770.015741\n",
      "9981   739.418018\n",
      "9982   623.526119\n",
      "9983   506.625454\n",
      "9984   875.241640\n",
      "9985   842.949429\n",
      "9986   401.332674\n",
      "9987  1092.906583\n",
      "9988     0.000000\n",
      "9989   999.281112\n",
      "9990   372.379239\n",
      "9991   658.799558\n",
      "9992  1111.647317\n",
      "9993   938.836241\n",
      "9994   172.412987\n",
      "9995   711.555020\n",
      "9996   757.962918\n",
      "9997   845.411989\n",
      "9998  1569.009053\n",
      "9999   200.922183\n",
      "\n",
      "[10000 rows x 1 columns]\n",
      "          balance\n",
      "0      729.526495\n",
      "1      817.180407\n",
      "2     1073.549164\n",
      "3      529.250605\n",
      "4      785.655883\n",
      "5      919.588530\n",
      "6      825.513331\n",
      "7      808.667504\n",
      "8     1161.057854\n",
      "9        0.000000\n",
      "10       0.000000\n",
      "11    1220.583753\n",
      "12     237.045114\n",
      "13     606.742343\n",
      "14    1112.968401\n",
      "15     286.232560\n",
      "16       0.000000\n",
      "17     527.540184\n",
      "18     485.936864\n",
      "19    1095.072735\n",
      "20     228.952550\n",
      "21     954.261793\n",
      "22    1055.956605\n",
      "23     641.984389\n",
      "24     773.211725\n",
      "25     855.008523\n",
      "26     642.999739\n",
      "27    1454.863272\n",
      "28     615.704277\n",
      "29    1119.569353\n",
      "...           ...\n",
      "9970  1294.500408\n",
      "9971   180.620128\n",
      "9972   755.432801\n",
      "9973   876.119027\n",
      "9974   933.332025\n",
      "9975   908.315934\n",
      "9976   218.417559\n",
      "9977   915.439827\n",
      "9978  2202.462395\n",
      "9979   173.249172\n",
      "9980   770.015741\n",
      "9981   739.418018\n",
      "9982   623.526119\n",
      "9983   506.625454\n",
      "9984   875.241640\n",
      "9985   842.949429\n",
      "9986   401.332674\n",
      "9987  1092.906583\n",
      "9988     0.000000\n",
      "9989   999.281112\n",
      "9990   372.379239\n",
      "9991   658.799558\n",
      "9992  1111.647317\n",
      "9993   938.836241\n",
      "9994   172.412987\n",
      "9995   711.555020\n",
      "9996   757.962918\n",
      "9997   845.411989\n",
      "9998  1569.009053\n",
      "9999   200.922183\n",
      "\n",
      "[10000 rows x 1 columns]\n",
      "          balance\n",
      "0      729.526495\n",
      "1      817.180407\n",
      "2     1073.549164\n",
      "3      529.250605\n",
      "4      785.655883\n",
      "5      919.588530\n",
      "6      825.513331\n",
      "7      808.667504\n",
      "8     1161.057854\n",
      "9        0.000000\n",
      "10       0.000000\n",
      "11    1220.583753\n",
      "12     237.045114\n",
      "13     606.742343\n",
      "14    1112.968401\n",
      "15     286.232560\n",
      "16       0.000000\n",
      "17     527.540184\n",
      "18     485.936864\n",
      "19    1095.072735\n",
      "20     228.952550\n",
      "21     954.261793\n",
      "22    1055.956605\n",
      "23     641.984389\n",
      "24     773.211725\n",
      "25     855.008523\n",
      "26     642.999739\n",
      "27    1454.863272\n",
      "28     615.704277\n",
      "29    1119.569353\n",
      "...           ...\n",
      "9970  1294.500408\n",
      "9971   180.620128\n",
      "9972   755.432801\n",
      "9973   876.119027\n",
      "9974   933.332025\n",
      "9975   908.315934\n",
      "9976   218.417559\n",
      "9977   915.439827\n",
      "9978  2202.462395\n",
      "9979   173.249172\n",
      "9980   770.015741\n",
      "9981   739.418018\n",
      "9982   623.526119\n",
      "9983   506.625454\n",
      "9984   875.241640\n",
      "9985   842.949429\n",
      "9986   401.332674\n",
      "9987  1092.906583\n",
      "9988     0.000000\n",
      "9989   999.281112\n",
      "9990   372.379239\n",
      "9991   658.799558\n",
      "9992  1111.647317\n",
      "9993   938.836241\n",
      "9994   172.412987\n",
      "9995   711.555020\n",
      "9996   757.962918\n",
      "9997   845.411989\n",
      "9998  1569.009053\n",
      "9999   200.922183\n",
      "\n",
      "[10000 rows x 1 columns]\n",
      "          balance\n",
      "0      729.526495\n",
      "1      817.180407\n",
      "2     1073.549164\n",
      "3      529.250605\n",
      "4      785.655883\n",
      "5      919.588530\n",
      "6      825.513331\n",
      "7      808.667504\n",
      "8     1161.057854\n",
      "9        0.000000\n",
      "10       0.000000\n",
      "11    1220.583753\n",
      "12     237.045114\n",
      "13     606.742343\n",
      "14    1112.968401\n",
      "15     286.232560\n",
      "16       0.000000\n",
      "17     527.540184\n",
      "18     485.936864\n",
      "19    1095.072735\n",
      "20     228.952550\n",
      "21     954.261793\n",
      "22    1055.956605\n",
      "23     641.984389\n",
      "24     773.211725\n",
      "25     855.008523\n",
      "26     642.999739\n",
      "27    1454.863272\n",
      "28     615.704277\n",
      "29    1119.569353\n",
      "...           ...\n",
      "9970  1294.500408\n",
      "9971   180.620128\n",
      "9972   755.432801\n",
      "9973   876.119027\n",
      "9974   933.332025\n",
      "9975   908.315934\n",
      "9976   218.417559\n",
      "9977   915.439827\n",
      "9978  2202.462395\n",
      "9979   173.249172\n",
      "9980   770.015741\n",
      "9981   739.418018\n",
      "9982   623.526119\n",
      "9983   506.625454\n",
      "9984   875.241640\n",
      "9985   842.949429\n",
      "9986   401.332674\n",
      "9987  1092.906583\n",
      "9988     0.000000\n",
      "9989   999.281112\n",
      "9990   372.379239\n",
      "9991   658.799558\n",
      "9992  1111.647317\n",
      "9993   938.836241\n",
      "9994   172.412987\n",
      "9995   711.555020\n",
      "9996   757.962918\n",
      "9997   845.411989\n",
      "9998  1569.009053\n",
      "9999   200.922183\n",
      "\n",
      "[10000 rows x 1 columns]\n"
     ]
    }
   ],
   "source": [
    "p_order = np.arange(1,4)\n",
    "r_state = np.arange(0,3)\n",
    "\n",
    "X, Y = np.meshgrid(p_order, r_state, indexing='ij')\n",
    "print(X)\n",
    "plt.plot(X, Y, marker='.', color='k', linestyle='none')\n",
    "plt.show()\n",
    "Z = np.zeros((p_order.size,r_state.size))\n",
    "print(Z)\n",
    "\n",
    "# Generate 3 random splits of the dataset\n",
    "for (i,j),v in np.ndenumerate(Z):\n",
    "    poly = PolynomialFeatures(int(X[i,j]))\n",
    "    x = dataset.iloc[:,3:4]\n",
    "    print(x)\n",
    "    y = dataset.default\n",
    "    labelencoder_y = LabelEncoder()\n",
    "    y = labelencoder_y.fit_transform(y)\n",
    " #   pred = regr.predict(X_test)\n",
    " #   Z[i,j]= mean_squared_error(y_test, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "X_train, X_test, y_train, y_test = train_test_split(x, y,\n",
    "                               test_size=0.5, random_state=0)                                                                 \n",
    " #regr.fit(X_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 10000 entries, 0 to 9999\n",
      "Data columns (total 5 columns):\n",
      "Unnamed: 0    10000 non-null int64\n",
      "default       10000 non-null object\n",
      "student       10000 non-null object\n",
      "balance       10000 non-null float64\n",
      "income        10000 non-null float64\n",
      "dtypes: float64(2), int64(1), object(2)\n",
      "memory usage: 468.8+ KB\n"
     ]
    }
   ],
   "source": [
    "dataset.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 0 0 ..., 0 0 0]\n"
     ]
    }
   ],
   "source": [
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def calc_train_error(X_train, y_train, model):\n",
    "    '''returns in-sample error for already fit model.'''\n",
    "    predictions = model.predict(X_train)\n",
    "    mse = mean_squared_error(y_train, predictions)\n",
    "    rmse = np.sqrt(mse)\n",
    "    return mse\n",
    "    \n",
    "def calc_validation_error(X_test, y_test, model):\n",
    "    '''returns out-of-sample error for already fit model.'''\n",
    "    predictions = model.predict(X_test)\n",
    "    mse = mean_squared_error(y_test, predictions)\n",
    "    rmse = np.sqrt(mse)\n",
    "    return mse\n",
    "    \n",
    "def calc_metrics(X_train, y_train, X_test, y_test, model):\n",
    "    '''fits model and returns the RMSE for in-sample error and out-of-sample error'''\n",
    "    model.fit(X_train, y_train)\n",
    "    train_error = calc_train_error(X_train, y_train, model)\n",
    "    validation_error = calc_validation_error(X_test, y_test, model)\n",
    "    return train_error, validation_error"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train error: 0.0254 | test error: 0.0312\n"
     ]
    }
   ],
   "source": [
    "lr = LinearRegression(fit_intercept=True)\n",
    "train_error, test_error = calc_metrics(X_train, y_train, X_test, y_test, lr)\n",
    "train_error, test_error = round(train_error, 4), round(test_error, 4)\n",
    "print('train error: {} | test error: {}'.format(train_error, test_error))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "p_order = np.arange(1,11)\n",
    "r_state = np.arange(0,10)\n",
    "\n",
    "# LeaveOneOut CV\n",
    "lr = LinearRegression(fit_intercept=True)\n",
    "loo = LeaveOneOut()\n",
    "loo.get_n_splits(dataset)\n",
    "scores = list()\n",
    "for i in p_order:\n",
    "    poly = PolynomialFeatures(i)\n",
    "    x = dataset.iloc[:,3:4]\n",
    "    y = dataset.default\n",
    "    labelencoder_y = LabelEncoder()\n",
    "    y = labelencoder_y.fit_transform(y)\n",
    "    score = cross_val_score(lr, x, y, cv=loo, scoring='neg_mean_squared_error').mean()\n",
    "    scores.append(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-0.028270522423\n"
     ]
    }
   ],
   "source": [
    "print(score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# k-fold CV\n",
    "folds = 5\n",
    "elements = len(dataset.index)\n",
    "\n",
    "X, Y = np.meshgrid(p_order, r_state, indexing='ij')\n",
    "Z = np.zeros((p_order.size,r_state.size))\n",
    "\n",
    "lr = LinearRegression(fit_intercept=True)\n",
    "\n",
    "for (i,j),v in np.ndenumerate(Z):\n",
    "    poly = PolynomialFeatures(X[i,j])\n",
    "    x = dataset.iloc[:,3:4]\n",
    "    y = dataset.default\n",
    "    labelencoder_y = LabelEncoder()\n",
    "    y = labelencoder_y.fit_transform(y)\n",
    "    kf_5 = KFold(n_splits=folds, random_state=Y[i,j])\n",
    "    Z[i,j] = cross_val_score(lr, x, y, cv=kf_5, scoring='neg_mean_squared_error').mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KFold(n_splits=5, random_state=9, shuffle=False)\n",
      "[[-0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515\n",
      "  -0.02825515 -0.02825515 -0.02825515 -0.02825515]\n",
      " [-0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515\n",
      "  -0.02825515 -0.02825515 -0.02825515 -0.02825515]\n",
      " [-0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515\n",
      "  -0.02825515 -0.02825515 -0.02825515 -0.02825515]\n",
      " [-0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515\n",
      "  -0.02825515 -0.02825515 -0.02825515 -0.02825515]\n",
      " [-0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515\n",
      "  -0.02825515 -0.02825515 -0.02825515 -0.02825515]\n",
      " [-0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515\n",
      "  -0.02825515 -0.02825515 -0.02825515 -0.02825515]\n",
      " [-0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515\n",
      "  -0.02825515 -0.02825515 -0.02825515 -0.02825515]\n",
      " [-0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515\n",
      "  -0.02825515 -0.02825515 -0.02825515 -0.02825515]\n",
      " [-0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515\n",
      "  -0.02825515 -0.02825515 -0.02825515 -0.02825515]\n",
      " [-0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515 -0.02825515\n",
      "  -0.02825515 -0.02825515 -0.02825515 -0.02825515]]\n"
     ]
    }
   ],
   "source": [
    "print(kf_5)\n",
    "print(Z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}