variousCFmethods.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!pip install neo4j\n",
    "#! pip install surprise\n",
    "from surprise import Reader\n",
    "from surprise import Dataset\n",
    "from surprise.model_selection import cross_validate\n",
    "from surprise import KNNBaseline\n",
    "from surprise import SVDpp\n",
    "from surprise import CoClustering\n",
    "from surprise import SlopeOne\n",
    "from surprise.model_selection import RandomizedSearchCV\n",
    "from surprise.model_selection import GridSearchCV\n",
    "from surprise.model_selection import cross_validate\n",
    "from neo4j import GraphDatabase\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import networkx as nx\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "from statistics import mean, median\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "uri = \"bolt://3.220.233.169:7687\"\n",
    "driver = GraphDatabase.driver(uri, auth=(\"neo4j\", \"i-0e23d19f0d8795714\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "with driver.session() as session:\n",
    "    result = session.run('\\\n",
    "    MATCH (u:User)-->(r:Review)-->(b:Business)-->(:City)-->(s:State {name:\"IL\"})\\\n",
    "    RETURN u.id, r.stars, b.id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "PA_reviews=result.values()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "PA_reviewsdf=pd.DataFrame(PA_reviews)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>I530_QvrsKeVedKUE0vGtQ</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8iXSfNmDjiZEhZp2am8qmQ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>IqIdDTzBKJDxnPegaUBvEw</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8iXSfNmDjiZEhZp2am8qmQ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>W-FnSpt7r2mMt5DN4z4rEw</td>\n",
       "      <td>5.0</td>\n",
       "      <td>8iXSfNmDjiZEhZp2am8qmQ</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>_xMbeumYOT4eZ5WQaVt7Sg</td>\n",
       "      <td>1.0</td>\n",
       "      <td>iKH0466yPmflu8oPhspusw</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>TUXYiutwnAnxO47yf-QEZA</td>\n",
       "      <td>5.0</td>\n",
       "      <td>iKH0466yPmflu8oPhspusw</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        0    1                       2\n",
       "0  I530_QvrsKeVedKUE0vGtQ  5.0  8iXSfNmDjiZEhZp2am8qmQ\n",
       "1  IqIdDTzBKJDxnPegaUBvEw  5.0  8iXSfNmDjiZEhZp2am8qmQ\n",
       "2  W-FnSpt7r2mMt5DN4z4rEw  5.0  8iXSfNmDjiZEhZp2am8qmQ\n",
       "3  _xMbeumYOT4eZ5WQaVt7Sg  1.0  iKH0466yPmflu8oPhspusw\n",
       "4  TUXYiutwnAnxO47yf-QEZA  5.0  iKH0466yPmflu8oPhspusw"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PA_reviewsdf.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(42378, 3)"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "PA_reviewsdf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "reader = Reader(rating_scale=(1, 5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "PA_reviewsdf.columns=['userID', 'rating','itemID']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "PA_reviewsdf.to_pickle('PA_reviews')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [],
   "source": [
    "#PA_reviewsdf="
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = Dataset.load_from_df(PA_reviewsdf[['userID', 'itemID', 'rating']], reader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "sim_options = {'name': 'pearson_baseline',\n",
    "               'user_based': False,\n",
    "               'min_support' : 8\n",
    "              }\n",
    "\n",
    "bsl_options = {'method': 'als',\n",
    "               'n_epochs': 10,\n",
    "               'reg_u': 5,\n",
    "               'reg_i': 1\n",
    "               }\n",
    "\n",
    "\n",
    "algo=KNNBaseline(sim_options=sim_options, bsl_options=bsl_options, k=20, min_k=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating RMSE, MAE of algorithm KNNBaseline on 3 split(s).\n",
      "\n",
      "                  Fold 1  Fold 2  Fold 3  Mean    Std     \n",
      "RMSE (testset)    1.2431  1.2394  1.2448  1.2424  0.0022  \n",
      "MAE (testset)     0.9859  0.9832  0.9876  0.9856  0.0018  \n",
      "Fit time          11.02   9.40    6.27    8.90    1.97    \n",
      "Test time         4.92    3.82    2.96    3.90    0.80    \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'test_rmse': array([1.24313092, 1.23940257, 1.24478002]),\n",
       " 'test_mae': array([0.98590472, 0.9832448 , 0.98764356]),\n",
       " 'fit_time': (11.021914005279541, 9.395414352416992, 6.273874282836914),\n",
       " 'test_time': (4.916727304458618, 3.823942184448242, 2.9577760696411133)}"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_validate(algo, data, cv=3, verbose=True, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = {'bsl_options': {'method': ['als'],\n",
    "                              'learning_rate': [.005, .01],\n",
    "                              'reg': [.2,.1,.05],\n",
    "                              'reg_i': [1,2,3,4],\n",
    "                              'reg_u': [2,3,4,5],\n",
    "                              'n_epochs':[15,20]\n",
    "                             },\n",
    "              \n",
    "              'k': [20,30,40],\n",
    "              'min_k':[10,15],\n",
    "              'sim_options': {'name': ['pearson_baseline'],\n",
    "                              'min_support': [5, 10],\n",
    "                              'user_based': [True],\n",
    "                              'shrinkage': [50,100]\n",
    "                             }\n",
    "              }\n",
    "\n",
    "gs = RandomizedSearchCV(KNNBaseline, param_grid, n_iter=10, measures=['rmse'], cv=3, n_jobs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 194,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n"
     ]
    }
   ],
   "source": [
    "gs.fit(data)\n",
    "results_df = pd.DataFrame.from_dict(gs.cv_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 181,
   "metadata": {},
   "outputs": [],
   "source": [
    "sortedresults.to_pickle(\"KNNB_random2_il\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth', 100)\n",
    "sortedresults=results_df.sort_values('rank_test_rmse')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>split0_test_rmse</th>\n",
       "      <th>split1_test_rmse</th>\n",
       "      <th>split2_test_rmse</th>\n",
       "      <th>mean_test_rmse</th>\n",
       "      <th>std_test_rmse</th>\n",
       "      <th>rank_test_rmse</th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_test_time</th>\n",
       "      <th>std_test_time</th>\n",
       "      <th>params</th>\n",
       "      <th>param_bsl_options</th>\n",
       "      <th>param_k</th>\n",
       "      <th>param_min_k</th>\n",
       "      <th>param_sim_options</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.308330</td>\n",
       "      <td>1.301990</td>\n",
       "      <td>1.304702</td>\n",
       "      <td>1.305008</td>\n",
       "      <td>0.002597</td>\n",
       "      <td>1</td>\n",
       "      <td>5.690586</td>\n",
       "      <td>0.097362</td>\n",
       "      <td>0.620896</td>\n",
       "      <td>0.007401</td>\n",
       "      <td>{'bsl_options': {'method': 'als', 'learning_rate': 0.005, 'reg': 0.1, 'reg_i': 2, 'reg_u': 5, 'n...</td>\n",
       "      <td>{'method': 'als', 'learning_rate': 0.005, 'reg': 0.1, 'reg_i': 2, 'reg_u': 5, 'n_epochs': 15}</td>\n",
       "      <td>30</td>\n",
       "      <td>10</td>\n",
       "      <td>{'name': 'pearson_baseline', 'min_support': 10, 'user_based': True, 'shrinkage': 100}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.308280</td>\n",
       "      <td>1.302266</td>\n",
       "      <td>1.305252</td>\n",
       "      <td>1.305266</td>\n",
       "      <td>0.002455</td>\n",
       "      <td>2</td>\n",
       "      <td>5.709057</td>\n",
       "      <td>0.095400</td>\n",
       "      <td>0.763704</td>\n",
       "      <td>0.166784</td>\n",
       "      <td>{'bsl_options': {'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 2, 'reg_u': 4, '...</td>\n",
       "      <td>{'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 2, 'reg_u': 4, 'n_epochs': 20}</td>\n",
       "      <td>40</td>\n",
       "      <td>15</td>\n",
       "      <td>{'name': 'pearson_baseline', 'min_support': 10, 'user_based': True, 'shrinkage': 50}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.308282</td>\n",
       "      <td>1.302266</td>\n",
       "      <td>1.305252</td>\n",
       "      <td>1.305267</td>\n",
       "      <td>0.002456</td>\n",
       "      <td>3</td>\n",
       "      <td>5.740249</td>\n",
       "      <td>0.105940</td>\n",
       "      <td>0.619192</td>\n",
       "      <td>0.001107</td>\n",
       "      <td>{'bsl_options': {'method': 'als', 'learning_rate': 0.005, 'reg': 0.2, 'reg_i': 2, 'reg_u': 4, 'n...</td>\n",
       "      <td>{'method': 'als', 'learning_rate': 0.005, 'reg': 0.2, 'reg_i': 2, 'reg_u': 4, 'n_epochs': 15}</td>\n",
       "      <td>30</td>\n",
       "      <td>15</td>\n",
       "      <td>{'name': 'pearson_baseline', 'min_support': 10, 'user_based': True, 'shrinkage': 100}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1.309129</td>\n",
       "      <td>1.303536</td>\n",
       "      <td>1.306063</td>\n",
       "      <td>1.306243</td>\n",
       "      <td>0.002287</td>\n",
       "      <td>4</td>\n",
       "      <td>5.717280</td>\n",
       "      <td>0.091121</td>\n",
       "      <td>0.588458</td>\n",
       "      <td>0.006096</td>\n",
       "      <td>{'bsl_options': {'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 3, 'reg_u': 4, '...</td>\n",
       "      <td>{'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 3, 'reg_u': 4, 'n_epochs': 20}</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>{'name': 'pearson_baseline', 'min_support': 10, 'user_based': True, 'shrinkage': 100}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.309370</td>\n",
       "      <td>1.303621</td>\n",
       "      <td>1.306747</td>\n",
       "      <td>1.306579</td>\n",
       "      <td>0.002350</td>\n",
       "      <td>5</td>\n",
       "      <td>5.728159</td>\n",
       "      <td>0.097608</td>\n",
       "      <td>0.587565</td>\n",
       "      <td>0.008711</td>\n",
       "      <td>{'bsl_options': {'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 2, 'reg_u': 3, '...</td>\n",
       "      <td>{'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 2, 'reg_u': 3, 'n_epochs': 20}</td>\n",
       "      <td>20</td>\n",
       "      <td>15</td>\n",
       "      <td>{'name': 'pearson_baseline', 'min_support': 10, 'user_based': True, 'shrinkage': 50}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.309375</td>\n",
       "      <td>1.303629</td>\n",
       "      <td>1.306866</td>\n",
       "      <td>1.306623</td>\n",
       "      <td>0.002352</td>\n",
       "      <td>6</td>\n",
       "      <td>5.703381</td>\n",
       "      <td>0.101096</td>\n",
       "      <td>0.602577</td>\n",
       "      <td>0.005020</td>\n",
       "      <td>{'bsl_options': {'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 2, 'reg_u': 3, '...</td>\n",
       "      <td>{'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 2, 'reg_u': 3, 'n_epochs': 20}</td>\n",
       "      <td>20</td>\n",
       "      <td>15</td>\n",
       "      <td>{'name': 'pearson_baseline', 'min_support': 5, 'user_based': True, 'shrinkage': 50}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>1.311417</td>\n",
       "      <td>1.304982</td>\n",
       "      <td>1.308866</td>\n",
       "      <td>1.308422</td>\n",
       "      <td>0.002646</td>\n",
       "      <td>7</td>\n",
       "      <td>5.719638</td>\n",
       "      <td>0.082924</td>\n",
       "      <td>0.687753</td>\n",
       "      <td>0.139705</td>\n",
       "      <td>{'bsl_options': {'method': 'als', 'learning_rate': 0.01, 'reg': 0.2, 'reg_i': 1, 'reg_u': 3, 'n_...</td>\n",
       "      <td>{'method': 'als', 'learning_rate': 0.01, 'reg': 0.2, 'reg_i': 1, 'reg_u': 3, 'n_epochs': 20}</td>\n",
       "      <td>20</td>\n",
       "      <td>10</td>\n",
       "      <td>{'name': 'pearson_baseline', 'min_support': 10, 'user_based': True, 'shrinkage': 100}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1.311393</td>\n",
       "      <td>1.304993</td>\n",
       "      <td>1.309182</td>\n",
       "      <td>1.308523</td>\n",
       "      <td>0.002654</td>\n",
       "      <td>8</td>\n",
       "      <td>5.696199</td>\n",
       "      <td>0.097143</td>\n",
       "      <td>0.762162</td>\n",
       "      <td>0.145246</td>\n",
       "      <td>{'bsl_options': {'method': 'als', 'learning_rate': 0.01, 'reg': 0.05, 'reg_i': 1, 'reg_u': 3, 'n...</td>\n",
       "      <td>{'method': 'als', 'learning_rate': 0.01, 'reg': 0.05, 'reg_i': 1, 'reg_u': 3, 'n_epochs': 15}</td>\n",
       "      <td>40</td>\n",
       "      <td>15</td>\n",
       "      <td>{'name': 'pearson_baseline', 'min_support': 5, 'user_based': True, 'shrinkage': 50}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>1.312446</td>\n",
       "      <td>1.307620</td>\n",
       "      <td>1.310095</td>\n",
       "      <td>1.310054</td>\n",
       "      <td>0.001971</td>\n",
       "      <td>9</td>\n",
       "      <td>5.733105</td>\n",
       "      <td>0.102816</td>\n",
       "      <td>0.601119</td>\n",
       "      <td>0.008509</td>\n",
       "      <td>{'bsl_options': {'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 4, 'reg_u': 3, '...</td>\n",
       "      <td>{'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 4, 'reg_u': 3, 'n_epochs': 20}</td>\n",
       "      <td>20</td>\n",
       "      <td>15</td>\n",
       "      <td>{'name': 'pearson_baseline', 'min_support': 5, 'user_based': True, 'shrinkage': 100}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.314760</td>\n",
       "      <td>1.309868</td>\n",
       "      <td>1.312690</td>\n",
       "      <td>1.312439</td>\n",
       "      <td>0.002005</td>\n",
       "      <td>10</td>\n",
       "      <td>5.711052</td>\n",
       "      <td>0.101553</td>\n",
       "      <td>0.623533</td>\n",
       "      <td>0.005624</td>\n",
       "      <td>{'bsl_options': {'method': 'als', 'learning_rate': 0.005, 'reg': 0.1, 'reg_i': 3, 'reg_u': 2, 'n...</td>\n",
       "      <td>{'method': 'als', 'learning_rate': 0.005, 'reg': 0.1, 'reg_i': 3, 'reg_u': 2, 'n_epochs': 20}</td>\n",
       "      <td>30</td>\n",
       "      <td>15</td>\n",
       "      <td>{'name': 'pearson_baseline', 'min_support': 10, 'user_based': True, 'shrinkage': 100}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   split0_test_rmse  split1_test_rmse  split2_test_rmse  mean_test_rmse  \\\n",
       "0          1.308330          1.301990          1.304702        1.305008   \n",
       "1          1.308280          1.302266          1.305252        1.305266   \n",
       "3          1.308282          1.302266          1.305252        1.305267   \n",
       "7          1.309129          1.303536          1.306063        1.306243   \n",
       "6          1.309370          1.303621          1.306747        1.306579   \n",
       "4          1.309375          1.303629          1.306866        1.306623   \n",
       "8          1.311417          1.304982          1.308866        1.308422   \n",
       "5          1.311393          1.304993          1.309182        1.308523   \n",
       "9          1.312446          1.307620          1.310095        1.310054   \n",
       "2          1.314760          1.309868          1.312690        1.312439   \n",
       "\n",
       "   std_test_rmse  rank_test_rmse  mean_fit_time  std_fit_time  mean_test_time  \\\n",
       "0       0.002597               1       5.690586      0.097362        0.620896   \n",
       "1       0.002455               2       5.709057      0.095400        0.763704   \n",
       "3       0.002456               3       5.740249      0.105940        0.619192   \n",
       "7       0.002287               4       5.717280      0.091121        0.588458   \n",
       "6       0.002350               5       5.728159      0.097608        0.587565   \n",
       "4       0.002352               6       5.703381      0.101096        0.602577   \n",
       "8       0.002646               7       5.719638      0.082924        0.687753   \n",
       "5       0.002654               8       5.696199      0.097143        0.762162   \n",
       "9       0.001971               9       5.733105      0.102816        0.601119   \n",
       "2       0.002005              10       5.711052      0.101553        0.623533   \n",
       "\n",
       "   std_test_time  \\\n",
       "0       0.007401   \n",
       "1       0.166784   \n",
       "3       0.001107   \n",
       "7       0.006096   \n",
       "6       0.008711   \n",
       "4       0.005020   \n",
       "8       0.139705   \n",
       "5       0.145246   \n",
       "9       0.008509   \n",
       "2       0.005624   \n",
       "\n",
       "                                                                                                params  \\\n",
       "0  {'bsl_options': {'method': 'als', 'learning_rate': 0.005, 'reg': 0.1, 'reg_i': 2, 'reg_u': 5, 'n...   \n",
       "1  {'bsl_options': {'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 2, 'reg_u': 4, '...   \n",
       "3  {'bsl_options': {'method': 'als', 'learning_rate': 0.005, 'reg': 0.2, 'reg_i': 2, 'reg_u': 4, 'n...   \n",
       "7  {'bsl_options': {'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 3, 'reg_u': 4, '...   \n",
       "6  {'bsl_options': {'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 2, 'reg_u': 3, '...   \n",
       "4  {'bsl_options': {'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 2, 'reg_u': 3, '...   \n",
       "8  {'bsl_options': {'method': 'als', 'learning_rate': 0.01, 'reg': 0.2, 'reg_i': 1, 'reg_u': 3, 'n_...   \n",
       "5  {'bsl_options': {'method': 'als', 'learning_rate': 0.01, 'reg': 0.05, 'reg_i': 1, 'reg_u': 3, 'n...   \n",
       "9  {'bsl_options': {'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 4, 'reg_u': 3, '...   \n",
       "2  {'bsl_options': {'method': 'als', 'learning_rate': 0.005, 'reg': 0.1, 'reg_i': 3, 'reg_u': 2, 'n...   \n",
       "\n",
       "                                                                                param_bsl_options  \\\n",
       "0   {'method': 'als', 'learning_rate': 0.005, 'reg': 0.1, 'reg_i': 2, 'reg_u': 5, 'n_epochs': 15}   \n",
       "1  {'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 2, 'reg_u': 4, 'n_epochs': 20}   \n",
       "3   {'method': 'als', 'learning_rate': 0.005, 'reg': 0.2, 'reg_i': 2, 'reg_u': 4, 'n_epochs': 15}   \n",
       "7  {'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 3, 'reg_u': 4, 'n_epochs': 20}   \n",
       "6  {'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 2, 'reg_u': 3, 'n_epochs': 20}   \n",
       "4  {'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 2, 'reg_u': 3, 'n_epochs': 20}   \n",
       "8    {'method': 'als', 'learning_rate': 0.01, 'reg': 0.2, 'reg_i': 1, 'reg_u': 3, 'n_epochs': 20}   \n",
       "5   {'method': 'als', 'learning_rate': 0.01, 'reg': 0.05, 'reg_i': 1, 'reg_u': 3, 'n_epochs': 15}   \n",
       "9  {'method': 'als', 'learning_rate': 0.005, 'reg': 0.05, 'reg_i': 4, 'reg_u': 3, 'n_epochs': 20}   \n",
       "2   {'method': 'als', 'learning_rate': 0.005, 'reg': 0.1, 'reg_i': 3, 'reg_u': 2, 'n_epochs': 20}   \n",
       "\n",
       "   param_k  param_min_k  \\\n",
       "0       30           10   \n",
       "1       40           15   \n",
       "3       30           15   \n",
       "7       20           10   \n",
       "6       20           15   \n",
       "4       20           15   \n",
       "8       20           10   \n",
       "5       40           15   \n",
       "9       20           15   \n",
       "2       30           15   \n",
       "\n",
       "                                                                       param_sim_options  \n",
       "0  {'name': 'pearson_baseline', 'min_support': 10, 'user_based': True, 'shrinkage': 100}  \n",
       "1   {'name': 'pearson_baseline', 'min_support': 10, 'user_based': True, 'shrinkage': 50}  \n",
       "3  {'name': 'pearson_baseline', 'min_support': 10, 'user_based': True, 'shrinkage': 100}  \n",
       "7  {'name': 'pearson_baseline', 'min_support': 10, 'user_based': True, 'shrinkage': 100}  \n",
       "6   {'name': 'pearson_baseline', 'min_support': 10, 'user_based': True, 'shrinkage': 50}  \n",
       "4    {'name': 'pearson_baseline', 'min_support': 5, 'user_based': True, 'shrinkage': 50}  \n",
       "8  {'name': 'pearson_baseline', 'min_support': 10, 'user_based': True, 'shrinkage': 100}  \n",
       "5    {'name': 'pearson_baseline', 'min_support': 5, 'user_based': True, 'shrinkage': 50}  \n",
       "9   {'name': 'pearson_baseline', 'min_support': 5, 'user_based': True, 'shrinkage': 100}  \n",
       "2  {'name': 'pearson_baseline', 'min_support': 10, 'user_based': True, 'shrinkage': 100}  "
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sortedresults.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [],
   "source": [
    "sortedresults.to_pickle(\"SVDpp_random_user_il\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Computing the cosine similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Computing the cosine similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Computing the cosine similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Evaluating RMSE, MAE of algorithm KNNBasic on 3 split(s).\n",
      "\n",
      "                  Fold 1  Fold 2  Fold 3  Mean    Std     \n",
      "RMSE (testset)    1.4509  1.4407  1.4447  1.4454  0.0042  \n",
      "MAE (testset)     1.1601  1.1519  1.1536  1.1552  0.0035  \n",
      "Fit time          3.85    3.83    3.87    3.85    0.02    \n",
      "Test time         1.43    1.42    1.41    1.42    0.01    \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'test_rmse': array([1.45087577, 1.44067006, 1.44470049]),\n",
       " 'test_mae': array([1.16008751, 1.15186291, 1.15362173]),\n",
       " 'fit_time': (3.853111743927002, 3.8317019939422607, 3.874323606491089),\n",
       " 'test_time': (1.4318106174468994, 1.4176058769226074, 1.4055752754211426)}"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_validate(algo, data, cv=3, verbose=True, n_jobs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 67,
   "metadata": {},
   "outputs": [],
   "source": [
    "algo2=SVD()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating RMSE, MAE of algorithm SVD on 3 split(s).\n",
      "\n",
      "                  Fold 1  Fold 2  Fold 3  Mean    Std     \n",
      "RMSE (testset)    1.2743  1.2777  1.2788  1.2769  0.0019  \n",
      "MAE (testset)     1.0368  1.0406  1.0402  1.0392  0.0017  \n",
      "Fit time          3.48    3.51    3.52    3.51    0.02    \n",
      "Test time         0.22    0.22    0.23    0.22    0.00    \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'test_rmse': array([1.27427863, 1.27765008, 1.27884736]),\n",
       " 'test_mae': array([1.03682902, 1.0406015 , 1.04024859]),\n",
       " 'fit_time': (3.4833498001098633, 3.510716199874878, 3.522801637649536),\n",
       " 'test_time': (0.22365760803222656, 0.2214968204498291, 0.2267014980316162)}"
      ]
     },
     "execution_count": 68,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_validate(algo2, data, cv=3, verbose=True, n_jobs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating RMSE, MAE of algorithm CoClustering on 3 split(s).\n",
      "\n",
      "                  Fold 1  Fold 2  Fold 3  Mean    Std     \n",
      "RMSE (testset)    1.4013  1.4103  1.4071  1.4063  0.0037  \n",
      "MAE (testset)     1.0725  1.0814  1.0769  1.0769  0.0036  \n",
      "Fit time          2.92    2.95    2.93    2.93    0.01    \n",
      "Test time         0.16    0.16    0.16    0.16    0.00    \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'test_rmse': array([1.40134973, 1.41032607, 1.40712141]),\n",
       " 'test_mae': array([1.07253277, 1.0814266 , 1.07687604]),\n",
       " 'fit_time': (2.923271656036377, 2.9544529914855957, 2.9256558418273926),\n",
       " 'test_time': (0.15801429748535156, 0.15717101097106934, 0.15750336647033691)}"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "algo3=CoClustering()\n",
    "cross_validate(algo3, data, cv=3, verbose=True, n_jobs=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = {'n_epochs': [10,20,40], \n",
    "              'n_cltr_u': [2, 3, 6, 8, 10],\n",
    "              'n_cltr_i': [2, 3, 6, 8, 10]\n",
    "              \n",
    "              }\n",
    "\n",
    "gs = GridSearchCV(CoClustering, param_grid, measures=['rmse', 'mae'], cv=3, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Evaluating RMSE, MAE of algorithm SlopeOne on 3 split(s).\n",
      "\n",
      "                  Fold 1  Fold 2  Fold 3  Mean    Std     \n",
      "RMSE (testset)    1.4392  1.4331  1.4368  1.4364  0.0025  \n",
      "MAE (testset)     1.0948  1.0900  1.0941  1.0930  0.0022  \n",
      "Fit time          4.83    6.08    4.56    5.16    0.66    \n",
      "Test time         3.58    2.40    2.33    2.77    0.57    \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'test_rmse': array([1.43916394, 1.43308728, 1.43682704]),\n",
       " 'test_mae': array([1.09484418, 1.08995855, 1.09411492]),\n",
       " 'fit_time': (4.8323187828063965, 6.075841426849365, 4.561625957489014),\n",
       " 'test_time': (3.581873893737793, 2.40161395072937, 2.3327174186706543)}"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cv=cross_validate(algo, data, measures=['rmse', 'mae'], cv=3, verbose=True, n_jobs=-1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 182,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = {'n_epochs': [15], 'lr_all': [0.02,0.03,0.04,0.05],\n",
    "              'reg_all': [0.2, 0.4, 0.6, 0.1]}\n",
    "\n",
    "gs = RandomizedSearchCV(SVDpp, param_grid, n_iter=16, measures=['rmse'], cv=3, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 186,
   "metadata": {},
   "outputs": [],
   "source": [
    "param_grid = {'n_epochs': [10,20,40], \n",
    "              'n_cltr_u': [2, 3, 6, 8, 10],\n",
    "              'n_cltr_i': [2, 3, 6, 8, 10]\n",
    "              \n",
    "              }\n",
    "\n",
    "gs = GridSearchCV(CoClustering, param_grid, measures=['rmse'], cv=3, n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "gs.fit(data)\n",
    "results_df = pd.DataFrame.from_dict(gs.cv_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>split0_test_rmse</th>\n",
       "      <th>split1_test_rmse</th>\n",
       "      <th>split2_test_rmse</th>\n",
       "      <th>mean_test_rmse</th>\n",
       "      <th>std_test_rmse</th>\n",
       "      <th>rank_test_rmse</th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_test_time</th>\n",
       "      <th>std_test_time</th>\n",
       "      <th>params</th>\n",
       "      <th>param_n_epochs</th>\n",
       "      <th>param_n_cltr_u</th>\n",
       "      <th>param_n_cltr_i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.448626</td>\n",
       "      <td>1.460587</td>\n",
       "      <td>1.433926</td>\n",
       "      <td>1.447713</td>\n",
       "      <td>0.010904</td>\n",
       "      <td>1</td>\n",
       "      <td>0.589236</td>\n",
       "      <td>0.016796</td>\n",
       "      <td>0.088030</td>\n",
       "      <td>0.016316</td>\n",
       "      <td>{'n_epochs': 10, 'n_cltr_u': 2, 'n_cltr_i': 3}</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.449093</td>\n",
       "      <td>1.462656</td>\n",
       "      <td>1.434196</td>\n",
       "      <td>1.448648</td>\n",
       "      <td>0.011623</td>\n",
       "      <td>2</td>\n",
       "      <td>0.616062</td>\n",
       "      <td>0.019827</td>\n",
       "      <td>0.072324</td>\n",
       "      <td>0.006089</td>\n",
       "      <td>{'n_epochs': 10, 'n_cltr_u': 2, 'n_cltr_i': 6}</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>1.452879</td>\n",
       "      <td>1.460183</td>\n",
       "      <td>1.433562</td>\n",
       "      <td>1.448874</td>\n",
       "      <td>0.011231</td>\n",
       "      <td>3</td>\n",
       "      <td>1.359499</td>\n",
       "      <td>0.014995</td>\n",
       "      <td>0.064327</td>\n",
       "      <td>0.001978</td>\n",
       "      <td>{'n_epochs': 20, 'n_cltr_u': 2, 'n_cltr_i': 8}</td>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53</th>\n",
       "      <td>1.458611</td>\n",
       "      <td>1.458016</td>\n",
       "      <td>1.432130</td>\n",
       "      <td>1.449586</td>\n",
       "      <td>0.012345</td>\n",
       "      <td>4</td>\n",
       "      <td>4.165110</td>\n",
       "      <td>0.059674</td>\n",
       "      <td>0.112467</td>\n",
       "      <td>0.000214</td>\n",
       "      <td>{'n_epochs': 40, 'n_cltr_u': 2, 'n_cltr_i': 8}</td>\n",
       "      <td>40</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>1.456223</td>\n",
       "      <td>1.456655</td>\n",
       "      <td>1.437458</td>\n",
       "      <td>1.450112</td>\n",
       "      <td>0.008950</td>\n",
       "      <td>5</td>\n",
       "      <td>3.564505</td>\n",
       "      <td>0.369151</td>\n",
       "      <td>0.112882</td>\n",
       "      <td>0.000642</td>\n",
       "      <td>{'n_epochs': 40, 'n_cltr_u': 2, 'n_cltr_i': 6}</td>\n",
       "      <td>40</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>1.455439</td>\n",
       "      <td>1.460436</td>\n",
       "      <td>1.434830</td>\n",
       "      <td>1.450235</td>\n",
       "      <td>0.011083</td>\n",
       "      <td>6</td>\n",
       "      <td>3.436473</td>\n",
       "      <td>0.243173</td>\n",
       "      <td>0.134450</td>\n",
       "      <td>0.028115</td>\n",
       "      <td>{'n_epochs': 40, 'n_cltr_u': 2, 'n_cltr_i': 3}</td>\n",
       "      <td>40</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54</th>\n",
       "      <td>1.456404</td>\n",
       "      <td>1.459923</td>\n",
       "      <td>1.435946</td>\n",
       "      <td>1.450758</td>\n",
       "      <td>0.010571</td>\n",
       "      <td>7</td>\n",
       "      <td>4.373627</td>\n",
       "      <td>0.067229</td>\n",
       "      <td>0.148535</td>\n",
       "      <td>0.037580</td>\n",
       "      <td>{'n_epochs': 40, 'n_cltr_u': 2, 'n_cltr_i': 10}</td>\n",
       "      <td>40</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>1.458104</td>\n",
       "      <td>1.460158</td>\n",
       "      <td>1.434595</td>\n",
       "      <td>1.450952</td>\n",
       "      <td>0.011597</td>\n",
       "      <td>8</td>\n",
       "      <td>1.566577</td>\n",
       "      <td>0.103460</td>\n",
       "      <td>0.079372</td>\n",
       "      <td>0.023619</td>\n",
       "      <td>{'n_epochs': 20, 'n_cltr_u': 2, 'n_cltr_i': 10}</td>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>1.458790</td>\n",
       "      <td>1.459007</td>\n",
       "      <td>1.435569</td>\n",
       "      <td>1.451122</td>\n",
       "      <td>0.010998</td>\n",
       "      <td>9</td>\n",
       "      <td>1.550124</td>\n",
       "      <td>0.274397</td>\n",
       "      <td>0.080563</td>\n",
       "      <td>0.022802</td>\n",
       "      <td>{'n_epochs': 20, 'n_cltr_u': 3, 'n_cltr_i': 10}</td>\n",
       "      <td>20</td>\n",
       "      <td>3</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>1.458649</td>\n",
       "      <td>1.460672</td>\n",
       "      <td>1.434195</td>\n",
       "      <td>1.451172</td>\n",
       "      <td>0.012033</td>\n",
       "      <td>10</td>\n",
       "      <td>3.720443</td>\n",
       "      <td>0.129308</td>\n",
       "      <td>0.111984</td>\n",
       "      <td>0.002334</td>\n",
       "      <td>{'n_epochs': 40, 'n_cltr_u': 3, 'n_cltr_i': 3}</td>\n",
       "      <td>40</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    split0_test_rmse  split1_test_rmse  split2_test_rmse  mean_test_rmse  \\\n",
       "1           1.448626          1.460587          1.433926        1.447713   \n",
       "2           1.449093          1.462656          1.434196        1.448648   \n",
       "28          1.452879          1.460183          1.433562        1.448874   \n",
       "53          1.458611          1.458016          1.432130        1.449586   \n",
       "52          1.456223          1.456655          1.437458        1.450112   \n",
       "51          1.455439          1.460436          1.434830        1.450235   \n",
       "54          1.456404          1.459923          1.435946        1.450758   \n",
       "29          1.458104          1.460158          1.434595        1.450952   \n",
       "34          1.458790          1.459007          1.435569        1.451122   \n",
       "56          1.458649          1.460672          1.434195        1.451172   \n",
       "\n",
       "    std_test_rmse  rank_test_rmse  mean_fit_time  std_fit_time  \\\n",
       "1        0.010904               1       0.589236      0.016796   \n",
       "2        0.011623               2       0.616062      0.019827   \n",
       "28       0.011231               3       1.359499      0.014995   \n",
       "53       0.012345               4       4.165110      0.059674   \n",
       "52       0.008950               5       3.564505      0.369151   \n",
       "51       0.011083               6       3.436473      0.243173   \n",
       "54       0.010571               7       4.373627      0.067229   \n",
       "29       0.011597               8       1.566577      0.103460   \n",
       "34       0.010998               9       1.550124      0.274397   \n",
       "56       0.012033              10       3.720443      0.129308   \n",
       "\n",
       "    mean_test_time  std_test_time  \\\n",
       "1         0.088030       0.016316   \n",
       "2         0.072324       0.006089   \n",
       "28        0.064327       0.001978   \n",
       "53        0.112467       0.000214   \n",
       "52        0.112882       0.000642   \n",
       "51        0.134450       0.028115   \n",
       "54        0.148535       0.037580   \n",
       "29        0.079372       0.023619   \n",
       "34        0.080563       0.022802   \n",
       "56        0.111984       0.002334   \n",
       "\n",
       "                                             params  param_n_epochs  \\\n",
       "1    {'n_epochs': 10, 'n_cltr_u': 2, 'n_cltr_i': 3}              10   \n",
       "2    {'n_epochs': 10, 'n_cltr_u': 2, 'n_cltr_i': 6}              10   \n",
       "28   {'n_epochs': 20, 'n_cltr_u': 2, 'n_cltr_i': 8}              20   \n",
       "53   {'n_epochs': 40, 'n_cltr_u': 2, 'n_cltr_i': 8}              40   \n",
       "52   {'n_epochs': 40, 'n_cltr_u': 2, 'n_cltr_i': 6}              40   \n",
       "51   {'n_epochs': 40, 'n_cltr_u': 2, 'n_cltr_i': 3}              40   \n",
       "54  {'n_epochs': 40, 'n_cltr_u': 2, 'n_cltr_i': 10}              40   \n",
       "29  {'n_epochs': 20, 'n_cltr_u': 2, 'n_cltr_i': 10}              20   \n",
       "34  {'n_epochs': 20, 'n_cltr_u': 3, 'n_cltr_i': 10}              20   \n",
       "56   {'n_epochs': 40, 'n_cltr_u': 3, 'n_cltr_i': 3}              40   \n",
       "\n",
       "    param_n_cltr_u  param_n_cltr_i  \n",
       "1                2               3  \n",
       "2                2               6  \n",
       "28               2               8  \n",
       "53               2               8  \n",
       "52               2               6  \n",
       "51               2               3  \n",
       "54               2              10  \n",
       "29               2              10  \n",
       "34               3              10  \n",
       "56               3               3  "
      ]
     },
     "execution_count": 188,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.max_colwidth', 100)\n",
    "sortedresults=results_df.sort_values('rank_test_rmse')\n",
    "sortedresults.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 185,
   "metadata": {},
   "outputs": [],
   "source": [
    "sortedresults.to_pickle(\"svdpp_grid5_il\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "from surprise import AlgoBase\n",
    "from surprise import Dataset\n",
    "from surprise.model_selection import cross_validate\n",
    "from statistics import mean, median\n",
    "\n",
    "\n",
    "class ensemble(AlgoBase):\n",
    "\n",
    "    def __init__(self, models):\n",
    "\n",
    "        # Always call base method before doing anything.\n",
    "        AlgoBase.__init__(self)\n",
    "        self.models=models\n",
    "\n",
    "    def estimate(self, u, i):\n",
    "        predictions=[]\n",
    "        for model in self.models:\n",
    "            predictions.append(model.predict(u,i).est)\n",
    "        return median(predictions)\n",
    "\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>split0_test_rmse</th>\n",
       "      <th>split1_test_rmse</th>\n",
       "      <th>split2_test_rmse</th>\n",
       "      <th>mean_test_rmse</th>\n",
       "      <th>std_test_rmse</th>\n",
       "      <th>rank_test_rmse</th>\n",
       "      <th>split0_test_mae</th>\n",
       "      <th>split1_test_mae</th>\n",
       "      <th>split2_test_mae</th>\n",
       "      <th>mean_test_mae</th>\n",
       "      <th>std_test_mae</th>\n",
       "      <th>rank_test_mae</th>\n",
       "      <th>mean_fit_time</th>\n",
       "      <th>std_fit_time</th>\n",
       "      <th>mean_test_time</th>\n",
       "      <th>std_test_time</th>\n",
       "      <th>params</th>\n",
       "      <th>param_n_epochs</th>\n",
       "      <th>param_n_cltr_u</th>\n",
       "      <th>param_n_cltr_i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>52</th>\n",
       "      <td>1.376336</td>\n",
       "      <td>1.367321</td>\n",
       "      <td>1.373840</td>\n",
       "      <td>1.372499</td>\n",
       "      <td>0.003801</td>\n",
       "      <td>1</td>\n",
       "      <td>1.045111</td>\n",
       "      <td>1.040399</td>\n",
       "      <td>1.041934</td>\n",
       "      <td>1.042481</td>\n",
       "      <td>0.001962</td>\n",
       "      <td>3</td>\n",
       "      <td>25.690547</td>\n",
       "      <td>0.551379</td>\n",
       "      <td>0.974465</td>\n",
       "      <td>0.028390</td>\n",
       "      <td>{'n_epochs': 40, 'n_cltr_u': 2, 'n_cltr_i': 6}</td>\n",
       "      <td>40</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1.377034</td>\n",
       "      <td>1.372635</td>\n",
       "      <td>1.371597</td>\n",
       "      <td>1.373755</td>\n",
       "      <td>0.002357</td>\n",
       "      <td>2</td>\n",
       "      <td>1.044305</td>\n",
       "      <td>1.041201</td>\n",
       "      <td>1.040129</td>\n",
       "      <td>1.041878</td>\n",
       "      <td>0.001771</td>\n",
       "      <td>1</td>\n",
       "      <td>3.842801</td>\n",
       "      <td>0.061902</td>\n",
       "      <td>0.601545</td>\n",
       "      <td>0.009810</td>\n",
       "      <td>{'n_epochs': 10, 'n_cltr_u': 3, 'n_cltr_i': 3}</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>1.375079</td>\n",
       "      <td>1.372864</td>\n",
       "      <td>1.373620</td>\n",
       "      <td>1.373854</td>\n",
       "      <td>0.000919</td>\n",
       "      <td>3</td>\n",
       "      <td>1.044187</td>\n",
       "      <td>1.042759</td>\n",
       "      <td>1.041861</td>\n",
       "      <td>1.042936</td>\n",
       "      <td>0.000958</td>\n",
       "      <td>6</td>\n",
       "      <td>10.003242</td>\n",
       "      <td>2.136677</td>\n",
       "      <td>0.777977</td>\n",
       "      <td>0.139789</td>\n",
       "      <td>{'n_epochs': 20, 'n_cltr_u': 2, 'n_cltr_i': 6}</td>\n",
       "      <td>20</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.376752</td>\n",
       "      <td>1.372959</td>\n",
       "      <td>1.373106</td>\n",
       "      <td>1.374272</td>\n",
       "      <td>0.001755</td>\n",
       "      <td>4</td>\n",
       "      <td>1.045384</td>\n",
       "      <td>1.043071</td>\n",
       "      <td>1.041457</td>\n",
       "      <td>1.043304</td>\n",
       "      <td>0.001612</td>\n",
       "      <td>9</td>\n",
       "      <td>3.920249</td>\n",
       "      <td>0.112717</td>\n",
       "      <td>0.614963</td>\n",
       "      <td>0.035695</td>\n",
       "      <td>{'n_epochs': 10, 'n_cltr_u': 2, 'n_cltr_i': 6}</td>\n",
       "      <td>10</td>\n",
       "      <td>2</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>1.377125</td>\n",
       "      <td>1.372202</td>\n",
       "      <td>1.374192</td>\n",
       "      <td>1.374506</td>\n",
       "      <td>0.002022</td>\n",
       "      <td>5</td>\n",
       "      <td>1.045421</td>\n",
       "      <td>1.041955</td>\n",
       "      <td>1.042449</td>\n",
       "      <td>1.043275</td>\n",
       "      <td>0.001531</td>\n",
       "      <td>7</td>\n",
       "      <td>25.639561</td>\n",
       "      <td>0.924668</td>\n",
       "      <td>0.990361</td>\n",
       "      <td>0.032554</td>\n",
       "      <td>{'n_epochs': 40, 'n_cltr_u': 3, 'n_cltr_i': 2}</td>\n",
       "      <td>40</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    split0_test_rmse  split1_test_rmse  split2_test_rmse  mean_test_rmse  \\\n",
       "52          1.376336          1.367321          1.373840        1.372499   \n",
       "6           1.377034          1.372635          1.371597        1.373755   \n",
       "27          1.375079          1.372864          1.373620        1.373854   \n",
       "2           1.376752          1.372959          1.373106        1.374272   \n",
       "55          1.377125          1.372202          1.374192        1.374506   \n",
       "\n",
       "    std_test_rmse  rank_test_rmse  split0_test_mae  split1_test_mae  \\\n",
       "52       0.003801               1         1.045111         1.040399   \n",
       "6        0.002357               2         1.044305         1.041201   \n",
       "27       0.000919               3         1.044187         1.042759   \n",
       "2        0.001755               4         1.045384         1.043071   \n",
       "55       0.002022               5         1.045421         1.041955   \n",
       "\n",
       "    split2_test_mae  mean_test_mae  std_test_mae  rank_test_mae  \\\n",
       "52         1.041934       1.042481      0.001962              3   \n",
       "6          1.040129       1.041878      0.001771              1   \n",
       "27         1.041861       1.042936      0.000958              6   \n",
       "2          1.041457       1.043304      0.001612              9   \n",
       "55         1.042449       1.043275      0.001531              7   \n",
       "\n",
       "    mean_fit_time  std_fit_time  mean_test_time  std_test_time  \\\n",
       "52      25.690547      0.551379        0.974465       0.028390   \n",
       "6        3.842801      0.061902        0.601545       0.009810   \n",
       "27      10.003242      2.136677        0.777977       0.139789   \n",
       "2        3.920249      0.112717        0.614963       0.035695   \n",
       "55      25.639561      0.924668        0.990361       0.032554   \n",
       "\n",
       "                                            params  param_n_epochs  \\\n",
       "52  {'n_epochs': 40, 'n_cltr_u': 2, 'n_cltr_i': 6}              40   \n",
       "6   {'n_epochs': 10, 'n_cltr_u': 3, 'n_cltr_i': 3}              10   \n",
       "27  {'n_epochs': 20, 'n_cltr_u': 2, 'n_cltr_i': 6}              20   \n",
       "2   {'n_epochs': 10, 'n_cltr_u': 2, 'n_cltr_i': 6}              10   \n",
       "55  {'n_epochs': 40, 'n_cltr_u': 3, 'n_cltr_i': 2}              40   \n",
       "\n",
       "    param_n_cltr_u  param_n_cltr_i  \n",
       "52               2               6  \n",
       "6                3               3  \n",
       "27               2               6  \n",
       "2                2               6  \n",
       "55               3               2  "
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_pickle(\"CoCluster_grid12_pa\").sort_values('rank_test_rmse').head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<surprise.prediction_algorithms.matrix_factorization.SVDpp at 0x7fef623f24a8>"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "svd=SVDpp(n_factors=15, n_epochs=15, lr_all=.02, reg_all=.2)\n",
    "svd.fit(data.build_full_trainset())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<surprise.prediction_algorithms.knns.KNNBaseline at 0x7fef38d9dc88>"
      ]
     },
     "execution_count": 208,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sim_options = {'name': 'pearson_baseline',\n",
    "               'user_based': False,\n",
    "               'min_support': 10,\n",
    "               'shrinkage': 50\n",
    "              }\n",
    "\n",
    "bsl_options = {'method': 'als',\n",
    "               'n_epochs': 15,\n",
    "               'reg_u': 5,\n",
    "               'reg_i': 2\n",
    "               }\n",
    "\n",
    "\n",
    "knnbi=KNNBaseline(sim_options=sim_options, bsl_options=bsl_options, k=30, min_k=15)\n",
    "knnbi.fit(data.build_full_trainset())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<surprise.prediction_algorithms.knns.KNNBaseline at 0x7fef38d9db00>"
      ]
     },
     "execution_count": 209,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sim_options = {'name': 'pearson_baseline',\n",
    "               'user_based': True,\n",
    "               'min_support': 10,\n",
    "               'shrinkage': 100\n",
    "              }\n",
    "\n",
    "bsl_options = {'method': 'als',\n",
    "               'n_epochs': 15,\n",
    "               'reg_u': 5,\n",
    "               'reg_i': 2\n",
    "               }\n",
    "\n",
    "\n",
    "knnbu=KNNBaseline(sim_options=sim_options, bsl_options=bsl_options, k=30, min_k=10)\n",
    "knnbu.fit(data.build_full_trainset())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<surprise.prediction_algorithms.co_clustering.CoClustering at 0x7fef445a3358>"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coclu=CoClustering(n_cltr_u=2, n_cltr_i=6, n_epochs=40)\n",
    "coclu.fit(data.build_full_trainset())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 213,
   "metadata": {},
   "outputs": [],
   "source": [
    "trained_models=[svd, knnbi, knnbu, coclu]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Evaluating RMSE, MAE of algorithm KNNBaseline on 5 split(s).\n",
      "\n",
      "                  Fold 1  Fold 2  Fold 3  Fold 4  Fold 5  Mean    Std     \n",
      "RMSE (testset)    1.2879  1.3041  1.2946  1.2840  1.2982  1.2938  0.0072  \n",
      "MAE (testset)     1.0500  1.0620  1.0536  1.0462  1.0561  1.0536  0.0054  \n",
      "Fit time          7.37    7.34    7.47    6.91    7.06    7.23    0.21    \n",
      "Test time         0.45    0.44    0.44    0.42    0.43    0.43    0.01    \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'test_rmse': array([1.28793246, 1.3040596 , 1.29462692, 1.28397782, 1.29819576]),\n",
       " 'test_mae': array([1.05003505, 1.06195386, 1.05359803, 1.04619711, 1.05614585]),\n",
       " 'fit_time': (7.3743884563446045,\n",
       "  7.344173908233643,\n",
       "  7.47336483001709,\n",
       "  6.91268253326416,\n",
       "  7.055426120758057),\n",
       " 'test_time': (0.44682812690734863,\n",
       "  0.43537044525146484,\n",
       "  0.43678951263427734,\n",
       "  0.42450737953186035,\n",
       "  0.429858922958374)}"
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "algo = ensemble(models=trained_models)\n",
    "\n",
    "cross_validate(knnbu, data, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Estimating biases using als...\n",
      "Computing the pearson_baseline similarity matrix...\n",
      "Done computing similarity matrix.\n",
      "Evaluating RMSE, MAE of algorithm KNNBaseline on 5 split(s).\n",
      "\n",
      "                  Fold 1  Fold 2  Fold 3  Fold 4  Fold 5  Mean    Std     \n",
      "RMSE (testset)    1.2992  1.2946  1.2900  1.2954  1.2999  1.2958  0.0036  \n",
      "MAE (testset)     1.0535  1.0520  1.0481  1.0598  1.0595  1.0546  0.0045  \n",
      "Fit time          0.21    0.22    0.22    0.22    0.22    0.22    0.00    \n",
      "Test time         0.16    0.16    0.17    0.16    0.16    0.16    0.00    \n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'test_rmse': array([1.29917076, 1.2945688 , 1.29001234, 1.29538176, 1.29993233]),\n",
       " 'test_mae': array([1.05346315, 1.05198029, 1.04812665, 1.05978126, 1.05953583]),\n",
       " 'fit_time': (0.21462154388427734,\n",
       "  0.2196192741394043,\n",
       "  0.22432518005371094,\n",
       "  0.22433209419250488,\n",
       "  0.21837711334228516),\n",
       " 'test_time': (0.16272544860839844,\n",
       "  0.16253876686096191,\n",
       "  0.16622257232666016,\n",
       "  0.16334915161132812,\n",
       "  0.16075921058654785)}"
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cross_validate(knnbi, data, verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}