local_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import glob\n",
    "import math\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook we show to locally evaluate a submission generated from a subset of the csv's from your training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = 'DATASET-PATH' # TODO: point this to your data folder\n",
    "submission_path = data_path + 'submissions/'\n",
    "training_path = data_path + 'training_set/'\n",
    "input_logs = sorted(glob.glob(training_path + \"log_mini*.csv\")) # TODO: point this to a subset of csv's in your training set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_ground_truth(input_logs):\n",
    "    ground_truths = []     \n",
    "    for i,f in enumerate(input_logs):\n",
    "        df = pd.read_csv(f)\n",
    "        # Below we keep only the relevant columns of the second half of the session for saving the ground truth\n",
    "        df = df[['session_id','skip_2','session_position','session_length']].loc[df['session_position']*2 > df['session_length']]\n",
    "        df = df.reset_index()\n",
    "        current_index = 0\n",
    "        # Here we process each session, saving a list containing the \n",
    "        while current_index < len(df):\n",
    "            partial_length = df['session_length'].iloc[current_index]-df['session_position'].iloc[current_index]+1\n",
    "            session_skips = list(df.loc[current_index:current_index+partial_length-1, 'skip_2'])\n",
    "            ground_truths.append(session_skips)\n",
    "            current_index += partial_length \n",
    "    return ground_truths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "ground_truth = get_ground_truth(input_logs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def generate_random_submission(input_logs):\n",
    "    output = []\n",
    "    for i,f in enumerate(input_logs):\n",
    "        df = pd.read_csv(f)\n",
    "        print('file {} read'.format(i))\n",
    "        # For this random submission we just need to know the length of each session\n",
    "        df = df.drop_duplicates(subset='session_id', keep='first', inplace=False)\n",
    "        # For each session we generate a random prediction of the required length\n",
    "        for j in df.index:\n",
    "            sess_len = df.loc[j,'session_length']\n",
    "            partial_len = math.ceil(sess_len/2)\n",
    "            current_preds = np.random.choice([0, 1], size=(partial_len,))\n",
    "            output.append(current_preds)\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = generate_random_submission(input_logs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluate(submission,groundtruth):\n",
    "    ap_sum = 0.0\n",
    "    first_pred_acc_sum = 0.0\n",
    "    counter = 0\n",
    "    for sub, tru in zip(submission, groundtruth):\n",
    "        if len(sub) != len(tru):\n",
    "            raise Exception('Line {} should contain {} predictions, but instead contains '\n",
    "                            '{}'.format(counter+1,len(tru),len(sub)))\n",
    "        ap_sum += ave_pre(sub,tru,counter)\n",
    "        first_pred_acc_sum += sub[0] == tru[0]\n",
    "        counter+=1\n",
    "    ap = ap_sum/counter\n",
    "    first_pred_acc = first_pred_acc_sum/counter\n",
    "    return ap,first_pred_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ave_pre(submission,groundtruth,counter):\n",
    "    s = 0.0\n",
    "    t = 0.0\n",
    "    c = 1.0\n",
    "    for x, y in zip(submission, groundtruth):\n",
    "        if x != 0 and x != 1:\n",
    "            raise Exception('Invalid prediction in line {}, should be 0 or 1'.format(counter))\n",
    "        if x==y:\n",
    "            s += 1.0\n",
    "            t += s / c\n",
    "        c += 1\n",
    "    return t/len(groundtruth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "ap,first_pred_acc = evaluate(submission,ground_truth)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "average precision: 0.33620825485008854\n",
      "first prediction accuracy: 0.5048\n"
     ]
    }
   ],
   "source": [
    "print('average precision: {}'.format(ap))\n",
    "print('first prediction accuracy: {}'.format(first_pred_acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "name": "python",
   "version": "3.10.9 (main, Dec 19 2022, 17:35:49) [GCC 12.2.0]"
  },
  "vscode": {
   "interpreter": {
    "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}