cs125_recommendation_system.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ded9fb94",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From C:\\Users\\lmm20\\anaconda3\\lib\\site-packages\\keras\\src\\losses.py:2976: The name tf.losses.sparse_softmax_cross_entropy is deprecated. Please use tf.compat.v1.losses.sparse_softmax_cross_entropy instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Import all the frameworks\n",
    "import xml.etree.ElementTree as ET\n",
    "import pandas as pd\n",
    "import os\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "import tensorflow as tf\n",
    "from sklearn.preprocessing import StandardScaler"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9ae26f14",
   "metadata": {},
   "source": [
    "# Getting Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4131a14f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[<Element 'HealthData' at 0x000001BDC482F1F0>, <Element 'HealthData' at 0x000001BDC482F100>, <Element 'HealthData' at 0x000001BDC482F2E0>, <Element 'HealthData' at 0x000001BE4E2C0630>, <Element 'HealthData' at 0x000001BDC438AE30>, <Element 'HealthData' at 0x000001BE00DF0CC0>, <Element 'HealthData' at 0x000001BDC482F330>, <Element 'HealthData' at 0x000001BE6FACFEC0>]\n"
     ]
    }
   ],
   "source": [
    "# First to get all the healthkit data from dataset, healthkit data is exported in xml file\n",
    "root_list = []\n",
    "\n",
    "path = 'C:/Users/lmm20/Downloads/healthkit_data'\n",
    "for file in os.listdir(path):\n",
    "    if file.endswith('.xml'):\n",
    "        file_name = path + '/' +file\n",
    "        tree = ET.parse(file_name)\n",
    "        root = tree.getroot()\n",
    "        root_list.append(root)\n",
    "print(root_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "13887801",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      user_id        date  steps\n",
      "0       user0  2017-02-23    550\n",
      "1       user0  2017-02-24   6819\n",
      "2       user0  2017-02-25    133\n",
      "3       user0  2017-02-26    320\n",
      "4       user0  2017-02-27    676\n",
      "...       ...         ...    ...\n",
      "15898   user7  2024-02-15   7872\n",
      "15899   user7  2024-02-16   7772\n",
      "15900   user7  2024-02-17   4646\n",
      "15901   user7  2024-02-18   4868\n",
      "15902   user7  2024-02-19    209\n",
      "\n",
      "[15903 rows x 3 columns]\n"
     ]
    }
   ],
   "source": [
    "# In this part, we are loading all the stepcount data and store it in pd.dataframe with three columns(user_id, date, steps)\n",
    "\n",
    "stepcount = []\n",
    "count = 0\n",
    "\n",
    "for i in root_list:\n",
    "    for record in i.findall(\".//Record[@type='HKQuantityTypeIdentifierStepCount']\"):\n",
    "        small = []\n",
    "        user_id = \"user\"+str(count)\n",
    "        small.append(user_id)\n",
    "        stepcount.append(small)\n",
    "        small.append(record.get('startDate'))\n",
    "        small.append(record.get('value'))\n",
    "    count += 1\n",
    "\n",
    "# set the format of datetime\n",
    "date_format = \"%Y-%m-%d %H:%M:%S\"\n",
    "\n",
    "df = pd.DataFrame(stepcount, columns=['user_id', 'date', 'steps'])\n",
    "\n",
    "# Convert 'date' string to datetime with specified format\n",
    "df['date'] = pd.to_datetime(df['date'], format=date_format, errors='coerce')\n",
    "\n",
    "# Check for any rows where 'date' conversion resulted in NaN\n",
    "if df['date'].isna().any():\n",
    "    print(\"Warning: Some 'date' values failed to convert and are NaN.\")\n",
    "    \n",
    "# Only proceed if there are no NaT values, or handle them accordingly\n",
    "if not df['date'].isna().any():\n",
    "    # Safely extract the date component\n",
    "    df['date'] = df['date'].apply(lambda x: x.date())\n",
    "else:\n",
    "    # Handle NaT values as needed\n",
    "    # For example, you might choose to drop these rows or fill them with a placeholder\n",
    "    df = df.dropna(subset=['date'])\n",
    "\n",
    "# Ensure 'steps' is numeric for summation\n",
    "df['steps'] = pd.to_numeric(df['steps'], errors='coerce')\n",
    "\n",
    "# Group by 'user_id' and 'date', then sum steps\n",
    "df_steps = df.groupby(['user_id', 'date'], as_index=False)['steps'].sum()\n",
    "\n",
    "\n",
    "print(df_steps)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "1873c1ff",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   user_id        date    weight unit\n",
      "0    user0  2019-04-21  65.00000   kg\n",
      "1    user0  2017-06-30  59.00000   kg\n",
      "2    user0  2020-08-23  70.00000   kg\n",
      "3    user0  2021-01-11  67.00000   kg\n",
      "4    user0  2023-11-01  72.00000   kg\n",
      "..     ...         ...       ...  ...\n",
      "89   user5  2021-08-11  67.00000   kg\n",
      "90   user5  2023-01-18  69.00000   kg\n",
      "91   user6  2023-12-28  76.00000   kg\n",
      "92   user7  2023-09-28  90.71840   kg\n",
      "93   user7  2019-09-11  86.18248   kg\n",
      "\n",
      "[94 rows x 4 columns]\n"
     ]
    }
   ],
   "source": [
    "# In this part, we are loading all the weights data and store it in pd.dataframe with three columns(user_id, date, weight, unit)\n",
    "\n",
    "\n",
    "weights = []\n",
    "user_count = 0 \n",
    "# Loop through the XML to find weight records\n",
    "for i in root_list:\n",
    "    for record in i.findall(\".//Record[@type='HKQuantityTypeIdentifierBodyMass']\"):\n",
    "        # Extract the relevant attributes for each weight record\n",
    "        date = record.get('startDate')  # The date and time when the weight was recorded\n",
    "        #value = 0\n",
    "        value = int(float(record.get('value')))  # The weight value\n",
    "        unit = record.get('unit') \n",
    "        \n",
    "        # We need to make sure all the weight values are stored with the same unit. If the weight unit is lb, we convert it to\n",
    "        # kg by multiply lb by 0.453592\n",
    "        if unit == 'lb':\n",
    "            value = value * 0.453592\n",
    "            unit = 'kg'\n",
    "        user_id = 'user'+str(user_count)\n",
    "        # Store the extracted data in the list\n",
    "        weights.append({'user_id':str(user_id), 'date': date, 'weight': value, 'unit': unit})\n",
    "    user_count += 1\n",
    "date_format = \"%Y-%m-%d\"\n",
    "\n",
    "df_weights = pd.DataFrame(weights)\n",
    "\n",
    "df_weights['date'] = pd.to_datetime(df_weights['date'], utc=True).dt.tz_convert(None).dt.date\n",
    "\n",
    "print(df_weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "7901d17f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      user_id        date  steps  weight unit\n",
      "0       user0  2017-02-23    550     NaN  NaN\n",
      "1       user0  2017-02-24   6819     NaN  NaN\n",
      "2       user0  2017-02-25    133     NaN  NaN\n",
      "3       user0  2017-02-26    320     NaN  NaN\n",
      "4       user0  2017-02-27    676     NaN  NaN\n",
      "...       ...         ...    ...     ...  ...\n",
      "15927   user7  2024-02-15   7872     NaN  NaN\n",
      "15928   user7  2024-02-16   7772     NaN  NaN\n",
      "15929   user7  2024-02-17   4646     NaN  NaN\n",
      "15930   user7  2024-02-18   4868     NaN  NaN\n",
      "15931   user7  2024-02-19    209     NaN  NaN\n",
      "\n",
      "[15932 rows x 5 columns]\n"
     ]
    }
   ],
   "source": [
    "# Mergde the dataframes of steps and weights according to the user_id and date, but there are many rows with no weight because \n",
    "# users do not store the change of weight everyday. \n",
    "\n",
    "merged_df = pd.merge(df_steps, df_weights, on=['user_id', 'date'], how='outer')\n",
    "\n",
    "print(merged_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "458ee796",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# This function aims to fill in the blank of weight data. Suppose one user do not have the data of weight on 2018-5-7, but the\n",
    "# system read that the weight on 2018-5-30 is 70kg so the algorithm automatically fill the weight on 2018-5-7 with 70kg. \n",
    "\n",
    "\n",
    "for i, row in merged_df.iterrows():\n",
    "    if pd.isna(row['unit']):  # Check if 'unit' is NaN\n",
    "        # Initialize variables to track the closest weight and its date difference\n",
    "        temp_weight = None\n",
    "        temp_date_difference = pd.Timedelta.max  # Use max timedelta for initial comparison\n",
    "\n",
    "        # Find the closest weight record in 'df_weights' for the same 'user_id'\n",
    "        for _, weight_row in df_weights[df_weights['user_id'] == row['user_id']].iterrows():\n",
    "            weight_date = pd.to_datetime(weight_row['date'])\n",
    "            steps_date = pd.to_datetime(row['date'])\n",
    "\n",
    "            # Calculate the absolute difference in days (or any other suitable measure)\n",
    "            diff = abs(weight_date - steps_date)\n",
    "\n",
    "            # Update if this record is closer to the 'steps' record date\n",
    "            if diff < temp_date_difference:\n",
    "                temp_weight = weight_row['weight']\n",
    "                temp_date_difference = diff\n",
    "\n",
    "        # Update the 'merged_df' DataFrame directly\n",
    "        if temp_weight is not None:\n",
    "            merged_df.loc[i, 'weight'] = temp_weight\n",
    "            merged_df.loc[i, 'unit'] = 'kg'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "953fc6a9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>steps</th>\n",
       "      <th>weight</th>\n",
       "      <th>unit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>user0</td>\n",
       "      <td>2017-02-23</td>\n",
       "      <td>550</td>\n",
       "      <td>59.0000</td>\n",
       "      <td>kg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>user0</td>\n",
       "      <td>2017-02-24</td>\n",
       "      <td>6819</td>\n",
       "      <td>59.0000</td>\n",
       "      <td>kg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>user0</td>\n",
       "      <td>2017-02-25</td>\n",
       "      <td>133</td>\n",
       "      <td>59.0000</td>\n",
       "      <td>kg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>user0</td>\n",
       "      <td>2017-02-26</td>\n",
       "      <td>320</td>\n",
       "      <td>59.0000</td>\n",
       "      <td>kg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>user0</td>\n",
       "      <td>2017-02-27</td>\n",
       "      <td>676</td>\n",
       "      <td>59.0000</td>\n",
       "      <td>kg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15927</th>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-15</td>\n",
       "      <td>7872</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>kg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15928</th>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-16</td>\n",
       "      <td>7772</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>kg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15929</th>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-17</td>\n",
       "      <td>4646</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>kg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15930</th>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-18</td>\n",
       "      <td>4868</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>kg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15931</th>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-19</td>\n",
       "      <td>209</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>kg</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15932 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      user_id        date  steps   weight unit\n",
       "0       user0  2017-02-23    550  59.0000   kg\n",
       "1       user0  2017-02-24   6819  59.0000   kg\n",
       "2       user0  2017-02-25    133  59.0000   kg\n",
       "3       user0  2017-02-26    320  59.0000   kg\n",
       "4       user0  2017-02-27    676  59.0000   kg\n",
       "...       ...         ...    ...      ...  ...\n",
       "15927   user7  2024-02-15   7872  90.7184   kg\n",
       "15928   user7  2024-02-16   7772  90.7184   kg\n",
       "15929   user7  2024-02-17   4646  90.7184   kg\n",
       "15930   user7  2024-02-18   4868  90.7184   kg\n",
       "15931   user7  2024-02-19    209  90.7184   kg\n",
       "\n",
       "[15932 rows x 5 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "61dca519",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "     user_id        date  calories_burned\n",
      "0      user0  2021-01-11           34.000\n",
      "1      user0  2021-01-12          239.000\n",
      "2      user0  2021-01-23            4.000\n",
      "3      user0  2022-11-28           61.564\n",
      "4      user0  2022-11-29           37.894\n",
      "...      ...         ...              ...\n",
      "5055   user7  2024-02-18           75.383\n",
      "5056   user7  2024-02-19          239.965\n",
      "5057   user7  2024-02-20          165.445\n",
      "5058   user7  2024-02-21          299.957\n",
      "5059   user7  2024-02-22           33.190\n",
      "\n",
      "[5060 rows x 3 columns]\n"
     ]
    }
   ],
   "source": [
    "# In this part, we are loading all the calorie consumption data and store it in pd.dataframe with \n",
    "# three columns(user_id, date, weight, unit)\n",
    "\n",
    "\n",
    "calorie_data = []\n",
    "user_id = 0\n",
    "# Loop through the XML to find calorie consumption records\n",
    "for i in root_list:\n",
    "    for record in i.findall(\".//Record[@type='HKQuantityTypeIdentifierActiveEnergyBurned']\"):\n",
    "        # Extract the relevant attributes\n",
    "        date = record.get('startDate')  # The date and time the calories were burned\n",
    "        calories_burned = float(record.get('value'))  # The amount of calories burned\n",
    "        unit = record.get('unit')  # The unit of measurement for the calories burned\n",
    "        user = 'user' + str(user_id)\n",
    "        # Append the data to the list\n",
    "        calorie_data.append({'user_id':user, 'date': date, 'calories_burned': calories_burned, 'unit': unit})\n",
    "    user_id += 1\n",
    "df_calories = pd.DataFrame(calorie_data)\n",
    "df_calories['date'] = pd.to_datetime(df_calories['date'], utc=True).dt.tz_convert(None).dt.date\n",
    "df_calories = df_calories.groupby(['user_id', 'date'], as_index=False)['calories_burned'].sum()\n",
    "print(df_calories)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "8b470dd7",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Merging the dataframe of calorie consumption with weight and steps\n",
    "merged_df = pd.merge(merged_df, df_calories, on=['user_id', 'date'], how='outer')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "fe6c4cd3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>steps</th>\n",
       "      <th>weight</th>\n",
       "      <th>unit</th>\n",
       "      <th>calories_burned</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>user0</td>\n",
       "      <td>2017-02-23</td>\n",
       "      <td>550.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>kg</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>user0</td>\n",
       "      <td>2017-02-24</td>\n",
       "      <td>6819.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>kg</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>user0</td>\n",
       "      <td>2017-02-25</td>\n",
       "      <td>133.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>kg</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>user0</td>\n",
       "      <td>2017-02-26</td>\n",
       "      <td>320.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>kg</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>user0</td>\n",
       "      <td>2017-02-27</td>\n",
       "      <td>676.0</td>\n",
       "      <td>59.0</td>\n",
       "      <td>kg</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15957</th>\n",
       "      <td>user7</td>\n",
       "      <td>2023-11-02</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>436.823</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15958</th>\n",
       "      <td>user7</td>\n",
       "      <td>2023-11-03</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>222.403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15959</th>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-20</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>165.445</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15960</th>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-21</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>299.957</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15961</th>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-22</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>33.190</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>15962 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      user_id        date   steps  weight unit  calories_burned\n",
       "0       user0  2017-02-23   550.0    59.0   kg              NaN\n",
       "1       user0  2017-02-24  6819.0    59.0   kg              NaN\n",
       "2       user0  2017-02-25   133.0    59.0   kg              NaN\n",
       "3       user0  2017-02-26   320.0    59.0   kg              NaN\n",
       "4       user0  2017-02-27   676.0    59.0   kg              NaN\n",
       "...       ...         ...     ...     ...  ...              ...\n",
       "15957   user7  2023-11-02     NaN     NaN  NaN          436.823\n",
       "15958   user7  2023-11-03     NaN     NaN  NaN          222.403\n",
       "15959   user7  2024-02-20     NaN     NaN  NaN          165.445\n",
       "15960   user7  2024-02-21     NaN     NaN  NaN          299.957\n",
       "15961   user7  2024-02-22     NaN     NaN  NaN           33.190\n",
       "\n",
       "[15962 rows x 6 columns]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "effc3142",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10874\n"
     ]
    }
   ],
   "source": [
    "count = 0 \n",
    "for i, j in merged_df.iterrows():\n",
    "    if pd.isna(j['calories_burned']):\n",
    "        count += 1\n",
    "print(count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "2781bc19",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lmm20\\AppData\\Local\\Temp\\ipykernel_23004\\2017924147.py:4: UserWarning: Boolean Series key will be reindexed to match DataFrame index.\n",
      "  df_not_nan = df_not_nan[merged_df['steps'].notna()]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>steps</th>\n",
       "      <th>weight</th>\n",
       "      <th>unit</th>\n",
       "      <th>calories_burned</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>979</td>\n",
       "      <td>user0</td>\n",
       "      <td>2021-01-11</td>\n",
       "      <td>10065.0</td>\n",
       "      <td>67.0000</td>\n",
       "      <td>kg</td>\n",
       "      <td>34.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>980</td>\n",
       "      <td>user0</td>\n",
       "      <td>2021-01-12</td>\n",
       "      <td>1464.0</td>\n",
       "      <td>67.0000</td>\n",
       "      <td>kg</td>\n",
       "      <td>239.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>991</td>\n",
       "      <td>user0</td>\n",
       "      <td>2021-01-23</td>\n",
       "      <td>2588.0</td>\n",
       "      <td>67.0000</td>\n",
       "      <td>kg</td>\n",
       "      <td>4.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1665</td>\n",
       "      <td>user0</td>\n",
       "      <td>2022-11-28</td>\n",
       "      <td>3364.0</td>\n",
       "      <td>72.0000</td>\n",
       "      <td>kg</td>\n",
       "      <td>61.564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1666</td>\n",
       "      <td>user0</td>\n",
       "      <td>2022-11-29</td>\n",
       "      <td>3824.0</td>\n",
       "      <td>72.0000</td>\n",
       "      <td>kg</td>\n",
       "      <td>37.894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5053</th>\n",
       "      <td>15927</td>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-15</td>\n",
       "      <td>7872.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>kg</td>\n",
       "      <td>342.828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5054</th>\n",
       "      <td>15928</td>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-16</td>\n",
       "      <td>7772.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>kg</td>\n",
       "      <td>277.667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5055</th>\n",
       "      <td>15929</td>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-17</td>\n",
       "      <td>4646.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>kg</td>\n",
       "      <td>349.917</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5056</th>\n",
       "      <td>15930</td>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-18</td>\n",
       "      <td>4868.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>kg</td>\n",
       "      <td>75.383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5057</th>\n",
       "      <td>15931</td>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-19</td>\n",
       "      <td>209.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>kg</td>\n",
       "      <td>239.965</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5058 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index user_id        date    steps   weight unit  calories_burned\n",
       "0       979   user0  2021-01-11  10065.0  67.0000   kg           34.000\n",
       "1       980   user0  2021-01-12   1464.0  67.0000   kg          239.000\n",
       "2       991   user0  2021-01-23   2588.0  67.0000   kg            4.000\n",
       "3      1665   user0  2022-11-28   3364.0  72.0000   kg           61.564\n",
       "4      1666   user0  2022-11-29   3824.0  72.0000   kg           37.894\n",
       "...     ...     ...         ...      ...      ...  ...              ...\n",
       "5053  15927   user7  2024-02-15   7872.0  90.7184   kg          342.828\n",
       "5054  15928   user7  2024-02-16   7772.0  90.7184   kg          277.667\n",
       "5055  15929   user7  2024-02-17   4646.0  90.7184   kg          349.917\n",
       "5056  15930   user7  2024-02-18   4868.0  90.7184   kg           75.383\n",
       "5057  15931   user7  2024-02-19    209.0  90.7184   kg          239.965\n",
       "\n",
       "[5058 rows x 7 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Some users do not have the data of calories burned on a specific date, so we just delete the row with no data\n",
    "\n",
    "df_not_nan = merged_df[merged_df['calories_burned'].notna()]\n",
    "df_not_nan = df_not_nan[merged_df['steps'].notna()]\n",
    "df_not_nan = df_not_nan.reset_index()\n",
    "df_not_nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f0108cd7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lmm20\\AppData\\Local\\Temp\\ipykernel_23004\\1687584561.py:5: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_not_nan['calories_difference'][0] = 0\n",
      "C:\\Users\\lmm20\\AppData\\Local\\Temp\\ipykernel_23004\\1687584561.py:6: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  df_not_nan['steps_difference'][0] = 0\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>steps</th>\n",
       "      <th>weight</th>\n",
       "      <th>unit</th>\n",
       "      <th>calories_burned</th>\n",
       "      <th>steps_difference</th>\n",
       "      <th>calories_difference</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>979</td>\n",
       "      <td>user0</td>\n",
       "      <td>2021-01-11</td>\n",
       "      <td>10065.0</td>\n",
       "      <td>67.0000</td>\n",
       "      <td>kg</td>\n",
       "      <td>34.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>980</td>\n",
       "      <td>user0</td>\n",
       "      <td>2021-01-12</td>\n",
       "      <td>1464.0</td>\n",
       "      <td>67.0000</td>\n",
       "      <td>kg</td>\n",
       "      <td>239.000</td>\n",
       "      <td>-8601.0</td>\n",
       "      <td>205.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>991</td>\n",
       "      <td>user0</td>\n",
       "      <td>2021-01-23</td>\n",
       "      <td>2588.0</td>\n",
       "      <td>67.0000</td>\n",
       "      <td>kg</td>\n",
       "      <td>4.000</td>\n",
       "      <td>1124.0</td>\n",
       "      <td>-235.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1665</td>\n",
       "      <td>user0</td>\n",
       "      <td>2022-11-28</td>\n",
       "      <td>3364.0</td>\n",
       "      <td>72.0000</td>\n",
       "      <td>kg</td>\n",
       "      <td>61.564</td>\n",
       "      <td>776.0</td>\n",
       "      <td>57.564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1666</td>\n",
       "      <td>user0</td>\n",
       "      <td>2022-11-29</td>\n",
       "      <td>3824.0</td>\n",
       "      <td>72.0000</td>\n",
       "      <td>kg</td>\n",
       "      <td>37.894</td>\n",
       "      <td>460.0</td>\n",
       "      <td>-23.670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5053</th>\n",
       "      <td>15927</td>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-15</td>\n",
       "      <td>7872.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>kg</td>\n",
       "      <td>342.828</td>\n",
       "      <td>-3842.0</td>\n",
       "      <td>-103.408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5054</th>\n",
       "      <td>15928</td>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-16</td>\n",
       "      <td>7772.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>kg</td>\n",
       "      <td>277.667</td>\n",
       "      <td>-100.0</td>\n",
       "      <td>-65.161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5055</th>\n",
       "      <td>15929</td>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-17</td>\n",
       "      <td>4646.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>kg</td>\n",
       "      <td>349.917</td>\n",
       "      <td>-3126.0</td>\n",
       "      <td>72.250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5056</th>\n",
       "      <td>15930</td>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-18</td>\n",
       "      <td>4868.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>kg</td>\n",
       "      <td>75.383</td>\n",
       "      <td>222.0</td>\n",
       "      <td>-274.534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5057</th>\n",
       "      <td>15931</td>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-19</td>\n",
       "      <td>209.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>kg</td>\n",
       "      <td>239.965</td>\n",
       "      <td>-4659.0</td>\n",
       "      <td>164.582</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5058 rows × 9 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index user_id        date    steps   weight unit  calories_burned  \\\n",
       "0       979   user0  2021-01-11  10065.0  67.0000   kg           34.000   \n",
       "1       980   user0  2021-01-12   1464.0  67.0000   kg          239.000   \n",
       "2       991   user0  2021-01-23   2588.0  67.0000   kg            4.000   \n",
       "3      1665   user0  2022-11-28   3364.0  72.0000   kg           61.564   \n",
       "4      1666   user0  2022-11-29   3824.0  72.0000   kg           37.894   \n",
       "...     ...     ...         ...      ...      ...  ...              ...   \n",
       "5053  15927   user7  2024-02-15   7872.0  90.7184   kg          342.828   \n",
       "5054  15928   user7  2024-02-16   7772.0  90.7184   kg          277.667   \n",
       "5055  15929   user7  2024-02-17   4646.0  90.7184   kg          349.917   \n",
       "5056  15930   user7  2024-02-18   4868.0  90.7184   kg           75.383   \n",
       "5057  15931   user7  2024-02-19    209.0  90.7184   kg          239.965   \n",
       "\n",
       "      steps_difference  calories_difference  \n",
       "0                  0.0                0.000  \n",
       "1              -8601.0              205.000  \n",
       "2               1124.0             -235.000  \n",
       "3                776.0               57.564  \n",
       "4                460.0              -23.670  \n",
       "...                ...                  ...  \n",
       "5053           -3842.0             -103.408  \n",
       "5054            -100.0              -65.161  \n",
       "5055           -3126.0               72.250  \n",
       "5056             222.0             -274.534  \n",
       "5057           -4659.0              164.582  \n",
       "\n",
       "[5058 rows x 9 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add two more columns with the difference of steps and calorie consumption between two days\n",
    "\n",
    "df_not_nan['steps_difference'] = df_not_nan['steps'].diff()\n",
    "df_not_nan['calories_difference'] = df_not_nan['calories_burned'].diff()\n",
    "df_not_nan['calories_difference'][0] = 0\n",
    "df_not_nan['steps_difference'][0] = 0\n",
    "df_not_nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "6fbb9cdf",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>index</th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>steps</th>\n",
       "      <th>weight</th>\n",
       "      <th>calories_burned</th>\n",
       "      <th>steps_difference</th>\n",
       "      <th>calories_difference</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>979</td>\n",
       "      <td>user0</td>\n",
       "      <td>2021-01-11</td>\n",
       "      <td>10065.0</td>\n",
       "      <td>67.0000</td>\n",
       "      <td>34.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>980</td>\n",
       "      <td>user0</td>\n",
       "      <td>2021-01-12</td>\n",
       "      <td>1464.0</td>\n",
       "      <td>67.0000</td>\n",
       "      <td>239.000</td>\n",
       "      <td>-8601.0</td>\n",
       "      <td>205.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>991</td>\n",
       "      <td>user0</td>\n",
       "      <td>2021-01-23</td>\n",
       "      <td>2588.0</td>\n",
       "      <td>67.0000</td>\n",
       "      <td>4.000</td>\n",
       "      <td>1124.0</td>\n",
       "      <td>-235.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1665</td>\n",
       "      <td>user0</td>\n",
       "      <td>2022-11-28</td>\n",
       "      <td>3364.0</td>\n",
       "      <td>72.0000</td>\n",
       "      <td>61.564</td>\n",
       "      <td>776.0</td>\n",
       "      <td>57.564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1666</td>\n",
       "      <td>user0</td>\n",
       "      <td>2022-11-29</td>\n",
       "      <td>3824.0</td>\n",
       "      <td>72.0000</td>\n",
       "      <td>37.894</td>\n",
       "      <td>460.0</td>\n",
       "      <td>-23.670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5053</th>\n",
       "      <td>15927</td>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-15</td>\n",
       "      <td>7872.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>342.828</td>\n",
       "      <td>-3842.0</td>\n",
       "      <td>-103.408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5054</th>\n",
       "      <td>15928</td>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-16</td>\n",
       "      <td>7772.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>277.667</td>\n",
       "      <td>-100.0</td>\n",
       "      <td>-65.161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5055</th>\n",
       "      <td>15929</td>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-17</td>\n",
       "      <td>4646.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>349.917</td>\n",
       "      <td>-3126.0</td>\n",
       "      <td>72.250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5056</th>\n",
       "      <td>15930</td>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-18</td>\n",
       "      <td>4868.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>75.383</td>\n",
       "      <td>222.0</td>\n",
       "      <td>-274.534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5057</th>\n",
       "      <td>15931</td>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-19</td>\n",
       "      <td>209.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>239.965</td>\n",
       "      <td>-4659.0</td>\n",
       "      <td>164.582</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5058 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      index user_id        date    steps   weight  calories_burned  \\\n",
       "0       979   user0  2021-01-11  10065.0  67.0000           34.000   \n",
       "1       980   user0  2021-01-12   1464.0  67.0000          239.000   \n",
       "2       991   user0  2021-01-23   2588.0  67.0000            4.000   \n",
       "3      1665   user0  2022-11-28   3364.0  72.0000           61.564   \n",
       "4      1666   user0  2022-11-29   3824.0  72.0000           37.894   \n",
       "...     ...     ...         ...      ...      ...              ...   \n",
       "5053  15927   user7  2024-02-15   7872.0  90.7184          342.828   \n",
       "5054  15928   user7  2024-02-16   7772.0  90.7184          277.667   \n",
       "5055  15929   user7  2024-02-17   4646.0  90.7184          349.917   \n",
       "5056  15930   user7  2024-02-18   4868.0  90.7184           75.383   \n",
       "5057  15931   user7  2024-02-19    209.0  90.7184          239.965   \n",
       "\n",
       "      steps_difference  calories_difference  \n",
       "0                  0.0                0.000  \n",
       "1              -8601.0              205.000  \n",
       "2               1124.0             -235.000  \n",
       "3                776.0               57.564  \n",
       "4                460.0              -23.670  \n",
       "...                ...                  ...  \n",
       "5053           -3842.0             -103.408  \n",
       "5054            -100.0              -65.161  \n",
       "5055           -3126.0               72.250  \n",
       "5056             222.0             -274.534  \n",
       "5057           -4659.0              164.582  \n",
       "\n",
       "[5058 rows x 8 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# drop the column of unit because it does not help with training \n",
    "df_not_nan.drop('unit',  axis=1, inplace=True)\n",
    "df_not_nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "111ac9ed",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>date</th>\n",
       "      <th>steps</th>\n",
       "      <th>weight</th>\n",
       "      <th>calories_burned</th>\n",
       "      <th>steps_difference</th>\n",
       "      <th>calories_difference</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>user0</td>\n",
       "      <td>2021-01-11</td>\n",
       "      <td>10065.0</td>\n",
       "      <td>67.0000</td>\n",
       "      <td>34.000</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>user0</td>\n",
       "      <td>2021-01-12</td>\n",
       "      <td>1464.0</td>\n",
       "      <td>67.0000</td>\n",
       "      <td>239.000</td>\n",
       "      <td>-8601.0</td>\n",
       "      <td>205.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>user0</td>\n",
       "      <td>2021-01-23</td>\n",
       "      <td>2588.0</td>\n",
       "      <td>67.0000</td>\n",
       "      <td>4.000</td>\n",
       "      <td>1124.0</td>\n",
       "      <td>-235.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>user0</td>\n",
       "      <td>2022-11-28</td>\n",
       "      <td>3364.0</td>\n",
       "      <td>72.0000</td>\n",
       "      <td>61.564</td>\n",
       "      <td>776.0</td>\n",
       "      <td>57.564</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>user0</td>\n",
       "      <td>2022-11-29</td>\n",
       "      <td>3824.0</td>\n",
       "      <td>72.0000</td>\n",
       "      <td>37.894</td>\n",
       "      <td>460.0</td>\n",
       "      <td>-23.670</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5053</th>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-15</td>\n",
       "      <td>7872.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>342.828</td>\n",
       "      <td>-3842.0</td>\n",
       "      <td>-103.408</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5054</th>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-16</td>\n",
       "      <td>7772.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>277.667</td>\n",
       "      <td>-100.0</td>\n",
       "      <td>-65.161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5055</th>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-17</td>\n",
       "      <td>4646.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>349.917</td>\n",
       "      <td>-3126.0</td>\n",
       "      <td>72.250</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5056</th>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-18</td>\n",
       "      <td>4868.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>75.383</td>\n",
       "      <td>222.0</td>\n",
       "      <td>-274.534</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5057</th>\n",
       "      <td>user7</td>\n",
       "      <td>2024-02-19</td>\n",
       "      <td>209.0</td>\n",
       "      <td>90.7184</td>\n",
       "      <td>239.965</td>\n",
       "      <td>-4659.0</td>\n",
       "      <td>164.582</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5058 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "     user_id        date    steps   weight  calories_burned  steps_difference  \\\n",
       "0      user0  2021-01-11  10065.0  67.0000           34.000               0.0   \n",
       "1      user0  2021-01-12   1464.0  67.0000          239.000           -8601.0   \n",
       "2      user0  2021-01-23   2588.0  67.0000            4.000            1124.0   \n",
       "3      user0  2022-11-28   3364.0  72.0000           61.564             776.0   \n",
       "4      user0  2022-11-29   3824.0  72.0000           37.894             460.0   \n",
       "...      ...         ...      ...      ...              ...               ...   \n",
       "5053   user7  2024-02-15   7872.0  90.7184          342.828           -3842.0   \n",
       "5054   user7  2024-02-16   7772.0  90.7184          277.667            -100.0   \n",
       "5055   user7  2024-02-17   4646.0  90.7184          349.917           -3126.0   \n",
       "5056   user7  2024-02-18   4868.0  90.7184           75.383             222.0   \n",
       "5057   user7  2024-02-19    209.0  90.7184          239.965           -4659.0   \n",
       "\n",
       "      calories_difference  \n",
       "0                   0.000  \n",
       "1                 205.000  \n",
       "2                -235.000  \n",
       "3                  57.564  \n",
       "4                 -23.670  \n",
       "...                   ...  \n",
       "5053             -103.408  \n",
       "5054              -65.161  \n",
       "5055               72.250  \n",
       "5056             -274.534  \n",
       "5057              164.582  \n",
       "\n",
       "[5058 rows x 7 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_not_nan.drop('index',  axis=1, inplace=True)\n",
    "df_not_nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "963f3e87",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = df_not_nan"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "cbc10663",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Implement my own accuracy function, if (predicted-true_value) <= 1000, counts as correct \n",
    "def calculate_accuracy(real_value, predicted):\n",
    "    count = 0\n",
    "    for i in range(len(predicted)):\n",
    "        if abs(real_value[i] - predicted[i])<=1000:\n",
    "            count +=1\n",
    "    return count/len(predicted)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "6f1e42d7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KNN predicted step category: 2169.0\n",
      "Logistic Regression predicted step category: 4483.0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\lmm20\\anaconda3\\lib\\site-packages\\sklearn\\linear_model\\_logistic.py:458: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n",
      "C:\\Users\\lmm20\\anaconda3\\lib\\site-packages\\sklearn\\base.py:420: UserWarning: X does not have valid feature names, but KNeighborsClassifier was fitted with feature names\n",
      "  warnings.warn(\n",
      "C:\\Users\\lmm20\\anaconda3\\lib\\site-packages\\sklearn\\base.py:420: UserWarning: X does not have valid feature names, but LogisticRegression was fitted with feature names\n",
      "  warnings.warn(\n"
     ]
    }
   ],
   "source": [
    "# Training the model of KNN anb Logistic Regression \n",
    "\n",
    "# Define step range categories\n",
    "# bins = [0, 2000, 4000, 6000, 8000, 10000, float('inf')]\n",
    "# labels = ['0-2000', '2001-4000', '4001-6000', '6001-8000', '8001-10000', '10000+']\n",
    "# df['step_category'] = pd.cut(df['steps'], bins=bins, labels=labels)\n",
    "\n",
    "# Feature and target variable\n",
    "X = df[['calories_burned', 'weight']]  # Using only calories_burned and weight as the features\n",
    "y = df['steps']\n",
    "\n",
    "# Encoding the target variable\n",
    "le = LabelEncoder()\n",
    "y_encoded = le.fit_transform(y)\n",
    "\n",
    "# Split the data\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y_encoded, test_size=0.2, random_state=42)\n",
    "\n",
    "# KNN model\n",
    "knn = KNeighborsClassifier()\n",
    "knn.fit(X_train, y_train)\n",
    "\n",
    "# Logistic Regression model\n",
    "log_reg = LogisticRegression(max_iter=200)\n",
    "log_reg.fit(X_train, y_train)\n",
    "\n",
    "# Example prediction with 200 calorie consumption and 70kg weight\n",
    "calorie_goal = [[200, 70]]  # Calorie goal as input\n",
    "predicted_step_category_knn = knn.predict(calorie_goal)\n",
    "predicted_step_category_log_reg = log_reg.predict(calorie_goal)\n",
    "\n",
    "# Translate prediction back to label\n",
    "print(\"KNN predicted step category:\", le.inverse_transform(predicted_step_category_knn)[0])\n",
    "print(\"Logistic Regression predicted step category:\", le.inverse_transform(predicted_step_category_log_reg)[0])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "46081891",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy of KNN model: 0.48122529644268774\n",
      "Accuracy of Logistic Regression model: 0.541501976284585\n"
     ]
    }
   ],
   "source": [
    "# Calculate the accuracy of two models\n",
    "y_pred_knn = knn.predict(X_test)\n",
    "y_pred_log_reg = log_reg.predict(X_test)\n",
    "\n",
    "# Calculating accuracy\n",
    "accuracy_knn = calculate_accuracy(y_test, y_pred_knn)\n",
    "accuracy_log_reg = calculate_accuracy(y_test, y_pred_log_reg)\n",
    "\n",
    "# Displaying accuracy\n",
    "print(\"Accuracy of KNN model:\", accuracy_knn)\n",
    "print(\"Accuracy of Logistic Regression model:\", accuracy_log_reg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "48b70595",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Conclusion: The accuracy right now is not satisfactory enough. I think there are two reasons. First, we need more factors\n",
    "# that having impact to stepcounts other than weigth and calorie consumption. Second, the data loaded from healthkit is not \n",
    "# good enough. For example, user0 with 67kg had 10065 steps on 2021-01-11, but only 34 calorie consumption. Hence, we have \n",
    "# reason to say the data is not convincing. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "811114da",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}