instacart_pre.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_orders = pd.read_csv(\"instacart/orders.csv\", usecols = [\"order_id\", \"user_id\", \"eval_set\"])\n",
    "f_order_product_train = pd.read_csv(\"instacart/order_products__prior.csv\", usecols = [\"order_id\", \"product_id\"])\n",
    "f_order_product_test = pd.read_csv(\"instacart/order_products__train.csv\", usecols = [\"order_id\", \"product_id\"])\n",
    "\n",
    "user_order = f_orders.groupby([\"user_id\"])[\"order_id\"].apply(list)\n",
    "order_product_train = f_order_product_train.groupby([\"order_id\"])[\"product_id\"].apply(list)\n",
    "order_product_test = f_order_product_test.groupby([\"order_id\"])[\"product_id\"].apply(list)\n",
    "\n",
    "order_product_list_order_train = order_product_train.keys().tolist()\n",
    "order_product_list_product_train = order_product_train.values.tolist()\n",
    "\n",
    "order_product_list_order_test = order_product_test.keys().tolist()\n",
    "order_product_list_product_test = order_product_test.values.tolist()\n",
    "\n",
    "data = f_orders.values.tolist()  #order_id, user_id, eval_set\n",
    "train_data = list(filter(lambda x: x[2] == \"prior\", data))\n",
    "test_data = list(filter(lambda x: x[2] == \"train\", data))\n",
    "\n",
    "order_train_dict = dict()\n",
    "for i, key in enumerate(order_product_list_order_train):\n",
    "    order_train_dict[key] = order_product_list_product_train[i]\n",
    "\n",
    "order_test_dict = dict()\n",
    "for i, key in enumerate(order_product_list_order_test):\n",
    "    order_test_dict[key] = order_product_list_product_test[i]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "56305\n"
     ]
    }
   ],
   "source": [
    "num_user = 200000\n",
    "\n",
    "raw_train_dict = dict()\n",
    "flag = 0\n",
    "for line in train_data:\n",
    "    key = line[1]\n",
    "    order_id = line[0]\n",
    "    if key not in raw_train_dict.keys():\n",
    "        flag += 1\n",
    "        if flag > num_user:\n",
    "            break\n",
    "        raw_train_dict[key] = [order_id]\n",
    "    else:\n",
    "        raw_train_dict[key].append(order_id)\n",
    "\n",
    "filter_train_dict = dict()        \n",
    "for (user, orders) in raw_train_dict.items():\n",
    "    if len(orders) >= 10:\n",
    "        filter_train_dict[user] = orders\n",
    "        \n",
    "\n",
    "test_dict = dict()\n",
    "flag = 0\n",
    "for line in test_data:\n",
    "    key = line[1]\n",
    "    order_id = line[0]\n",
    "    if key not in test_dict.keys() and key in filter_train_dict.keys():\n",
    "        test_dict[key] = []\n",
    "        test_dict[key] += list(set(order_test_dict[order_id]))\n",
    "    if key in test_dict.keys() and key in filter_train_dict.keys():\n",
    "        test_dict[key] += list(set(order_test_dict[order_id]))\n",
    "\n",
    "\n",
    "\n",
    "pre_train_dict = dict()\n",
    "train_dict = dict()\n",
    "train_bpr_dict = dict()\n",
    "count = 0\n",
    "for (user, orders) in filter_train_dict.items():\n",
    "    if user in test_dict.keys():\n",
    "        filter_orders = []\n",
    "        for order in orders:\n",
    "            if (len(order_train_dict[order])) > 2:\n",
    "                filter_orders.append(order)\n",
    "        if (len(filter_orders)) >=10:\n",
    "            pre_train_dict[user] = [len(filter_orders[:-1])]\n",
    "            train_bpr_dict[user] = []\n",
    "            for order_id in filter_orders[:-1]:\n",
    "                pre_train_dict[user] += list(set(order_train_dict[order_id]))\n",
    "                train_bpr_dict[user] += list(set(order_train_dict[order_id]))\n",
    "            train_dict[user] = list(set(order_train_dict[filter_orders[-3]]))\n",
    "            train_dict[user] += list(set(order_train_dict[filter_orders[-2]]))\n",
    "            train_dict[user] += list(set(order_train_dict[filter_orders[-1]]))\n",
    "            train_bpr_dict[user] += list(set(order_train_dict[filter_orders[-1]]))\n",
    "\n",
    "delete_key = []\n",
    "for key in test_dict.keys():\n",
    "    if key not in train_dict.keys():\n",
    "        delete_key.append(key)\n",
    "\n",
    "for key in delete_key:\n",
    "    del test_dict[key]\n",
    "        \n",
    "\n",
    "ori_dict = dict() # user id starts from 0\n",
    "flag = 0\n",
    "for key in test_dict.keys():\n",
    "    ori_dict[key] = flag\n",
    "    flag += 1\n",
    "\n",
    "new_pre_train_dict = dict((ori_dict[key], value) for (key, value) in pre_train_dict.items())\n",
    "new_train_dict = dict((ori_dict[key], value) for (key, value) in train_dict.items())\n",
    "new_test_dict = dict((ori_dict[key], value) for (key, value) in test_dict.items())\n",
    "new_train_bpr_dict = dict((ori_dict[key], value) for (key, value) in train_bpr_dict.items())\n",
    "\n",
    "print(len(train_dict.keys()))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_train = open(\"./Data/instacart/train.txt\", \"w\")\n",
    "for key in new_train_dict.keys():\n",
    "    line = str(key)\n",
    "    for product in new_train_dict[key]:\n",
    "        line += ' ' + str(product)\n",
    "    line += '\\n'\n",
    "    f_train.write(line)\n",
    "f_train.close()\n",
    "\n",
    "f_train_bpr = open(\"./Data/instacart/train_bpr.txt\", \"w\")\n",
    "for key in new_train_bpr_dict.keys():\n",
    "    line = str(key)\n",
    "    for product in new_train_bpr_dict[key]:\n",
    "        line += ' ' + str(product)\n",
    "    line += '\\n'\n",
    "    f_train_bpr.write(line)\n",
    "f_train_bpr.close()\n",
    "\n",
    "f_test = open(\"./Data/instacart/test.txt\", \"w\")\n",
    "for key in new_test_dict.keys():\n",
    "    line = str(key)\n",
    "    for product in new_test_dict[key]:\n",
    "        line += ' ' + str(product)\n",
    "    line += '\\n'\n",
    "    f_test.write(line)\n",
    "f_test.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_train_mf = open(\"./Data/instacart/train_mf.txt\",\"w\")\n",
    "\n",
    "for (u_id,ps) in new_pre_train_dict.items():\n",
    "    n_order = ps[0]\n",
    "    items = ps[1:]\n",
    "    u_fre = dict()\n",
    "    for p in items:\n",
    "        if p not in u_fre.keys():\n",
    "            u_fre[p] = 1\n",
    "        else:\n",
    "            u_fre[p] += 1\n",
    "    for p in u_fre.keys():\n",
    "        fre = \"{:.3f}\".format(u_fre[p]/float(n_order))\n",
    "        line = str(u_id)+ ' ' + str(p) + ' ' + str(fre) + '\\n'\n",
    "        f_train_mf.write(line)\n",
    "f_train_mf.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_fre_test = open(\"./Data/instacart/fre_test.txt\",\"w\")\n",
    "for (u_id,ps) in new_pre_train_dict.items():\n",
    "    n_order = ps[0]\n",
    "    items = ps[1:]\n",
    "    u_fre = dict()\n",
    "    for p in items:\n",
    "        if p not in u_fre.keys():\n",
    "            u_fre[p] = 1\n",
    "        else:\n",
    "            u_fre[p] += 1\n",
    "    for p in new_train_dict[u_id]:\n",
    "        if p not in u_fre.keys():\n",
    "            u_fre[p] = 1\n",
    "        else:\n",
    "            u_fre[p] += 1\n",
    "    for p in u_fre.keys():\n",
    "        fre = \"{:.3f}\".format(u_fre[p]/float(n_order+1))\n",
    "        line = str(u_id)+ ' ' + str(p) + ' ' + str(fre) + '\\n'\n",
    "        f_fre_test.write(line)\n",
    "f_fre_test.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}