execute_with_averaged_eval.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "view-in-github"
   },
   "source": [
    "<a href=\"https://colab.research.google.com/github/upriyam-cmu/EDGE-Rec/blob/main/execute.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "9Q6JZwINFIas"
   },
   "source": [
    "# Install project code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "T8zKX7jqFIax"
   },
   "outputs": [],
   "source": [
    "!pip uninstall -y edge-rec\n",
    "!pip install -e ."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "YyB0la9eFIa2"
   },
   "source": [
    "# Train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "qHy9cdt2FIa3"
   },
   "outputs": [],
   "source": [
    "from edge_rec.gaussian_diffusion import GaussianDiffusion, Trainer\n",
    "from edge_rec.model import GraphReconstructionModel\n",
    "\n",
    "results_folder = \"./results\"  # Replace this to load trained models\n",
    "\n",
    "model = GraphReconstructionModel.default()\n",
    "\n",
    "diffusion_model = GaussianDiffusion(model, image_size=50)\n",
    "trainer = Trainer(\n",
    "    diffusion_model, folder=\"./movie_lens\", train_num_steps=int(1e4),\n",
    "    use_alternate_dense_dataset=True, save_and_sample_every=1000,\n",
    "    train_on_binary_targets=False,\n",
    "    train_mask_unknown_ratings=False,\n",
    "    alternate_dense_dataset_density=0.7,\n",
    "    results_folder=results_folder\n",
    "    # If the patch_size is large:\n",
    "    # train_batch_size=8,\n",
    "    # gradient_accumulate_every=2,\n",
    ")\n",
    "print(\"Using device:\", trainer.device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"num params:\", sum(param.numel() for param in model.parameters()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "9fJB149xFIa5"
   },
   "outputs": [],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "d5_rCKjIFIa6"
   },
   "source": [
    "# Sample ratings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "YIsW331MFIa6"
   },
   "outputs": [],
   "source": [
    "d70_subgraph, train_ratings, test_ratings = trainer.ds.dataset.get_subgraph(\n",
    "    subgraph_size=50,\n",
    "    target_density=0.5,\n",
    "    include_separate_train_test_ratings=True,\n",
    ")\n",
    "d70_subgraph = d70_subgraph[:-1]\n",
    "sampled_ratings = trainer.eval(milestone=5000, full_graph=d70_subgraph, subgraph_size=80)[0, 0, :, :].cpu().detach()\n",
    "sampled_ratings = (sampled_ratings - 0.2) * 2.5 + 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampled_ratings.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import pandas as pd\n",
    "from edge_rec.evaluate import get_metrics\n",
    "\n",
    "torch.manual_seed(1000)\n",
    "iters = 30\n",
    "density = 0.5\n",
    "patch_size = 50\n",
    "for milestone in [10000]:\n",
    "    sampled_metrics = 0\n",
    "    random_metrics = 0\n",
    "    for iter in range(iters):\n",
    "        d70_subgraph, train_ratings, test_ratings = trainer.ds.dataset.get_subgraph(\n",
    "            subgraph_size=patch_size,\n",
    "            target_density=density,\n",
    "            include_separate_train_test_ratings=True\n",
    "        )\n",
    "        d70_subgraph = d70_subgraph[:-1]\n",
    "        sampled_ratings = trainer.eval(milestone=milestone, full_graph=d70_subgraph, subgraph_size=patch_size)[0, 0, :,\n",
    "                          :].cpu().detach()\n",
    "        sampled_ratings = (sampled_ratings - 0.2) * 2.5 + 3\n",
    "\n",
    "        random_ratings = torch.rand(sampled_ratings.shape) * 5\n",
    "\n",
    "        train_edges = train_ratings.long().to_sparse_coo()\n",
    "        train_edges = torch.cat([\n",
    "            train_edges.indices(),\n",
    "            train_edges.values().unsqueeze(dim=0)\n",
    "        ], dim=0)\n",
    "        test_edges = test_ratings.long().to_sparse_coo()\n",
    "        test_edges = torch.cat([\n",
    "            test_edges.indices(),\n",
    "            test_edges.values().unsqueeze(dim=0)\n",
    "        ], dim=0)\n",
    "\n",
    "        metrics = pd.DataFrame(get_metrics(train_edges, test_edges, sampled_ratings))\n",
    "        sampled_metrics = sampled_metrics + metrics\n",
    "        random_metrics = random_metrics + pd.DataFrame(get_metrics(train_edges, test_edges, random_ratings))\n",
    "\n",
    "    avg_sampled_metrics, avg_random_metrics = sampled_metrics / iters, random_metrics / iters\n",
    "    avg_sampled_metrics.to_csv(\n",
    "        f\"metrics/avg_sampled_metrics_patch_size_{patch_size}_density_{density}_milestone_{milestone}.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.ticker import (MultipleLocator, AutoMinorLocator)\n",
    "\n",
    "mpl.rcParams.update({'figure.dpi': 300, 'font.size': 4})\n",
    "plt.style.use('bmh')\n",
    "fig, axs = plt.subplots(1, 5, figsize=[10, 2.4])\n",
    "ks = (1, 5, 10, 20, 30, 40, 50)\n",
    "\n",
    "for density in [0.5, 0.7]:\n",
    "    for patch_size in [50, 80]:\n",
    "        d = pd.read_csv(f\"metrics/avg_sampled_metrics_patch_size_{patch_size}_density_{density}_milestone_10000.csv\")\n",
    "        for ax, name, alias in zip(axs, ['precision', 'recall', 'ndcg', 'mean_reciprocal_rank', 'hit_rate'],\n",
    "                                   ['Precision', 'Recall', 'NDCG', 'MRR', 'HR']):\n",
    "            ax.set_title(alias)\n",
    "            ax.xaxis.grid(True, which='major')\n",
    "            ax.xaxis.set_major_locator(MultipleLocator(10))\n",
    "            ax.xaxis.set_major_formatter('{x:.0f}')\n",
    "            ax.plot(ks, d[name], '.-', linewidth=1.0, markersize=4.0,\n",
    "                    label=f\"Density = {density}, Patch Size = {patch_size}x{patch_size}\")\n",
    "            if name == 'precision':\n",
    "                ax.set_ylabel(\"ML-100k, Patch Sampling, GDiT-400k\")\n",
    "            elif name == 'hit_rate':\n",
    "                ax.legend()\n",
    "            ax.set_xlabel(\"Top-K\")\n",
    "            #ax.set_aspect(50 / (d[name].max() - d[name].min()))\n",
    "\n",
    "plt.tight_layout(h_pad=-25.0)\n",
    "plt.savefig(\"ml100k_large_model_results.png\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "uoaM5uiZFIa7"
   },
   "source": [
    "# Evaluate metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from edge_rec.evaluate import get_metrics\n",
    "import torch\n",
    "\n",
    "train_edges = train_ratings.long().to_sparse_coo()\n",
    "train_edges = torch.cat([\n",
    "    train_edges.indices(),\n",
    "    train_edges.values().unsqueeze(dim=0)\n",
    "], dim=0)\n",
    "test_edges = test_ratings.long().to_sparse_coo()\n",
    "test_edges = torch.cat([\n",
    "    test_edges.indices(),\n",
    "    test_edges.values().unsqueeze(dim=0)\n",
    "], dim=0)\n",
    "\n",
    "get_metrics(train_edges, test_edges, sampled_ratings)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "RpzXY9IfFIa-"
   },
   "source": [
    "# Display sampled ratings distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "37uiTgF2FIa_"
   },
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "plt.hist(sampled_ratings.numpy().flatten(), bins=5)\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "include_colab_link": true,
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}