multithumos_dense.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1c78a322",
   "metadata": {},
   "source": [
    "## MultiTHUMOS Segmentation-map Eval"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "63f5e99e",
   "metadata": {},
   "source": [
    "### Setup Environment"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1d7b1d85-31f6-43e0-a771-8b9c6b9e49d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import math\n",
    "import torch\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import json\n",
    "from sklearn.metrics import average_precision_score\n",
    "from scipy.stats import norm\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "52a4faa9-f0e3-4a7e-b558-239815b6aaa4",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_json(file):\n",
    "    with open(file) as json_file:\n",
    "        data = json.load(json_file)\n",
    "        return data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9688c4cf",
   "metadata": {},
   "source": [
    "### Init Seg-mAP Evaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "16fdc82d-aeca-4f49-8046-1102dbe20098",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/data/miniconda3/envs/env-3.6.8/lib/python3.6/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "class Meter(object):\n",
    "    def reset(self):\n",
    "        pass\n",
    "\n",
    "    def add(self):\n",
    "        pass\n",
    "\n",
    "    def value(self):\n",
    "        pass\n",
    "\n",
    "\n",
    "class APMeter(Meter):\n",
    "    \"\"\"\n",
    "    The APMeter measures the average precision per class.\n",
    "    The APMeter is designed to operate on `NxK` Tensors `output` and\n",
    "    `target`, and optionally a `Nx1` Tensor weight where (1) the `output`\n",
    "    contains model output scores for `N` examples and `K` classes that ought to\n",
    "    be higher when the model is more convinced that the example should be\n",
    "    positively labeled, and smaller when the model believes the example should\n",
    "    be negatively labeled (for instance, the output of a sigmoid function); (2)\n",
    "    the `target` contains only values 0 (for negative examples) and 1\n",
    "    (for positive examples); and (3) the `weight` ( > 0) represents weight for\n",
    "    each sample.\n",
    "    \"\"\"\n",
    "    def __init__(self,weighted=False):\n",
    "        super(APMeter, self).__init__()\n",
    "        self.reset()\n",
    "        self.weighted=weighted\n",
    "\n",
    "    def reset(self):\n",
    "        \"\"\"Resets the meter with empty member variables\"\"\"\n",
    "        self.scores = torch.FloatTensor(torch.FloatStorage())\n",
    "        self.targets = torch.LongTensor(torch.LongStorage())\n",
    "        self.weights = torch.FloatTensor(torch.FloatStorage())\n",
    "\n",
    "    def add(self, output, target, weight=None):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            output (Tensor): NxK tensor that for each of the N examples\n",
    "                indicates the probability of the example belonging to each of\n",
    "                the K classes, according to the model. The probabilities should\n",
    "                sum to one over all classes\n",
    "            target (Tensor): binary NxK tensort that encodes which of the K\n",
    "                classes are associated with the N-th input\n",
    "                    (eg: a row [0, 1, 0, 1] indicates that the example is\n",
    "                         associated with classes 2 and 4)\n",
    "            weight (optional, Tensor): Nx1 tensor representing the weight for\n",
    "                each example (each weight > 0)\n",
    "        \"\"\"\n",
    "        if not torch.is_tensor(output):\n",
    "            output = torch.from_numpy(output)\n",
    "        if not torch.is_tensor(target):\n",
    "            target = torch.from_numpy(target)\n",
    "\n",
    "        if weight is not None:\n",
    "            if not torch.is_tensor(weight):\n",
    "                weight = torch.from_numpy(weight)\n",
    "            weight = weight.squeeze()\n",
    "        if output.dim() == 1:\n",
    "            output = output.view(-1, 1)\n",
    "        else:\n",
    "            assert output.dim() == 2, \\\n",
    "                'wrong output size (should be 1D or 2D with one column \\\n",
    "                per class)'\n",
    "        if target.dim() == 1:\n",
    "            target = target.view(-1, 1)\n",
    "        else:\n",
    "            assert target.dim() == 2, \\\n",
    "                'wrong target size (should be 1D or 2D with one column \\\n",
    "                per class)'\n",
    "        if weight is not None:\n",
    "            assert weight.dim() == 1, 'Weight dimension should be 1'\n",
    "            assert weight.numel() == target.size(0), \\\n",
    "                'Weight dimension 1 should be the same as that of target'\n",
    "            assert torch.min(weight) >= 0, 'Weight should be non-negative only'\n",
    "        assert torch.equal(target**2, target), \\\n",
    "            'targets should be binary (0 or 1)'\n",
    "        if self.scores.numel() > 0:\n",
    "            assert target.size(1) == self.targets.size(1), \\\n",
    "                'dimensions for output should match previously added examples.'\n",
    "\n",
    "        # make sure storage is of sufficient size\n",
    "        if self.scores.storage().size() < self.scores.numel() + output.numel():\n",
    "            new_size = math.ceil(self.scores.storage().size() * 1.5)\n",
    "            new_weight_size = math.ceil(self.weights.storage().size() * 1.5)\n",
    "            self.scores.storage().resize_(int(new_size + output.numel()))\n",
    "            self.targets.storage().resize_(int(new_size + output.numel()))\n",
    "            if weight is not None:\n",
    "                self.weights.storage().resize_(int(new_weight_size\n",
    "                                               + output.size(0)))\n",
    "\n",
    "        # store scores and targets\n",
    "        offset = self.scores.size(0) if self.scores.dim() > 0 else 0\n",
    "        self.scores.resize_(offset + output.size(0), output.size(1))\n",
    "        self.targets.resize_(offset + target.size(0), target.size(1))\n",
    "        self.scores.narrow(0, offset, output.size(0)).copy_(output)\n",
    "        self.targets.narrow(0, offset, target.size(0)).copy_(target)\n",
    "\n",
    "        if weight is not None:\n",
    "            self.weights.resize_(offset + weight.size(0))\n",
    "            self.weights.narrow(0, offset, weight.size(0)).copy_(weight)\n",
    "\n",
    "    def value(self):\n",
    "        \"\"\"Returns the model's average precision for each class\n",
    "        Return:\n",
    "            ap (FloatTensor): 1xK tensor, with avg precision for each class k\n",
    "        \"\"\"\n",
    "\n",
    "        if self.scores.numel() == 0:\n",
    "            return 0\n",
    "        ap = torch.zeros(self.scores.size(1))\n",
    "\n",
    "        rg = torch.arange(1, self.scores.size(0)+1).float()\n",
    "        if self.weights.numel() > 0:\n",
    "            weight = self.weights.new(self.weights.size())\n",
    "            weighted_truth = self.weights.new(self.weights.size())\n",
    "\n",
    "        # compute average precision for each class\n",
    "        # print(self.scores.size(1))\n",
    "        for k in range(self.scores.size(1)):\n",
    "            # sort scores\n",
    "            scores = self.scores[:, k]\n",
    "            targets = self.targets[:, k]\n",
    "            scores_2, sortind = torch.sort(scores, 0, True)\n",
    "            \n",
    "            truth = targets[sortind]\n",
    "            # if k==35:\n",
    "            #     print(scores_2)\n",
    "            if self.weights.numel() > 0:\n",
    "                weight = self.weights[sortind]\n",
    "                weighted_truth = truth.float() * weight\n",
    "                rg = weight.cumsum(0)\n",
    "\n",
    "            # compute true positive sums\n",
    "            if self.weights.numel() > 0:\n",
    "                tp = weighted_truth.cumsum(0)\n",
    "            else:\n",
    "                tp = truth.float().cumsum(0)\n",
    "\n",
    "            # compute precision curve\n",
    "            precision = tp.div(rg)\n",
    "\n",
    "            # compute average precision\n",
    "            # print(truth)\n",
    "            ap[k] = precision[truth.bool()].sum() / max(truth.sum(), 1)\n",
    "\n",
    "        return ap"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "40876044",
   "metadata": {},
   "source": [
    "### Read Sparse Predictions and Dense Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "8ea1e531-266e-4f3e-8f7e-4b752c354aa0",
   "metadata": {},
   "outputs": [],
   "source": [
    "pred = pd.read_csv('/path/to/Multithumos_results_eval.csv')\n",
    "dense_segs = os.listdir('/path/to/dense_results')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "212fd657-85e4-4d07-8f75-40abe5ae61ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "dense_segments = {}\n",
    "for i in dense_segs:\n",
    "    groups = i.split('_')\n",
    "    vid = groups[0]+'_'+groups[1]+'_'+groups[2]\n",
    "    base = float(groups[3])\n",
    "    # print(vid, base)\n",
    "    if vid not in dense_segments.keys():\n",
    "        dense_segments[vid] = []\n",
    "    dense_segments[vid].append((base, torch.load(f'/path/to/dense_results/{i}', map_location='cpu')))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64c70ffe",
   "metadata": {},
   "source": [
    "### Evaluate Dense Scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "7eba0541-0c71-4602-86bf-d49b41164c4a",
   "metadata": {},
   "outputs": [],
   "source": [
    "frame_dict = load_json('/path/to/multithumos_frames.json')\n",
    "annotations = load_json('/path/to/multithumos.json')\n",
    "base_scores = {}\n",
    "gt_dense = {}\n",
    "\n",
    "# Merge sliding windows\n",
    "n_feature_per_window = 64\n",
    "interval = 4\n",
    "for vid in dense_segments.keys():\n",
    "    num_frames = int(frame_dict[vid]) // interval\n",
    "    num_features = (num_frames // n_feature_per_window + 1) * n_feature_per_window\n",
    "    fps = float(int(frame_dict[vid]) / annotations[vid]['duration'])\n",
    "    scores = np.zeros((num_features, 65))\n",
    "    gt = np.zeros((num_features, 65))\n",
    "    for (base, seg) in dense_segments[vid]:\n",
    "        win_size = seg.shape[0]\n",
    "        scores[int(base//interval):int(base//interval) + win_size,:] = seg\n",
    "    base_scores[vid] = scores[:num_frames,:]\n",
    "    for anno in annotations[vid]['actions']:\n",
    "        start, end = int(anno[1]*fps), int(anno[2]*fps)\n",
    "        label = anno[0]-1\n",
    "        gt[int(start//interval):int(end//interval)+1,label] = 1\n",
    "    gt_dense[vid] = gt[:num_frames,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "d6b566a9-6b7f-429d-94aa-d975e7785f8f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Test Frame-based map tensor(34.4330)\n"
     ]
    }
   ],
   "source": [
    "apm = APMeter()\n",
    "for vid in base_scores.keys():\n",
    "    logit = base_scores[vid]\n",
    "    apm.add(logit, gt_dense[vid])\n",
    "val_map = 100 * apm.value().mean()\n",
    "print (\"Test Frame-based map\", val_map)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "098e964a",
   "metadata": {},
   "source": [
    "### Transform Sparse Preds to Dense Predictions with Gaussian Kernels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "54776829-363e-4ae8-982d-4190521ee06f",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sparse2dense(prediction, annotations, n_classes, scale=3,ratio=5, thresh=0.01):\n",
    "    y_preds = {}\n",
    "    y_gts = {}\n",
    "    for vid in tqdm(base_scores.keys()):\n",
    "        num_features = base_scores[vid].shape[0]\n",
    "        fps = num_features / annotations[vid]['duration']\n",
    "        vlen = num_features\n",
    "        x_d = np.linspace(1, vlen, vlen)\n",
    "        preds = prediction[prediction['video-id']==vid]\n",
    "\n",
    "        y_true = np.zeros((vlen, 65))\n",
    "        vdata = annotations[vid]\n",
    "        actions = vdata['actions']\n",
    "        for act in actions:\n",
    "            label = act[0]-1\n",
    "            start = int(act[1]*fps)\n",
    "            end = int(act[2]*fps)\n",
    "            for f in range(start, end+1):\n",
    "                if f < vlen:\n",
    "                    y_true[f,label] = 1\n",
    "        \n",
    "        temp = {}\n",
    "        for label in range(n_classes):\n",
    "            temp[label] = {}\n",
    "            \n",
    "        for i, p in preds.iterrows():\n",
    "            label = int(p['label'])\n",
    "            score = float(p['score'])\n",
    "            if score > thresh:\n",
    "                for loc in range(int(p['t-start']*fps), int(p['t-end']*fps)):\n",
    "                    if loc in temp[label]:\n",
    "                        temp[label][loc] = max(temp[label][loc], score)\n",
    "                    else:\n",
    "                        temp[label][loc] = score\n",
    "        y_scores = []\n",
    "        for label in range(n_classes):\n",
    "            if len(temp[label]) == 0:\n",
    "                y_scores.append(np.zeros((vlen,1)))\n",
    "            else:\n",
    "                y_scores.append(np.array(sum(norm(loc=loc, scale=scale).pdf(x_d)*ratio*temp[label][loc] for loc in temp[label])).reshape(-1,1))\n",
    "        y_scores = np.concatenate(y_scores, axis=-1)\n",
    "        y_preds[vid]=y_scores\n",
    "        y_gts[vid]=y_true\n",
    "    return y_preds, y_gts\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "c2f9a551-e39a-47c7-ac9e-1be088b78d78",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 213/213 [05:08<00:00,  1.45s/it]\n"
     ]
    }
   ],
   "source": [
    "extend_logits, gt_extend = sparse2dense(pred, annotations, 65)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c056c05",
   "metadata": {},
   "source": [
    "### Fuse Sparse2Dense Scores and Dense Scores, and Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "f0b45c1a-0331-47cc-92b9-3a00fd0bddb1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0\n",
      "Test Frame-based map tensor(34.4291)\n",
      "0.1\n",
      "Test Frame-based map tensor(40.6733)\n",
      "0.2\n",
      "Test Frame-based map tensor(41.1709)\n",
      "0.30000000000000004\n",
      "Test Frame-based map tensor(40.6506)\n",
      "0.4\n",
      "Test Frame-based map tensor(39.9089)\n",
      "0.5\n",
      "Test Frame-based map tensor(39.2042)\n",
      "0.6000000000000001\n",
      "Test Frame-based map tensor(38.6092)\n",
      "0.7000000000000001\n",
      "Test Frame-based map tensor(38.0807)\n",
      "0.8\n",
      "Test Frame-based map tensor(37.6205)\n",
      "0.9\n",
      "Test Frame-based map tensor(37.2111)\n",
      "1.0\n",
      "Test Frame-based map tensor(36.3598)\n"
     ]
    }
   ],
   "source": [
    "for theta in np.arange(0,1.1,0.1):\n",
    "    print(theta)\n",
    "    apm = APMeter()\n",
    "    for vid in extend_logits.keys():\n",
    "        logit = theta * extend_logits[vid] + (1-theta) * base_scores[vid]\n",
    "        apm.add(logit, gt_extend[vid])\n",
    "    val_map = 100 * apm.value().mean()\n",
    "    print (\"Test Frame-based map\", val_map)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}