interpret.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from run import *\n",
    "import tint, gc, os\n",
    "import pandas as pd\n",
    "from exp.config import FeatureFiles\n",
    "from utils.interpreter import *\n",
    "\n",
    "from tint.attr import (\n",
    "    AugmentedOcclusion,\n",
    "    DynaMask,\n",
    "    Occlusion, \n",
    "    FeatureAblation\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Arguments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "parser = get_parser()\n",
    "argv = \"\"\"\n",
    "  --model DLinear --result_path scratch --data_path Top_20.csv\n",
    "\"\"\".split()\n",
    "args = parser.parse_args(argv)\n",
    "\n",
    "initial_setup(args)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Experiment "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Disable cudnn if using cuda accelerator.\n",
    "# Please see https://captum.ai/docs/faq#how-can-i-resolve-cudnn-rnn-backward-error-for-rnn-or-lstm-network\n",
    "# args.no_gpu = True\n",
    "\n",
    "setting = stringify_setting(args)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting experiment. Result folder scratch\\DLinear_Top_20.\n",
      "Use GPU: cuda:0\n",
      "adding time index columns TimeFromStart\n",
      "added time encoded known reals ['month', 'day', 'weekday'].\n",
      "\n",
      "Train samples 12740, validation samples 560, test samples 560\n",
      "637 days of training, 14 days of validation data, 14 days of test data.\n",
      "\n",
      "Fitting scalers on train data\n",
      "Loading dataset from ./dataset/processed\\Top_20\\train.pt\n",
      "Loading dataset from ./dataset/processed\\Top_20\\val.pt\n",
      "Loading dataset from ./dataset/processed\\Top_20\\test.pt\n",
      "loading best model from scratch\\DLinear_Top_20\\checkpoint.pth\n"
     ]
    }
   ],
   "source": [
    "exp = Exp_Forecast(args, setting)  # set experiments\n",
    "exp.load_model()\n",
    "model = exp.model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# align importance along their time axis with the input data\n",
    "features = DataConfig.static_reals + DataConfig.observed_reals\n",
    "age_features = DataConfig.static_reals"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Interpret"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculate Attribute"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "flag = 'test'\n",
    "dataset, dataloader = exp.get_data(flag)\n",
    "df = exp.data_map[flag]\n",
    "df.sort_values(by=['Date', 'FIPS'], inplace=True)\n",
    "df.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 382/382 [00:23<00:00, 15.99it/s]\n"
     ]
    }
   ],
   "source": [
    "explainer = FeatureAblation(model)\n",
    "attr = batch_compute_attr(dataloader, exp, explainer, add_x_mark=False)\n",
    "\n",
    "# batch x pred_len x seq_len x features -> batch x pred_len x features\n",
    "attr = attr.mean(axis=2)\n",
    "# batch x features x pred_len\n",
    "attr = attr.permute(0, 2, 1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get ground truth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Load ground truth\n",
    "group_cases = pd.read_csv(\n",
    "    os.path.join(FeatureFiles.root_folder, 'Cases by age groups.csv')\n",
    ")\n",
    "group_cases['end_of_week'] = pd.to_datetime(group_cases['end_of_week'])\n",
    "\n",
    "population = pd.read_csv('dataset/raw/Population.csv')\n",
    "population = population[['FIPS', 'POPESTIMATE']]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Aggregate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2020-03-15 00:00:00 2021-11-27 00:00:00\n",
      "               FIPS   UNDER5   AGE517  AGE1829  AGE3039  AGE4049  AGE5064  \\\n",
      "count  12460.000000  12460.0  12460.0  12460.0  12460.0  12460.0  12460.0   \n",
      "mean   22995.050000      0.0      0.0      0.0      0.0      0.0      0.0   \n",
      "std    18249.847559      0.0      0.0      0.0      0.0      0.0      0.0   \n",
      "min     2261.000000      0.0      0.0      0.0      0.0      0.0      0.0   \n",
      "25%     6069.500000      0.0      0.0      0.0      0.0      0.0      0.0   \n",
      "50%    14558.500000      0.0      0.0      0.0      0.0      0.0      0.0   \n",
      "75%    39068.000000      0.0      0.0      0.0      0.0      0.0      0.0   \n",
      "max    53033.000000      0.0      0.0      0.0      0.0      0.0      0.0   \n",
      "\n",
      "       AGE6574  AGE75PLUS  VaccinationFull    Cases  \n",
      "count  12460.0    12460.0          12460.0  12460.0  \n",
      "mean       0.0        0.0              0.0      1.0  \n",
      "std        0.0        0.0              0.0      0.0  \n",
      "min        0.0        0.0              0.0      1.0  \n",
      "25%        0.0        0.0              0.0      1.0  \n",
      "50%        0.0        0.0              0.0      1.0  \n",
      "75%        0.0        0.0              0.0      1.0  \n",
      "max        0.0        0.0              0.0      1.0  \n"
     ]
    }
   ],
   "source": [
    "attr_numpy = attr.detach().cpu().numpy()\n",
    "# taking absolute since we want the magnitude of feature importance only\n",
    "attr_numpy  = np.abs(attr_numpy)\n",
    "\n",
    "attr_df = align_interpretation(\n",
    "    ranges=dataset.ranges,\n",
    "    attr=attr_numpy,\n",
    "    features=features,\n",
    "    min_date=df['Date'].min(),\n",
    "    seq_len=args.seq_len, pred_len=args.pred_len\n",
    ")\n",
    "print(attr_df.describe())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "attr_by_date = attr_df.groupby('Date')[\n",
    "    age_features\n",
    "].aggregate('sum').reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Weighted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "weights = df.groupby('FIPS').first()[age_features].reset_index()\n",
    "groups = []\n",
    "\n",
    "for FIPS, group_df in attr_df.groupby('FIPS'):\n",
    "    county_age_weights = weights[weights['FIPS']==FIPS][age_features].values\n",
    "    total_population = population[\n",
    "        population['FIPS']==FIPS]['POPESTIMATE'].values[0]\n",
    "    group_df[age_features] *= county_age_weights * total_population\n",
    "    # group_df[age_features] *= total_population\n",
    "    groups.append(group_df)\n",
    "    \n",
    "groups = pd.concat(groups, axis=0)\n",
    "weighted_attr_df = groups[['FIPS', 'Date']+age_features].reset_index(drop=True)\n",
    "\n",
    "weighted_attr_by_date = weighted_attr_df.groupby('Date')[\n",
    "    age_features].aggregate('sum').reset_index()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate globally"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found first common date 2020-03-21T00:00:00.000000000.\n",
      "Found last common date 2021-11-27T00:00:00.000000000.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age_group</th>\n",
       "      <th>cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UNDER5</td>\n",
       "      <td>14060.865169</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AGE517</td>\n",
       "      <td>69605.303371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AGE1829</td>\n",
       "      <td>112643.910112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AGE3039</td>\n",
       "      <td>87244.471910</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AGE4049</td>\n",
       "      <td>76051.707865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AGE5064</td>\n",
       "      <td>99080.303371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AGE6574</td>\n",
       "      <td>36918.449438</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AGE75PLUS</td>\n",
       "      <td>28115.640449</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age_group          cases\n",
       "0     UNDER5   14060.865169\n",
       "1     AGE517   69605.303371\n",
       "2    AGE1829  112643.910112\n",
       "3    AGE3039   87244.471910\n",
       "4    AGE4049   76051.707865\n",
       "5    AGE5064   99080.303371\n",
       "6    AGE6574   36918.449438\n",
       "7  AGE75PLUS   28115.640449"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dates = attr_by_date['Date'].values\n",
    "first_common_date = find_first_common_date(group_cases, dates)\n",
    "last_common_date = find_last_common_date(group_cases, dates)\n",
    "\n",
    "summed_ground_truth = group_cases[\n",
    "    (group_cases['end_of_week']>=first_common_date) &\n",
    "    (group_cases['end_of_week']<=last_common_date)\n",
    "][age_features].mean(axis=0).T.reset_index()\n",
    "summed_ground_truth.columns = ['age_group', 'cases']\n",
    "summed_ground_truth"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unweighted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age_group</th>\n",
       "      <th>attr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UNDER5</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AGE517</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AGE1829</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AGE3039</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AGE4049</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AGE5064</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AGE6574</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AGE75PLUS</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age_group  attr\n",
       "0     UNDER5   0.0\n",
       "1     AGE517   0.0\n",
       "2    AGE1829   0.0\n",
       "3    AGE3039   0.0\n",
       "4    AGE4049   0.0\n",
       "5    AGE5064   0.0\n",
       "6    AGE6574   0.0\n",
       "7  AGE75PLUS   0.0"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summed_attr = attr_df[\n",
    "    (attr_df['Date']>=(first_common_date-pd.to_timedelta(6, unit='D'))) &\n",
    "    (attr_df['Date']<=last_common_date)\n",
    "][age_features].mean(axis=0).T.reset_index()\n",
    "summed_attr.columns = ['age_group', 'attr']\n",
    "summed_attr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age_group</th>\n",
       "      <th>cases</th>\n",
       "      <th>attr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UNDER5</td>\n",
       "      <td>2.684803</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AGE517</td>\n",
       "      <td>13.290540</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AGE1829</td>\n",
       "      <td>21.508396</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AGE3039</td>\n",
       "      <td>16.658589</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AGE4049</td>\n",
       "      <td>14.521426</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AGE5064</td>\n",
       "      <td>18.918540</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AGE6574</td>\n",
       "      <td>7.049264</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AGE75PLUS</td>\n",
       "      <td>5.368442</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age_group      cases  attr\n",
       "0     UNDER5   2.684803   0.0\n",
       "1     AGE517  13.290540   0.0\n",
       "2    AGE1829  21.508396   0.0\n",
       "3    AGE3039  16.658589   0.0\n",
       "4    AGE4049  14.521426   0.0\n",
       "5    AGE5064  18.918540   0.0\n",
       "6    AGE6574   7.049264   0.0\n",
       "7  AGE75PLUS   5.368442   0.0"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "merged = summed_ground_truth.merge(\n",
    "    summed_attr, on='age_group', how='inner'\n",
    ") \n",
    "merged[['cases', 'attr']] = merged[['cases', 'attr']].truediv(\n",
    "    merged[['cases', 'attr']].sum(axis=0)/100, \n",
    "    axis=1).fillna(0) \n",
    "merged"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Weighted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>age_group</th>\n",
       "      <th>attr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>UNDER5</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AGE517</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AGE1829</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AGE3039</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AGE4049</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>AGE5064</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>AGE6574</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>AGE75PLUS</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   age_group  attr\n",
       "0     UNDER5   0.0\n",
       "1     AGE517   0.0\n",
       "2    AGE1829   0.0\n",
       "3    AGE3039   0.0\n",
       "4    AGE4049   0.0\n",
       "5    AGE5064   0.0\n",
       "6    AGE6574   0.0\n",
       "7  AGE75PLUS   0.0"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summed_weighted_attr = weighted_attr_df[\n",
    "    (weighted_attr_df['Date']>=(first_common_date-pd.to_timedelta(6, unit='D'))) &\n",
    "    (weighted_attr_df['Date']<=last_common_date)\n",
    "][age_features].mean(axis=0).T.reset_index()\n",
    "summed_weighted_attr.columns = ['age_group', 'attr']\n",
    "summed_weighted_attr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "   age_group      cases  attr  cases_rank  attr_rank\n",
      "0     UNDER5   2.684803   0.0         8.0        4.5\n",
      "1     AGE517  13.290540   0.0         5.0        4.5\n",
      "2    AGE1829  21.508396   0.0         1.0        4.5\n",
      "3    AGE3039  16.658589   0.0         3.0        4.5\n",
      "4    AGE4049  14.521426   0.0         4.0        4.5\n",
      "5    AGE5064  18.918540   0.0         2.0        4.5\n",
      "6    AGE6574   7.049264   0.0         6.0        4.5\n",
      "7  AGE75PLUS   5.368442   0.0         7.0        4.5\n"
     ]
    }
   ],
   "source": [
    "global_rank = summed_ground_truth.merge(\n",
    "    summed_weighted_attr, on='age_group', how='inner'\n",
    ") \n",
    "global_rank[['cases', 'attr']] = global_rank[['cases', 'attr']].div(\n",
    "    global_rank[['cases', 'attr']].sum(axis=0)/100, axis=1).fillna(0)\n",
    "\n",
    "global_rank['cases_rank'] = global_rank['cases'].rank(\n",
    "    axis=0, ascending=False\n",
    ")\n",
    "global_rank['attr_rank'] = global_rank['attr'].rank(\n",
    "    axis=0, ascending=False\n",
    ")\n",
    "print(global_rank)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "global_rank.to_csv(\n",
    "    os.path.join(\n",
    "        exp.output_folder, \n",
    "        f'{flag}_global_rank_{explainer.get_name()}.csv'\n",
    "    ), \n",
    "    index=False\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluate Local Interpretation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Unweighted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found first common date 2021-12-18T00:00:00.000000000.\n",
      "Rank mae: 0.32812, rmse: 0.4239, ndcg: 0.82107\n",
      "Normalized mae: 0.061398, rmse: 0.074441, ndcg: 0.82411\n"
     ]
    }
   ],
   "source": [
    "# since age group ground truth is weekly aggregated\n",
    "# do the same for predicted importance\n",
    "weekly_agg_scores = aggregate_importance_by_window(\n",
    "    attr_by_date, age_features, first_common_date\n",
    ")\n",
    "result_df = evaluate_interpretation(\n",
    "    group_cases, weekly_agg_scores, age_features\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Weighted"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found first common date 2021-12-18T00:00:00.000000000.\n",
      "Rank mae: 0.125, rmse: 0.1875, ndcg: 0.98915\n",
      "Normalized mae: 0.039509, rmse: 0.047049, ndcg: 0.9155\n"
     ]
    }
   ],
   "source": [
    "# since age group ground truth is weekly aggregated\n",
    "# do the same for predicted importance\n",
    "weekly_agg_scores_df = aggregate_importance_by_window(\n",
    "    weighted_attr_by_date, age_features, first_common_date\n",
    ")\n",
    "result_df = evaluate_interpretation(\n",
    "    group_cases, weekly_agg_scores_df, age_features\n",
    ")\n",
    "# result_df.to_csv(\n",
    "#     os.path.join(\n",
    "#         exp.output_folder, \n",
    "#         f'{flag}_int_metrics_{explainer.get_name()}.csv'\n",
    "#     ), \n",
    "#     index=False\n",
    "# )"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}