bbh.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Big Bench Hard"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from datasets import load_dataset\n",
    "import tqdm\n",
    "from torch.utils.data import DataLoader\n",
    "import sys\n",
    "import pickle as pkl\n",
    "sys.path.append(\"../\")\n",
    "\n",
    "from train import train_ensemble\n",
    "\n",
    "\n",
    "hf_token = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "bbh_ds_names = [\"boolean_expressions\",  \"causal_judgement\",  \"date_understanding\",  \"disambiguation_qa\",  \"formal_fallacies\",  \"geometric_shapes\",  \"hyperbaton\",  \"logical_deduction_five_objects\",  \"logical_deduction_seven_objects\",  \"logical_deduction_three_objects\",  \"movie_recommendation\",  \"navigate\",  \"object_counting\",  \"penguins_in_a_table\",  \"reasoning_about_colored_objects\",  \"ruin_names\",  \"salient_translation_error_detection\",  \"snarks\",  \"sports_understanding\",  \"temporal_sequences\",  \"tracking_shuffled_objects_five_objects\",  \"tracking_shuffled_objects_seven_objects\",  \"tracking_shuffled_objects_three_objects\",  \"web_of_lies\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_model_data = {\n",
    "    \"meta-llama__Llama-2-70b-chat-hf\": {},\n",
    "    \"mistralai__Mixtral-8x7B-Instruct-v0.1\" : {},\n",
    "    \"google__gemma-7b\": {},\n",
    "    \"microsoft__phi-2\": {},\n",
    "    \"mistralai__Mistral-7B-v0.1\": {},\n",
    "}\n",
    "\n",
    "model_names = list(all_model_data.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "meta-llama__Llama-2-70b-chat-hf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24/24 [00:24<00:00,  1.02s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mistralai__Mixtral-8x7B-Instruct-v0.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24/24 [00:24<00:00,  1.01s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "google__gemma-7b\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24/24 [00:25<00:00,  1.08s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "microsoft__phi-2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24/24 [00:24<00:00,  1.02s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mistralai__Mistral-7B-v0.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24/24 [00:26<00:00,  1.10s/it]\n"
     ]
    }
   ],
   "source": [
    "def load_data(model_name, ds_name):\n",
    "    data = load_dataset(\n",
    "    f\"HuggingFaceEvalInternal/{model_name}-details-private\",\n",
    "    name=f\"{model_name}__leaderboard_bbh_{ds_name}\",\n",
    "    split=\"latest\",\n",
    "    token=hf_token)\n",
    "\n",
    "    idx = np.argsort(data[\"doc_id\"])\n",
    "\n",
    "    probs = np.array(data[\"resps\"]).squeeze()[:, :, 0].astype(float)\n",
    "    labels = np.array([data[\"doc\"][i][\"target\"] for i in range(len(probs))])\n",
    "    choices = sorted(list(set(labels)))\n",
    "    labels_list = [choices.index(lbl) for lbl in labels]\n",
    "    label_arr = np.array(labels_list).astype(int)\n",
    "\n",
    "    return probs[idx], label_arr[idx]\n",
    "\n",
    "for mn in list(all_model_data.keys()):\n",
    "    print(mn)\n",
    "    for ds_name in tqdm.tqdm(bbh_ds_names):\n",
    "        x, y = load_data(mn, ds_name)\n",
    "        all_model_data[mn][ds_name] = {\n",
    "            \"data\" : x,\n",
    "            \"label\": y\n",
    "        }\n",
    "\n",
    "with open(\"bbh_data.pkl\", \"wb\") as f:\n",
    "    pkl.dump(model_names, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "meta-llama__Llama-2-70b-chat-hf\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24/24 [00:23<00:00,  1.01it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mistralai__Mixtral-8x7B-Instruct-v0.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24/24 [00:24<00:00,  1.03s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "google__gemma-7b\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24/24 [00:25<00:00,  1.08s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "microsoft__phi-2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24/24 [00:24<00:00,  1.02s/it]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mistralai__Mistral-7B-v0.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24/24 [00:25<00:00,  1.04s/it]\n"
     ]
    }
   ],
   "source": [
    "with open(\"bbh_data.pkl\", \"rb\") as f:\n",
    "    all_model_data = pkl.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/24 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/stekin6/sft/inference_llm/other_results/../train.py:77: UserWarning: nn.init.xavier_uniform is now deprecated in favor of nn.init.xavier_uniform_.\n",
      "  torch.nn.init.xavier_uniform(m.weight)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.6600\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.9400\n",
      "google__gemma-7b: Acc: 0.8200\n",
      "microsoft__phi-2: Acc: 0.9200\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.8200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|▍         | 1/24 [00:06<02:26,  6.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No improvement in 200 epochs, breaking\n",
      "Novel Acc = 94.2308 +- 5.73\n",
      "94.23076923076923\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.3684\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.3158\n",
      "google__gemma-7b: Acc: 0.2895\n",
      "microsoft__phi-2: Acc: 0.4474\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.4211\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/stekin6/sft/inference_llm/other_results/../train.py:77: UserWarning: nn.init.xavier_uniform is now deprecated in favor of nn.init.xavier_uniform_.\n",
      "  torch.nn.init.xavier_uniform(m.weight)\n",
      "  8%|▊         | 2/24 [00:12<02:17,  6.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Novel Acc = 62.5000 +- 15.88\n",
      "62.5\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.1800\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.5600\n",
      "google__gemma-7b: Acc: 0.4000\n",
      "microsoft__phi-2: Acc: 0.4200\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.4800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|█▎        | 3/24 [00:20<02:28,  7.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Novel Acc = 59.6154 +- 11.36\n",
      "59.61538461538461\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.2600\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.5600\n",
      "google__gemma-7b: Acc: 0.5600\n",
      "microsoft__phi-2: Acc: 0.8200\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.4200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 4/24 [00:28<02:29,  7.47s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Novel Acc = 76.9231 +- 12.46\n",
      "76.92307692307692\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.4600\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.4400\n",
      "google__gemma-7b: Acc: 0.5000\n",
      "microsoft__phi-2: Acc: 0.5200\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.4800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|██        | 5/24 [00:36<02:26,  7.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Novel Acc = 57.6923 +- 9.81\n",
      "57.69230769230769\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.1200\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.0000\n",
      "google__gemma-7b: Acc: 0.0600\n",
      "microsoft__phi-2: Acc: 0.0000\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.0400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 6/24 [00:42<02:07,  7.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No improvement in 200 epochs, breaking\n",
      "Novel Acc = 36.5385 +- 15.65\n",
      "36.53846153846154\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.5200\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.8200\n",
      "google__gemma-7b: Acc: 0.6200\n",
      "microsoft__phi-2: Acc: 0.8400\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.5800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|██▉       | 7/24 [00:49<01:57,  6.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No improvement in 200 epochs, breaking\n",
      "Novel Acc = 90.3846 +- 6.61\n",
      "90.38461538461539\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.2600\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.5200\n",
      "google__gemma-7b: Acc: 0.2000\n",
      "microsoft__phi-2: Acc: 0.5000\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.4400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 8/24 [00:57<01:56,  7.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Novel Acc = 46.1538 +- 16.73\n",
      "46.15384615384615\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.1400\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.3000\n",
      "google__gemma-7b: Acc: 0.3200\n",
      "microsoft__phi-2: Acc: 0.5200\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.2600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 9/24 [01:05<01:52,  7.53s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Novel Acc = 30.7692 +- 9.47\n",
      "30.76923076923077\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.4200\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.5000\n",
      "google__gemma-7b: Acc: 0.5600\n",
      "microsoft__phi-2: Acc: 0.7600\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.6200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|████▏     | 10/24 [01:12<01:45,  7.57s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No improvement in 200 epochs, breaking\n",
      "Novel Acc = 76.9231 +- 11.26\n",
      "76.92307692307692\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.1400\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.6600\n",
      "google__gemma-7b: Acc: 0.5600\n",
      "microsoft__phi-2: Acc: 0.7000\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.6800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████▌     | 11/24 [01:21<01:40,  7.73s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Novel Acc = 73.0769 +- 15.51\n",
      "73.07692307692308\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.5200\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.2600\n",
      "google__gemma-7b: Acc: 0.4400\n",
      "microsoft__phi-2: Acc: 0.4000\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.5000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 12/24 [01:29<01:33,  7.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Novel Acc = 71.1538 +- 12.89\n",
      "71.15384615384616\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.0400\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.0600\n",
      "google__gemma-7b: Acc: 0.1200\n",
      "microsoft__phi-2: Acc: 0.0000\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.1200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████▍    | 13/24 [01:35<01:19,  7.27s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No improvement in 200 epochs, breaking\n",
      "Novel Acc = 23.0769 +- 9.92\n",
      "23.076923076923077\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.2333\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.5000\n",
      "google__gemma-7b: Acc: 0.4667\n",
      "microsoft__phi-2: Acc: 0.4667\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.4333\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 14/24 [01:39<01:05,  6.50s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Novel Acc = 37.5000 +- 12.25\n",
      "37.5\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.0800\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.5800\n",
      "google__gemma-7b: Acc: 0.3400\n",
      "microsoft__phi-2: Acc: 0.3400\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.2800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▎   | 15/24 [01:45<00:55,  6.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No improvement in 200 epochs, breaking\n",
      "Novel Acc = 15.3846 +- 10.03\n",
      "15.384615384615385\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.2200\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.5600\n",
      "google__gemma-7b: Acc: 0.3400\n",
      "microsoft__phi-2: Acc: 0.4000\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.5600\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 16/24 [01:53<00:54,  6.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Novel Acc = 55.7692 +- 14.26\n",
      "55.76923076923077\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.1200\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.4000\n",
      "google__gemma-7b: Acc: 0.2200\n",
      "microsoft__phi-2: Acc: 0.4200\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.4400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71%|███████   | 17/24 [02:01<00:50,  7.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Novel Acc = 44.2308 +- 10.86\n",
      "44.23076923076923\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.4444\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.6944\n",
      "google__gemma-7b: Acc: 0.6389\n",
      "microsoft__phi-2: Acc: 0.6389\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.5833\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 18/24 [02:06<00:39,  6.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No improvement in 200 epochs, breaking\n",
      "Novel Acc = 72.2222 +- 9.25\n",
      "72.22222222222223\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.1200\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.2400\n",
      "google__gemma-7b: Acc: 0.1200\n",
      "microsoft__phi-2: Acc: 0.4000\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.2400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|███████▉  | 19/24 [02:12<00:30,  6.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No improvement in 200 epochs, breaking\n",
      "Novel Acc = 86.5385 +- 10.14\n",
      "86.53846153846153\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.3000\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.4600\n",
      "google__gemma-7b: Acc: 0.1400\n",
      "microsoft__phi-2: Acc: 0.1800\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.1400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 20/24 [02:20<00:27,  6.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Novel Acc = 50.0000 +- 11.92\n",
      "50.0\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.1200\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.1400\n",
      "google__gemma-7b: Acc: 0.0800\n",
      "microsoft__phi-2: Acc: 0.1600\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.0400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 21/24 [02:28<00:21,  7.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Novel Acc = 23.0769 +- 13.55\n",
      "23.076923076923077\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.1200\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.1600\n",
      "google__gemma-7b: Acc: 0.0800\n",
      "microsoft__phi-2: Acc: 0.1600\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.0400\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████▏| 22/24 [02:36<00:14,  7.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Novel Acc = 5.7692 +- 5.73\n",
      "5.769230769230769\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.3200\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.3200\n",
      "google__gemma-7b: Acc: 0.3800\n",
      "microsoft__phi-2: Acc: 0.2600\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.3200\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▌| 23/24 [02:44<00:07,  7.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Novel Acc = 42.3077 +- 15.44\n",
      "42.30769230769231\n",
      "meta-llama__Llama-2-70b-chat-hf: Acc: 0.5600\n",
      "mistralai__Mixtral-8x7B-Instruct-v0.1: Acc: 0.6000\n",
      "google__gemma-7b: Acc: 0.4400\n",
      "microsoft__phi-2: Acc: 0.4200\n",
      "mistralai__Mistral-7B-v0.1: Acc: 0.5800\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 24/24 [02:52<00:00,  7.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Novel Acc = 65.3846 +- 12.54\n",
      "65.38461538461539\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "results = {}\n",
    "for ds_name in tqdm.tqdm(bbh_ds_names):\n",
    "    results[ds_name] = {}\n",
    "    all_x, all_y = [], []\n",
    "    for mn in all_model_data.keys():\n",
    "        x, y = all_model_data[mn][ds_name].values()\n",
    "        all_x.append(x)\n",
    "        all_y.append(y)\n",
    "    all_x = np.concatenate(all_x, axis=1)\n",
    "\n",
    "    try:\n",
    "        for y_idx in all_y:\n",
    "            for i in range(len(y_idx)):\n",
    "                assert(y_idx[i] == y[i])\n",
    "    except AssertionError as e:\n",
    "        print(e)\n",
    "        print(f\"skipping ... {ds_name}\")\n",
    "\n",
    "    concat_data = np.concatenate([all_x, y[:, None]], axis=1)\n",
    "\n",
    "    train_size = int(len(x) * 0.8)\n",
    "    train_data, test_data = concat_data[:train_size], concat_data[train_size:]\n",
    "\n",
    "    val_size = int(len(train_data) * 0.2)\n",
    "    train_size = len(train_data) - val_size\n",
    "\n",
    "    novel_data = test_data[:, :-1]\n",
    "    novel_label = test_data[:, -1]\n",
    "    for i, novel_arr in enumerate(np.split(novel_data, len(model_names), axis=1)):\n",
    "        pred = novel_arr.argmax(axis=1)\n",
    "        acc = np.mean(novel_label == pred)\n",
    "        print(f\"{model_names[i]}: Acc: {acc:.4f}\")\n",
    "        results[ds_name][model_names[i]] = acc * 100\n",
    "\n",
    "\n",
    "    batch_size = 4\n",
    "    train_loader = DataLoader(train_data, batch_size=batch_size, shuffle=True)\n",
    "    val_loader = DataLoader(train_data[train_size:], batch_size=batch_size, shuffle=True)\n",
    "    novel_loader = DataLoader(test_data, batch_size=batch_size, shuffle=True)\n",
    "\n",
    "    space_size = all_x[0].shape[0] // len(model_names)\n",
    "\n",
    "    exp_result = train_ensemble(model_names, train_loader, val_loader, novel_loader,\n",
    "                    n_epochs=300, save_dir=\".\",\n",
    "                    space_size=space_size, verbose=False)\n",
    "    results[ds_name][\"TOPLA\"] = exp_result[\"test_acc\"]\n",
    "    print(exp_result[\"test_acc\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "avg_scores = {mn: 0 for mn in model_names}\n",
    "avg_scores[\"TOPLA\"] = 0\n",
    "for ds_name, res in results.items():\n",
    "    for mn, acc in res.items():\n",
    "        avg_scores[mn] += acc\n",
    "\n",
    "for mn in model_names:\n",
    "    avg_scores[mn] = avg_scores[mn] / len(results.keys())\n",
    "avg_scores[\"TOPLA\"] /= len(results.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'meta-llama__Llama-2-70b-chat-hf': 28.025828460038984,\n",
       " 'mistralai__Mixtral-8x7B-Instruct-v0.1': 44.1259746588694,\n",
       " 'google__gemma-7b': 36.22928849902534,\n",
       " 'microsoft__phi-2': 44.553849902534104,\n",
       " 'mistralai__Mistral-7B-v0.1': 39.65716374269007,\n",
       " 'TOPLA': 54.050925925925924}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "avg_scores"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAGkCAYAAADXDuRQAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/TGe4hAAAACXBIWXMAAA9hAAAPYQGoP6dpAABprUlEQVR4nO3dd1gU1/s28HvpHQSpCoJKRAmWYENjiyjGEguJDRXU2LFEsX1jQWOPKPYWxBIQNZbYe1fsNRGxocQCqAiISt15//Blfq5L2YVFYL0/17WXzpkzM8/M7s4+nHNmRiIIggAiIiIiNaVR0gEQERERFScmO0RERKTWmOwQERGRWmOyQ0RERGqNyQ4RERGpNSY7REREpNaY7BAREZFaY7JDREREao3JDhEREak1JjullEQiQWBgYEmH8VkFBgZCIpHg5cuXJR1KkeXsCxW/R48eQSKRYN26dUove+LECUgkEpw4cULlcRWXouwvqUZqaip+/vln2NjYQCKRYNSoUSUdEhWAyc5nsm7dOkgkEpmXlZUVWrRogf379xf79v38/GS2raWlBXt7e3Tv3h23b98u9u1/Dn5+fjAyMspzvkQigb+//2eMqHByfoA/fpmbm6Nhw4YICwsr6fDylJPgaWho4L///pObn5KSAn19/TLzPpRljo6Ocp+h3F4fJ0xv377Fb7/9hpo1a8LAwACmpqZo0qQJNmzYgNyeKvTxejQ0NGBnZ4fWrVvLJY6Ojo5o3769wrHv27cPEokEdnZ2kEqlCi/36TnOxMQEtWrVQlBQENLT0xVejyJmzZqFdevWYciQIdi4cSN69+6t0vWT6mmVdABfmunTp8PJyQmCICA+Ph7r1q1D27ZtsXv3bpkTwvv376Glpdq3R1dXF3/88QcAICsrCw8ePMDKlStx4MAB3L59G3Z2dirdHhXNiBEjUK9ePQDAq1evsHnzZvTq1QtJSUkYNmxYCUeXN11dXWzatAnjxo2TKd++fXsJRfTlCQ4ORmpqqji9b98+bNq0CQsXLkT58uXF8kaNGgEA4uPj0bJlS0RFRaF79+7w9/dHWloatm3bBl9fX+zbtw9hYWHQ1NSU2U6rVq3Qp08fCIKAmJgYLF++HN999x327t2L77//vlCxh4WFwdHREY8ePcKxY8fg6emp8LIfn+OSkpKwbds2BAQE4NKlS4iIiChUPLk5duwYGjZsiKlTp6psnVTMBPosQkNDBQDCpUuXZMoTExMFbW1toWfPnsW6fV9fX8HQ0FCufM+ePQIAYfXq1cW6fUVMnTpVACC8ePGiUMvntY85AAjDhg0rbHgyMjMzhfT09Dzn5+xLYRw/flwAIGzdulWmPD09XahQoYLQqFGjQq23uOXsc5cuXYTatWvLzW/VqpXg7e2t0vdBEAQhJiZGACCEhoYqvWzOsT5+/LjK4iluhdnf33//XQAgxMTE5Drfy8tL0NDQEP7++2+5eQEBAQIAYc6cOTLlub2PN2/eFAAIrVu3FssqVaoktGvXTqE4U1NTBUNDQ2Hx4sVCnTp1BD8/P4WWE4Tcv//Z2dlC3bp1BQDC06dPFV5XbrKzs4X3798LgiAITk5OCu+TIgo6n1DRsRurhJmZmUFfX1+uFSe3MTsnTpxA3bp1oaenhypVqmDVqlVFHhtiY2MDAHLbT0pKwqhRo2Bvbw9dXV1UrVoVc+fOlWlWzhk7MH/+fKxevRpVqlSBrq4u6tWrh0uXLslt686dO+jatSssLS2hr6+PatWq4ddff5Wrl5SUBD8/P5iZmcHU1BR9+/bFu3fvCr2PecnIyMCUKVPg7u4OU1NTGBoaokmTJjh+/LhMvY/3Mzg4WNzPnO6/M2fOoF69ejLvS3HQ0dFBuXLl5N6r0NBQfPfdd7CysoKuri5q1KiBFStWyC1/+fJleHl5oXz58tDX14eTkxP69esnU0cqlSI4OBiurq7Q09ODtbU1Bg0ahNevXyscZ8+ePXH9+nXcuXNHLIuLi8OxY8fQs2fPXJdJSEhA//79YW1tDT09PdSqVQvr16+Xq5fz2TA1NYWZmRl8fX2RlJSU6zrv3LmDH3/8Eebm5tDT00PdunWxa9euAuO/d+8evL29YWNjAz09PVSsWBHdu3dHcnJyvsudPn0aP/30ExwcHKCrqwt7e3v88ssveP/+vUy9nO7Wp0+folOnTjAyMoKlpSUCAgKQnZ1d6P0trPPnz+PgwYPw8/PDDz/8IDd/9uzZcHZ2xty5c+X25VNubm4oX748YmJiChXLjh078P79e/z000/o3r07tm/fjrS0tEKtCwA0NDTQvHlzAB++xwCQnp6OqVOnomrVquL7NG7cOLmurpzu1rCwMLi6ukJXVxcHDhyARCJBTEwM9u7dK3aZ5axbkc9xfueTnPP53bt30atXL5iamsLS0hKTJ0+GIAj477//0LFjR5iYmMDGxgZBQUEy6y7MOU1V5+6nT5+iX79+sLa2hq6uLlxdXbF27Vpl37Jiw26szyw5ORkvX76EIAhISEjAkiVLkJqail69euW73LVr19CmTRvY2tpi2rRpyM7OxvTp02FpaanU9nMG/2ZnZ+Phw4cYP348LCwsZLrQ3r17h2bNmuHp06cYNGgQHBwccO7cOUycOBHPnz9HcHCwzDrDw8Px5s0bDBo0CBKJBPPmzUOXLl3w8OFDaGtrAwBu3ryJJk2aQFtbGwMHDoSjoyMePHiA3bt3Y+bMmTLr69q1K5ycnDB79mxcvXoVf/zxB6ysrDB37lyl9rEgKSkp+OOPP9CjRw8MGDAAb968QUhICLy8vHDx4kXUrl1bpn5oaCjS0tIwcOBA6OrqwtzcHLdu3ULr1q1haWmJwMBAZGVlYerUqbC2tlYohvy8efNG3JfExESEh4fjn3/+QUhIiEy9FStWwNXVFT/88AO0tLSwe/duDB06FFKpVOzuSkhIEOOcMGECzMzM8OjRI7mupUGDBmHdunXo27cvRowYgZiYGCxduhTXrl3D2bNnxfczP02bNkXFihURHh6O6dOnAwA2b94MIyMjtGvXTq7++/fv0bx5c9y/fx/+/v5wcnLC1q1b4efnh6SkJIwcORIAIAgCOnbsiDNnzmDw4MGoXr06duzYAV9fX7l1/vvvv2jcuDEqVKiACRMmwNDQEFu2bEGnTp2wbds2dO7cOdfYMzIy4OXlhfT0dAwfPhw2NjZ4+vQp9uzZg6SkJJiamua531u3bsW7d+8wZMgQWFhY4OLFi1iyZAmePHmCrVu3ytTNzs6Gl5cXGjRogPnz5+PIkSMICgpClSpVMGTIEKX3tyh2794NAOjTp0+u87W0tNCzZ09MmzYNZ8+ezbdb6fXr13j9+jWqVq1aqFjCwsLQokUL2NjYoHv37pgwYQJ2796Nn376qVDrA4AHDx4AACwsLCCVSvHDDz/gzJkzGDhwIKpXr45bt25h4cKFuHv3Lnbu3Cmz7LFjx7Blyxb4+/ujfPnysLW1xcaNG/HLL7+gYsWKGDNmDADA0tJS4c9xjtzOJzm6deuG6tWrY86cOdi7dy9mzJgBc3NzrFq1Ct999x3mzp2LsLAwBAQEoF69emjatCkA5c9pqjp3x8fHo2HDhmKCaGlpif3796N///5ISUkpHQO4S7Zh6cuR04316UtXV1dYt26dXH0AwtSpU8XpDh06CAYGBjJNsffu3RO0tLQU6i7x9fXNdfsVKlQQrly5IlP3t99+EwwNDYW7d+/KlE+YMEHQ1NQUYmNjBUH4v+Z0CwsLITExUaz3999/CwCE3bt3i2VNmzYVjI2NhcePH8usUyqViv/P6Qbp16+fTJ3OnTsLFhYWhd7Hj18fN7tnZWXJNR2/fv1asLa2lokhZz9NTEyEhIQEmfqdOnUS9PT0ZPbr9u3bgqamZpG7sT59aWhoCDNnzpSr/+7dO7kyLy8voXLlyuL0jh07cu1G/djp06cFAEJYWJhM+YEDB3It/9TH3ZABAQFC1apVxXn16tUT+vbtKwiCfPdHcHCwAED4888/xbKMjAzBw8NDMDIyElJSUgRBEISdO3cKAIR58+aJ9bKysoQmTZrIdeu0bNlScHNzE9LS0sQyqVQqNGrUSHB2dhbLPu3GunbtWq5diIrI7X2YPXu2IJFIZD4fOZ/T6dOny9StU6eO4O7uLk4rs78Fya8bq1OnTgIA4fXr13kuv337dgGAsHjxYrEMgNC/f3/hxYsXQkJCgnDhwgWhZcuWAgAhKChIrKdoN1Z8fLygpaUlrFmzRixr1KiR0LFjR4X2Macb68WLF8KLFy+E+/fvC7NmzRIkEolQs2ZNQRAEYePGjYKGhoZw+vRpmWVXrlwpABDOnj0rs38aGhrCv//+K7et3PZJ0c9xfueTnO/QwIEDxbKsrCyhYsWKgkQikelKfP36taCvry/4+vrK1FXmnKaqc3f//v0FW1tb4eXLlzJ1unfvLpiamub63fjc2I31mS1btgyHDx/G4cOH8eeff6JFixb4+eef8x28mZ2djSNHjqBTp04yg4irVq2q1CBAPT09cdsHDx7EqlWrYGRkhLZt2+Lu3btiva1bt6JJkyYoV64cXr58Kb48PT2RnZ2NU6dOyay3W7duKFeunDjdpEkTAMDDhw8BAC9evMCpU6fQr18/ODg4yCybWxfc4MGDZaabNGmCV69eISUlRal9/PT1KU1NTejo6AD40H2TmJiIrKws1K1bF1evXpWr7+3tLdOSlp2djYMHD6JTp04y+1W9enV4eXkVGGtBpkyZIsa+efNm9OjRA7/++isWLVokU09fX1/8f07LYbNmzfDw4UOx68XMzAwAsGfPHmRmZua6va1bt8LU1BStWrWSed/d3d1hZGQk1xSen549e+L+/fu4dOmS+G9eXVj79u2DjY0NevToIZZpa2tjxIgRSE1NxcmTJ8V6WlpaYssH8OE9HD58uMz6EhMTcezYMXTt2lVsHXv58iVevXoFLy8v3Lt3D0+fPs01lpyWm4MHDyrddfrx+/D27Vu8fPkSjRo1giAIuHbtmlz93D7nOd8ZZfa3qN68eQMAMDY2zrNOzrxPv4MhISGwtLSElZUVGjRogLNnz2L06NGF+ks+IiICGhoa8Pb2Fst69OiB/fv3K9yN+vbtW1haWsLS0hJVq1bF//73P3h4eGDHjh0APnzGq1evDhcXF5nP+HfffQcAcp/xZs2aoUaNGgptW9HPcY5Pzycf+/nnn8X/a2pqom7duhAEAf379xfLzczMUK1aNZnPjLLnNFWcuwVBwLZt29ChQwcIgiBzXL28vJCcnJzrtj83dmN9ZvXr10fdunXF6R49eqBOnTrw9/dH+/btxQ/qxxISEvD+/ftcm4Y/LUtMTERGRoY4ra+vL57ANTU15Zqg27ZtC2dnZ0ycOBHbtm0D8GHMws2bN/P8IiYkJMhMf/olyPny5Jygcr44X3/9da7r+1R+6zMxMcl32dz2MT/r169HUFAQ7ty5I5MEODk5ydX9tOzFixd4//49nJ2d5epWq1YN+/btUziO3Li5ucnsS9euXZGcnIwJEyagZ8+e4vtz9uxZTJ06FZGRkXI/0MnJyTA1NUWzZs3g7e2NadOmYeHChWjevDk6deqEnj17QldXF8CH9z05ORlWVla5xvPp+56fOnXqwMXFBeHh4TAzM4ONjY34g/Kpx48fw9nZGRoasn97Va9eXZyf86+tra3c7QWqVasmM33//n0IgoDJkydj8uTJee5LhQoV5MqdnJwwevRoLFiwAGFhYWjSpAl++OEHcfxEfmJjYzFlyhTs2rVL7sf50/E+enp6ct+vcuXKySyn6P4WVU4i8+bNGzEp/lReCVHHjh3h7+8PiUQCY2NjuLq6wtDQsFBx/Pnnn6hfvz5evXqFV69eAfjwOcrIyMDWrVsxcODAAtehp6cndsvp6urCyckJFStWFOffu3cPUVFRCp/bcjsP5EXRz7Ei6/70HGhqago9PT2ZK+lyynOOVQ5lzmmqOHe/ePECSUlJWL16NVavXp1rHWXOHcWFyU4J09DQQIsWLbBo0SLcu3cPrq6uRVpfly5dZP6C8PX1zffmYxUrVkS1atVkWmukUilatWold+lwjq+++kpm+tPLUXMIudybQxGqXl9e/vzzT/j5+aFTp04YO3YsrKysoKmpidmzZ4v9/B/7+C/3ktKyZUvs2bMHFy9eRLt27fDgwQO0bNkSLi4uWLBgAezt7aGjo4N9+/Zh4cKF4oByiUSCv/76C+fPn8fu3btx8OBB9OvXD0FBQTh//jyMjIwglUphZWWV5718lB0f1rNnT6xYsQLGxsbo1q2b3I9AccnZ54CAgDxb2PIbUxIUFAQ/Pz/8/fffOHToEEaMGIHZs2fj/PnzMj+cH8vOzkarVq2QmJiI8ePHw8XFBYaGhnj69Cn8/Pzk7heT12e8JFSvXh07d+7EzZs3xbEfn7p58yYAyLVyVKxYUak/LvJy7949cWBsbn88hIWFKZTsFPTHjlQqhZubGxYsWJDrfHt7e5np4vzO57fu3D4fipwXlT2nqeJcm/PZ7tWrV57jyWrWrKnw+ooLk51SICsrCwBk7ovxMSsrK+jp6eH+/fty8z4tCwoKkvnrUJF752RlZclsu0qVKkhNTVXJSQwAKleuDAD4559/VLI+Vfnrr79QuXJlbN++XaY7TdF7Z+RcmXDv3j25edHR0SqL82OfflZ2796N9PR07Nq1S+avtLy6nBo2bIiGDRti5syZCA8Ph4+PDyIiIvDzzz+jSpUqOHLkCBo3bqySk3zPnj0xZcoUPH/+HBs3bsyzXqVKlXDz5k1IpVKZhCjnaq5KlSqJ/x49ehSpqakyrR2fHuucz5u2tnahP8Nubm5wc3PDpEmTcO7cOTRu3BgrV67EjBkzcq1/69Yt3L17F+vXr5cZ6Jtb96miFN3fomrfvj1mz56NDRs25JrsZGdnIzw8HOXKlUPjxo1Vuu0cYWFh0NbWxsaNG+V+gM+cOYPFixcjNjZWriVCWVWqVMGNGzfQsmVLld/hXNHPcXEq6jntU4qcuy0tLWFsbIzs7GyV/WYUB47ZKWGZmZk4dOgQdHR0xObOT+X8tbJz5048e/ZMLL9//77c3Zfd3d3h6ekpvgrqb7579y6io6NRq1Ytsaxr166IjIzEwYMH5eonJSWJP7iKsrS0RNOmTbF27VrExsbKzFN1a40yck6qH8dw4cIFREZGKry8l5cXdu7cKbNfUVFRuR47VdizZw8AiO9XbvuQnJyM0NBQmeVev34td6xzrszIueS2a9euyM7Oxm+//Sa33aysLKUvea5SpQqCg4Mxe/Zs1K9fP896bdu2RVxcHDZv3iyzvSVLlsDIyAjNmjUT62VlZclcVp+dnY0lS5bIrM/KygrNmzfHqlWr8Pz5c7ntvXjxIs9YUlJS5D7fbm5u0NDQyPcuvLm9D4IgyI2vUoai+1tUjRo1gqenJ0JDQ8XP18d+/fVX3L17F+PGjSu2lo6cLsNu3brhxx9/lHmNHTsWALBp06Yib6dr1654+vQp1qxZIzfv/fv3ePv2baHXrejnuDgV9Zz2KUXO3ZqamvD29sa2bdtyTYry+759TmzZ+cz2798vZvoJCQkIDw/HvXv3MGHChHzHowQGBuLQoUNo3LgxhgwZguzsbCxduhRff/01rl+/rtC2s7Ky8OeffwL40PT46NEjrFy5ElKpVCbzHzt2LHbt2oX27dvDz88P7u7uePv2LW7duoW//voLjx49kus7LsjixYvx7bff4ptvvsHAgQPh5OSER48eYe/evQrHr2rt27fH9u3b0blzZ7Rr1w4xMTFYuXIlatSokWcr26emTZuGAwcOoEmTJhg6dKh4cnN1dRWb/nMEBgZi2rRpOH78uHjvj/ycPn1avMdIYmIidu3ahZMnT6J79+5wcXEBALRu3Ro6Ojro0KEDBg0ahNTUVKxZswZWVlYyP/Tr16/H8uXL0blzZ1SpUgVv3rzBmjVrYGJigrZt2wL4MBhz0KBBmD17Nq5fv47WrVtDW1sb9+7dw9atW7Fo0SL8+OOPCh2XHJ9ebpubgQMHYtWqVfDz88OVK1fg6OiIv/76C2fPnkVwcLA4TqRDhw5o3LgxJkyYgEePHqFGjRrYvn17rve/WbZsGb799lu4ublhwIABqFy5MuLj4xEZGYknT57gxo0bucZy7Ngx+Pv746effsJXX32FrKwssbXh44Gzn3JxcUGVKlUQEBCAp0+fwsTEBNu2bVPq/kSfUmZ/i2rDhg1o2bIlOnbsiJ49e6JJkyZIT0/H9u3bceLECXTr1k1MOgrj/v37ubaK1alTB+XLlxcv185NhQoV8M033yAsLAzjx48vdAwA0Lt3b2zZsgWDBw/G8ePH0bhxY2RnZ+POnTvYsmULDh48KDOmUhmKfo6LkyrOaZ9S5Nw9Z84cHD9+HA0aNMCAAQNQo0YNJCYm4urVqzhy5AgSExNVuJeF9Nmv//pC5XbpuZ6enlC7dm1hxYoVMpfxCYL8peeCIAhHjx4V6tSpI+jo6AhVqlQR/vjjD2HMmDGCnp5egdvP7bJsExMToWXLlsKRI0fk6r9580aYOHGiULVqVUFHR0coX7680KhRI2H+/PlCRkaGIAj/d/ni77//Lrd8bvH/888/QufOnQUzMzNBT09PqFatmjB58mRxfl53UM45dnnd/fXjfVTmDspSqVSYNWuWUKlSJUFXV1eoU6eOsGfPHsHX11eoVKmSWC+//RQEQTh58qTg7u4u6OjoCJUrVxZWrlyZ6x2Ux4wZI0gkEiEqKirf/cjt0nMdHR3BxcVFmDlzpnj8c+zatUuoWbOmoKenJzg6Ogpz584V1q5dK3PMrl69KvTo0UNwcHAQdHV1BSsrK6F9+/bC5cuX5ba/evVqwd3dXdDX1xeMjY0FNzc3Ydy4ccKzZ8/yjVvRO2B/+j4IwofLjvv27SuUL19e0NHREdzc3HK9tPrVq1dC7969BRMTE8HU1FTo3bu3eLn4p/UfPHgg9OnTR7CxsRG0tbWFChUqCO3btxf++usvsc6nl54/fPhQ6Nevn1ClShVBT09PMDc3F1q0aJHrd+RTt2/fFjw9PQUjIyOhfPnywoABA4QbN27IxZbX5zS3z4wy+5ufgu6gLAgfvvOBgYGCq6ur+N43btxYWLdundz5SRAUvyN5pUqV8rwVRP/+/YXhw4cLAIQHDx7kuY7AwEABgHDjxo086xT0/c+RkZEhzJ07V3B1dRV0dXWFcuXKCe7u7sK0adOE5ORkhfYvr8vpFfkc53c+yes7lNe+NWvWTHB1dRWnVXFOK8y5O2ffhw0bJtjb2wva2tqCjY2N0LJly1Jxd35BEASJIJRgPwIVWadOnfDvv//mOm6ESpf69eujUqVKcjeYIyKi4sUxO2XIp7dqv3fvHvbt26dQlwiVrJSUFNy4cUO8ozAREX0+bNkpQ2xtbeHn54fKlSvj8ePHWLFiBdLT03Ht2rVcL9ckIiIiDlAuU9q0aYNNmzYhLi4Ourq68PDwwKxZs5joEBER5aNEu7FOnTqFDh06wM7ODhKJRO4hbIIgYMqUKbC1tYW+vj48PT3lxqYkJibCx8cHJiYmMDMzQ//+/Qs96ry0Cw0NxaNHj5CWlobk5GQcOHAA33zzTUmHRUREVKqVaLLz9u1b1KpVC8uWLct1/rx587B48WKsXLkSFy5cgKGhIby8vMTLcQHAx8cH//77Lw4fPow9e/bg1KlTCt1pk4iIiL4MpWbMjkQiwY4dO9CpUycAH1p17OzsMGbMGAQEBAD4cLM0a2trrFu3Dt27d0dUVBRq1KiBS5cuifdGOHDgANq2bYsnT54odPdgIiIiUm+ldsxOTEwM4uLiZG4/bWpqigYNGiAyMhLdu3dHZGQkzMzMZG4C5enpCQ0NDVy4cAGdO3fOdd3p6ekyd0PNeTqshYWFym8hTkRERMVDEAS8efMGdnZ2+T5/r9QmO3FxcQAAa2trmXJra2txXlxcnNwTmrW0tGBubi7Wyc3s2bMxbdo0FUdMREREJeG///7L80G9QClOdorTxIkTMXr0aHE6OTkZDg4OiImJ+Sy39CYiIqKie/PmDZycnAr87S61yY6NjQ0AID4+Hra2tmJ5fHy8+ABDGxsbJCQkyCyXlZWFxMREcfnc6OrqQldXV67c3Nw83+dTERERUemhra0NAAUOQSm1d1B2cnKCjY0Njh49KpalpKTgwoUL8PDwAAB4eHggKSkJV65cEescO3YMUqkUDRo0+OwxExERUelToi07qampuH//vjgdExOD69evw9zcHA4ODhg1ahRmzJgBZ2dnODk5YfLkybCzsxOv2KpevTratGmDAQMGYOXKlcjMzIS/vz+6d+/OK7GIiIgIQAknO5cvX0aLFi3E6ZxxNL6+vli3bh3GjRuHt2/fYuDAgUhKSsK3336LAwcOQE9PT1wmLCwM/v7+aNmyJTQ0NODt7Y3Fixd/9n0hIiKi0qnU3GenJKWkpMDU1BTJycl5jtmRSqXIyMj4zJERUW50dHTyvcyUiL4Mivx+A6V4gHJpkpGRgZiYGEil0pIOhYgAaGhowMnJCTo6OiUdChGVAUx2CiAIAp4/fw5NTU3Y29vzr0miEiaVSvHs2TM8f/4cDg4OvBEoERWIyU4BsrKy8O7dO9jZ2cHAwKCkwyEiAJaWlnj27BmysrLES0+JiPLCZooCZGdnAwCby4lKkZzvY873k4goP0x2FMSmcqLSg99HIlIGkx0iIiJSa0x2vkASiQQ7d+4s6TBKnXXr1sHMzEzl6/2cx/vEiROQSCRISkrKt97q1avFAffBwcGfJTYiopLCAcqF5Dhh72fd3qM57ZSq7+fnh6SkpDKX1Gzfvh0rVqzA9evXkZ6eDldXVwQGBsLLyyvf5fLq1pg3bx7Gjh0LAEhMTMTw4cOxe/du8QaUixYtgpGRkcr3Q9VU+X6mpKTA398fCxYsgLe3N0xNTYseIBFRKcaWHSpVTp06hVatWmHfvn24cuUKWrRogQ4dOuDatWv5Lvf8+XOZ19q1ayGRSODt7S3W8fHxwb///ovDhw9jz549OHXqFAYOHFjcu1TqxMbGIjMzE+3atYOtrS2vMiQitcdkhzB+/Hh89dVXMDAwQOXKlTF58mRkZmaK8wMDA1G7dm2sXbsWDg4OMDIywtChQ5GdnY158+bBxsYGVlZWmDlzpsx6FyxYADc3NxgaGsLe3h5Dhw5FampqvrEEBwdj3LhxqFevHpydnTFr1iw4Oztj9+7d+S5nY2Mj8/r777/RokULVK5cGQAQFRWFAwcO4I8//kCDBg3w7bffYsmSJYiIiMCzZ89k1rVz5044OztDT08PXl5e+O+//wo8hmvXroWrqyt0dXVha2sLf39/mfkvX75E586dYWBgAGdnZ+zatUucl52djf79+8PJyQn6+vqoVq0aFi1aJM4PDAzE+vXr8ffff0MikUAikeDEiRP5xnPlyhXUrVsXBgYGaNSoEaKjowF86Kpzc3MDAFSuXBkSiQSPHj0qcP+IiMoyJjsEY2NjrFu3Drdv38aiRYuwZs0aLFy4UKbOgwcPsH//fhw4cACbNm1CSEgI2rVrhydPnuDkyZOYO3cuJk2ahAsXLojLaGhoYPHixfj333+xfv16HDt2DOPGjVMqNqlUijdv3sDc3FzhZeLj47F37170799fLIuMjISZmRnq1q0rlnl6ekJDQ0Mm5nfv3mHmzJnYsGEDzp49i6SkJHTv3j3f7a1YsQLDhg3DwIEDcevWLezatQtVq1aVqTNt2jR07doVN2/eRNu2beHj44PExERxHytWrIitW7fi9u3bmDJlCv73v/9hy5YtAICAgAB07doVbdq0EVuuGjVqlG9Mv/76K4KCgnD58mVoaWmhX79+AIBu3brhyJEjAICLFy/i+fPnsLe3L+iQEhGVaRyzQ5g0aZL4f0dHRwQEBCAiIkImMZFKpVi7di2MjY1Ro0YNtGjRAtHR0di3bx80NDRQrVo1zJ07F8ePH0eDBg0AAKNGjZJZ74wZMzB48GAsX75c4djmz5+P1NRUdO3aVeFl1q9fD2NjY3Tp0kUsi4uLg5WVlUw9LS0tmJubIy4uTizLzMzE0qVLxX1Yv349qlevjosXL6J+/fq5bm/GjBkYM2YMRo4cKZbVq1dPpo6fnx969OgBAJg1axYWL16Mixcvok2bNtDW1sa0adPEuk5OToiMjMSWLVvQtWtXGBkZQV9fH+np6bCxsVHoGMycORPNmjUDAEyYMAHt2rVDWloa9PX1YWFhAeDDjfkUXR8RUVnGlh3C5s2b0bhxY9jY2MDIyAiTJk1CbGysTB1HR0cYGxuL09bW1qhRo4bM4zOsra2RkJAgTh85cgQtW7ZEhQoVYGxsjN69e+PVq1d49+4dAMDIyEh8DR48WC6u8PBwTJs2DVu2bBETlbCwMJnlTp8+Lbfc2rVr4ePjAz09PaWPhZaWlkyi4uLiAjMzM0RFRSE2NlZm27NmzUJCQgKePXuGli1b5rvemjVriv83NDSEiYmJzLFatmwZ3N3dYWlpCSMjI6xevVruPfjU999/L8bi6uqa5/ZsbW0BQGZ7RERfErbsfOEiIyPh4+ODadOmwcvLC6ampoiIiEBQUJBMvU9vyS+RSHIty3lY6qNHj9C+fXsMGTIEM2fOhLm5Oc6cOYP+/fsjIyMDBgYGuH79urjsp0+rjYiIwM8//4ytW7fC09NTLP/hhx/EVhcAqFChgsxyp0+fRnR0NDZv3ixTbmNjI/djn5WVhcTERIVbN+zs7GRiNjc3V/hRBfkdq4iICAQEBCAoKAgeHh4wNjbG77//LtO9lps//vgD79+/z3X9H0/nXKnGB9kS0ZeKyc4X7ty5c6hUqRJ+/fVXsezx48dFXu+VK1cglUoRFBQktv7kjEHJ8em4lhybNm1Cv379EBERgXbtZC+5NzY2lmlh+lRISAjc3d1Rq1YtmXIPDw8kJSXhypUrcHd3BwAcO3YMUqlUJnnKysrC5cuXxS6r6OhoJCUloXr16tDS0so1ZkdHRxw9ehQtWrTIM678nD17Fo0aNcLQoUPFsgcPHsjU0dHRkXs0wqeJHhER5Y7JjhpLTk6WaYkAII7XyOHs7IzY2FhERESgXr162Lt3L3bs2FHkbVetWhWZmZlYsmQJOnTogLNnz2LlypUFLhceHg5fX18sWrQIDRo0EMfT6OvrF3g/mJSUFGzdulWuVQoAqlevjjZt2mDAgAFYuXIlMjMz4e/vj+7du8POzk6sp62tjeHDh2Px4sXQ0tKCv78/GjZsmOd4HeDD1VKDBw+GlZUVvv/+e7x58wZnz57F8OHDC9xf4MN7sGHDBhw8eBBOTk7YuHEjLl26BCcnJ7GOo6MjDh48iOjoaFhYWMDU1JQPwCQiUhCTnUJS9iZ/JeHEiROoU6eOTNnHVygBH7qFfvnlF/j7+yM9PR3t2rXD5MmTERgYWKRt16pVCwsWLMDcuXMxceJENG3aFLNnz0afPn3yXW716tXIysrCsGHDMGzYMLHc19cX69aty3fZiIgICIIgDgT+VFhYGPz9/dGyZUvxpoKLFy+WqWNgYIDx48ejZ8+eePr0KZo0aYKQkJB8t+vr64u0tDQsXLgQAQEBKF++PH788cd8l/nYoEGDcO3aNXTr1g0SiQQ9evTA0KFDsX//frHOgAEDcOLECdStWxepqak4fvw4mjdvrvA2iIi+ZBJBEISSDqKkpaSkwNTUFMnJyXJjR9LS0hATEwMnJ6dCDXglItXj95KIgPx/vz/Gq7GIiIhIrTHZISIiIrXGZIeIiIjUGpMdIiIiUmtMdoiIiEitMdkhIiIitcZkh4iIiNQakx0iIiJSa0x2iIiISK0x2SE0b94co0aN+mK3T0RE6o3PxiqswPwfSqn67SUrVd3Pzw/r16/HoEGD5B7AOWzYMCxfvlx83tT27duVeqikn58fkpKSsHPnTqViKoq4uDiMHTsWhw8fxps3b1CtWjX8+uuv8Pb2VngdEokk1/J58+Zh7NixcnU0NTVhZ2eHH3/8EbNnz4aurm6e6w4MDMTOnTvlHrxaWM2bN0ft2rURHByskvXl59GjR3BycsK1a9dQu3btIq1LEARMnToVa9asQVJSEho3bowVK1bA2dk5z2VOnTqF33//HVeuXMHz58+xY8cOdOrUqUhxEBF9jC07asze3h4RERF4//69WJaWlobw8HA4ODiIZebm5jA2Nlb59jMzM1W2rj59+iA6Ohq7du3CrVu30KVLF3Tt2hXXrl1TeB3Pnz+Xea1duxYSiUQuYQoNDcXz588RExOD5cuXY+PGjZgxY4bK9kVVBEFAVlZWSYchY968eVi8eDFWrlyJCxcuwNDQEF5eXkhLS8tzmbdv36JWrVpYtmzZZ4yUiL4kTHbU2DfffAN7e3ts375dLNu+fTscHBxknob+cTfSnTt3YGBggPDwcHH+li1boK+vj9u3byMwMBDr16/H33//DYlEAolEghMnTuDRo0eQSCTYvHkzmjVrBj09PYSFheHVq1fo0aMHKlSoAAMDA7i5uWHTpk1K78u5c+cwfPhw1K9fH5UrV8akSZNgZmaGK1euAAA2bNgAIyMj3Lt3T1xm6NChcHFxwbt37wAANjY2Mq+///4bLVq0QOXKlWW2ZWZmBhsbG9jb26N9+/bo2LEjrl69qlS8fn5+6NSpE+bPnw9bW1tYWFhg2LBhMgng8uXL4ezsDD09PVhbW4tPSvfz88PJkyexaNEi8Rg/evQIJ06cgEQiwf79++Hu7g5dXV2cOXNG3NbHRo0aJfNUdKlUinnz5qFq1arQ1dWFg4MDZs6cCQBwcnICANSpUwcSiSTPp6k3atQI48ePlyl78eIFtLW1cerUKQiCgODgYEyaNAkdO3ZEzZo1sWHDBjx79izfVsDvv/8eM2bMQOfOnRU8ukREymGyo+b69euH0NBQcXrt2rXo27dvnvVdXFwwf/58DB06FLGxsXjy5AkGDx6MuXPnokaNGggICEDXrl3Rpk0bsYWkUaNG4vITJkzAyJEjERUVJf5F7+7ujr179+Kff/7BwIED0bt3b1y8eFGp/WjUqBE2b96MxMRESKVSREREIC0tTfxh7tOnD9q2bQsfHx9kZWVh7969+OOPPxAWFgYDAwO59cXHx2Pv3r3o379/vtu9e/cujh07hgYNGigVLwAcP34cDx48wPHjx7F+/XqsW7cO69atAwBcvnwZI0aMwPTp0xEdHY0DBw6gadOmAIBFixbBw8MDAwYMEI+xvb29uN4JEyZgzpw5iIqKQs2aNRWKZeLEiZgzZw4mT56M27dvIzw8HNbW1gAgvhdHjhzB8+fPZZLjj/n4+CAiIgKCIIhlmzdvhp2dHZo0aYKYmBjExcXB09NTnG9qaooGDRogMjJS8QNHRKRiHLOj5nr16oWJEyfi8ePHAICzZ88iIiICJ06cyHOZoUOHYt++fejVqxd0dHRQr149DB8+HABgZGQEfX19pKenw8bGRm7ZUaNGoUuXLjJlAQEB4v+HDx+OgwcPYsuWLahfv77C+7FlyxZ069YNFhYW0NLSgoGBAXbs2IGqVauKdVatWoWaNWtixIgR2L59OwIDA+Hu7p7r+tavXw9jY2O5WAGgR48e0NTURFZWFtLT09G+fXtMnDhR4VhzlCtXDkuXLoWmpiZcXFzQrl07HD16FAMGDEBsbCwMDQ3Rvn17GBsbo1KlSmJrm6mpKXR0dGBgYJDrMZ4+fTpatWqlcBxv3rzBokWLsHTpUvj6+gIAqlSpgm+//RYAYGlpCQCwsLDIdXs5unbtilGjRuHMmTNo0qQJACA8PBw9evSARCJBXFwcAIhJVA5ra2txHhFRSWDLjpqztLREu3btsG7dOoSGhqJdu3YoX758gcutXbsWN2/exNWrV7Fu3bo8B/d+qm7dujLT2dnZ+O233+Dm5gZzc3MYGRnh4MGDiI2NzXX5WbNmwcjISHzl1Js8eTKSkpJw5MgRXL58GaNHj0bXrl1x69Ytcdly5cohJCQEK1asQJUqVTBhwoR898/Hxwd6enpy8xYuXIjr16/jxo0b2LNnD+7evYvevXsDAGJjY2XimzVrVp7bcHV1haampjhta2uLhIQEAECrVq1QqVIlVK5cGb1790ZYWJjY3VaQT49xQaKiopCeno6WLVsqvMzp06dl9jMsLAyWlpZo3bo1wsLCAAAxMTGIjIyEj4+PUvEQEX1ubNn5AvTr1w/+/v4AoPAg0Bs3buDt27fQ0NDA8+fPYWtrq9ByhoaGMtO///47Fi1ahODgYLi5ucHQ0BCjRo1CRkZGrssPHjwYXbt2Faft7Ozw4MEDLF26FP/88w9cXV0BALVq1cLp06exbNkymavNTp06BU1NTTx//hxv377NdeD16dOnER0djc2bN+cag42NjdhiVK1aNbx58wY9evTAjBkz4OjoKHPFlbm5eZ7H4tMr3CQSCaRSKQDA2NgYV69exYkTJ3Do0CFMmTIFgYGBuHTpEszMzPJcJyB/jDU0NGS6lgDZweH6+vr5ri83devWldnPnNYaHx8fjBgxAkuWLEF4eDjc3Nzg5uYGAGKrUHx8vMznJT4+vshXeRERFQVbdr4Abdq0QUZGBjIzM+Hl5VVg/cTERPj5+eHXX3+Fn58ffHx8ZK7o0tHRQXZ2tkLbPnv2LDp27IhevXqhVq1aqFy5Mu7evZtnfXNzc1StWlV8aWlpiS0eGhqyH1dNTU0xeQA+DGKeO3cudu/eDSMjIzHB+1RISAjc3d1Rq1YthfYhp3Xm/fv30NLSkokvv2SnIFpaWvD09MS8efNw8+ZNPHr0CMeOHQOg3DG2tLTE8+fPZco+TlScnZ2hr6+Po0eP5rq8jo4OAMhsT19fX2Y/c5LGjh07Ii0tDQcOHEB4eLhMq46TkxNsbGxktpOSkoILFy7Aw8NDoX0hIioOTHa+AJqamoiKisLt27dlulXyMnjwYNjb22PSpElYsGABsrOzZcbdODo64ubNm4iOjsbLly/zvcTc2dkZhw8fxrlz5xAVFYVBgwYhPj5eqfhdXFxQtWpVDBo0CBcvXsSDBw8QFBSEw4cPi1chvXnzBr1798aIESPw/fffIywsDJs3b8Zff/0ls66UlBRs3boVP//8c57bS0pKQlxcHJ49e4aTJ09i+vTp+Oqrr1C9enWl4s7Pnj17sHjxYly/fh2PHz/Ghg0bIJVKUa1aNQAfjvGFCxfw6NEjvHz5Uiap+9R3332Hy5cvY8OGDbh37x6mTp2Kf/75R5yvp6eH8ePHY9y4cdiwYQMePHiA8+fPIyQkBABgZWUFfX19HDhwAPHx8UhOzvueToaGhujUqRMmT56MqKgo9OjRQ5wnkUgwatQozJgxQ7xFQJ8+fWBnZydztVjLli2xdOlScTo1NRXXr18XE7SYmBhcv349z65OIiKlCSQkJycLAITk5GS5ee/fvxdu374tvH//vgQiKzxfX1+hY8eOec7v2LGj4OvrKwiCIDRr1kwYOXKkIAiCsH79esHQ0FC4e/euWPfChQuCtra2sG/fPkEQBCEhIUFo1aqVYGRkJAAQjh8/LsTExAgAhGvXrsls59WrV0LHjh0FIyMjwcrKSpg0aZLQp08fmdg+3n5e7t69K3Tp0kWwsrISDAwMhJo1awobNmwQ5/ft21dwc3MT0tLSxLKgoCDB3NxcePLkiVi2atUqQV9fX0hKSsp1OwDEl0QiEWxtbYVu3boJDx48yDe+qVOnCrVq1RKnczv+I0eOFJo1ayYIgiCcPn1aaNasmVCuXDlBX19fqFmzprB582axbnR0tNCwYUNBX19fACDExMQIx48fFwAIr1+/ltv+lClTBGtra8HU1FT45ZdfBH9/f3FbgiAI2dnZwowZM4RKlSoJ2tragoODgzBr1ixx/po1awR7e3tBQ0NDZrnc7Nu3TwAgNG3aVG6eVCoVJk+eLFhbWwu6urpCy5YthejoaJk6lSpVEqZOnSpO5+zXp6+cz2duyur3kohUK7/f749JBOGTzv4vUEpKCkxNTZGcnAwTExOZeWlpaYiJiYGTk1Oug1mJ6PPj95KIgPx/vz/GbiwiIiJSa0x2iIiISK0x2SEiIiK1xmSHiIiI1BqTHSIiIlJrTHaIiIhIrTHZISIiIrXGZIeIiIjUGpMdIiIiUmtMdkitPXr0CBKJRObBmERE9GXRKukAyiq39W6fdXu3fG991u2VFRKJJNfyefPmYezYsZ85GiIiKo2Y7FCZ9vz5c5np/fv3o3///vD29i6hiIiIqLRhN5Yae/PmDXx8fGBoaAhbW1ssXLgQzZs3x6hRowAA6enpCAgIQIUKFWBoaIgGDRrgxIkT4vLr1q2DmZkZ9uzZg2rVqsHAwAA//vgj3r17h/Xr18PR0RHlypXDiBEjkJ2dLS7n6OiIGTNmoE+fPjAyMkKlSpWwa9cuvHjxAh07doSRkRFq1qyJy5cvi8u8evUKPXr0QIUKFWBgYAA3Nzds2rSpwH20sbGRef39999o0aIFKleuLFPvzp07aNSoEfT09PD111/j5MmTRTu4RERUZjDZUWOjR4/G2bNnsWvXLhw+fBinT5/G1atXxfn+/v6IjIxEREQEbt68iZ9++glt2rTBvXv3xDrv3r3D4sWLERERgQMHDuDEiRPo3Lkz9u3bh3379mHjxo1YtWoV/vrrL5ltL1y4EI0bN8a1a9fQrl079O7dG3369EGvXr1w9epVVKlSBX369IEgCAA+PMXa3d0de/fuxT///IOBAweid+/euHjxosL7Gx8fj71796J///5y88aOHYsxY8bg2rVr8PDwQIcOHfDq1StlDykREZVBTHbU1Js3b7B+/XrMnz8fLVu2xNdff43Q0FCxBSY2NhahoaHYunUrmjRpgipVqiAgIADffvstQkNDxfVkZmZixYoVqFOnDpo2bYoff/wRZ86cQUhICGrUqIH27dujRYsWOH78uMz227Zti0GDBsHZ2RlTpkxBSkoK6tWrh59++glfffUVxo8fj6ioKMTHxwMAKlSogICAANSuXRuVK1fG8OHD0aZNG2zZskXhfV6/fj2MjY3RpUsXuXn+/v7w9vZG9erVsWLFCpiamiIkJKQwh5aIiMoYJjtq6uHDh8jMzET9+vXFMlNTU1SrVg0AcOvWLWRnZ+Orr76CkZGR+Dp58iQePHggLmNgYIAqVaqI09bW1nB0dISRkZFMWUJCgsz2a9asKTMfANzc3OTKcpbLzs7Gb7/9Bjc3N5ibm8PIyAgHDx5EbGwsACAsLEwmztOnT8vt89q1a+Hj4wM9PT25eR4eHuL/tbS0ULduXURFReV5/IiISH1wgPIXKjU1FZqamrhy5Qo0NTVl5n2cyGhra8vMk0gkuZZJpVKZso/r5FwxlVtZznK///47Fi1ahODgYLi5ucHQ0BCjRo1CRkYGAOCHH35AgwYNxOUrVKggs73Tp08jOjoamzdvVmDviYjoS8JkR01VrlwZ2trauHTpEhwcHAAAycnJuHv3Lpo2bYo6deogOzsbCQkJaNKkSQlHC5w9exYdO3ZEr169AHxIgu7evYsaNWoAAIyNjWFsbJzn8iEhIXB3d0etWrVynX/+/Hk0bdoUAJCVlYUrV67A399fxXtBRESlEZMdNWVsbAxfX1+MHTsW5ubmsLKywtSpU6GhoQGJRIKvvvoKPj4+6NOnD4KCglCnTh28ePECR48eRc2aNdGuXbvPGq+zszP++usvnDt3DuXKlcOCBQsQHx8vJjv5SUlJwdatWxEUFJRnnWXLlsHZ2RnVq1fHwoUL8fr1a/Tr10+Vu0BERKUUk51CKgs3+VuwYAEGDx6M9u3bw8TEBOPGjcN///0njmkJDQ3FjBkzMGbMGDx9+hTly5dHw4YN0b59+88e66RJk/Dw4UN4eXnBwMAAAwcORKdOnZCcnFzgshERERAEAT169Mizzpw5czBnzhxcv34dVatWxa5du1C+fHlV7gIREZVSEiHn2t8vWEpKCkxNTZGcnAwTExOZeWlpaYiJiYGTk1OuA1/Lkrdv36JChQoICgrK9fJsorJCnb6XRFR4+f1+f4wtO2rs2rVruHPnDurXr4/k5GRMnz4dANCxY8cSjoyIiOjzYbKj5ubPn4/o6Gjo6OjA3d0dp0+fZvcNERF9UZjsqLE6dergypUrJR0GERFRiSrVNxXMzs7G5MmT4eTkBH19fVSpUgW//fYbPh5mJAgCpkyZAltbW+jr68PT01PmcQdERET0ZSvVyc7cuXOxYsUKLF26FFFRUZg7dy7mzZuHJUuWiHXmzZuHxYsXY+XKlbhw4QIMDQ3h5eWFtLS0EoyciIiISotS3Y117tw5dOzYUbzni6OjIzZt2iQ+HFIQBAQHB2PSpEnioNsNGzbA2toaO3fuRPfu3UssdiIiIiodSnWy06hRI6xevRp3797FV199hRs3buDMmTNYsGABACAmJgZxcXHw9PQUlzE1NUWDBg0QGRmZZ7KTnp6O9PR0cTolJQXAh4deZmZmytTNzMyEIAiQSqVyj0QgopIhlUohCAIyMzPlHndCRF+OT3+z81Kqk50JEyYgJSUFLi4u0NTURHZ2NmbOnAkfHx8AQFxcHID/e6hkDmtra3FebmbPno1p06bJlR86dAgGBgYyZVpaWrCxsUFqaqr4nCYiKlkZGRl4//49Tp06haysrJIOh4hKyLt37xSqV6qTnS1btiAsLAzh4eFwdXXF9evXMWrUKNjZ2cHX17fQ6504cSJGjx4tTqekpMDe3h6tW7fO9aaC//33H4yMjHjzMqJSIi0tDfr6+mjatCm/l0RfsJyemYKU6mRn7NixmDBhgtgd5ebmhsePH2P27Nnw9fWFjY0NACA+Ph62trbicvHx8ahdu3ae69XV1YWurq5cuba2ttwTvbOzsyGRSKChoQENjVI9nlspjo6OGDVqFEaNGpXr/EePHsHJyQnXrl3L91gSlYScZ7zl9p0loi+Hot//Up3svHv3Ti7B0NTUFMfOODk5wcbGBkePHhV/kFNSUnDhwgUMGTKkWGOLcqlerOv/VPU7UZ91e/b29nj+/Hm+NyA8ceIEFi5ciIsXLyIlJQXOzs4YO3as2M1IRERUGpTqZKdDhw6YOXMmHBwc4OrqimvXrmHBggXi06olEglGjRqFGTNmwNnZGU5OTpg8eTLs7OzQqVOnkg2+jNPU1BRbzvJy7tw51KxZE+PHj4e1tTX27NmDPn36wNTUtEQeJkpERJSbUp3sLFmyBJMnT8bQoUORkJAAOzs7DBo0CFOmTBHrjBs3Dm/fvsXAgQORlJSEb7/9FgcOHPji+/GbN2+Or7/+GgCwceNGaGtrY8iQIZg+fTokEgmADy1n/fr1w9atW1GuXDlMmjQJAwcOBKBYN9b//vc/memRI0fi0KFD2L59O5MdIiIqNUr1IBRjY2MEBwfj8ePHeP/+PR48eIAZM2ZAR0dHrCORSDB9+nTExcUhLS0NR44cwVdffVWCUZce69evh5aWFi5evIhFixZhwYIF+OOPP8T5QUFBqFu3Lq5du4ahQ4diyJAhiI6OLtI2k5OTYW5uXtTQiYiIVKZUJztUNPb29li4cCGqVasGHx8fDB8+HAsXLhTnt23bFkOHDkXVqlUxfvx4lC9fHsePHy/09rZs2YJLly6hb9++qgifiIhIJZjsqLGGDRuKXVYA4OHhgXv37iE7OxsAULNmTXGeRCKBjY0NEhIScl2Xq6srjIyMYGRkhO+//15u/vHjx9G3b1+sWbMGrq6uKt4TIiKiwivVY3aoeH16yZ5EIsnzLtH79u0T71Spr68vM+/kyZPo0KEDFi5ciD59+hRPsERERIXEZEeNXbhwQWb6/PnzcHZ2LtTt9StVqpRr+YkTJ9C+fXvMnTtXHNxMRERUmrAbS43FxsZi9OjRiI6OxqZNm7BkyRKMHDlSZes/fvw42rVrhxEjRsDb2xtxcXGIi4tDYmKiyrZBRERUVGzZKaTPfZO/wujTpw/ev3+P+vXrQ1NTEyNHjlRp68v69evx7t07zJ49G7NnzxbLmzVrhhMnTqhsO0REREUhEQRBKOkgSlpKSgpMTU2RnJyc67OxYmJi4OTkVKbu3dO8eXPUrl0bwcHBJR0KkcqV1e8lEalWfr/fH2M3FhEREak1JjtERESk1jhmR01xzAwREdEHbNkhIiIitcZkh4iIiNQakx0iIiJSa0x2iIiISK0x2SEiIiK1xmSHiIiI1BovPS+kZYOPfdbtDVv5XbGtuyTvtuzo6IhRo0Zh1KhRn33bRET0ZWDLjpry8/ODRCLB4MGD5eYNGzYMEokEfn5+AIDt27fjt99+U2i9zZs3/6yJyYkTJyCRSHJ9Xbp0Kdc6+vr6cHV1xerVq4u8/bS0NAwbNgwWFhYwMjKCt7c34uPj811m+/btaN26NSwsLCCRSHD9+vUix0FERIXHZEeN2dvbIyIiAu/fvxfL0tLSEB4eDgcHB7HM3NwcxsbGKtuuIAjIyspSyboaNWqE58+fy7x+/vlnODk5oW7dujJ1o6Oj8fz5c9y+fRuDBg3CkCFDcPTo0SJt/5dffsHu3buxdetWnDx5Es+ePUOXLl3yXebt27f49ttvMXfu3CJtm4iIVIPJjhr75ptvYG9vj+3bt4tl27dvh4ODA+rUqSOWfdpas3z5cjg7O0NPTw/W1tb48ccfAXxoLTp58iQWLVoktqI8evRIbFnZv38/3N3doaurizNnzuDBgwfo2LEjrK2tYWRkhHr16uHIkSNK7YOOjg5sbGzEl4WFBf7++2/07dsXEolEpq6VlRVsbGzg5OSEESNGwMnJCVevXs11vVKpFBUrVsSKFStkyq9duwYNDQ08fvwYycnJCAkJwYIFC/Ddd9/B3d0doaGhOHfuHM6fP59nzL1798aUKVPg6emp1L4SEVHxYLKj5vr164fQ0FBxeu3atejbt2+e9S9fvowRI0Zg+vTpiI6OxoEDB9C0aVMAwKJFi+Dh4YEBAwaIrSz29vbishMmTMCcOXMQFRWFmjVrIjU1FW3btsXRo0dx7do1tGnTBh06dEBsbGyh92fXrl149epVvvsgCAIOHDiA2NhYNGjQINc6Ghoa6NGjB8LDw2XKw8LC0LhxY1SqVAlXrlxBZmamTNLi4uICBwcHREZGFnofiIjo8+IAZTXXq1cvTJw4EY8fPwYAnD17FhEREXk+Oys2NhaGhoZo3749jI2NUalSJbEVyNTUFDo6OjAwMICNjY3cstOnT0erVq3EaXNzc9SqVUuc/u2337Bjxw7s2rUL/v7+hdqfkJAQeHl5oWLFinLzcsrS09MhlUoxffp0MVHLjY+PD4KCghAbGwsHBwdIpVJERERg0qRJAIC4uDjo6OjAzMxMZjlra2vExcUVKn4iIvr82LKj5iwtLdGuXTusW7cOoaGhaNeuHcqXL59n/VatWqFSpUqoXLkyevfujbCwMLx7906hbX06hiY1NRUBAQGoXr06zMzMYGRkhKioqDxbdgYPHgwjIyPx9aknT57g4MGD6N+/f67Lnz59GtevX8f169fxxx9/YNasWWI3VVhYmMy6T58+jdq1a6N69epi687JkyeRkJCAn376SaH9JSKisoEtO1+Afv36iS0py5Yty7eusbExrl69ihMnTuDQoUOYMmUKAgMDcenSJbkWjk8ZGhrKTAcEBODw4cOYP38+qlatCn19ffz444/IyMjIdfnp06cjICAgz/WHhobCwsICP/zwQ67znZycxBhdXV1x4cIFzJw5E0OGDMEPP/wg06VVoUIFAB9ad8LDwzFhwgSEh4ejTZs2sLCwAADY2NggIyMDSUlJMvseHx+fa8sWERGVTmzZ+QK0adMGGRkZyMzMhJeXV4H1tbS04OnpiXnz5uHmzZt49OgRjh37cF8hHR0dZGdnK7Tds2fPws/PD507d4abmxtsbGzw6NGjPOtbWVmhatWq4utjgiAgNDQUffr0gba2tkLb19TUFK9EMzY2llm3vr4+AKBnz574559/cOXKFfz111/w8fERl3d3d4e2trbMFV3R0dGIjY2Fh4eHQjEQEVHJY8vOF0BTUxNRUVHi//OzZ88ePHz4EE2bNkW5cuWwb98+SKVSVKtWDcCHmwBeuHABjx49gpGREczNzfNcl7OzM7Zv344OHTpAIpFg8uTJkEqlhdqHY8eOISYmBj///HOedRISEpCWlob09HRcvHgRGzduFK8ky4ujoyMaNWqE/v37Izs7W6bVyNTUFP3798fo0aNhbm4OExMTDB8+HB4eHmjYsKFYz8XFBbNnz0bnzp0BAImJiYiNjcWzZ88AfEiQAIhXlBER0efFZKeQivOOxsXBxMREoXpmZmbYvn07AgMDkZaWBmdnZ2zatAmurq4APnRN+fr6okaNGnj//j1iYmLyXNeCBQvQr18/NGrUCOXLl8f48eORkpJSqPhDQkLQqFEjuLi45FknJyHT0tKCvb09Bg0ahMDAwALX7ePjg6FDh6JPnz5ii0+OhQsXQkNDA97e3khPT4eXlxeWL18uUyc6OhrJycni9K5du2SuFuvevTsAYOrUqQrFQ0REqiURBEEo6SBKWkpKCkxNTZGcnCyXFKSlpSEmJgZOTk7Q09MroQiJ6GP8XhIRkP/v98c4ZoeIiIjUGpMdIiIiUmtMdoiIiEitMdkhIiIitcZkh4iIiNRaoS49z8zMRFxcHN69ewdLS8t877VCREREVJIUbtl58+YNVqxYgWbNmsHExASOjo6oXr06LC0tUalSJQwYMACXLl0qzliJiIiIlKZQsrNgwQI4OjoiNDQUnp6e2LlzJ65fv467d+8iMjISU6dORVZWFlq3bo02bdrg3r17xR03ERERkUIU6sa6dOkSTp06Jd5F91P169dHv379sHLlSoSGhuL06dNwdnZWaaBEREREhaFQsrNp0yaFVqarq4vBgwcXKaCyIqhb+8+6vTGb93zW7REREamLIl2NlZmZiX///Rc3b95Eenq6qmKiIpJIJPm+Pn4+0/r161GvXj0YGBjA2NgYzZo1w549sonViRMnZJa3traGt7c3Hj58KNZxdHREcHBwgbFt2rQJmpqaGDZsmKp2l4iIKF+FTnZOnz4NR0dHtGjRAs2bN4e9vT0OHDigytiokJ4/fy6+goODYWJiIlMWEBAA4MNDPQcNGoRu3brh5s2buHjxIr799lt07NgRS5culVtvdHQ0nj17hq1bt+Lff/9Fhw4dkJ2drVRsISEhGDduHDZt2oS0tDSV7C8REVF+FL70XCqVQkPj/3KjUaNGISwsDM2bNwcArF69GkOGDMn3Kdj0edjY2Ij/NzU1hUQikSkDgPPnzyMoKAiLFy/G8OHDxfKZM2ciLS0No0ePRseOHWFvby/Os7KygpmZGWxtbTFlyhT4+Pjg/v374tPGCxITE4Nz585h27ZtOH78OLZv346ePXsWcW+JiIjyp3DLToMGDXD16lVxOiMjAw4ODuK0g4MD/1IvQzZt2gQjIyMMGjRIbt6YMWOQmZmJbdu25bm8vr4+gA+fA0WFhoaiXbt2MDU1Ra9evRASEqJ84EREREpSONlZunQpfv75Z/zyyy94+/Ytpk6dCnd3dzRs2BDu7u7w9vbGzJkzizNWUqG7d++iSpUq0NHRkZtnZ2cHExMT3L17N9dlnz9/jvnz56NChQoKt+pIpVKsW7cOvXr1AgB0794dZ86cYUsgEREVO6Vadi5dugQrKyu4u7tDR0cH0dHR+PXXXzF58mTcvXsX/fr1K85YScUEQVCqfsWKFWFoaAg7Ozu8ffsW27ZtyzVZys3hw4fx9u1btG3bFgBQvnx5tGrVCmvXrlU6biIiImUo9bgITU1NTJw4EV27dsXgwYOxfv16LFmyBHZ2dsUVHxWTr776CmfOnEFGRoZcwvLs2TOkpKTgq6++kik/ffo0TExMYGVlBWNjY6W2FxISgsTERLH7C/jQ2nPz5k1MmzZNZjwYERGRKin1C/Pvv/9i27ZtyM7OxuHDh/HDDz+gSZMmWL58eXHFR8Wke/fuSE1NxapVq+TmzZ8/H9ra2vD29pYpd3JyQpUqVZROdF69eoW///4bERERuH79uvi6du0aXr9+jUOHDhVpX4iIiPKjcMvOggULMGnSJNSsWRP37t3DnDlzMGDAALRr1w6jR4/Gxo0bsXr1ari5uRVnvKQiHh4eGDlyJMaOHYuMjAx06tQJmZmZ+PPPP7Fo0SIEBwfLXImliKdPn+L69esyZZUqVcLGjRthYWGBrl27QiKRyMxv27YtQkJC0KZNm6LuEhERUe4EBVlbWwvHjh0TBEEQHj16JLi4uMjMP3TokFxZWZGcnCwAEJKTk+XmvX//Xrh9+7bw/v37Eois6EJDQwVTU9M854eEhAju7u6Cnp6eYGhoKDRp0kTYtWuXTJ3jx48LAITXr1/nuZ5KlSoJAOReGzduFNzc3IShQ4fmutzmzZsFHR0d4cWLF4XZPfpClfXvJRGpRn6/3x+TCIJio1Stra2xZcsWNGvWDE+ePIGnpyfu3LkjUyctLQ16enoqTcY+h5SUFJiamiI5ORkmJiYy89LS0hATEwMnJ6cyuW9E6ojfSyIC8v/9/pjC3Vhjx45F27ZtUatWLdy9exezZs2Sq8OTDhEREZU2Cic7AQEB8PLywp07d+Dm5gYXF5fijIuIiIhIJZS69NzNzY0DkImIiKhMUejS8zlz5uDdu3cKrfDChQvYu3dvkYIiIiIiUhWFkp3bt2+jUqVKGDp0KPbv348XL16I87KysnDz5k0sX74cjRo1Qrdu3ZS+D0tZoOA4biL6DPh9JCJlKNSNtWHDBty4cQNLly5Fz549kZKSAk1NTejq6ootPnXq1MHPP/8MPz8/tRqorKmpCeDDAy8/vvsvEZWcnAfQ5nw/iYjyo/Cl5zlybvH/+PFjvH//HuXLl0ft2rVRvnz54oqx2OV36ZogCIiNjUVmZibs7Oz4WAOiEiaVSvHs2TNoa2vDwcFB7kaVRPTlUPTSc6WTHXVU0MHKyMhATEwMpFJpCURHRJ/S0NCAk5OTwg+iJSL1pPL77HzJdHR04OzsLDadE1HJ0tHRYSsrESmMyY6CNDQ01GosEhER0ZeCfxoRERGRWmOyQ0RERGpN6WQnOTkZiYmJcuWJiYlISUlRSVBEREREqqJ0stO9e3dERETIlW/ZsgXdu3dXSVBEREREqqJ0snPhwgW0aNFCrrx58+a4cOGCSoL62NOnT9GrVy9YWFhAX18fbm5uuHz5sjhfEARMmTIFtra20NfXh6enJ+7du6fyOIiIiKhsUjrZSU9PR1ZWllx5ZmYm3r9/r5Kgcrx+/RqNGzeGtrY29u/fj9u3byMoKAjlypUT68ybNw+LFy/GypUrceHCBRgaGsLLywtpaWkqjYWIiIjKJqVvKtiiRQt8/fXXWLJkiUz5sGHDcPPmTZw+fVplwU2YMAFnz57Nc52CIMDOzg5jxoxBQEAAgA9jiqytrbFu3TqFu9UUvSkRERERlR7FdlPBGTNmwNPTEzdu3EDLli0BAEePHsWlS5dw6NChwkeci127dsHLyws//fQTTp48iQoVKmDo0KEYMGAAACAmJgZxcXHw9PQUlzE1NUWDBg0QGRmZZ7KTnp6O9PR0cTpnYHVmZiYyMzNVug9ERERUPBT9zVY62WncuDEiIyPx+++/Y8uWLdDX10fNmjUREhICZ2dnpQPNz8OHD7FixQqMHj0a//vf/3Dp0iWMGDECOjo68PX1RVxcHADA2tpaZjlra2txXm5mz56NadOmyZUfOnQIBgYGKt0HIiIiKh45DyMvSKl+NpaOjg7q1q2Lc+fOiWUjRozApUuXEBkZiXPnzqFx48Z49uwZbG1txTpdu3aFRCLB5s2bc11vbi079vb2ePnyJbuxiIiIyoiUlBSUL19e9d1Y+/btg6amJry8vGTKDx48CKlUiu+//175aPNga2uLGjVqyJRVr14d27ZtAwDY2NgAAOLj42WSnfj4eNSuXTvP9erq6kJXV1euXFtbG9ra2iqInIiIiIqbor/ZSl+NNWHCBGRnZ8uVC4KACRMmKLu6fDVu3BjR0dEyZXfv3kWlSpUAAE5OTrCxscHRo0fF+SkpKbhw4QI8PDxUGgsRERGVTUq37Ny7d0+utQUAXFxccP/+fZUEleOXX35Bo0aNMGvWLHTt2hUXL17E6tWrsXr1agCARCLBqFGjMGPGDDg7O8PJyQmTJ0+GnZ0dOnXqpNJYiIiIqGxSOtkxNTXFw4cP4ejoKFN+//59GBoaqiouAEC9evWwY8cOTJw4EdOnT4eTkxOCg4Ph4+Mj1hk3bhzevn2LgQMHIikpCd9++y0OHDjAJ5QTERERgEIMUB40aBAiIyOxY8cOVKlSBcCHRMfb2xv16tXDH3/8USyBFifeZ4eIiKjsUfT3W+kxO/PmzYOhoSFcXFzg5OQEJycnVK9eHRYWFvj999+LFDQRERGRqhWqG+vcuXM4fPgwbty4Id5np2nTpsURHxEREVGRqOQ+O4Ig4MCBAwgJCcFff/2lirg+K3ZjERERlT3F1o31sZiYGEyePBkODg7o3LkzH75JREREpY7S3Vjp6en466+/EBISgjNnziA7Oxvz589H//792SpCREREpY7CLTtXrlzB0KFDYWNjg+DgYHTq1An//fcfNDQ04OXlxUSHiIiISiWFW3YaNGiA4cOH4/z586hWrVpxxkRERESkMgonOy1btkRISAgSEhLQu3dveHl5QSKRFGdsREREREWmcDfWwYMH8e+//6JatWoYMmQIbG1tMXLkSABg0kNERESlllJXY9nb22PKlCmIiYnBxo0b8eLFC2hpaaFjx4743//+h6tXrxZXnERERESFUuT77Lx+/Rp//vkn1q5di5s3b+b6RPTSjvfZISIiKnsU/f1WyU0Fc1y9ehXffPONqlb32TDZISIiKns+y00FP1UWEx0iIiJSbypNdoiIiIhKGyY7REREpNaUflwEERERlR5B3dqXdAgFGrN5T4luv1AtO1lZWThy5AhWrVqFN2/eAACePXuG1NRUlQZHREREVFRKt+w8fvwYbdq0QWxsLNLT09GqVSsYGxtj7ty5SE9Px8qVK4sjTiIiIqJCUbplZ+TIkahbty5ev34NfX19sbxz5844evSoSoMjIiIiKiqlW3ZOnz6Nc+fOQUdHR6bc0dERT58+VVlgRERERKqgdMuOVCrN9S7JT548gbGxsUqCIiIiIlIVpZOd1q1bIzg4WJyWSCRITU3F1KlT0bZtW1XGRkRERFRkSndjBQUFwcvLCzVq1EBaWhp69uyJe/fuoXz58ti0aVNxxEhERERUaEonOxUrVsSNGzewefNm3LhxA6mpqejfvz98fHxkBiwTERERlQaFuqmglpYWfHx84OPjo+p4iIiIiFRK6TE7s2fPxtq1a+XK165di7lz56okKCIiIiJVUTrZWbVqFVxcXOTKXV1deUNBIiIiKnWUTnbi4uJga2srV25paYnnz5+rJCgiIiIiVVE62bG3t8fZs2flys+ePQs7OzuVBEVERESkKkoPUB4wYABGjRqFzMxMfPfddwCAo0ePYty4cRgzZozKAyQiIiIqCqWTnbFjx+LVq1cYOnQoMjIyAAB6enoYP348Jk6cqPIAiYiIiIpC6WRHIpFg7ty5mDx5MqKioqCvrw9nZ2fo6uoWR3xERERERVKo++wAgJGREerVq6fKWIiIiIhUTulk5+3bt5gzZw6OHj2KhIQESKVSmfkPHz5UWXBERERERaV0svPzzz/j5MmT6N27N2xtbSGRSIojLiIiIiKVUDrZ2b9/P/bu3YvGjRsXRzxEREREKqX0fXbKlSsHc3Pz4oiFiIiISOWUTnZ+++03TJkyBe/evSuOeIiIiIhUSulurKCgIDx48ADW1tZwdHSEtra2zPyrV6+qLDgiIiKiolI62enUqVMxhEFERERUPJROdqZOnVoccRAREREVC6XH7BARERGVJUq37GRnZ2PhwoXYsmULYmNjxedj5UhMTFRZcERERERFpXTLzrRp07BgwQJ069YNycnJGD16NLp06QINDQ0EBgYWQ4hEREREhad0shMWFoY1a9ZgzJgx0NLSQo8ePfDHH39gypQpOH/+fHHESERERFRoSic7cXFxcHNzA/DhYaDJyckAgPbt22Pv3r2qjY6IiIioiJROdipWrIjnz58DAKpUqYJDhw4BAC5dugRdXV3VRkdERERUREonO507d8bRo0cBAMOHD8fkyZPh7OyMPn36oF+/fioPkIiIiKgolL4aa86cOeL/u3XrBgcHB0RGRsLZ2RkdOnRQaXBERERERaV0svMpDw8PeHh4qCIWIiIiIpUrVLLz7NkznDlzBgkJCZBKpTLzRowYoZLAiIhKqyiX6iUdQoGq34kq6RCISg2lk51169Zh0KBB0NHRgYWFBSQSiThPIpEw2aHiEWha0hEULDC5pCMgIqJcKJ3sTJ48GVOmTMHEiROhocGnTRAREVHppnS28u7dO3Tv3p2JDhEREZUJSmcs/fv3x9atW4sjFiIiIiKVU7oba/bs2Wjfvj0OHDgANzc3aGtry8xfsGCByoIjIiIiKqpCJTsHDx5EtWrVAEBugDIRERFRaaJ0shMUFIS1a9fCz8+vGMIhIiIiUi2lx+zo6uqicePGxRELERERkcopneyMHDkSS5YsKY5YiIiIiFRO6W6sixcv4tixY9izZw9cXV3lBihv375dZcERERERFZXSyY6ZmRm6dOlSHLEQERERqZxSyU5WVhZatGiB1q1bw8bGprhiIiIiIlIZpcbsaGlpYfDgwUhPTy+uePI1Z84cSCQSjBo1SixLS0vDsGHDYGFhASMjI3h7eyM+Pr5E4iMiIqLSR+kByvXr18e1a9eKI5Z8Xbp0CatWrULNmjVlyn/55Rfs3r0bW7duxcmTJ/Hs2TN2sxEREZFI6TE7Q4cOxZgxY/DkyRO4u7vD0NBQZv6nyYgqpKamwsfHB2vWrMGMGTPE8uTkZISEhCA8PBzfffcdACA0NBTVq1fH+fPn0bBhQ5XHQkRERGWL0slO9+7dAQAjRowQyyQSCQRBgEQiQXZ2tuqi+/+GDRuGdu3awdPTUybZuXLlCjIzM+Hp6SmWubi4wMHBAZGRkXkmO+np6TJdcSkpKQCAzMxMZGZmqjx+UgENvZKOoGD87HwxsnV1SzqEAvFc9uWQaGkXXKmEFdfnUdH1Kp3sxMTEKB1MUURERODq1au4dOmS3Ly4uDjo6OjAzMxMptza2hpxcXF5rnP27NmYNm2aXPmhQ4dgYGBQ5JipGNRaXdIRFGzfvpKOgD6X6fLnj9LmAT+PX4wqXf1KOoQC7Sumz+O7d+8Uqqd0slOpUiWlgyms//77DyNHjsThw4ehp6e6v+wnTpyI0aNHi9MpKSmwt7dH69atYWJiorLtkArNrljSERRs4pOSjoA+k+i69Uo6hAJVuyz/B2Jps+aXUyUdQoEGLGxa0iEUaIlf15IOoUDD120plvXm9MwUROlkBwAePHiA4OBgREVFAQBq1KiBkSNHokqVKoVZXZ6uXLmChIQEfPPNN2JZdnY2Tp06haVLl+LgwYPIyMhAUlKSTOtOfHx8vpfG6+rqQjeXZmhtbW25myRSKSFNK+kICsbPzhdDs4SuSFVGWTiXCVml/+HRZeM4lv4uy+I6joquV+mrsQ4ePIgaNWrg4sWLqFmzJmrWrIkLFy7A1dUVhw8fVjrQ/LRs2RK3bt3C9evXxVfdunXh4+Mj/l9bWxtHjx4Vl4mOjkZsbCw8PDxUGgsRERGVTUq37EyYMAG//PIL5syZI1c+fvx4tGrVSmXBGRsb4+uvv5YpMzQ0hIWFhVjev39/jB49Gubm5jAxMcHw4cPh4eHBK7GIiIgIQCGSnaioKGzZIt/31q9fPwQHB6siJqUsXLgQGhoa8Pb2Rnp6Ory8vLB8+fLPHgcRERGVTkonO5aWlrh+/TqcnZ1lyq9fvw4rKyuVBZaXEydOyEzr6elh2bJlWLZsWbFvm4iIiMoepZOdAQMGYODAgXj48CEaNWoEADh79izmzp0rc4UTERERUWmgdLIzefJkGBsbIygoCBMnTgQA2NnZITAwUOZGg0RERESlgUJXY+3atUu8S6FEIsEvv/yCJ0+eIDk5GcnJyXjy5AlGjhwJiaT0X0ZIREREXxaFkp3OnTsjKSkJAKCpqYmEhAQAH66WMjY2LrbgiIiIiIpKoWTH0tIS58+fBwDxGVhEREREZYFCY3YGDx6Mjh07QiKRQCKR5Ht34uJ4ECgRERFRYSmU7AQGBqJ79+64f/8+fvjhB4SGhso9fJOIiIioNFL4aiwXFxdUq1YNvr6+8Pb2hpGRUXHGRUTFwG29W0mHUKBbvrdKOgQiUjNKPRtLEASEhYXh+fPnxRUPERERkUoplexoaGjA2dkZr169Kq54iIiIiFRK6aeez5kzB2PHjsU///xTHPEQERERqZTSd1Du06cP3r17h1q1akFHRwf6+voy8xMTE1UWHBEREVFRKZ3slMSTzYmIiIgKS+lkx9fXtzjiICIiIioWSo/ZAYAHDx5g0qRJ6NGjh/joiP379+Pff/9VaXBERERERaV0snPy5Em4ubnhwoUL2L59O1JTUwEAN27cwNSpU1UeIBEREVFRKJ3sTJgwATNmzMDhw4eho6Mjln/33Xfi87OIiIiISgulk51bt26hc+fOcuVWVlZ4+fKlSoIiIiIiUhWlkx0zM7Nc76B87do1VKhQQSVBEREREamK0slO9+7dMX78eMTFxUEikUAqleLs2bMICAhAnz59iiNGIiIiokJTOtmZNWsWXFxcYG9vj9TUVNSoUQNNmzZFo0aNMGnSpOKIkYiIiKjQlL7Pjo6ODtasWYMpU6bg1q1bSE1NRZ06deDs7Fwc8REREREVicLJjlQqxe+//45du3YhIyMDLVu2xNSpU+UeF0FERERUmijcjTVz5kz873//g5GRESpUqIBFixZh2LBhxRkbERERUZEpnOxs2LABy5cvx8GDB7Fz507s3r0bYWFhkEqlxRkfERERUZEonOzExsaibdu24rSnpyckEgmePXtWLIERERERqYLCyU5WVhb09PRkyrS1tZGZmanyoIiIiIhUReEByoIgwM/PD7q6umJZWloaBg8eDENDQ7Fs+/btqo2QiIiIqAgUTnZ8fX3lynr16qXSYIiIiIhUTeFkJzQ0tDjjICIiIioWSt9BmYiIiKgsYbJDREREao3JDhEREak1JjtERESk1pjsEBERkVpjskNERERqjckOERERqTWF77NDheM4YW9Jh1CgR3PalXQIRERExYYtO0RERKTWmOwQERGRWmOyQ0RERGqNyQ4RERGpNSY7REREpNaY7BAREZFaY7JDREREao3JDhEREak1JjtERESk1pjsEBERkVpjskNERERqjckOERERqTUmO0RERKTWmOwQERGRWmOyQ0RERGqNyQ4RERGpNSY7REREpNaY7BAREZFaY7JDREREao3JDhEREak1JjtERESk1pjsEBERkVpjskNERERqjckOERERqTUmO0RERKTWSnWyM3v2bNSrVw/GxsawsrJCp06dEB0dLVMnLS0Nw4YNg4WFBYyMjODt7Y34+PgSipiIiIhKm1Kd7Jw8eRLDhg3D+fPncfjwYWRmZqJ169Z4+/atWOeXX37B7t27sXXrVpw8eRLPnj1Dly5dSjBqIiIiKk20SjqA/Bw4cEBmet26dbCyssKVK1fQtGlTJCcnIyQkBOHh4fjuu+8AAKGhoahevTrOnz+Phg0blkTYREREVIqU6mTnU8nJyQAAc3NzAMCVK1eQmZkJT09PsY6LiwscHBwQGRmZZ7KTnp6O9PR0cTolJQUAkJmZiczMTJXGrKspqHR9xUHV+1wsNPRKOoKClYHjqAvdkg6hQGXh85ity+OoChItnh9VQaKlXdIhFKi4jqOi65UIglD6P20ApFIpfvjhByQlJeHMmTMAgPDwcPTt21cmcQGA+vXro0WLFpg7d26u6woMDMS0adPkysPDw2FgYKD64ImIiEjl3r17h549eyI5ORkmJiZ51iszLTvDhg3DP//8IyY6RTFx4kSMHj1anE5JSYG9vT1at26d78EqjK8DD6p0fcXhn0Cvkg6hYLMrlnQEBZv4pKQjKJBHuEdJh1CgyJ6RJR1CgaLr1ivpEApU7fKlkg6hQGt+OVXSIRRowMKmJR1CgZb4dS3pEAo0fN2WYllvTs9MQcpEsuPv7489e/bg1KlTqFjx/370bGxskJGRgaSkJJiZmYnl8fHxsLGxyXN9urq60M2lGVpbWxva2qptDkzPlqh0fcVB1ftcLKRpJR1BwcrAcUxHesGVSlhZ+DxqpvM4qoKQxfOjKghZpb+rrbiOo6LrLdVXYwmCAH9/f+zYsQPHjh2Dk5OTzHx3d3doa2vj6NGjYll0dDRiY2Ph4VH6/4IlIiKi4leqW3aGDRuG8PBw/P333zA2NkZcXBwAwNTUFPr6+jA1NUX//v0xevRomJubw8TEBMOHD4eHhwevxCIiIiIApTzZWbFiBQCgefPmMuWhoaHw8/MDACxcuBAaGhrw9vZGeno6vLy8sHz58s8cKREREZVWpTrZUeRCMT09PSxbtgzLli37DBERERFRWVOqx+wQERERFRWTHSIiIlJrTHaIiIhIrTHZISIiIrXGZIeIiIjUGpMdIiIiUmtMdoiIiEitMdkhIiIitcZkh4iIiNQakx0iIiJSa0x2iIiISK0x2SEiIiK1xmSHiIiI1BqTHSIiIlJrTHaIiIhIrTHZISIiIrXGZIeIiIjUGpMdIiIiUmtMdoiIiEitMdkhIiIitcZkh4iIiNQakx0iIiJSa0x2iIiISK0x2SEiIiK1xmSHiIiI1BqTHSIiIlJrTHaIiIhIrTHZISIiIrXGZIeIiIjUGpMdIiIiUmtMdoiIiEitMdkhIiIitcZkh4iIiNQakx0iIiJSa0x2iIiISK0x2SEiIiK1xmSHiIiI1BqTHSIiIlJrTHaIiIhIrTHZISIiIrXGZIeIiIjUGpMdIiIiUmtMdoiIiEitMdkhIiIitcZkh4iIiNQakx0iIiJSa0x2iIiISK0x2SEiIiK1xmSHiIiI1BqTHSIiIlJrTHaIiIhIrTHZISIiIrXGZIeIiIjUGpMdIiIiUmtMdoiIiEitMdkhIiIitcZkh4iIiNQakx0iIiJSa0x2iIiISK0x2SEiIiK1xmSHiIiI1BqTHSIiIlJrapPsLFu2DI6OjtDT00ODBg1w8eLFkg6JiIiISgG1SHY2b96M0aNHY+rUqbh69Spq1aoFLy8vJCQklHRoREREVMLUItlZsGABBgwYgL59+6JGjRpYuXIlDAwMsHbt2pIOjYiIiEpYmU92MjIycOXKFXh6eoplGhoa8PT0RGRkZAlGRkRERKWBVkkHUFQvX75EdnY2rK2tZcqtra1x586dXJdJT09Henq6OJ2cnAwASExMRGZmpkrj08p6q9L1FYdXr16VdAgFy9Ap6QgKVgaOo9b70v+VLwufx2QtHkdVSJOmlnQIBSoLxzFdKOkIClZcx/HNmzcAAEHI/yCU/m9sMZg9ezamTZsmV+7k5FQC0ZS88kElHYGamF2+pCNQC+WH8DiqRHkeR1UIWFPSEaiHX7cW7+fxzZs3MDU1zXN+mU92ypcvD01NTcTHx8uUx8fHw8bGJtdlJk6ciNGjR4vTUqkUiYmJsLCwgEQiKdZ4iyolJQX29vb477//YGJiUtLhlFk8jqrB46gaPI6qweOoGmXpOAqCgDdv3sDOzi7femU+2dHR0YG7uzuOHj2KTp06AfiQvBw9ehT+/v65LqOrqwtdXV2ZMjMzs2KOVLVMTExK/YewLOBxVA0eR9XgcVQNHkfVKCvHMb8WnRxlPtkBgNGjR8PX1xd169ZF/fr1ERwcjLdv36Jv374lHRoRERGVMLVIdrp164YXL15gypQpiIuLQ+3atXHgwAG5QctERET05VGLZAcA/P398+y2Uie6urqYOnWqXDccKYfHUTV4HFWDx1E1eBxVQx2Po0Qo6HotIiIiojKszN9UkIiIiCg/THaIiIhIrTHZISIiIrXGZIeIiIjUGpMdIiIiUmtMdoiIiEitMdkhIiIitcZkh4iIiNQakx0iIiJSa0x2iIiISK0x2SEiIiK19v8Ab/IErOw3iasAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "\n",
    "width = 0.5  # the width of the bars\n",
    "multiplier = 0\n",
    "mstrip_names = []\n",
    "x = np.arange(len(avg_scores.keys()))\n",
    "for i, (mn, acc) in enumerate(avg_scores.items()):\n",
    "    strip_name = mn.split(\"__\")\n",
    "    strip_name = strip_name[1] if len(strip_name) == 2 else strip_name[0]\n",
    "    offset = width * multiplier\n",
    "    rects = ax.bar(x[i] + offset, acc, width, label=strip_name, color=f\"C{i}\")\n",
    "    # mstrip_names.append(strip_name)\n",
    "\n",
    "ax.set_ylabel('Performance Acc (%)')\n",
    "ax.set_title('Big-Bench Hard, Base Models and TOPLA Performance')\n",
    "ax.set_xticks(x, mstrip_names, rotation=10)\n",
    "ax.legend(loc='upper left')\n",
    "ax.set_ylim(0, 100)\n",
    "ax.yaxis.grid(zorder=0)\n",
    "plt.show()\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "mistral",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.19"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}