conditional_bert_contextual_augmentation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "lexical_substitution_with_bert.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/tkmaroon/use-cases-of-bert/blob/master/conditional_bert_contextual_augmentation.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "i7LF71CEeB0M",
        "colab_type": "text"
      },
      "source": [
        "# Conditional BERT Contextual Augmentation\n",
        "## Overview\n",
        "This notebook performs lexical substitution using  BERT of Hugging face.\n",
        "It shows how to constrain substitution candidates following the *Conditional BERT Contextual Augmentation*.\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DpxaNWMmdkO7",
        "colab_type": "text"
      },
      "source": [
        "## Settings"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZMX_w_VydE7j",
        "colab_type": "code",
        "cellView": "form",
        "outputId": "bafe7f61-7338-43f4-bf81-f0fbe43dc6e6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "#@title Setup environment\n",
        "!pip install --quiet pytorch-transformers\n",
        "!pip install --quiet pytorch-nlp\n",
        "!pip install --quiet tqdm"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[?25l\r\u001b[K     |█▉                              | 10kB 31.2MB/s eta 0:00:01\r\u001b[K     |███▊                            | 20kB 6.7MB/s eta 0:00:01\r\u001b[K     |█████▋                          | 30kB 9.3MB/s eta 0:00:01\r\u001b[K     |███████▍                        | 40kB 6.4MB/s eta 0:00:01\r\u001b[K     |█████████▎                      | 51kB 7.8MB/s eta 0:00:01\r\u001b[K     |███████████▏                    | 61kB 9.3MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 71kB 10.6MB/s eta 0:00:01\r\u001b[K     |██████████████▉                 | 81kB 11.8MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 92kB 13.1MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 102kB 10.3MB/s eta 0:00:01\r\u001b[K     |████████████████████▍           | 112kB 10.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 122kB 10.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████▏       | 133kB 10.3MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 143kB 10.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 153kB 10.3MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▊  | 163kB 10.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▋| 174kB 10.3MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 184kB 10.3MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.0MB 54.1MB/s \n",
            "\u001b[K     |████████████████████████████████| 870kB 77.6MB/s \n",
            "\u001b[?25h  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "\u001b[K     |████████████████████████████████| 92kB 6.8MB/s \n",
            "\u001b[?25h"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ckESRGmKdMrh",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Setup common imports\n",
        "import random\n",
        "import math\n",
        "from collections import OrderedDict\n",
        "from tqdm import tqdm\n",
        "\n",
        "import torch\n",
        "from torch.nn import CrossEntropyLoss\n",
        "import torch.nn.functional as F\n",
        "\n",
        "from torchnlp.datasets import smt_dataset\n",
        "\n",
        "from pytorch_transformers import (\n",
        "    BertConfig,\n",
        "    BertTokenizer,\n",
        "    BertForMaskedLM,\n",
        "    BertForTokenClassification,\n",
        "    AdamW,\n",
        "    WarmupLinearSchedule,\n",
        ")\n",
        "\n",
        "import matplotlib.pyplot as plt\n",
        "% matplotlib inline"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fWT62eJMJYHF",
        "colab_type": "text"
      },
      "source": [
        "## Examples\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C3Ju8s5k4dCs",
        "colab_type": "text"
      },
      "source": [
        "### Conditional BERT Contextual Augmentation\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "84InK9iVi4K4",
        "colab_type": "code",
        "cellView": "both",
        "colab": {}
      },
      "source": [
        "#@title setup common functions\n",
        "class BatchIterator(object):\n",
        "    def __init__(self, tokenizer, data, batchsize, device, shuffle=True, repeat=False):\n",
        "        self.tokenizer = tokenizer\n",
        "        self.pad_idx = self.tokenizer.convert_tokens_to_ids('[PAD]')\n",
        "        self._numericalize(data)\n",
        "        self.batchsize = batchsize\n",
        "        self.device = device\n",
        "        self.shuffle = shuffle\n",
        "        self.repeat = repeat\n",
        "                    \n",
        "    def __len__(self):\n",
        "        return math.ceil(len(self.data)/self.batchsize)\n",
        "\n",
        "    def __iter__(self):\n",
        "        while True:\n",
        "            self._init_batches()\n",
        "            for batch in self.batches:\n",
        "                yield batch.to(self.device)\n",
        "            if not self.repeat:\n",
        "                return\n",
        "            \n",
        "    def _numericalize(self, data):\n",
        "        self.data = [self.tokenizer.encode(s) for s in data]\n",
        "\n",
        "    def _init_batches(self):\n",
        "        data = random.sample(self.data, len(self.data)) if self.shuffle else self.data\n",
        "        self.batches = [self._padding(data[i:i+self.batchsize]) for i in range(0, len(self.data), self.batchsize)]\n",
        "    \n",
        "    def _padding(self, batch):\n",
        "        maxlen = max([len(b) for b in batch])\n",
        "        return torch.tensor([b + [self.pad_idx for _ in range(maxlen-len(b))] for b in batch])    \n",
        "    \n",
        "    \n",
        "class Perturbator(object):\n",
        "    def __init__(self, tokenizer, vocab_range, sampling_rate=0.15, \\\n",
        "                 masking_ratio=0.8, replacing_ratio=0.1, unchanging_ratio=0.1):\n",
        "        self.mask_idx = tokenizer.mask_token_id\n",
        "        self.pad_idx = tokenizer.pad_token_id\n",
        "        self.vocab_range = vocab_range\n",
        "        self.sampling_rate = sampling_rate\n",
        "        self.masking_ratio = masking_ratio\n",
        "        self.replacing_ratio = replacing_ratio\n",
        "        self.unchanging_ratio = unchanging_ratio\n",
        "        assert (self.masking_ratio + self.replacing_ratio + self.unchanging_ratio) == 1.0, \\\n",
        "                '`masking_ratio + replacing_ratio + unchanging_ratio` must be 1.0'\n",
        "\n",
        "    def __call__(self, batch):\n",
        "        device = batch.device\n",
        "        bsz, slen = batch.size()\n",
        "        batch = batch.to(torch.device('cpu'))\n",
        "        sampler = (torch.rand((bsz, slen)).le(self.sampling_rate)) & batch.ne(self.pad_idx) # [PAD] tokens are not sampled\n",
        "        sampler[:, 0] = 0 # [CLS] tokens are not sampled\n",
        "        \n",
        "        masked_lm_labels = torch.where(\n",
        "            sampler,\n",
        "            batch,\n",
        "            torch.ones_like(batch) * -100\n",
        "        )\n",
        "\n",
        "        rnd = torch.rand((bsz, slen))\n",
        "        batch = torch.where(\n",
        "            (self.masking_ratio >= rnd) & sampler,\n",
        "            torch.ones_like(batch) * self.mask_idx,\n",
        "            batch\n",
        "        )\n",
        "        \n",
        "        th = self.replacing_ratio + self.masking_ratio\n",
        "        batch = torch.where(\n",
        "            ((th >= rnd) & (rnd > self.masking_ratio) & sampler), \n",
        "            torch.randint_like(batch, self.vocab_range[0], self.vocab_range[1]),\n",
        "            batch,\n",
        "        )\n",
        "        return batch.to(device), masked_lm_labels.to(device)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VTV29b1g4lpf",
        "colab_type": "code",
        "cellView": "both",
        "outputId": "e77630f4-5d63-4a17-be4f-f80b0d1d5629",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "#@title Load training data and preprocessing\n",
        "\n",
        "train_smt = smt_dataset(train=True)\n",
        "\n",
        "print('----- DETAILS OF TRAINING DATA -----')\n",
        "print('* Data Summary')\n",
        "for i in range(3):\n",
        "    print(train_smt[i]['label'] + '\\t' + train_smt[i]['text'][:100] + '...')\n",
        "print('...')\n",
        "print(f'* Train data size: {len(train_smt)}')\n",
        "\n",
        "unique_labels = set([data['label'] for data in train_smt])\n",
        "print(f'* class: {unique_labels}')\n",
        "\n",
        "train_data = ['[' + data['label'].upper() + '] ' + data['text'] for data in train_smt]\n"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----- DETAILS OF TRAINING DATA -----\n",
            "* Data Summary\n",
            "positive\tThe Rock is destined to be the 21st Century 's new `` Conan '' and that he 's going to make a splash...\n",
            "positive\tThe gorgeously elaborate continuation of `` The Lord of the Rings '' trilogy is so huge that a colum...\n",
            "positive\tSinger\\/composer Bryan Adams contributes a slew of songs -- a few potential hits , a few more simply...\n",
            "...\n",
            "* Train data size: 8544\n",
            "* class: {'negative', 'positive', 'neutral'}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SKdU3O3l5faT",
        "colab_type": "code",
        "cellView": "form",
        "colab": {}
      },
      "source": [
        "#@title Setup tokenizer and iterator\n",
        "\n",
        "# Set tokenizer    \n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')\n",
        "added_special_tokens = [ '[' + label.upper() + ']' for label in iter(unique_labels)]\n",
        "tokenizer.add_tokens(added_special_tokens)\n",
        "\n",
        "batch_size =  32#@param {type:\"integer\"}\n",
        "assert batch_size > 0, 'Please set `batch_size` value more than zero'\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "train_iter = BatchIterator(tokenizer, train_data, batch_size, device)\n",
        "#train_iter = BatchIterator(tokenizer, small_train, batch_size, device)\n"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JahA0MmVibbg",
        "colab_type": "code",
        "cellView": "both",
        "outputId": "8bf68a8e-9073-401a-c3dc-4c5aeefa0b80",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#@title Build a model\n",
        "config = BertConfig.from_pretrained('bert-base-uncased', vocab_size=len(tokenizer))\n",
        "model = BertForMaskedLM(config)\n",
        "model.to(device)\n",
        "\n",
        "# freeze bert\n",
        "# for param in model.bert.parameters():\n",
        "#    param.requires_grad = False\n",
        "\n",
        "print('Built a model!')"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Built a model!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_HoYYzjNJVfK",
        "colab_type": "code",
        "cellView": "both",
        "outputId": "a3066600-f88a-4e84-c80c-ce5dc421e5f9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "#@title Fine-tuning\n",
        "n_epochs = 10 #@param {type:\"integer\"}\n",
        "assert n_epochs > 0, 'Please set `n_epochs` value more than zero'\n",
        "\n",
        "learning_rate = 0.25 #@param\n",
        "num_total_steps = n_epochs * len(train_iter)\n",
        "num_warmup_steps = len(train_iter)\n",
        "\n",
        "# optimizer = AdamW(model.parameters(), lr=learning_rate, correct_bias=False)\n",
        "optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)\n",
        "#scheduler = WarmupLinearSchedule(optimizer, warmup_steps=num_warmup_steps, t_total=num_total_steps)\n",
        "\n",
        "vocab_range = (max(tokenizer.all_special_ids), min(tokenizer.convert_tokens_to_ids(added_special_tokens)))\n",
        "\n",
        "sampling_rate = 0.2 #@param {type:\"slider\", min:0.1, max:1.0, step:0.05}\n",
        "perturbator = Perturbator(tokenizer, vocab_range, sampling_rate)\n",
        "\n",
        "loss_fn = CrossEntropyLoss(ignore_index=-100)\n",
        "\n",
        "for epoch in range(1, n_epochs+1):  \n",
        "    with tqdm(train_iter, dynamic_ncols=True) as pbar:\n",
        "        train_loss = 0.0\n",
        "        for batch in pbar: \n",
        "            bsz, slen = batch.size()\n",
        "            srcs, masked_lm_labels = perturbator(batch)\n",
        "\n",
        "            outputs = model(srcs)[0].view(bsz*slen, -1)\n",
        "            loss = loss_fn(outputs, masked_lm_labels.view(-1))\n",
        "            pbar.set_description(f'epoch {str(epoch).zfill(3)}')\n",
        "            progress_state = OrderedDict(\n",
        "                loss=loss.item(),\n",
        "                bsz=bsz,\n",
        "            )\n",
        "            pbar.set_postfix(progress_state)\n",
        "            loss.backward()\n",
        "            torch.nn.utils.clip_grad_norm_(model.parameters(), 0.1)\n",
        "\n",
        "            optimizer.step()\n",
        "            # scheduler.step()\n",
        "            "
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "epoch 001: 100%|██████████| 267/267 [01:46<00:00,  2.64it/s, loss=7.96, bsz=32]\n",
            "epoch 002: 100%|██████████| 267/267 [01:46<00:00,  2.38it/s, loss=7.32, bsz=32]\n",
            "epoch 003: 100%|██████████| 267/267 [01:47<00:00,  2.44it/s, loss=7.38, bsz=32]\n",
            "epoch 004: 100%|██████████| 267/267 [01:47<00:00,  2.68it/s, loss=7.04, bsz=32]\n",
            "epoch 005: 100%|██████████| 267/267 [01:47<00:00,  2.60it/s, loss=6.6, bsz=32]\n",
            "epoch 006: 100%|██████████| 267/267 [01:46<00:00,  2.57it/s, loss=7.32, bsz=32]\n",
            "epoch 007: 100%|██████████| 267/267 [01:47<00:00,  2.56it/s, loss=7.19, bsz=32]\n",
            "epoch 008: 100%|██████████| 267/267 [01:47<00:00,  2.18it/s, loss=7.45, bsz=32]\n",
            "epoch 009: 100%|██████████| 267/267 [01:47<00:00,  2.54it/s, loss=6.8, bsz=32]\n",
            "epoch 010: 100%|██████████| 267/267 [01:46<00:00,  2.32it/s, loss=6.47, bsz=32]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Oj3OF1hg_Ouw",
        "colab_type": "code",
        "cellView": "form",
        "outputId": "a00fe629-0c44-4e3b-d41f-d66ab5bd435f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "#@title Select condition\n",
        "example = \"The actors are fantastic .\"\n",
        "\n",
        "sentiment_label = '[NEGATIVE]' #@param [\"[NEUTRAL]\", \"[NEGATIVE]\", \"[POSITIVE]\"]\n",
        "conditional_example = sentiment_label + example\n",
        "tokenized_example = tokenizer.tokenize(conditional_example)\n",
        "print(tokenized_example)\n"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['[NEGATIVE]', 'the', 'actors', 'are', 'fantastic', '.']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GxcUMyPf_yo3",
        "colab_type": "code",
        "outputId": "93b7bbea-e8c2-4640-9ec4-299ae1babad3",
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "#@title Output top10 candidates\n",
        "masked_position = 4 #@param {type:\"integer\"}\n",
        "tokenized_example[masked_position] = '[MASK]'\n",
        "print(f'Input: {tokenized_example}')\n",
        "\n",
        "input_tensor = torch.tensor([tokenizer.convert_tokens_to_ids(tokenized_example)]).to(device)\n",
        "outputs = model(input_tensor)[0]\n",
        "\n",
        "topk_score, topk_index = torch.topk(outputs[0, masked_position], 10)\n",
        "topk_tokens = tokenizer.convert_ids_to_tokens(topk_index.tolist())\n",
        "print(topk_tokens)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Input: ['[NEGATIVE]', 'the', 'actors', 'are', '[MASK]', '.']\n",
            "[',', '-', 'the', \"'\", 'of', 'a', 's', 'and', 'is', 'to']\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TNWIfSGPFFG8",
        "colab_type": "code",
        "outputId": "d1875231-7da2-4b4d-9102-41288fd596c5",
        "cellView": "form",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 276
        }
      },
      "source": [
        "#@title Visualize output probabilities\n",
        "plt.bar(topk_tokens, torch.softmax(topk_score, 0).tolist())\n",
        "plt.xticks(rotation=70)\n",
        "plt.ylabel('Probability')\n",
        "plt.show()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEDCAYAAAAvNJM9AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0\ndHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAU40lEQVR4nO3de7ydVX3n8c+XcBUEEeILSyhBiJdU\nKXYCdMaWXhQIUsFRKNHaItWhzpSpl9E21Xpp7KvSVlsvxUEULMIoUNROWqIUFJ2OFkwQhEKlRkQJ\nY9tIkHEUueU3fzxP7PFkkbOR8+x9OPm8X6/zOs9+Lmf9Aufs737WWnvtVBWSJE23w6QLkCTNTQaE\nJKnJgJAkNRkQkqQmA0KS1LTjpAuYLfvuu28tXrx40mVI0qPKtdde+62qWtg6Nm8CYvHixaxbt27S\nZUjSo0qSrz/UMbuYJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTfPmndSP\n1OKVlw3exm1nHj94G5I0W7yDkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAk\nNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU0GhCSpyYCQJDUNGhBJlie5Jcn6JCsbx1+T5OYk\nNyT5VJIDpxw7NclX+q9Th6xTkrS1wQIiyQLgLOA4YCnwoiRLp512HbCsqg4FLgX+uL/28cCbgSOB\nI4A3J9l7qFolSVsb8g7iCGB9Vd1aVfcBFwEnTj2hqq6qqu/1D68GFvXbxwJXVNWmqroLuAJYPmCt\nkqRphgyI/YHbpzze0O97KC8DPvFwrk1yepJ1SdZt3LjxEZYrSZpqTgxSJ3kJsAz4k4dzXVWdU1XL\nqmrZwoULhylOkrZTQwbEHcABUx4v6vf9kCTPAd4AnFBV9z6cayVJwxkyINYCS5IclGRnYAWweuoJ\nSZ4JvI8uHP51yqHLgWOS7N0PTh/T75MkjcmOQ/3gqnogyRl0T+wLgPOq6qYkq4B1VbWarktpD+Av\nkwB8o6pOqKpNSd5KFzIAq6pq01C1SpK2NlhAAFTVGmDNtH1vmrL9nG1cex5w3nDVSZK2ZU4MUkuS\n5h4DQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlq\nMiAkSU0GhCSpyYCQJDUZEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYD\nQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJgJAkNRkQkqQmA0KS1GRASJKadpx0AYLFKy8bvI3b\nzjx+8DYkzS/eQUiSmgwISVLToAGRZHmSW5KsT7KycfyoJF9M8kCSk6YdezDJ9f3X6iHrlCRtbbAx\niCQLgLOAo4ENwNokq6vq5imnfQN4KfDaxo+4p6oOG6o+SdK2DTlIfQSwvqpuBUhyEXAi8IOAqKrb\n+mObB6xDkvQjGLKLaX/g9imPN/T7RrVrknVJrk7y/NYJSU7vz1m3cePGR1KrJGmauTxIfWBVLQNe\nDLwzycHTT6iqc6pqWVUtW7hw4fgrlKR5bMiAuAM4YMrjRf2+kVTVHf33W4HPAM+czeIkSds2ZECs\nBZYkOSjJzsAKYKTZSEn2TrJLv70v8CymjF1IkoY3WEBU1QPAGcDlwD8Cl1TVTUlWJTkBIMnhSTYA\nJwPvS3JTf/nTgHVJvgRcBZw5bfaTJGlggy61UVVrgDXT9r1pyvZauq6n6dd9HnjGkLVJkrZtpDuI\nJB9LcnySuTyoLUmaRaM+4b+XbjbRV5KcmeQpA9YkSZoDRgqIqrqyqn4F+CngNuDKJJ9PclqSnYYs\nUJI0GSN3GSXZh25ZjJcD1wHvoguMKwapTJI0USMNUif5OPAU4ALgeVX1zf7QxUnWDVWcJGlyRp3F\n9P5+RtIPJNmlqu7t3+0sSZpnRu1i+oPGvr+fzUIkSXPLNu8gkuxHt8DebkmeCaQ/tCfwmIFrkyRN\n0ExdTMfSDUwvAv50yv7vAK8fqCZJ0hywzYCoqvOB85O8sKo+OqaaJElzwExdTC+pqguBxUleM/14\nVf1p4zJJ0jwwUxfT7v33PYYuRJI0t8zUxfS+/vvvj6ccSdJcMVMX07u3dbyqfmt2y5EkzRUzdTFd\nO5YqJElzziizmCRJ26GZupjeWVWvSvLXQE0/XlUnDFaZxmLxyssGb+O2M48fvA1Js2+mLqYL+u9v\nH7oQSdLcMlMX07X9988m2Rl4Kt2dxC1Vdd8Y6pMkTcioy30fD5wNfJVuPaaDkvxGVX1iyOIkSZMz\n6nLf7wB+oarWAyQ5GLgMMCAkaZ4adbnv72wJh96tdAv2SZLmqZlmMb2g31yXZA1wCd0YxMnA2oFr\nkyRN0ExdTM+bsv0vwM/12xuB3QapSJI0J8w0i+m0cRUiSZpbRp3FtCvwMuAngF237K+qXx+oLknS\nhI06SH0BsB/dJ8x9lu4T5hyklqR5bNSAOKSq3gh8t1+f6XjgyOHKkiRN2qgBcX///dtJng7sBTxh\nmJIkSXPBqG+UOyfJ3sAbgdV0nzD3xsGqkiRN3EgBUVUf6Dc/CzxpuHIkSXPFSF1MSfZJ8p4kX0xy\nbZJ3Jtln6OIkSZMz6hjERcC/Ai8ETgK+BVw8VFGSpMkbdQziiVX11imP/yDJKUMUJEmaG0a9g/jb\nJCuS7NB//TJw+ZCFSZIma6bF+r5DtzhfgFcBF/aHdgD+H/DaQauTJE3MTGsxPXZchUiS5pZRu5hI\nckKSt/dfvzTiNcuT3JJkfZKVjeNH9TOjHkhy0rRjpyb5Sv916qh1SpJmx6jTXM8EXgnc3H+9Msnb\nZrhmAXAWcBywFHhRkqXTTvsG8FLgw9OufTzwZrrlPI4A3ty/UU+SNCajzmJ6LnBYVW0GSHI+cB3w\nu9u45ghgfVXd2l9zEXAiXcAAUFW39cc2T7v2WOCKqtrUH78CWA58ZMR6JUmP0MhdTMDjpmzvNcL5\n+wO3T3m8od83ipGuTXJ6knVJ1m3cuHHEHy1JGsWodxBvA65LchXdjKajgK3GFMatqs4BzgFYtmxZ\nTbgcSZpXZgyIJAH+N/DTwOH97t+pqn+e4dI7gAOmPF7U7xvFHcDPT7v2MyNeK0maBTN2MVVVAWuq\n6ptVtbr/mikcANYCS5IclGRnYAXdSrCjuBw4Jsne/eD0MfjGPEkaq1HHIL6Y5PCZT/s3VfUAcAbd\nE/s/ApdU1U1JViU5ASDJ4Uk2ACcD70tyU3/tJuCtdCGzFli1ZcBakjQeo45BHAm8JMltwHfpxiGq\nqg7d1kVVtQZYM23fm6Zsr6XrPmpdex5w3oj1SZJm2agBceygVUiS5pyZ1mLaFXgFcAhwI3Bu33Uk\nSZrnZhqDOB9YRhcOxwHvGLwiSdKcMFMX09KqegZAknOBLwxfkiRpLpjpDuL+LRt2LUnS9mWmO4if\nTPJ/++0Au/WPt8xi2nPQ6iRJEzPT50EsGFch2v4sXnnZ4G3cdubxg7chzVcPZ7E+SdJ2xICQJDUZ\nEJKkJgNCktRkQEiSmgwISVKTASFJajIgJElNoy73Lc0rvklPmpkBIY2Z4aRHC7uYJElNBoQkqcmA\nkCQ1GRCSpCYHqaXtiAPkeji8g5AkNRkQkqQmA0KS1GRASJKaDAhJUpOzmCSNhTOoHn0MCEnznuH0\no7GLSZLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJaa6SNKBH8xRb7yAkSU2DBkSS5UluSbI+ycrG\n8V2SXNwfvybJ4n7/4iT3JLm+/zp7yDolSVsbrIspyQLgLOBoYAOwNsnqqrp5ymkvA+6qqkOSrAD+\nCDilP/bVqjpsqPokSds25B3EEcD6qrq1qu4DLgJOnHbOicD5/falwLOTZMCaJEkjGjIg9gdun/J4\nQ7+veU5VPQDcDezTHzsoyXVJPpvkZ1sNJDk9ybok6zZu3Di71UvSdm6uDlJ/E/jxqnom8Brgw0n2\nnH5SVZ1TVcuqatnChQvHXqQkzWdDBsQdwAFTHi/q9zXPSbIjsBdwZ1XdW1V3AlTVtcBXgScPWKsk\naZohA2ItsCTJQUl2BlYAq6edsxo4td8+Cfh0VVWShf0gN0meBCwBbh2wVknSNIPNYqqqB5KcAVwO\nLADOq6qbkqwC1lXVauBc4IIk64FNdCECcBSwKsn9wGbgFVW1aahaJUlbG/Sd1FW1Blgzbd+bpmx/\nHzi5cd1HgY8OWZskadvm6iC1JGnCDAhJUpMBIUlqMiAkSU0GhCSpyYCQJDUZEJKkJgNCktRkQEiS\nmgwISVKTASFJajIgJElNBoQkqcmAkCQ1GRCSpCYDQpLUZEBIkpoMCElSkwEhSWoyICRJTQaEJKnJ\ngJAkNRkQkqQmA0KS1GRASJKaDAhJUpMBIUlqMiAkSU0GhCSpyYCQJDUZEJKkJgNCktRkQEiSmgwI\nSVKTASFJajIgJElNBoQkqcmAkCQ1DRoQSZYnuSXJ+iQrG8d3SXJxf/yaJIunHPvdfv8tSY4dsk5J\n0tYGC4gkC4CzgOOApcCLkiyddtrLgLuq6hDgz4A/6q9dCqwAfgJYDry3/3mSpDEZ8g7iCGB9Vd1a\nVfcBFwEnTjvnROD8fvtS4NlJ0u+/qKruraqvAev7nydJGpNU1TA/ODkJWF5VL+8f/ypwZFWdMeWc\nf+jP2dA//ipwJPAW4OqqurDffy7wiaq6dFobpwOn9w+fAtwyyD+mbV/gW2Nsz7Zt27a3n/bH2faB\nVbWwdWDHMRUwiKo6BzhnEm0nWVdVy2zbtm17/rU96fYn/W/fYsgupjuAA6Y8XtTva56TZEdgL+DO\nEa+VJA1oyIBYCyxJclCSnekGnVdPO2c1cGq/fRLw6er6vFYDK/pZTgcBS4AvDFirJGmawbqYquqB\nJGcAlwMLgPOq6qYkq4B1VbUaOBe4IMl6YBNdiNCfdwlwM/AA8JtV9eBQtf6IJtK1Zdu2bdvbRfuT\n/rcDAw5SS5Ie3XwntSSpyYCQZtC/N0fa7hgQUkOSpyXZB6Dshx27JDsYzJNnQOhh207+cN8L/FiS\nI5LsN+lixinJ7kl+KcluE2o/VbV5SzAnWbCd/M7Nub8tA+JRKMlz+6nD4253JxjfK+ok70pyaZKj\nxtHelHYfC9xDN8vvD4F/6feP7Y83yW5JTkvygSQvT3Jokp3HVMMZwNFVdU+SJyd5c5JTxtDuFtcl\n+UySowGq6sGqqnQGec7a8nOT/FiSRZN6op5rd6sGxCPQv0dj3G0+FnhHv77VONt9LvD6JBcmecKY\nmn018CfAfxpzIN4LXAJcARxOt9DkE8f8x/tq4FC6kFoFnA18HHjeGNpeDrw7ySLgrXTLPjxrjHdS\nvwBcBVyaZEOSc5L8ZHU2D9z2fwcO7QNpSZLnj+v3PcmyJCuTvCDJnFh77lG91MYkJXkccEWSa4E3\nVNX6MTX9U8B7xtFQf6tfSZ4GvIFuxd1TgLuS/DhwIPD5od6j0j8ZXNN/jU0fvn+R5ABgJ+A3gFVJ\nrgY+CVwyhoB+NvArwErgvwI3An9Ft6rAYJLsBXwb+FXg5+nm418C/D3wBOCfB25/p6q6K8lVdM9P\ndwMHA1cm2QS8t6reNdvtVtXmJAuBRVW1JslhdKtRfxnYDfjIbLcJP/Q3tgJ4Md2d6wHA45JcWVUf\nHKLdUXkH8SOqqm/TLWP+eaC50NVArqd7g+E4bLnN/nXgA3S/vF+qqvuBxcAr5uAbGB+RLV0LSfYA\nvgfsA/wP4C+ArwCvp1tQcsga9gFu7dt/BnBNVf0T3ZPVp4Zsu6ruBn4P2Ax8sqo+TPeiZHNV3TBk\n23379/eb7wauqKq3V9V/Bn4R+Drdcjyzakp30pHAhn5h0VfSBcSHgN+a7TYbXgy8p6pOobt7/Eu6\n1SQOGUPbD8k7iEegqu5LMpZX81PavHuMbW25nf8HuleVpwFn9vt+me4Ja77ZAXiQ7lX7U+leADwI\nPBH4WlVN/0yTWVdVd9J1q+0A/C1wYb/awKKqGnzF4n4lg5v7V7Y7A/+O8b0ooW/zc0y5W6qqG5Ns\nBGb9FfWWrsOq+pskzwBeQLd69IeTvBa4erbbnNp2kscA3wCO6Bfpuwv4qySvprubGFfvxFYMiEdo\nDH2ic8Gn6F5FHw5ck2QJ8NNs/fkej3pT7oieBfxeVV0PkOSTwPuTfKGq/teYatmc5M+B/wPsQfdq\ndiymPGnel+SDwP0zXDKbbd+X5ELgvCS/Rte9cyDw9Kq6fTbbSvJzdL/HbwB2r6q3TTm2EPgPdOMw\nQ/oZuhdgB9PdNXwf2A/YoaquGrjtbXKpDY2kn/J4NHAMsDvwzqr60mSrGka6Ty/8beDfA78P3Ng/\naV0PnFZV1020wO1A//9gR+DXgGOBK4HPVdWNA7S1E3AY3cD41cD5VXVBf+zAqvr6bLfZ/+z/SBcE\np9F1426i6166h64L90NVdeUQbY/KgNBD6rs4fgd4PF130tfoPsjpexMtbAz62WK/TdfltB/dTJ7v\n933EmmVJdujvmA6lGyDfk+4Jcw2wtqq+P6Y6ngy8CnghcAPw0qqa9Y8a6Mc9ltJ9zPJRdAH4IWBN\n/2LkXOAts33H9HAZEHpISV4E/CbdL+9edLM57gZuAi6Ya3O2Z1vfN3w43UD1/XRPVIPO4tleTZnN\ncwHdf+u/Bvan+1z6J9INmJ89xnoWAD8L3FBVmwZs56l0A/B30t0tHQxsAHatqp8Zqt1RGRDaSpLX\n0d1urwA+XlWfS7IL3WDlc4BvVtX7J1mj5p/+VfVbgD+uqu8meTxdOCwFvjxE99Jc0t+xPwlYBtwy\nF7oyDQj9kCR70g3YLQH2Br5DN1h7w5RzFsy36a2avCTPBz4GnF1V/2XS9ciA0DRTbvVX0nWv3EPX\nvXQ38HfAx6pq4yRr1PyV5GS6gdqnA58GPlhV/3PL7+Vkq9v+OM1V053c30WcALy8qm7u31X6FOC/\nATszpndya/6b8oLkMXQDw0+mmzkWundyfyTJ8ZOe7rm98p3U+oG+D/RmujcKPR34wyQvAf6pqi6m\nu5vwD1Wzactz0KvoxrgOAi4AXgc8DniO4TA5djFpK1NmVnyLbsrhEuB2YEFV/eIka9P8lORT/Nva\nU39HN1Pub4A/q6qzJlnb9swuJm2lqr5Mv4xGkkvpZlYcDgy+zIO2P421p95eVRuS3EA3xVoT4h2E\npDmh7+J8HXAc3fpDh1bVnFj2entlQEiaM5LsTjcGtgdw07jWvVKbASFJanIWkySpyYCQJDUZEJKk\nJgNCktRkQEiSmgwISVLT/wep0icpcMB2GQAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}