conv_bn_act_output_variance.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "conv_bn_act_output_variance.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyNliEhgMWHxYff27Hom9Nqq",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/hihunjin/Code-snippet-for-everything/blob/main/conv_bn_act_output_variance.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "twkXeQooqx82"
      },
      "outputs": [],
      "source": [
        "%%capture\n",
        "!pip install timm"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "import timm"
      ],
      "metadata": {
        "id": "66svo3yaqy-O"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = timm.create_model(\"resnet18\")\n",
        "model"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iY_TNpG0q1FI",
        "outputId": "1cd323f7-5b3f-4135-f13d-16e5b2a4a373"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ResNet(\n",
              "  (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
              "  (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "  (act1): ReLU(inplace=True)\n",
              "  (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
              "  (layer1): Sequential(\n",
              "    (0): BasicBlock(\n",
              "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (drop_block): Identity()\n",
              "      (act1): ReLU(inplace=True)\n",
              "      (aa): Identity()\n",
              "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (act2): ReLU(inplace=True)\n",
              "    )\n",
              "    (1): BasicBlock(\n",
              "      (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (drop_block): Identity()\n",
              "      (act1): ReLU(inplace=True)\n",
              "      (aa): Identity()\n",
              "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (act2): ReLU(inplace=True)\n",
              "    )\n",
              "  )\n",
              "  (layer2): Sequential(\n",
              "    (0): BasicBlock(\n",
              "      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (drop_block): Identity()\n",
              "      (act1): ReLU(inplace=True)\n",
              "      (aa): Identity()\n",
              "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (act2): ReLU(inplace=True)\n",
              "      (downsample): Sequential(\n",
              "        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): BasicBlock(\n",
              "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (drop_block): Identity()\n",
              "      (act1): ReLU(inplace=True)\n",
              "      (aa): Identity()\n",
              "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (act2): ReLU(inplace=True)\n",
              "    )\n",
              "  )\n",
              "  (layer3): Sequential(\n",
              "    (0): BasicBlock(\n",
              "      (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (drop_block): Identity()\n",
              "      (act1): ReLU(inplace=True)\n",
              "      (aa): Identity()\n",
              "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (act2): ReLU(inplace=True)\n",
              "      (downsample): Sequential(\n",
              "        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): BasicBlock(\n",
              "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (drop_block): Identity()\n",
              "      (act1): ReLU(inplace=True)\n",
              "      (aa): Identity()\n",
              "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (act2): ReLU(inplace=True)\n",
              "    )\n",
              "  )\n",
              "  (layer4): Sequential(\n",
              "    (0): BasicBlock(\n",
              "      (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (drop_block): Identity()\n",
              "      (act1): ReLU(inplace=True)\n",
              "      (aa): Identity()\n",
              "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (act2): ReLU(inplace=True)\n",
              "      (downsample): Sequential(\n",
              "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): BasicBlock(\n",
              "      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (drop_block): Identity()\n",
              "      (act1): ReLU(inplace=True)\n",
              "      (aa): Identity()\n",
              "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (act2): ReLU(inplace=True)\n",
              "    )\n",
              "  )\n",
              "  (global_pool): SelectAdaptivePool2d (pool_type=avg, flatten=Flatten(start_dim=1, end_dim=-1))\n",
              "  (fc): Linear(in_features=512, out_features=1000, bias=True)\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x = torch.rand(1, 3, 64, 64)"
      ],
      "metadata": {
        "id": "25k0ch11q1To"
      },
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "activation = {}\n",
        "def get_activation(name):\n",
        "    def hook(model, input, output):\n",
        "        activation[name] = output.detach()\n",
        "    return hook\n",
        "model.layer1[1].conv2.register_forward_hook(get_activation('layer1_1_conv2'))\n",
        "model.layer1[1].bn2.register_forward_hook(get_activation('layer1_1_bn2'))\n",
        "model.layer1[1].act2.register_forward_hook(get_activation('layer1_1_act2'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Hc3XI4xFq3MS",
        "outputId": "7eb8600e-fb81-4957-fdc3-12224952f895"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<torch.utils.hooks.RemovableHandle at 0x7f8342fbe4d0>"
            ]
          },
          "metadata": {},
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model(x);"
      ],
      "metadata": {
        "id": "FkX3QcSjrCw5"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## 결과\n",
        "* bn2(BatchNorm2D)의 mean이 작아서 bn2의 Ou\\tput과 act2의 Output이 같다.\n",
        "  * bn2의 결과가 모두 양수이기 때문이다."
      ],
      "metadata": {
        "id": "O4l0OBZr1VFG"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for layer_name in [\"layer1_1_conv2\", \"layer1_1_bn2\", \"layer1_1_act2\"]:\n",
        "    print(layer_name)\n",
        "    print(\"mean: \", activation[layer_name].mean())\n",
        "    print(\"var : \", activation[layer_name].var())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i5kYrMmOrGRm",
        "outputId": "3aa1e1dd-e474-452b-d123-94d01c51dcfa"
      },
      "execution_count": 73,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "layer1_1_conv2\n",
            "mean:  tensor(0.0077)\n",
            "var :  tensor(0.8560)\n",
            "layer1_1_bn2\n",
            "mean:  tensor(1.4006)\n",
            "var :  tensor(0.3962)\n",
            "layer1_1_act2\n",
            "mean:  tensor(1.4006)\n",
            "var :  tensor(0.3962)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.layer1[1].bn2.running_mean.max(), model.layer1[1].bn2.running_var.mean()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MLWljV-dsFoi",
        "outputId": "b7cde012-6586-4630-b84b-9ce774e5dba4"
      },
      "execution_count": 70,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(tensor(0.4709), tensor(0.8244))"
            ]
          },
          "metadata": {},
          "execution_count": 70
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model.layer1[1].conv2.weight.var()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FrkmEuB4sVPP",
        "outputId": "f11bf4f4-8e14-42fa-af2c-c8c34ff29523"
      },
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor(0.0035, grad_fn=<VarBackward0>)"
            ]
          },
          "metadata": {},
          "execution_count": 57
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt\n",
        "plt.hist(model.layer1[1].conv2.weight.flatten().detach().numpy(), bins=30)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 491
        },
        "id": "KJ-C7ZPwyENl",
        "outputId": "717b64db-1561-4b58-e999-637d70d120cd"
      },
      "execution_count": 67,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(array([2.000e+00, 5.000e+00, 8.000e+00, 1.800e+01, 4.000e+01, 8.600e+01,\n",
              "        1.660e+02, 3.250e+02, 5.510e+02, 8.950e+02, 1.350e+03, 1.875e+03,\n",
              "        2.521e+03, 3.090e+03, 3.569e+03, 3.799e+03, 3.792e+03, 3.571e+03,\n",
              "        3.183e+03, 2.596e+03, 1.857e+03, 1.408e+03, 9.290e+02, 5.670e+02,\n",
              "        3.290e+02, 1.690e+02, 8.300e+01, 4.700e+01, 2.000e+01, 1.300e+01]),\n",
              " array([-0.24808708, -0.23260975, -0.21713243, -0.2016551 , -0.18617778,\n",
              "        -0.17070046, -0.15522313, -0.1397458 , -0.12426848, -0.10879115,\n",
              "        -0.09331383, -0.07783651, -0.06235918, -0.04688185, -0.03140453,\n",
              "        -0.0159272 , -0.00044988,  0.01502745,  0.03050477,  0.0459821 ,\n",
              "         0.06145942,  0.07693674,  0.09241407,  0.1078914 ,  0.12336873,\n",
              "         0.13884604,  0.15432337,  0.1698007 ,  0.18527803,  0.20075534,\n",
              "         0.21623267], dtype=float32),\n",
              " <a list of 30 Patch objects>)"
            ]
          },
          "metadata": {},
          "execution_count": 67
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD4CAYAAAAAczaOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAASnklEQVR4nO3df6xf9X3f8eer/Eq1dgXKnefZzsxaTxWZVhPdGqZOaxYaMCDNVMoyIjW4EZI7CaRWq6Y6zR+0yZDcbS1rpBTNXbyYqi1htBFWoGWOg9RFGuBLSl0MpdwmzrDn4JuY0GRsbKbv/fH9eP3Oudf3e32/93ttf54P6at7zvt8zvl+zkG8vsef7/mek6pCktSH71rtDkiSJsfQl6SOGPqS1BFDX5I6YuhLUkcuXe0OnM0111xTGzduXO1uSNIF5bnnnvt6VU3Nt+y8Dv2NGzcyMzOz2t2QpAtKkq8utMzhHUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6sh5/Ytc6Xy2cefjY93ekV23j3V70nw805ekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjrivXekM4z7njrjfl/v0aPlWPRMP8k7kjyb5I+THE7yS63+6SRfSfJ8e21u9ST5RJLZJIeSvHtoW9uTvNJe21dutyRJ8xnlTP8t4L1V9e0klwFfTPL7bdm/rKpHz2h/K7CpvW4AHgRuSHI1cB8wDRTwXJJ9VfX6OHZEkrS4Rc/0a+Dbbfay9qqzrLINeKit9zRwZZK1wC3A/qo62YJ+P7B1ed2XJC3FSF/kJrkkyfPACQbB/UxbdH8bwnkgyRWttg54dWj1o622UP3M99qRZCbJzNzc3BJ3R5J0NiOFflW9XVWbgfXAliR/D/gI8EPAjwBXAz8/jg5V1e6qmq6q6ampqXFsUpLULOmSzar6JvAUsLWqjrchnLeA/whsac2OARuGVlvfagvVJUkTMsrVO1NJrmzT3w28D/jTNk5PkgB3AC+0VfYBd7WreG4E3qiq48CTwM1JrkpyFXBzq0mSJmSUq3fWAnuTXMLgQ+KRqvpcki8kmQICPA/889b+CeA2YBZ4E/gwQFWdTPJx4GBr97GqOjm+XZEkLWbR0K+qQ8D189Tfu0D7Au5ZYNkeYM8S+yhJGhNvwyBJHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXEB6NLFxgfoK7l8Exfkjpi6EtSRwx9SeqIoS9JHfGLXHVj1C9ApYuZZ/qS1JFRHoz+jiTPJvnjJIeT/FKrX5vkmSSzST6T5PJWv6LNz7blG4e29ZFWfznJLSu1U5Kk+Y1ypv8W8N6q+mFgM7A1yY3ALwMPVNUPAq8Dd7f2dwOvt/oDrR1JrgPuBN4FbAV+vT1sXZI0IYuGfg18u81e1l4FvBd4tNX3Ane06W1tnrb8piRp9Yer6q2q+gowC2wZy15IkkYy0ph+kkuSPA+cAPYDfw58s6pOtSZHgXVteh3wKkBb/gbw/cP1edYZfq8dSWaSzMzNzS19jyRJCxop9Kvq7araDKxncHb+QyvVoaraXVXTVTU9NTW1Um8jSV1a0tU7VfVN4CngHwBXJjl9yed64FibPgZsAGjLvw/4xnB9nnUkSRMwytU7U0mubNPfDbwPeIlB+L+/NdsOPNam97V52vIvVFW1+p3t6p5rgU3As+PaEUnS4kb5cdZaYG+70ua7gEeq6nNJXgQeTvKvgD8CPtXafwr4zSSzwEkGV+xQVYeTPAK8CJwC7qmqt8e7O5Kks1k09KvqEHD9PPUvM8/VN1X1v4B/usC27gfuX3o3JUnj4C9yJakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdGeXJWZIuQBt3Pj5y2yO7bl/Bnuh84pm+JHVklAejb0jyVJIXkxxO8jOt/otJjiV5vr1uG1rnI0lmk7yc5Jah+tZWm02yc2V2SZK0kFGGd04BP1dVX0ryvcBzSfa3ZQ9U1b8dbpzkOgYPQ38X8LeAzyf5u23xJ4H3AUeBg0n2VdWL49gRSdLiRnkw+nHgeJv+VpKXgHVnWWUb8HBVvQV8Jcksf/UA9dn2QHWSPNzaGvo6Z0sZt5a0xDH9JBuB64FnWuneJIeS7ElyVautA14dWu1oqy1UlyRNyMihn+R7gN8Ffraq/gJ4EPgBYDODfwn8yjg6lGRHkpkkM3Nzc+PYpCSpGSn0k1zGIPB/q6p+D6CqXquqt6vqL4Hf4K+GcI4BG4ZWX99qC9X/P1W1u6qmq2p6ampqqfsjSTqLUa7eCfAp4KWq+tWh+tqhZj8BvNCm9wF3JrkiybXAJuBZ4CCwKcm1SS5n8GXvvvHshiRpFKNcvfOjwIeAP0nyfKv9AvDBJJuBAo4APw1QVYeTPMLgC9pTwD1V9TZAknuBJ4FLgD1VdXiM+yJJWsQoV+98Ecg8i544yzr3A/fPU3/ibOtJklaWv8iVpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOrJo6CfZkOSpJC8mOZzkZ1r96iT7k7zS/l7V6knyiSSzSQ4leffQtra39q8k2b5yuyVJms8oZ/qngJ+rquuAG4F7klwH7AQOVNUm4ECbB7gV2NReO4AHYfAhAdwH3ABsAe47/UEhSZqMRUO/qo5X1Zfa9LeAl4B1wDZgb2u2F7ijTW8DHqqBp4Erk6wFbgH2V9XJqnod2A9sHeveSJLOaklj+kk2AtcDzwBrqup4W/Q1YE2bXge8OrTa0VZbqH7me+xIMpNkZm5ubindkyQtYuTQT/I9wO8CP1tVfzG8rKoKqHF0qKp2V9V0VU1PTU2NY5OSpGak0E9yGYPA/62q+r1Wfq0N29D+nmj1Y8CGodXXt9pCdUnShIxy9U6ATwEvVdWvDi3aB5y+Amc78NhQ/a52Fc+NwBttGOhJ4OYkV7UvcG9uNUnShFw6QpsfBT4E/EmS51vtF4BdwCNJ7ga+CnygLXsCuA2YBd4EPgxQVSeTfBw42Np9rKpOjmUvJEkjWTT0q+qLQBZYfNM87Qu4Z4Ft7QH2LKWDkqTx8Re5ktSRUYZ3pInbuPPx1e6CdFEy9CWN/CF7ZNftK9wTrTSHdySpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SerIKA9G35PkRJIXhmq/mORYkufb67ahZR9JMpvk5SS3DNW3ttpskp3j3xVJ0mJGOdP/NLB1nvoDVbW5vZ4ASHIdcCfwrrbOrye5JMklwCeBW4HrgA+2tpKkCRrlweh/mGTjiNvbBjxcVW8BX0kyC2xpy2ar6ssASR5ubV9cco8lSedsOWP69yY51IZ/rmq1dcCrQ22OttpC9e+QZEeSmSQzc3Nzy+ieJOlM5xr6DwI/AGwGjgO/Mq4OVdXuqpququmpqalxbVaSxDk+GL2qXjs9neQ3gM+12WPAhqGm61uNs9QlSRNyTmf6SdYOzf4EcPrKnn3AnUmuSHItsAl4FjgIbEpybZLLGXzZu+/cuy1JOheLnukn+R3gPcA1SY4C9wHvSbIZKOAI8NMAVXU4ySMMvqA9BdxTVW+37dwLPAlcAuypqsNj3xtJ0lmlqla7Dwuanp6umZmZ1e6GVsHGnY+vdhe0DEd23b7aXehakueqanq+Zf4iV5I6YuhLUkfO6eod6Vw5bCOtLs/0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6siioZ9kT5ITSV4Yql2dZH+SV9rfq1o9ST6RZDbJoSTvHlpne2v/SpLtK7M7kqSzGeVM/9PA1jNqO4EDVbUJONDmAW4FNrXXDuBBGHxIMHig+g3AFuC+0x8UkqTJWTT0q+oPgZNnlLcBe9v0XuCOofpDNfA0cGWStcAtwP6qOllVrwP7+c4PEknSCjvXMf01VXW8TX8NWNOm1wGvDrU72moL1b9Dkh1JZpLMzM3NnWP3JEnzWfYXuVVVQI2hL6e3t7uqpqtqempqalyblSRx7qH/Whu2of090erHgA1D7da32kJ1SdIEnWvo7wNOX4GzHXhsqH5Xu4rnRuCNNgz0JHBzkqvaF7g3t5okaYIuXaxBkt8B3gNck+Qog6twdgGPJLkb+Crwgdb8CeA2YBZ4E/gwQFWdTPJx4GBr97GqOvPLYUkXiY07Hx+p3ZFdt69wT3SmRUO/qj64wKKb5mlbwD0LbGcPsGdJvZMkjdWioS+NYtQzO0mry9swSFJHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkeW9eSsJEeAbwFvA6eqajrJ1cBngI3AEeADVfV6kgC/xuAZum8CP1VVX1rO+0u6sPks3ckbx+MS/3FVfX1ofidwoKp2JdnZ5n8euBXY1F43AA+2vzqP+RhE6eKyEsM724C9bXovcMdQ/aEaeBq4MsnaFXh/SdIClhv6BfznJM8l2dFqa6rqeJv+GrCmTa8DXh1a92irSZImZLnDO/+wqo4l+RvA/iR/OrywqipJLWWD7cNjB8A73/nOZXZPkjRsWWf6VXWs/T0BfBbYArx2etim/T3Rmh8DNgytvr7Vztzm7qqarqrpqamp5XRPknSGcw79JH8tyfeengZuBl4A9gHbW7PtwGNteh9wVwZuBN4YGgaSJE3AcoZ31gCfHVyJyaXAb1fVHyQ5CDyS5G7gq8AHWvsnGFyuOcvgks0PL+O9JUnn4JxDv6q+DPzwPPVvADfNUy/gnnN9P0nS8vmLXEnqiKEvSR0x9CWpI4a+JHVkHPfe0QXIe+roQuKN2cbHM31J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSR/xx1kXGH12pZ/6Ia3Ge6UtSRwx9SeqIoS9JHXFM/wLgOL00Xkv5f+piG//3TF+SOjLxM/0kW4FfAy4B/kNV7Zp0HyRpVBfbFUETDf0klwCfBN4HHAUOJtlXVS9Osh/nC4dtJE3apM/0twCzVfVlgCQPA9uACyL0DWlJCxl3PqzUvxwmHfrrgFeH5o8CNww3SLID2NFmv53k5Qn1bdyuAb6+2p04D3gcPAaneRyWcAzyy8t6n7+90ILz7uqdqtoN7F7tfixXkpmqml7tfqw2j4PH4DSPw/lxDCZ99c4xYMPQ/PpWkyRNwKRD/yCwKcm1SS4H7gT2TbgPktStiQ7vVNWpJPcCTzK4ZHNPVR2eZB8m6IIfohoTj4PH4DSPw3lwDFJVq90HSdKE+ItcSeqIoS9JHTH0xyTJ1Un2J3ml/b1qnjabk/zXJIeTHEryz1ajrytplOPQ2v1Bkm8m+dyk+7hSkmxN8nKS2SQ751l+RZLPtOXPJNk4+V6uvBGOwz9K8qUkp5K8fzX6uNJGOAb/IsmLLQcOJFnwuvpxM/THZydwoKo2AQfa/JneBO6qqncBW4F/l+TKCfZxEkY5DgD/BvjQxHq1woZuMXIrcB3wwSTXndHsbuD1qvpB4AFgeT+/OQ+NeBz+G/BTwG9PtneTMeIx+CNguqr+PvAo8K8n1T9Df3y2AXvb9F7gjjMbVNWfVdUrbfq/AyeAqYn1cDIWPQ4AVXUA+NakOjUB/+8WI1X1v4HTtxgZNnxsHgVuSpIJ9nESFj0OVXWkqg4Bf7kaHZyAUY7BU1X1Zpt9msFvlibC0B+fNVV1vE1/DVhztsZJtgCXA3++0h2bsCUdh4vIfLcYWbdQm6o6BbwBfP9Eejc5oxyHi91Sj8HdwO+vaI+GnHe3YTifJfk88DfnWfTR4ZmqqiQLXgubZC3wm8D2qrrgznbGdRyk3iX5SWAa+LFJvaehvwRV9eMLLUvyWpK1VXW8hfqJBdr9deBx4KNV9fQKdXVFjeM4XIRGucXI6TZHk1wKfB/wjcl0b2K81cqIxyDJjzM4UfqxqnprQn1zeGeM9gHb2/R24LEzG7RbT3wWeKiqHp1g3yZp0eNwkRrlFiPDx+b9wBfq4vt1pLdaGeEYJLke+PfAP6mqyZ4YVZWvMbwYjM0eAF4BPg9c3erTDJ4QBvCTwP8Bnh96bV7tvk/6OLT5/wLMAf+TwZjnLavd9zHs+23AnzH4nuajrfYxBv9jA7wD+E/ALPAs8HdWu8+rdBx+pP03/x8M/qVzeLX7vArH4PPAa0M5sG9SffM2DJLUEYd3JKkjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqyP8FK+EyW/l+8CYAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        ""
      ],
      "metadata": {
        "id": "bz3R4I1l0T8T"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}