PyTorch_Transfer_Learning_And_Other_Tricks.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "PyTorch_Transfer Learning And Other Tricks.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1mLm_TieMuzKgCMP21Uxl-o-eU-huVS3M",
      "authorship_tag": "ABX9TyMMYkZtJC5MIXWgReQJO39x",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Shrey-Viradiya/PyTorch_for_DL/blob/master/PyTorch_Transfer_Learning_And_Other_Tricks.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DVaKe7Y3xZ4K"
      },
      "source": [
        "# PyTorch_Transfer Learning And Other Tricks"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xa3Q9vYj-PEN",
        "outputId": "97cdce71-5a6d-443e-a3e2-40679338aa42",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 372
        }
      },
      "source": [
        "!nvidia-smi"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Fri Oct  2 08:02:26 2020       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 455.23.05    Driver Version: 418.67       CUDA Version: 10.1     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   35C    P0    25W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "|                               |                      |                 ERR! |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "72fI-NdS-QdK",
        "outputId": "baf8d63b-a259-4941-9961-67d7c641ff06",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "!unzip -n '/content/drive/My Drive/images.zip'"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Archive:  /content/drive/My Drive/images.zip\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_HxpkDdAxVUe"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "import torch.utils.data\n",
        "import torch.nn.functional as F\n",
        "import torchvision\n",
        "import torchvision.models as models\n",
        "from torchvision import transforms\n",
        "from PIL import Image\n",
        "import matplotlib.pyplot as plt\n",
        "import time\n",
        "import os\n",
        "import sys"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "td9-uo9IxfL0"
      },
      "source": [
        "transfer_model = models.resnet50(pretrained=True)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hVjcgnTw7nox",
        "outputId": "5e6c2d5e-b69f-44b5-90ee-ff7ebc5fb6c7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "print(transfer_model)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "ResNet(\n",
            "  (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
            "  (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "  (relu): ReLU(inplace=True)\n",
            "  (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
            "  (layer1): Sequential(\n",
            "    (0): Bottleneck(\n",
            "      (conv1): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "      (downsample): Sequential(\n",
            "        (0): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "    )\n",
            "    (1): Bottleneck(\n",
            "      (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "    )\n",
            "    (2): Bottleneck(\n",
            "      (conv1): Conv2d(256, 64, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv3): Conv2d(64, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn3): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "    )\n",
            "  )\n",
            "  (layer2): Sequential(\n",
            "    (0): Bottleneck(\n",
            "      (conv1): Conv2d(256, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "      (downsample): Sequential(\n",
            "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
            "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "    )\n",
            "    (1): Bottleneck(\n",
            "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "    )\n",
            "    (2): Bottleneck(\n",
            "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "    )\n",
            "    (3): Bottleneck(\n",
            "      (conv1): Conv2d(512, 128, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv3): Conv2d(128, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn3): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "    )\n",
            "  )\n",
            "  (layer3): Sequential(\n",
            "    (0): Bottleneck(\n",
            "      (conv1): Conv2d(512, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "      (downsample): Sequential(\n",
            "        (0): Conv2d(512, 1024, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
            "        (1): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "    )\n",
            "    (1): Bottleneck(\n",
            "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "    )\n",
            "    (2): Bottleneck(\n",
            "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "    )\n",
            "    (3): Bottleneck(\n",
            "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "    )\n",
            "    (4): Bottleneck(\n",
            "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "    )\n",
            "    (5): Bottleneck(\n",
            "      (conv1): Conv2d(1024, 256, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv3): Conv2d(256, 1024, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn3): BatchNorm2d(1024, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "    )\n",
            "  )\n",
            "  (layer4): Sequential(\n",
            "    (0): Bottleneck(\n",
            "      (conv1): Conv2d(1024, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "      (downsample): Sequential(\n",
            "        (0): Conv2d(1024, 2048, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
            "        (1): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      )\n",
            "    )\n",
            "    (1): Bottleneck(\n",
            "      (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "    )\n",
            "    (2): Bottleneck(\n",
            "      (conv1): Conv2d(2048, 512, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
            "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (conv3): Conv2d(512, 2048, kernel_size=(1, 1), stride=(1, 1), bias=False)\n",
            "      (bn3): BatchNorm2d(2048, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
            "      (relu): ReLU(inplace=True)\n",
            "    )\n",
            "  )\n",
            "  (avgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n",
            "  (fc): Linear(in_features=2048, out_features=1000, bias=True)\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MAiPrUsoxvar"
      },
      "source": [
        "## Freezing parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wwN4TG6sxuFH"
      },
      "source": [
        "for name, param in transfer_model.named_parameters():\n",
        "    if(\"bn\" not in name):\n",
        "        param.requires_grad = False"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YuHY1zEp7tzW"
      },
      "source": [
        "## Replacing the classifier"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Mw3rX5ft7wIy"
      },
      "source": [
        "transfer_model.fc = nn.Sequential(\n",
        "    nn.Linear(transfer_model.fc.in_features, 500),\n",
        "    nn.ReLU(),\n",
        "    nn.Dropout(),\n",
        "    nn.Linear(500, 2)\n",
        ")"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HLQ2QvjQ8DMg"
      },
      "source": [
        "## Training Again"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WlguBgwCEWQ4"
      },
      "source": [
        "def GPUinfo():\n",
        "    a  = os.popen('nvidia-smi').readlines()\n",
        "    mem = a[9][33:54]\n",
        "    util = a[9][58:66]\n",
        "    sys.stdout.write(f\"\\rGPU Memory: {mem} Utilization: {util}\")\n",
        "    sys.stdout.flush()"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2b2gB0g98Bup"
      },
      "source": [
        "def train(model, optimizer, loss_fn, train_loader, val_loader, epochs=20, device=\"cpu\"):\n",
        "    \n",
        "    for epoch in range(epochs):\n",
        "        start = time.time()\n",
        "        training_loss = 0.0\n",
        "        valid_loss = 0.0\n",
        "        model.train()\n",
        "        for batch in train_loader:\n",
        "            optimizer.zero_grad()\n",
        "            inputs, targets = batch\n",
        "            inputs = inputs.to(device)\n",
        "            GPUinfo()\n",
        "            targets = targets.to(device)\n",
        "            GPUinfo()\n",
        "            output = model(inputs)\n",
        "            GPUinfo()\n",
        "            loss = loss_fn(output, targets)\n",
        "            GPUinfo()\n",
        "            loss.backward()\n",
        "            GPUinfo()\n",
        "            optimizer.step()\n",
        "            GPUinfo()\n",
        "            training_loss += loss.item()\n",
        "            \n",
        "        training_loss /= len(train_loader.dataset)\n",
        "\n",
        "        \n",
        "        model.eval()\n",
        "        correct = 0 \n",
        "        total = 0\n",
        "        with torch.no_grad():\n",
        "            for batch in val_loader:\n",
        "                inputs, targets = batch\n",
        "                inputs = inputs.to(device)\n",
        "                GPUinfo()\n",
        "                output = model(inputs)\n",
        "                GPUinfo()\n",
        "                targets = targets.to(device)\n",
        "                GPUinfo()\n",
        "                loss = loss_fn(output,targets) \n",
        "                GPUinfo()\n",
        "                valid_loss += loss.item()\n",
        "                _, predicted = torch.max(output.data, 1)\n",
        "                total += targets.size(0)\n",
        "                correct += (predicted == targets).sum().item()\n",
        "                \n",
        "\n",
        "        valid_loss /= len(val_loader.dataset)\n",
        "        a  = os.popen('nvidia-smi').readlines()\n",
        "        mem = a[9][33:54]\n",
        "        util = a[9][58:66]\n",
        "        print(f'\\nEpoch: {epoch}, Training Loss: {training_loss:.2f}, Validation Loss: {valid_loss:.2f}, accuracy = {(correct / total):.2f}, time: {time.time() - start}')"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "szXBe2-D8Hwv"
      },
      "source": [
        "def check_image(path):\n",
        "    try:\n",
        "        im = Image.open(path)\n",
        "        return True\n",
        "    except:\n",
        "        return False\n",
        "img_transforms = transforms.Compose([\n",
        "    transforms.Resize((64,64)),    \n",
        "    transforms.ColorJitter(brightness=0, contrast=0, saturation=0, hue=0),\n",
        "    transforms.RandomHorizontalFlip(p=0.5),\n",
        "    transforms.RandomVerticalFlip(p=0.5),\n",
        "    transforms.RandomGrayscale(p=0.1),\n",
        "    transforms.RandomRotation((10,25), resample=False,expand=False, center=None),\n",
        "    transforms.ToTensor()\n",
        "    ])\n",
        "train_data_path = \"./train/\"\n",
        "train_data = torchvision.datasets.ImageFolder(root=train_data_path,transform=img_transforms, is_valid_file=check_image)\n",
        "val_data_path = \"./val/\"\n",
        "val_data = torchvision.datasets.ImageFolder(root=val_data_path,transform=img_transforms, is_valid_file=check_image)\n",
        "batch_size=64\n",
        "train_data_loader = torch.utils.data.DataLoader(train_data, batch_size=batch_size, shuffle=True)\n",
        "val_data_loader  = torch.utils.data.DataLoader(val_data, batch_size=batch_size, shuffle=True)\n",
        "if torch.cuda.is_available():\n",
        "    device = torch.device(\"cuda\") \n",
        "else:\n",
        "    device = torch.device(\"cpu\")"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eVDkni8w_E1F",
        "outputId": "880fed05-db87-4f44-f4d3-8c042f88e4f2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "print(len(val_data_loader.dataset))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "107\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B8EhH7SM_HBf"
      },
      "source": [
        "transfer_model.to(device)\n",
        "optimizer = optim.Adam(transfer_model.parameters(), lr=3e-4)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9DSLFyVf_NDu",
        "outputId": "1e0b80e2-2ae7-4373-8baf-6f972f544beb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        }
      },
      "source": [
        "train(transfer_model, optimizer,torch.nn.CrossEntropyLoss(), train_data_loader,val_data_loader, epochs=8, device=device)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "GPU Memory:    1413MiB / 16280MiB Utilization:     9%  \n",
            "Epoch: 0, Training Loss: 0.01, Validation Loss: 0.01, accuracy = 0.74, time: 9.629743576049805\n",
            "GPU Memory:    1413MiB / 16280MiB Utilization:     9%  \n",
            "Epoch: 1, Training Loss: 0.01, Validation Loss: 0.01, accuracy = 0.78, time: 9.622750043869019\n",
            "GPU Memory:    1413MiB / 16280MiB Utilization:     9%  \n",
            "Epoch: 2, Training Loss: 0.01, Validation Loss: 0.01, accuracy = 0.81, time: 9.634412288665771\n",
            "GPU Memory:    1413MiB / 16280MiB Utilization:     9%  \n",
            "Epoch: 3, Training Loss: 0.01, Validation Loss: 0.01, accuracy = 0.77, time: 9.682465314865112\n",
            "GPU Memory:    1413MiB / 16280MiB Utilization:     9%  \n",
            "Epoch: 4, Training Loss: 0.01, Validation Loss: 0.01, accuracy = 0.64, time: 9.733755826950073\n",
            "GPU Memory:    1413MiB / 16280MiB Utilization:     9%  \n",
            "Epoch: 5, Training Loss: 0.01, Validation Loss: 0.01, accuracy = 0.90, time: 9.634206295013428\n",
            "GPU Memory:    1413MiB / 16280MiB Utilization:     9%  \n",
            "Epoch: 6, Training Loss: 0.01, Validation Loss: 0.01, accuracy = 0.80, time: 9.582375764846802\n",
            "GPU Memory:    1413MiB / 16280MiB Utilization:     8%  \n",
            "Epoch: 7, Training Loss: 0.00, Validation Loss: 0.01, accuracy = 0.79, time: 9.593873977661133\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lVaEfVzgFID_"
      },
      "source": [
        "## LR Finder"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "358buNT0Ag0h"
      },
      "source": [
        "def find_lr(model, loss_fn, optimizer, train_loader, init_value=1e-8, final_value=10.0, device=\"cpu\"):\n",
        "    number_in_epoch = len(train_loader) - 1\n",
        "    update_step = (final_value / init_value) ** (1 / number_in_epoch)\n",
        "    lr = init_value\n",
        "    optimizer.param_groups[0][\"lr\"] = lr\n",
        "    best_loss = 0.0\n",
        "    batch_num = 0\n",
        "    losses = []\n",
        "    log_lrs = []\n",
        "    for data in train_loader:\n",
        "        batch_num += 1\n",
        "        inputs, targets = data\n",
        "        inputs = inputs.to(device)\n",
        "        targets = targets.to(device)\n",
        "        optimizer.zero_grad()\n",
        "        outputs = model(inputs)\n",
        "        loss = loss_fn(outputs, targets)\n",
        "\n",
        "        # Crash out if loss explodes\n",
        "\n",
        "        if batch_num > 1 and loss > 4 * best_loss:\n",
        "            if(len(log_lrs) > 20):\n",
        "                return log_lrs[10:-5], losses[10:-5]\n",
        "            else:\n",
        "                return log_lrs, losses\n",
        "\n",
        "        # Record the best loss\n",
        "\n",
        "        if loss < best_loss or batch_num == 1:\n",
        "            best_loss = loss\n",
        "\n",
        "        # Store the values\n",
        "        losses.append(loss.item())\n",
        "        log_lrs.append((lr))\n",
        "\n",
        "        # Do the backward pass and optimize\n",
        "\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "\n",
        "        # Update the lr for the next step and store\n",
        "\n",
        "        lr *= update_step\n",
        "        optimizer.param_groups[0][\"lr\"] = lr\n",
        "    if(len(log_lrs) > 20):\n",
        "        return log_lrs[10:-5], losses[10:-5]\n",
        "    else:\n",
        "        return log_lrs, losses"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kNOhBx-QFKgJ",
        "outputId": "8d4e8d14-b823-49bc-c733-b857542f5153",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        }
      },
      "source": [
        "(lrs, losses) = find_lr(transfer_model, torch.nn.CrossEntropyLoss(),optimizer, train_data_loader,device=device)\n",
        "plt.plot(lrs, losses)\n",
        "\n",
        "plt.xscale(\"log\")\n",
        "plt.xlabel(\"Learning rate\")\n",
        "plt.ylabel(\"Loss\")\n",
        "plt.show()"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEKCAYAAAAFJbKyAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxU1dnA8d+Tyb6TkLAFCIQ1QNjC6r4DKli3ular1mpfa1ut4tJqa11xq1r11Vpt61JF1IoKLlVQawIStrBDJuxbkgkQIHty3j9m4jvGQBbmzp2ZPN/PZz7MnLs9hyE8uefcc44YY1BKKaXaKszuAJRSSgUXTRxKKaXaRROHUkqpdtHEoZRSql00cSillGoXTRxKKaXaJdzuAPyha9euJjMz0+4wlFIqqCxdurTMGJPWvLxTJI7MzEwKCgrsDkMppYKKiGxtqVybqpRSSrWLJg6llFLtoolDKaVUu2jiUEop1S6aOJRSSrWLJg6llFLtoolDKaW8rN1VQW19o91hBDRNHEop5bH7QBXnPPM1D85bZ3coAc3SxCEiU0Rkg4gUicgdLWy/QURWicgKEfmviGR7ys8QkaWebUtF5FSvYxZ6zrnC80q3sg5Kqc4jr8hFo4HXFm1lc9lhu8MJWJYlDhFxAM8CU4Fs4NKmxODlDWPMCGPMKGAW8ISnvAw41xgzArgKeLXZcZcbY0Z5XiVW1UEp1bnkOV0kRocTGR7GrI/X2x1OwLLyjmM8UGSMKTbG1AJvAjO8dzDGVHh9jAOMp3y5MWaXp3wNECMiURbGqpTq5Iwx5DvLOH5gV244KYv5q/dQsKXc7rACkpWJoxew3evzDk/Z94jI/4iIE/cdx80tnOcCYJkxpsar7BVPM9XvRURauriIXC8iBSJSUFpa2vFaKKU6hW3llew6UM2krK5cd0I/uiVG8cC8dRhj7A4t4NjeOW6MedYYkwXMBH7nvU1EhgGPAD/3Kr7c04R1gud15RHO+6IxJtcYk5uW9oPJHTudzWWHqaptsDsMpQJWntMFwKT+qcRGhnPrGYNZvm0/81btsTmywGNl4tgJ9Pb6nOEpO5I3gfOaPohIBvAe8BNjjLOp3Biz0/PnQeAN3E1i6ijW7qrgjCe+5OpXvqW+QR8zVKoleU4X6QlRZKXFAXDB2AyGdE/gkY/XU1Ovv3R5szJxLAEGikg/EYkELgHmeu8gIgO9Pp4NbPKUJwMfAXcYY77x2j9cRLp63kcA5wCrLaxD0KtvaOT2d1YSFR7G4s3lPDxfO/yUas7dv+FiclYqTa3fjjDhrmlD2VZeyav5Lc4u3mlZljiMMfXATcAnwDpgtjFmjYjcJyLTPbvdJCJrRGQFcAvuJ6jwHDcAuKfZY7dRwCciUgiswH0H81er6hAKXvy6mNU7K3jsopFcPTmTl/67mQ9W7mr9QKU6kaKSQ5QdqmFSVur3yk8clMYJA7vyzBdFHKissym6wGPpQk7GmHnAvGZl93i9/9URjrsfuP8Ipx3rswBDXFHJIf78n01MHd6dqSN6cNrQbqzeeYDb5xQyqFsCg7sn2B2iUgGhqX9jclbXH2y7a9pQpj39NX9ZsIm7z24+oqBzsr1zXFmjodEw851CYiMd/HHGMAAiw8N47vIxxEeHc8NrS6mo1t+glALId7rI6BJD75TYH2wb2iORi8Zm8I+8rWwvr7QhusCjiSNE/TN/C0u37uOec7JJT4j+rjw9MZrnLx/D9vJKbnlrJY2N+qih6twaGw35xS4m9U894j63nDGYsDCY9ckGP0YWuDRxhKBtrkpmfbyBUwan8aPRPxg6Q25mCr8/J5v/rNvLswuKbIhQqcCxdncFB6rqmDzgyImje1I015/Qnw9W7mL5tn1+jC4waeIIMcYY7nyvEEeY8MCPRnCE8ZH8ZFJffjS6F0/8ZyMLN+isLarzyv9u/MYP+ze8XX9SFl3jo3hQBwVq4gg1by3ZzjdFLu6aNpSeyTFH3E9EePBHIxjSPZFfvblC225Vp5Vf7KJ/1zi6J0Ufdb/4qHB+c8ZAlmzZx6dr9/opusCkiSOE7D5QxQMfrWNS/1QuHd+71f1jIh28cMVYjDH8/NWlOrJcdTp1DY0sLnb94DHcI/lxbm8GpMfz8Pz11HXiwbSaOEKEMYbfvbeausZGHr7gyE1UzfVJjeWpS0ezbk8Fd/97Vae/BVedy6qdBzhc29DiY7gtCXeEcde0IWwuO8wbi7dZHF3g0sQRIt5fsYvP15dw21lD6Jsa165jTxmczq9PG8S7y3by2iIdIas6j6b+jYn9U9p8zCmD05nUP5WnPt/UaR9p18QRAkoP1vCHD9Ywpk8yV0/O7NA5fnnqAE4bks4fP1jL0q06lbTqHPKdLoZ0TyA1vu2rNogId589lPLDtTy/0Nn6ASFIE0cI+MPcNVTWNDDrwhwcYW1romouLEx44sejyOgSw42vLaPkYLWPo1QqsNTUN7BkS3mb+ze8De+VxPmje/Hyfzezc3+VBdEFNk0cQe7j1Xv4aNVufnX6QAakH9sUIkkxEfzvlWM5WF3PTa8v79Sdfyr0rdi2n5r6xjb3bzR361mDMcDjnXBQoCaOILa/spbfv7+aYT0Tuf7E/j4555DuiTx8wQi+3VLOg/PW+eScSgWiPKeLMIHx/drev+GtV3IM1x7fj3eX72T1zgM+ji6waeIIYn/6cB37Dtcy68IcIhy++ypnjOrFNcf145VvtvD+iqMtoaJU8Mp3uhjeK4mkmIgOn+PGk7NIiYvkgY8616BATRxBauGGEt5ZtoMbTspiWM8kn5//zmlDGN8vhZnvFLJud0XrBygVRKpqG1i+fV+H+je8JUZH8KvTBpJf7GJBJ5qBQRNHEDpYXcdd765iQHo8vzxtgCXXiHCE8ZfLRpMUE8ENry3lQFXnfOxQhaaCreXUNZijTmzYVpdN6EO/rnE8OG99p1lhUxNHEHrk4/Xsrqhm1oU5RIU7LLtOekI0z10+ll37q/jNWyt0Jl0VMvKcLsLDhHGZHevf8BbhCOOOqUMoKjnE7IIdPogu8GniCDL5ThevLdrGtcf1Y0yfLpZfb2zfLtxzTjZfrC/h6S82WX49pfwhz+liVO9k4qJ8s5bdmdndGJfZhSc+28ihmnqfnDOQaeIIIlW1DdzxbiF9U2O59czBfrvuFRP7csGYDJ76fBML1needlwVmiqq61i1Yz+Tj7F/w5uIe33yskM1vPhl6A8K1MQRRB7/dANbXZU8fH4OMZHWNVE1JyI88KPhZPdI5FdvLmer67Dfrq2Ury3ZXE6jgYk+TBwAo/t04dyRPXnx62L2HAjtAbSaOILE8m37ePmbzVw+oc8xPwnSEdERDv73irGIiM6kq4JantNFZHiYJU29t581mMZGeOKz0B4UaGniEJEpIrJBRIpE5I4Wtt8gIqtEZIWI/FdEsr223ek5boOInNXWc4aimvoGbp9TSPfEaO6YOsS2OHqnxPL0paPZsPcgd75b2KmeW1ehI9/pIrdvF6IjfH/X3jsllqsm9+XtpTtC+jF2yxKHiDiAZ4GpQDZwqXdi8HjDGDPCGDMKmAU84Tk2G7gEGAZMAZ4TEUcbzxly/vJFEZtKDvHA+SNIiO74YCVfOGlQGreeMYh/r9jFP/K22BqLUu2173Ata3dX+OQx3CO56ZSBJEZH8ND89ZZdw25W3nGMB4qMMcXGmFrgTWCG9w7GGO+UHAc0/Qo7A3jTGFNjjNkMFHnO1+o5Q82aXQd4fqGT88f04pTB6XaHA8AvTh7A6UO7cf9H61iyRWfSVcFjUbF7GvWjrS9+rJJiI/jlqQP4amMpX24stew6drIycfQCtnt93uEp+x4R+R8RceK+47i5lWPbdM5QUdfQyO1zCkmOjeSecwLnxso9k+5IeqfE8ovXl1FSEdodgSp05Be7iI10kJORbOl1rpzUlz4psTw0bx0NITj+yfbOcWPMs8aYLGAm8DtfnVdErheRAhEpKC0Nzqz/4lfFrNlVwf3nDSM5NtLucL4nMTqCF64cy+Gaen7x+jJq6zvHiFkV3PKcLsZlpvh0breWRIU7uH3KYNbvOcg7y0JvUKCVf3s7Ae+FrzM8ZUfyJnBeK8e2+ZzGmBeNMbnGmNy0tLR2hm6/opKDPPWfTUwb0Z0pw3vYHU6LBnVLYNaFORRs3ccDH621OxyljqqkopqikkM+Hb9xNGeP6MGo3sk8/ukGKmtDa1CglYljCTBQRPqJSCTuzu653juIyECvj2cDTUOT5wKXiEiUiPQDBgLftuWcoaCh0XD7nEJioxz8cfpwu8M5qnNyevKzE/rxj/ytvLc89H6zUqEjv6l/o4Prb7SXiPC7s4eyt6KGl77e7Jdr+otlicMYUw/cBHwCrANmG2PWiMh9IjLds9tNIrJGRFYAtwBXeY5dA8wG1gIfA/9jjGk40jmtqoNd/pG3hWXb9nPvudmkJbR9SUu7zJwyhIn9U7jz3VWs2dW51iVQwSPf6SIxOpzsnol+u2ZuZgpThnXnf790htSqmtIZnsXPzc01BQUFdofRJttclZz156+YlJXK367KRaRjS8H6W9mhGs55+r9EhAsf3HR8wPXJKHXirAUM7p7AX3+S69frbi47zBlPfMnF43rz4I9G+PXax0pElhpjfvAXZnvnuPp/xhhmvlNIeJh7io9gSRoAXeOjeP6KMew9UMOvdSZdFWB27KtkW3ml3/o3vPXrGscVE/vy5rfb2LT3oN+vbwVNHAHkX99uJ7/YxZ3ThtIjKcbucNptdJ8u3Ds9m4UbSvnz5zqTrgoc+U7/9m80d/NpA4mLDOfhEBkUqIkjQOzaX8WD89YxOSuVS8f3bv2AAHXZ+D5cnJvB059v4j9r99odjlKAO3GkxkUyqFu8LddPiYvkf04dwOfrS8grKrMlBl/SxBEAjDHc/d4qGhoND5+fE1RNVM2JCPfNGM6IXkn8ZvYKtpTpTLrKXsYY8pwuJmal2vqzdfXkTHolx/DAvHVB35SriSMA/HvFThZsKOW2swbTJzXW7nCOWXSEg+evGEN4mHsm3VB7hl0Fly2uSvZUVNvSv+EtOsLBbWcNZs2uCt5febQhbYFPE4fNSg/W8McP1jKmTzJXTc60Oxyfyejinkl3U8lBZr6zSmfSVbbJc7qbhqyc2LCtpo/syYheSTz68Qaq64J3aQJNHDa7d+5qKmsbmHXhSBxhwdtE1ZITBqbx27MG88HKXbz8zRa7w1GdVJ7TRffEaPp1jbM7FMLC3CsF7jpQzcvfBO+gQE0cNpq/ajfzVu3hV6cNZEC6PZ12VrvxpCzOGtaNB+et+25mUqX8xRjDIqeLyTb3b3iblJXK6UPTeX6BE9ehGrvD6RBNHDbZX1nL799fw7CeiVx/Yn+7w7GMiPDYRSPpmxrLTW8sC/klNVVg2bj3EK7DtT5fJvZY3TF1CJV1DTwdpI+ta+KwyX0frmV/ZS2zLsyxfKZOuyVER/DilWOpqm3gxteXUlMfvG27Krg09W/Y3THe3ID0BC4d35vXF2+juPSQ3eG0W2j/jxWgFqwv4d1lO7nx5CyG9UyyOxy/GJCewKMXjWT5tv386UOdSVf5R57TRZ+UWDK6BN7Tir86bRBR4WE88nHwDQrUxOFnB6vruOu9VQxMj+emUwfYHY5fTRvRg5+f1J/XFm1jzlKdSVdZq6HRsLjYFRBPU7UkLSGKG0/O4pM1e/l2c3CtpKmJw88emr+evRXVzLowh6hwh93h+N1tZw5mclYqd7+3itU7dSZdZZ21uyqoqK63dJnYY3Xt8f3pnhjNA/PWBdUj65o4/Cjf6eKNxdu49vh+jO7Txe5wbBHuCOOZS0eTGhfJz19dyr7DtXaHpEJUII3fOJKYSAe3njmIldv382HhbrvDaTNNHH5SWVvPzHcKyUyN5ZYzBtsdjq1S46N4/oqxlB6s4eY3l4fkmszKfvnFLgakx5OeGG13KEd1/pgMhvZI5JGP1wfNgyOaOPzk8U83sq28kocvyCEmsvM1UTU3sncy980Yxtebynjys412h6NCTF1DI99uLg/ou40mjjDh7mlD2bGvin/mbbU7nDbRxOEHy7bt4+VvNnPFxD5MDIJ/yP5yyfg+XDKuN39ZUMSna/bYHY4KIYU79lNZ2xBwj+EeyfEDu3LSoDSe+WIT+ysDv/lWE4fFauobuH1OIT0So5k5ZYjd4QScP0wfxsiMJG6dvTIon2dXgalp/Y1g+kXtrmlDOVRTzzNfFNkdSqs0cVjsmc+LKCo5xIPnjyAhOsLucAKOeybdsUSEh/HzV5dyuEZn0lXHLs/pYmiPRLrEBc8SxoO7J3Bxbm/+mb+Fra7AXo5AE4eFVu88wPNfOrlgTAYnD063O5yA1TM5hr9cOhpn6SFun1MYVI8lqsBTXddAwdZ9QdNM5e2WMwYRHhbGrE822B3KUWnisEhdQyO3zymkS2wkvz9nqN3hBLzJA7oyc8oQPlq1m5e+Dt5ZQ5X9lm/bT219Y1AmjvTEaK4/sT8fFe5m6dZ9dodzRJYmDhGZIiIbRKRIRO5oYfstIrJWRApF5HMR6espP0VEVni9qkXkPM+2v4vIZq9to6ysQ0e9+FUxa3dXcP95w0mODZ7bZTtdf2J/po3ozsMfr//uGXyl2ivfWUaYwLh+KXaH0iHXn9iftIQoHgzgQYGWJQ4RcQDPAlOBbOBSEclutttyINcYkwPMAWYBGGMWGGNGGWNGAacClcCnXsfd1rTdGLPCqjp01Ka9B3nqP5s4e0QPpgzvbnc4QUNEmHXhSPp1jeOXbyxn1/4qu0NSQSjP6WJERjKJQdqnGBcVzi1nDGLp1n18EqBPG1p5xzEeKDLGFBtjaoE3gRneO3gSRKXn4yIgo4XzXAjM99ovoDU0Gm5/p5C4KAd/mD7M7nCCTnxUOC9cOZaa+kZufH1Z0AyIUoHhcE09K7bvD8pmKm8Xjc1gULd4Hp6/ntr6RrvD+QErE0cvYLvX5x2esiO5FpjfQvklwL+alT3gad56UkSiWjqZiFwvIgUiUlBaWtqeuI/JK99sZvm2/dx77jDSEloMTbUiKy2exy4aycrt+/njBzqTrmq7gq37qG80QTHw72jCHWHcOW0oW1yVvL448AYFBkTnuIhcAeQCjzYr7wGMAD7xKr4TGAKMA1KAmS2d0xjzojEm1xiTm5aWZknczW11HeaxTzdw2pB0Zozq6Zdrhqopw7vzi5OzeGPxNmYv2d76AUrhnp8qwiHkZgb/XHAnD0rjuAGpPP35Jg5U1dkdzvdYmTh2Ar29Pmd4yr5HRE4H7gamG2Oar6N4MfCeMea7vzVjzG7jVgO8grtJzHaNjYaZ7xQSERbGAz8aETDLVAazW88czAkDu/K791dTuGO/3eGoIJDvdDG6dxdiI8PtDuWYibjXJ99fVcdzCwNrUKCViWMJMFBE+olIJO4mp7neO4jIaOAF3EmjpIVzXEqzZirPXQji/p/5PGC1BbG327+WbGNRcTl3nT2U7kmBPalasHCECU9dMpq0+ChufG0Z5TqTrjqKA1V1rN55gElB3r/hbVjPJM4fncEr32xhx77A6ea1LHEYY+qBm3A3M60DZhtj1ojIfSIy3bPbo0A88Lbn0drvEouIZOK+Y/my2alfF5FVwCqgK3C/VXVoq137q3ho3nqOG5DKJeN6t36AarOUuEj+94qxlB6q4eZ/6Uy66si+3VxOoyGkEgfAb88ahACPBdCgQEvv54wx84B5zcru8Xp/+lGO3UILnenGmFN9GOIxM8Zw13uraGg0PHx+jjZRWWBERhL3nzec2+cU8tinG3TOL9WiPGcZUeFhjO6TbHcoPtUjKYbrTujHswucXHN8P3Iy7K9fQHSOB7P3lu9k4YZSbp8ymN4pgbeucai4OLc3l03ow/MLnXy8OngWvFH+k+90MS4zJSRX1rzhpCxS4yJ54KPAGBSoieMYlBys5o8frGVs3y5cNSnT7nBC3r3nZjOqdzK3zl7J5rLAngRO+ZfrUA3r9xwMuWaqJgnREfz69IEs3lzO5+ta6g72L00cx+De99dQVdfAIxfkEBamTVRWiwp38PwVYwh3hHHr7BXa36G+s6i4HAi9/g1vl4zvQ/+0OB6cv466BnsHBWri6KB5q3Yzf/Uefn36QAakx9sdTqfRIymG+2YMY9m2/bzwldPucFSAyC8uIz4qnJxeSXaHYpkIRxh3Th1Kcelh3rR5bJMmjg7Yd7iWe95fzfBeiVx/Qn+7w+l0po/sybQR3Xnys42s31NhdzgqAOQ5XYzL7EK4I7T/Szt9aDrj+6Xw1H82crDavkGBof23bJH7PlzL/so6Zl0wMuT/oQYiEeH+80aQFBPJb95aGZBz+Sj/2XOgmuLSw0zO6mp3KJYTca9PXnaolhe+LLYtDv1fr50WrC/hveU7+cXJWWT3TLQ7nE4rJS6Sh88fwbrdFTz9+Sa7w1E2yi92T8Efyv0b3kb2Tmb6yJ789etidh+wZwZpTRztUFFdx13vrWJQt3j+59QBdofT6Z2e3Y2Lxmbw3MIilm8L3EVvlLXynS6SYiLI7tF5fpG77azBGAOPf7rRlutr4miHh+atZ29FNbMuHBmSz4oHo3vOzaZHUgy3zl5JVa1Owd4Z5TldTOyf0qmebOydEstPj8vknWU7WLPrgN+vr4mjjfKKyvjXt9u49vh+jOpt/8hN5ZYQHcGjF+ZQXHaYWZ+stzsc5WfbyyvZsa+qU/RvNPeLUwaQFBPBQ/PW+31QoCaONqisrWfmu4VkpsZyyxmD7Q5HNTN5QFeunpzJK99s0SVnO5l8pwsg6Bdu6oikmAhuPnUg/y0q48uN/ltzCDRxtMljn2xke3kVj1yQQ0ykNlEFoplThtC/axy3vV1o62OKyr/ynGV0jY/qtGOprpjYl76psTw4bx31fhwUqImjFUu3lvNK3maunNiXCUG+qlgoi4l08NjFI9l9oIo/fairBnYGxhjynC4mZaV22slFI8PDmDllCBv3HmLO0h1+u64mjqOormvg9jmF9EyKYeZUnZE10I3p04UbT85idsEO/rN2r93hKIsVlx2m5GBNp2ym8jZ1eHfG9Enmic82crim3i/X1MRxFM98sQln6WEePH8E8VHBv6JYZ3DzaQMZ0j2BO95dxT5d+Cmk5Xn6N4J9ffFjJSLcfXY2JQdr+OvX/hkUqInjKOobDD/O7c1Jg/yzZrk6dlHhDp788SgOVNXyu/cDYnFIZZF8Zxk9k6Lpm6rLGYzt24VpI7rzwpfFlFRUW349TRxHcee0oTx8wQi7w1DtNLRHIr8+fRAfFe5m7spddoejLNDYaFhUXM6krK6dtn+judvPGkJ9YyNP/sf6QYGaOFqh/yiD089P7M/oPsn8/t+r2euH38CUf23Ye5Dyw7WdZpqRtsjsGseVEzN5a8l2Nuw5aOm1NHGokBTuCOPxi0ZSU9/AzHcKA2LVNOU73/VvaOL4nl+eOoC4qHAemr/O0uto4lAhq39aPHdMGcLCDaW8ZfP6Bcq38p1lZKbG0is5xu5QAkqXuEh+eeoAFm4o5b+brBsMa2niEJEpIrJBRIpE5I4Wtt8iImtFpFBEPheRvl7bGkRkhec116u8n4gs9pzzLRGJtLIOKrj9ZFImk7NS+dOHa9leXml3OMoH6hsaWezp31A/9JNJmWR0ieGBeessWyXTssQhIg7gWWAqkA1cKiLZzXZbDuQaY3KAOcAsr21VxphRntd0r/JHgCeNMQOAfcC1VtVBBb+wMOHRi0YSJsJv315Joy43G/TW7KrgYE29NlMdQXSEg9vOGsy63RW8t3ynJddoU+IQkTgRCfO8HyQi00UkopXDxgNFxphiY0wt8CYww3sHY8wCY0zTr4GLgIxW4hDgVNxJBuAfwHltqYPqvHolx3DPudks3lzOy99stjscdYx0/Ebrzs3pyciMJB7/dIMls0a39Y7jKyBaRHoBnwJXAn9v5ZhegHfD8g5P2ZFcC8z3+hwtIgUiskhEmpJDKrDfGNM0PLK1cyoFwIVjMzh9aDdmfbKBohJrnzhR1sovdjGoWzxpCVF2hxKwwsKEu6YNRYAtrsO+P38b9xPPncH5wHPGmIuAYb4KQkSuAHKBR72K+xpjcoHLgD+LSFY7z3m9J/EUlJb6d+ZIFXhEhAfPH05cpINbZq+kzo8Twinfqa1vZMnmcr3baIMJ/VNZeNspDLVggas2Jw4RmQRcDnzkKWttmtidQG+vzxmesuYnPh24G5hujKlpKjfG7PT8WQwsBEYDLiBZRJrm/2jxnJ7jXjTG5BpjctPSdOS3gvSEaB740QgKdxzg+YVOu8NRHbByx36q6hq0Y7yNIsOt6cZu61l/DdwJvGeMWSMi/YEFrRyzBBjoeQoqErgEmOu9g4iMBl7AnTRKvMq7iEiU531X4DhgrXE/jL8AuNCz61XA+22sg1JMG9GDGaN68vTnm1i90/8rp6ljk+90IQIT+6fYHUqn1qbEYYz50hgz3RjziKeTvMwYc3Mrx9QDNwGfAOuA2Z6kc5+IND0l9SgQD7zd7LHboUCBiKzEnSgeNsY0zZU9E7hFRIpw93n8re3VVQrumz6c1PhIbpm9guo6XW42mOQ5y8jukUhyrD6Fb6c2TfkqIm8ANwANuO8kEkXkKWPMo0c7zhgzD5jXrOwer/enH+G4PKDFSaI8TVfj2xK3Ui1Jio3gkQtyuPqVJTz52UbunDbU7pBUG1TXNbBs636umty39Z2VpdraVJVtjKnA/ejrfKAf7ierlApKJw9O59LxfXjx62KWbCm3OxzVBku37qO2obFTri8eaNqaOCI84zbOA+YaY+oAHUmlgtrdZw8lo0sMt85e6bcFcFTH5TtdOMKEcf20f8NubU0cLwBbgDjgK8/UIBVWBaWUP8RHhfP4RaPYvq/S8knh1LHLc5aRk5Gki6oFgLZ2jj9tjOlljJlm3LYCp1gcm1KWG98vheuO78dri7bx1UYd7xOoDtXUs3LHgU6/TGygaOuUI0ki8kTTgDoReRz33YdSQe/WMwczID2e2+cUcqCyzu5wVAuWbCmnodFo/0aAaGtT1cvAQeBiz6sCeMWqoJTyp+gIB09cPIQszsoAABl2SURBVJLSQzX84YM1doejWpDvdBHpCGNs3y52h6Joe+LIMsbc65mwsNgY80egv5WBKeVPORnJ3HTKAN5bvpOPV++2OxzVTJ6zjNF9komOaG3CCuUPbU0cVSJyfNMHETkOqLImJKXscdOpAxjRK4m731tN2aGa1g9QfnGgso41uyq0mSqAtDVx3AA8KyJbRGQL8Bfg55ZFpZQNIhxhPHHxSA7W1HPXu6t0udkAsWizC2N0mdhA0tanqlYaY0YCOUCOMWY07nUxlAopA7slcNuZg/l07V7eXWbNIjiqffKdLqIjwhjVO9nuUJRHu6ZONMZUeEaQA9xiQTxK2e6a4/sxPjOFP8xdw6792iJrt3yni3GZKZbN9Kra71i+CfFZFEoFEEeY8NhFI2kwhtvnFGqTlY1KD9awYe9BbaYKMMeSOPSnSYWsPqmx3H32UP5bVMZri7baHU6ntajYvUysdowHlqMmDhE5KCIVLbwOAj39FKNStrhsfB9OGpTGg/PWs6XM98tvqtblOV0kRIUzvKfvV7FTHXfUxGGMSTDGJLbwSjDG6IQxKqSJCI9ckEOEQ7j17ZU0NOpNtr8tKnYxoX8K4Q7t3wgk+m0odRTdk6K5b8Zwlm7dx4tfFdsdTqeya38Vm8sOM1HXFw84mjiUasWMUT2ZOrw7T362kfV7dFJof8l3av9GoNLEoVQrRIT7zxtOYkw4t7y1ktr6RrtD6hTyi110iY1gSPcEu0NRzWjiUKoNUuOjeOj8HNburuCZLzbZHU7IM8aQ73QxsX8qYWH65H+g0cShVBudkd2NC8dm8NxCJyu277c7nJC2rbySnfurdP2NAKWJQ6l2uOfcbLolRHHL7BVU1zXYHU7IaurfmKT9GwHJ0sQhIlNEZIOIFInIHS1sv0VE1opIoYh87lmSFhEZJSL5IrLGs+3HXsf8XUQ2i8gKz2uUlXVQyltidASPXjSS4tLDPPLxervDCVl5ThdpCVFkpel6cYHIssQhIg7gWWAqkA1cKiLZzXZbDuQaY3KAOcAsT3kl8BNjzDBgCvBnEfGe4ew2Y8woz2uFVXVQqiXHDejKVZP68so3W8hzltkdTsgxxpDndDE5KxUR7d8IRFbecYwHijwLP9UCbwIzvHcwxiwwxlR6Pi4CMjzlG40xmzzvdwElQJqFsSrVLndMHUq/rnHc9nYhB6t1uVlfcpYeouxQjfZvBDArE0cvYLvX5x2esiO5FpjfvFBExgORgNOr+AFPE9aTIhLV0slE5PqmNdJLS0vbH71SRxET6eCxi0ay+0AV93+4zu5wQkpeU/9Gf+3fCFQB0TkuIlcAucCjzcp7AK8CPzXGND08fycwBBgHpAAzWzqnMeZFY0yuMSY3LU1vVpTvje3bhZ+flMVbBdv5Yv1eu8MJGXlFLnolx9A7JcbuUNQRWJk4dgK9vT5neMq+R0ROB+4GphtjarzKE4GPgLuNMYuayo0xu41bDfAK7iYxpWzx69MHMqR7AjPfWcW+w7V2hxP0GhsNizZr/0agszJxLAEGikg/EYkELgHmeu8gIqOBF3AnjRKv8kjgPeCfxpg5zY7p4flTgPOA1RbWQamjigp38MTFo9hfWcvv39d/isdq3Z4K9lfWMXmA9m8EMssShzGmHrgJ+ARYB8w2xqwRkftEZLpnt0eBeOBtz6O1TYnlYuBE4OoWHrt9XURWAauArsD9VtVBqbbI7pnIr08fxIeFu/lg5S67wwlq+dq/ERQsnRrdGDMPmNes7B6v96cf4bjXgNeOsE3XOlcB5+cn9ueztXv5/furmdAvhfTEaLtDCkp5Thf9u8bRPUn//gJZQHSOKxXswh1hPH7xSKrrGpj5ji432xH1DY18u7lcl4kNApo4lPKRrLR4Zk4ZwoINpby1ZHvrB6jvWbXzAIdq6jVxBAFNHEr50FWTMpnUP5U/fbiW7eWVrR+gvtM0fkMXbgp8mjiU8qGwMOHRi3IQEX779koadbnZNltU7GJI9wS6xrc4plcFEE0cSvlYRpdY7jknm8Wby3klb4vd4QSFmvoGlmwp17uNIKGJQykLXJSbwWlD0pn18XqKSg7ZHU7AW7FtP9V1jTo/VZDQxKGUBUSEhy4YQWykg1tnr6C+QZebPZr8YhdhAhP0jiMoaOJQyiLpCdHcf94IVu44wHMLna0f0InlOV0M65lEUkyE3aGoNtDEoZSFzs7pwfSRPXn6802s3nnA7nACUlVtA8u37dNmqiCiiUMpi903YxgpcZHcOnslNfW63GxzBVvLqWswOn4jiGjiUMpiybGRPHJBDhv2HuSJzzbaHU7AyXe6CA8TxmWm2B2KaiNNHEr5wSlD0rl0fG9e/KqYgi3ldocTUPKcLkb2TiYuytKp85QPaeJQyk/uPjubjC4x3Pr2Sg7X1NsdTkA4WF3Hqp0HtH8jyGjiUMpP4qPCefTCkWwrr+Th+evtDicgLNlSTkOj9m8EG00cSvnRxP6pXHNcP15dtJWvN5XaHY7t8opcRIaHMaZPF7tDUe2giUMpP7vtrMEMSI/ntrcLOVBVZ3c4tspzuhjbpwvREQ67Q1HtoIlDKT+LjnDwxMUjKT1Uw73vr+60a3fsO1zLuj0V2r8RhDRxKGWDnIxkbj51IP9esYvHPt1gdzi2WLzZhTFo/0YQ0ufflLLJzacNYE9FFc8ucBIT4eCmUwfaHZJf5TldxEY6yMlItjsU1U6aOJSyiYhw/3kjqK5r5LFPNxId4eC6E/rbHZbf5DtdjMtMITJcGz6CjaXfmIhMEZENIlIkIne0sP0WEVkrIoUi8rmI9PXadpWIbPK8rvIqHysiqzznfFpExMo6KGUlR5jw6IU5TB3enfs/Wsfri7faHZJflBysZlPJIW2mClKWJQ4RcQDPAlOBbOBSEclutttyINcYkwPMAWZ5jk0B7gUmAOOBe0Wk6Xm954GfAQM9rylW1UEpfwh3hPHUJaM5dUg6v/v3at5ZusPukCyX71kmVjvGg5OVdxzjgSJjTLExphZ4E5jhvYMxZoExpmlh5kVAhuf9WcBnxphyY8w+4DNgioj0ABKNMYuM+1GUfwLnWVgHpfwiMjyM5y4fw+SsVG6bs5KPCnfbHZKl8p0uEqLDGdYzye5QVAdYmTh6Adu9Pu/wlB3JtcD8Vo7t5Xnf1nMqFTSiIxz89Se5jOnThV+9uZzP1+21OyTL5Be7mNg/FUeYtjQHo4DolRKRK4Bc4FEfnvN6ESkQkYLSUh2hq4JDbGQ4L/90HNk9E7nxtWUhObp8x75KtroqmaSr/QUtKxPHTqC31+cMT9n3iMjpwN3AdGNMTSvH7uT/m7OOeE4AY8yLxphcY0xuWlpahyuhlL8lRkfwz2vG0z8tjp/9s4BvN4fWbLrf9W8M0MQRrKxMHEuAgSLST0QigUuAud47iMho4AXcSaPEa9MnwJki0sXTKX4m8IkxZjdQISITPU9T/QR438I6KGWL5NhIXr12Aj2TY7jm70tYsX2/3SH5TH6xi9S4SAalJ9gdiuogyxKHMaYeuAl3ElgHzDbGrBGR+0Rkume3R4F44G0RWSEicz3HlgN/wp18lgD3ecoAfgG8BBQBTv6/X0SpkJKWEMUb100kJS6Sn/xtMWt3Vdgd0jEzxpDvdPdvhGn/RtCSzjBPTm5urikoKLA7DKU6ZHt5JRe/kE9tfSNv/XwiA4L4N/XNZYc55bGF3H/ecK6Y2Lf1A5StRGSpMSa3eXlAdI4rpY6sd0osr183ARHhsr8uZqvrsN0hdZiO3wgNmjiUCgL90+J5/boJ1DU0ctlfF7Nzf5XdIXVInrOMbolR9OsaZ3co6hho4lAqSAzunsCr106gorqOy/+6iJKKartDahdjDIuKXUzO6orOFBTcNHEoFUSG90ri7z8dT8nBGi5/aTGuQzWtHxQgNpUcouxQrc5PFQI0cSgVZMb27cJLV+WyrbySK//2LQcqg2MVwbyiMkD7N0KBJg6lgtDkrK68cOVYNpUc5KpXvuVQTb3dIbUqz+mid0oMGV1i7Q5FHSNNHEoFqZMHp/PMpWNYtfMA1/59CVW1DXaHdEQNjZ7+jf5d7Q5F+YAmDqWC2JTh3Xni4pF8u6Wc618toKY+MJPHut0VVFTX6zQjIUITh1JBbsaoXjxyfg5fbyrjpjeWU9fQaHdIP5DndPdv6MSGoUETh1Ih4OJxvfnj9GF8tnYvv3lrBQ2NgTUjRJ7TRVZaHOmJ0XaHonxA1xxXKkRcNTmTqroGHp6/nugIB7MuyAmI+aDqGhpZsrmc88dktL6zCgqaOJQKITeclEVVbQNPfb6JmAgH980YZvtgu8IdBzhc26DjN0KIJg6lQsyvTx9IVV0DL35VTEykgzunDrE1eeR7+jcmav9GyNDEoVSIERHunDqEqlpP8ohw8JszBtkWT36xi6E9EkmJi7QtBuVbmjiUCkEiwh+nD6O6ztNsFenghpOy/B5HdV0DBVv2cfkEnUI9lGjiUCpEhYUJD1+QQ3V9Iw/PX09MhIOrJmf6NYbl2/ZTU9+o04yEGE0cSoUwR5jwxMUjqa5r4N65a4iJcHDxuN5+u35+sYswgfH9U/x2TWU9HcehVIiLcITxl8tGc+KgNGa+W8j7K3b67dr5zjJGZCSTGB3ht2sq62niUKoTiAp38MIVYxmfmcIts1fy8eo9ll+zsrae5dv262jxEKSJQ6lOIibSwd+uHkdORhK//NcyFmwosfR6S7bso77RaP9GCNLEoVQnEh8Vzt9/Op5B3RK44dWl380hZYV8p4sIh5Cb2cWyayh7WJo4RGSKiGwQkSIRuaOF7SeKyDIRqReRC73KTxGRFV6vahE5z7Pt7yKy2WvbKCvroFSoSYqJ4NVrJ9AnJZbr/lHA0q3lllwn31nGqN7JxEbqMzihxrLEISIO4FlgKpANXCoi2c122wZcDbzhXWiMWWCMGWWMGQWcClQCn3rtclvTdmPMCqvqoFSoSomL5PXrJpCeEMXVLy9h1Y4DPj1/RXUdq3YeYFKWrr8Riqy84xgPFBljio0xtcCbwAzvHYwxW4wxhcDR5oG+EJhvjKm0LlSlOp/0xGhe/9lEEmMiuPLlxWzYc9Bn5/62uJxGo8vEhiorE0cvYLvX5x2esva6BPhXs7IHRKRQRJ4UkaiWDhKR60WkQEQKSktLO3BZpUJfr+QY3vjZBKLCw7j8pcUUlx7yyXnznC6iwsMY3SfZJ+dTgSWgO8dFpAcwAvjEq/hOYAgwDkgBZrZ0rDHmRWNMrjEmNy0tzfJYlQpWfVPjeP26CRhjuPylxWwvP/ab+zxnGbmZXYgKd/ggQhVorEwcOwHvIaoZnrL2uBh4zxhT11RgjNlt3GqAV3A3iSmljsGA9ARevXYClbUNXPbSInYfqOrwucoP17J+z0Ema/9GyLIycSwBBopIPxGJxN3kNLed57iUZs1UnrsQxD1P9HnAah/EqlSnl90zkX9cM559h+u4/KXFlB6s6dB5FhW7AJ1GPZRZljiMMfXATbibmdYBs40xa0TkPhGZDiAi40RkB3AR8IKIrGk6XkQycd+xfNns1K+LyCpgFdAVuN+qOijV2YzqnczLV49j1/4qrvzbYvYdrm33OfKcZcRFOsjJSLIgQhUIxJjAWpvYCrm5uaagoMDuMJQKGv/dVMY1/1jCkO4JvHbdhHbNNXXq4wvpmxLLKz/VVuRgJyJLjTG5zcsDunNcKWWP4wd25fnLx7B2VwXXvLKEytr6Nh23t6Ka4tLD2r8R4jRxKKVadNrQbjx1yWiWbdvHdf8ooLquodVj8p3u/g1dXzy0aeJQSh3R2Tk9eOyikeQXu7jxtaXU1h9trK67fyMpJoKhPRL9FKGygyYOpdRRnT8mg/vPG86CDaX86s3l1DccOXnkF7uY2D8FR5j4MULlb5o4lFKtunxCX35/TjbzV+/ht2+vpKHxhw/VbC+vZHt5la6/0QnotJVKqTa59vh+VNXW89inG4mJdPDgj0bgHk7l1tS/MXmAdoyHOk0cSqk2u+nUgVTVNfDsAidR4Q7uPTf7u+SRX+yia3wkA9PjbY5SWU0Th1KqXX575mAqaxt45ZstxEY6uH3KEIwx5DnLmNg/9Xt3ISo0aeJQSrWLiHDPOdlU1zXy3EInsZEOpo7owd6KGh2/0Ulo4lBKtZuI8MB5w6mpa+CxTzeycIN76QJdf6Nz0MShlOqQsDBh1oU5VNc3MG/VHnokRdM3NdbusJQfaOJQSnVYuCOMP/94NEkxaxiQHq/9G52EJg6l1DGJDA/jofNH2B2G8iMdAKiUUqpdNHEopZRqF00cSiml2kUTh1JKqXbRxKGUUqpdNHEopZRqF00cSiml2kUTh1JKqXYRY364IEuoEZFSYKvnYxJwoIX33p+9y7sCZccYQvPrdGS/lra1pexI9W16H8j1a6m8vfWDwK5ja2VtqW8g16+lcv03+sP3gVrHvsaYtB/sZYzpVC/gxZbee39utk+BL6/Z0f1a2taWsiPV16uuAVu/1urTlvoFeh1bK2tLfQO5fr74DkO9fsFQx+avzthU9cER3nt/bl7uy2t2dL+WtrWl7Ej19WUdrapfS+V21K895/P1d9jW+h6rUP8OQ71+7TlfR+r4PZ2iqepYiEiBMSbX7jisEur1g9Cvo9Yv+AVbHTvjHUd7vWh3ABYL9fpB6NdR6xf8gqqOesehlFKqXfSOQymlVLto4lBKKdUumjiUUkq1iyaOYyAifUTk3yLysojcYXc8viYiJ4jI/4rISyKSZ3c8viYiYSLygIg8IyJX2R2PFUTkZBH52vM9nmx3PFYQkTgRKRCRc+yOxQoiMtTz/c0RkRvtjgc6ceLw/GdfIiKrm5VPEZENIlLUhmQwAphjjLkGGG1ZsB3gi/oZY742xtwAfAj8w8p428tH398MIAOoA3ZYFWtH+aiOBjgERBNgdfRR/QBmArOtifLY+OjncJ3n5/Bi4Dgr422rTvtUlYiciPsH6p/GmOGeMgewETgD9w/ZEuBSwAE81OwU1wANwBzcP5yvGmNe8U/0rfNF/YwxJZ7jZgPXGmMO+in8Vvno+7sG2GeMeUFE5hhjLvRX/G3hozqWGWMaRaQb8IQx5nJ/xd8aH9VvJJCKOzGWGWM+9E/0beOrn0MRmQ7ciPv/mTf8Ff+RhNsdgF2MMV+JSGaz4vFAkTGmGEBE3gRmGGMeAn5wGywivwXu9ZxrDhAwicMX9fPs0wc4EEhJA3z2/e0Aaj0fG6yLtmN89R167AOirIizo3z0HZ4MxAHZQJWIzDPGNFoZd3v46js0xswF5orIR4AmjgDTC9ju9XkHMOEo+38M/EFELgO2WBiXr7S3fgDXEkAJsRXtrd+7wDMicgLwlZWB+VC76igi5wNnAcnAX6wNzSfaVT9jzN0AInI1nrsrS6PzjfZ+hycD5+NO/PMsjayNNHEcA2PMaiCgmjd8zRhzr90xWMUYU4k7MYYsY8y7uBNkSDPG/N3uGKxijFkILLQ5jO/ptJ3jR7AT6O31OcNTFiq0fsEv1OsY6vWDEKijJo7vWwIMFJF+IhIJXALMtTkmX9L6Bb9Qr2Oo1w9CoI6dNnGIyL+AfGCwiOwQkWuNMfXATcAnwDpgtjFmjZ1xdpTWL7jrB6Ffx1CvH4RuHTvt47hKKaU6ptPecSillOoYTRxKKaXaRROHUkqpdtHEoZRSql00cSillGoXTRxKKaXaRROH6tRE5JCfr+fXdU1EJFlEfuHPa6rQp4lDKR8SkaPO/2aMmeznayYDmjiUT2niUKoZEckSkY9FZKm4V88b4ik/V0QWi8hyEfmPZ40LROQPIvKqiHwDvOr5/LKILBSRYhG52evchzx/nuzZPkdE1ovI6yIinm3TPGVLReRpEfnBGhMicrWIzBWRL4DPRSReRD4XkWUiskpEZnh2fRjIEpEVIvKo59jbRGSJiBSKyB+t/LtUoUlnx1Xqh14EbjDGbBKRCcBzwKnAf4GJxhgjItcBtwO3eo7JBo43xlSJyB+AIcApQAKwQUSeN8bUNbvOaGAYsAv4BjhORAqAF4ATjTGbPVNWHMkYIMcYU+656/iRMaZCRLoCi0RkLnAHMNwYMwpARM4EBuJeE0Jwr/FwojEmWKaVVwFAE4dSXkQkHpgMvO25AYD/XwApA3hLRHoAkcBmr0PnGmOqvD5/ZIypAWpEpAToxg+Xbv3WGLPDc90VQCbu1eKKjTFN5/4XcP0Rwv3MGFPeFDrwoLhXnGvEveZDtxaOOdPzWu75HI87kWjiUG2miUOp7wsD9jf9ht7MM7iXX53rWVznD17bDjfbt8brfQMt/6y1ZZ+j8b7m5UAaMNYYUyciW3Avp9qcAA8ZY15o57WU+o72cSjlxRhTAWwWkYsAxG2kZ3MS/79uwlUWhbAB6O+13OiP23hcElDiSRqnAH095QdxN5c1+QS4xnNnhYj0EpH0Y45adSp6x6E6u1hxrz3e5Ancv70/LyK/AyKAN4GVuO8w3haRfcAXQD9fB+PpI/kF8LGIHMa9dkNbvA58ICKrgAJgved8LhH5RkRWA/ONMbeJyFAg39MUdwi4AijxdV1U6NJp1ZUKMCISb4w55HnK6llgkzHmSbvjUqqJNlUpFXh+5uksX4O7CUr7I1RA0TsOpZRS7aJ3HEoppdpFE4dSSql20cShlFKqXTRxKKWUahdNHEoppdpFE4dSSql2+T8mkGZmzJN/6wAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tWjcQmKMFNHq"
      },
      "source": [
        "pytorch_total_params = sum(p.numel() for p in transfer_model.parameters())"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kMk_-MStQ7od"
      },
      "source": [
        "## Differential Learning Rates"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DACsXawtQUnc"
      },
      "source": [
        "found_lr = 3e-4\n",
        "optimizer = optim.Adam([\n",
        "{ 'params': transfer_model.layer4.parameters(), 'lr': found_lr /3},\n",
        "{ 'params': transfer_model.layer3.parameters(), 'lr': found_lr /9},\n",
        "], lr=found_lr)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H8YmHLCFRRjx"
      },
      "source": [
        "unfreeze_layers = [transfer_model.layer3, transfer_model.layer4]\n",
        "for layer in unfreeze_layers:\n",
        "    for param in layer.parameters():\n",
        "        param.requires_grad = True"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NceGT2V8Rgph",
        "outputId": "1a037edd-d610-496f-9069-98a7e4477fb8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 301
        }
      },
      "source": [
        "train(transfer_model, optimizer,torch.nn.CrossEntropyLoss(), train_data_loader,val_data_loader, epochs=8, device=device)"
      ],
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "GPU Memory:    1675MiB / 16280MiB Utilization:     8%  \n",
            "Epoch: 0, Training Loss: 0.00, Validation Loss: 0.00, accuracy = 0.90, time: 9.671537637710571\n",
            "GPU Memory:    1675MiB / 16280MiB Utilization:     9%  \n",
            "Epoch: 1, Training Loss: 0.00, Validation Loss: 0.01, accuracy = 0.84, time: 9.642489671707153\n",
            "GPU Memory:    1675MiB / 16280MiB Utilization:     9%  \n",
            "Epoch: 2, Training Loss: 0.00, Validation Loss: 0.00, accuracy = 0.93, time: 9.645704507827759\n",
            "GPU Memory:    1675MiB / 16280MiB Utilization:     9%  \n",
            "Epoch: 3, Training Loss: 0.00, Validation Loss: 0.00, accuracy = 0.89, time: 9.672334909439087\n",
            "GPU Memory:    1675MiB / 16280MiB Utilization:     9%  \n",
            "Epoch: 4, Training Loss: 0.00, Validation Loss: 0.01, accuracy = 0.87, time: 9.743113279342651\n",
            "GPU Memory:    1675MiB / 16280MiB Utilization:     9%  \n",
            "Epoch: 5, Training Loss: 0.00, Validation Loss: 0.01, accuracy = 0.83, time: 9.652405023574829\n",
            "GPU Memory:    1675MiB / 16280MiB Utilization:     9%  \n",
            "Epoch: 6, Training Loss: 0.00, Validation Loss: 0.01, accuracy = 0.83, time: 9.681614398956299\n",
            "GPU Memory:    1675MiB / 16280MiB Utilization:     9%  \n",
            "Epoch: 7, Training Loss: 0.00, Validation Loss: 0.01, accuracy = 0.88, time: 9.6589834690094\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}