LA-Transformer Training.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import print_function\n",
    "\n",
    "import os\n",
    "import time\n",
    "import random\n",
    "import zipfile\n",
    "from itertools import chain\n",
    "\n",
    "import timm\n",
    "import numpy as np\n",
    "from PIL import Image\n",
    "from tqdm.notebook import tqdm\n",
    "from collections import OrderedDict\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.nn import init\n",
    "import torch.optim as optim\n",
    "from torchvision import models\n",
    "import torch.nn.functional as F\n",
    "from torch.autograd import Variable\n",
    "from torch.optim.lr_scheduler import StepLR\n",
    "from torchvision import datasets, transforms\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "\n",
    "from LATransformer.model import ClassBlock, LATransformer\n",
    "from LATransformer.utils import save_network, update_summary\n",
    "\n",
    "os.environ['CUDA_VISIBLE_DEVICES']='1'\n",
    "device = \"cuda\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Set Config Parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "num_epochs = 30\n",
    "lr = 3e-4\n",
    "gamma = 0.7\n",
    "unfreeze_after=2\n",
    "lr_decay=.8\n",
    "lmbd = 8"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "transform_train_list = [\n",
    "    transforms.Resize((224,224), interpolation=3),\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]\n",
    "transform_val_list = [\n",
    "    transforms.Resize(size=(224,224),interpolation=3), #Image.BICUBIC\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ]\n",
    "data_transforms = {\n",
    "'train': transforms.Compose( transform_train_list ),\n",
    "'val': transforms.Compose(transform_val_list),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "751\n"
     ]
    }
   ],
   "source": [
    "image_datasets = {}\n",
    "data_dir = \"data/Market-Pytorch/Market/\"\n",
    "\n",
    "image_datasets['train'] = datasets.ImageFolder(os.path.join(data_dir, 'train'),\n",
    "                                          data_transforms['train'])\n",
    "image_datasets['val'] = datasets.ImageFolder(os.path.join(data_dir, 'val'),\n",
    "                                          data_transforms['val'])\n",
    "train_loader = DataLoader(dataset = image_datasets['train'], batch_size=batch_size, shuffle=True )\n",
    "valid_loader = DataLoader(dataset = image_datasets['val'], batch_size=batch_size, shuffle=True)\n",
    "# dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size=opt.batchsize,\n",
    "#                                              shuffle=True, num_workers=8, pin_memory=True) # 8 workers may work faster\n",
    "#               for x in ['train', 'val']}\n",
    "# dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val']}\n",
    "class_names = image_datasets['train'].classes\n",
    "print(len(class_names))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "VisionTransformer(\n",
       "  (patch_embed): PatchEmbed(\n",
       "    (proj): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16))\n",
       "  )\n",
       "  (pos_drop): Dropout(p=0.0, inplace=False)\n",
       "  (blocks): ModuleList(\n",
       "    (0): Block(\n",
       "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (attn): Attention(\n",
       "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "      (drop_path): Identity()\n",
       "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (mlp): Mlp(\n",
       "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        (act): GELU()\n",
       "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "        (drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "    )\n",
       "    (1): Block(\n",
       "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (attn): Attention(\n",
       "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "      (drop_path): Identity()\n",
       "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (mlp): Mlp(\n",
       "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        (act): GELU()\n",
       "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "        (drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "    )\n",
       "    (2): Block(\n",
       "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (attn): Attention(\n",
       "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "      (drop_path): Identity()\n",
       "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (mlp): Mlp(\n",
       "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        (act): GELU()\n",
       "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "        (drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "    )\n",
       "    (3): Block(\n",
       "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (attn): Attention(\n",
       "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "      (drop_path): Identity()\n",
       "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (mlp): Mlp(\n",
       "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        (act): GELU()\n",
       "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "        (drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "    )\n",
       "    (4): Block(\n",
       "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (attn): Attention(\n",
       "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "      (drop_path): Identity()\n",
       "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (mlp): Mlp(\n",
       "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        (act): GELU()\n",
       "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "        (drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "    )\n",
       "    (5): Block(\n",
       "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (attn): Attention(\n",
       "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "      (drop_path): Identity()\n",
       "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (mlp): Mlp(\n",
       "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        (act): GELU()\n",
       "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "        (drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "    )\n",
       "    (6): Block(\n",
       "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (attn): Attention(\n",
       "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "      (drop_path): Identity()\n",
       "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (mlp): Mlp(\n",
       "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        (act): GELU()\n",
       "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "        (drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "    )\n",
       "    (7): Block(\n",
       "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (attn): Attention(\n",
       "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "      (drop_path): Identity()\n",
       "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (mlp): Mlp(\n",
       "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        (act): GELU()\n",
       "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "        (drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "    )\n",
       "    (8): Block(\n",
       "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (attn): Attention(\n",
       "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "      (drop_path): Identity()\n",
       "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (mlp): Mlp(\n",
       "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        (act): GELU()\n",
       "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "        (drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "    )\n",
       "    (9): Block(\n",
       "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (attn): Attention(\n",
       "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "      (drop_path): Identity()\n",
       "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (mlp): Mlp(\n",
       "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        (act): GELU()\n",
       "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "        (drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "    )\n",
       "    (10): Block(\n",
       "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (attn): Attention(\n",
       "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "      (drop_path): Identity()\n",
       "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (mlp): Mlp(\n",
       "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        (act): GELU()\n",
       "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "        (drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "    )\n",
       "    (11): Block(\n",
       "      (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (attn): Attention(\n",
       "        (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
       "        (attn_drop): Dropout(p=0.0, inplace=False)\n",
       "        (proj): Linear(in_features=768, out_features=768, bias=True)\n",
       "        (proj_drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "      (drop_path): Identity()\n",
       "      (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "      (mlp): Mlp(\n",
       "        (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
       "        (act): GELU()\n",
       "        (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
       "        (drop): Dropout(p=0.0, inplace=False)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
       "  (head): Linear(in_features=768, out_features=751, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load pre-trained ViT\n",
    "vit_base = timm.create_model('vit_base_patch16_224', pretrained=True, num_classes=751)\n",
    "vit_base= vit_base.to(device)\n",
    "vit_base.eval()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "###  Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AverageMeter:\n",
    "    \"\"\"Computes and stores the average and current value\"\"\"\n",
    "    def __init__(self):\n",
    "        self.reset()\n",
    "\n",
    "    def reset(self):\n",
    "        self.val = 0\n",
    "        self.avg = 0\n",
    "        self.sum = 0\n",
    "        self.count = 0\n",
    "\n",
    "    def update(self, val, n=1):\n",
    "        self.val = val\n",
    "        self.sum += val * n\n",
    "        self.count += n\n",
    "        self.avg = self.sum / self.count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def validate(model, loader, loss_fn):\n",
    "    batch_time_m = AverageMeter()\n",
    "    losses_m = AverageMeter()\n",
    "    top1_m = AverageMeter()\n",
    "    top5_m = AverageMeter()\n",
    "\n",
    "    model.eval()\n",
    "    epoch_accuracy = 0\n",
    "    epoch_loss = 0\n",
    "    end = time.time()\n",
    "    last_idx = len(loader) - 1\n",
    "    \n",
    "    running_loss = 0.0\n",
    "    running_corrects = 0.0\n",
    "\n",
    "    with torch.no_grad():\n",
    "        for input, target in tqdm(loader):\n",
    "\n",
    "            input, target = input.to(device), target.to(device)\n",
    "            \n",
    "            output = model(input)\n",
    "            \n",
    "            score = 0.0\n",
    "            sm = nn.Softmax(dim=1)\n",
    "            for k, v in output.items():\n",
    "                score += sm(output[k])\n",
    "            _, preds = torch.max(score.data, 1)\n",
    "\n",
    "            loss = 0.0\n",
    "            for k,v in output.items():\n",
    "                loss += loss_fn(output[k], target)\n",
    "\n",
    "\n",
    "            batch_time_m.update(time.time() - end)\n",
    "            acc = (preds == target.data).float().mean()\n",
    "            epoch_loss += loss/len(loader)\n",
    "            epoch_accuracy += acc / len(loader)\n",
    "            \n",
    "            print(f\"Epoch : {epoch+1} - val_loss : {epoch_loss:.4f} - val_acc: {epoch_accuracy:.4f}\", end=\"\\r\")\n",
    "    print()    \n",
    "    metrics = OrderedDict([('val_loss', epoch_loss.data.item()), (\"val_accuracy\", epoch_accuracy.data.item())])\n",
    "\n",
    "\n",
    "    return metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_one_epoch(\n",
    "        epoch, model, loader, optimizer, loss_fn,\n",
    "        lr_scheduler=None, saver=None, output_dir='', \n",
    "        loss_scaler=None, model_ema=None, mixup_fn=None):\n",
    "\n",
    " \n",
    "\n",
    "    \n",
    "    batch_time_m = AverageMeter()\n",
    "    data_time_m = AverageMeter()\n",
    "    losses_m = AverageMeter()\n",
    "\n",
    "    model.train()\n",
    "    epoch_accuracy = 0\n",
    "    epoch_loss = 0\n",
    "    end = time.time()\n",
    "    last_idx = len(loader) - 1\n",
    "    num_updates = epoch * len(loader)\n",
    "    running_loss = 0.0\n",
    "    running_corrects = 0.0\n",
    "\n",
    "    for data, target in tqdm(loader):\n",
    "        data, target = data.to(device), target.to(device)\n",
    "\n",
    "            \n",
    "        data_time_m.update(time.time() - end)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        score = 0.0\n",
    "        sm = nn.Softmax(dim=1)\n",
    "        for k, v in output.items():\n",
    "            score += sm(output[k])\n",
    "        _, preds = torch.max(score.data, 1)\n",
    "        \n",
    "        loss = 0.0\n",
    "        for k,v in output.items():\n",
    "            loss += loss_fn(output[k], target)\n",
    "        loss.backward()\n",
    "\n",
    "        optimizer.step()\n",
    "\n",
    "        batch_time_m.update(time.time() - end)\n",
    "        \n",
    "#         print(preds, target.data)\n",
    "        acc = (preds == target.data).float().mean()\n",
    "        \n",
    "#         print(acc)\n",
    "        epoch_loss += loss/len(loader)\n",
    "        epoch_accuracy += acc / len(loader)\n",
    "#         if acc:\n",
    "#             print(acc, epreds, target.data)\n",
    "        print(\n",
    "    f\"Epoch : {epoch+1} - loss : {epoch_loss:.4f} - acc: {epoch_accuracy:.4f}\"\n",
    ", end=\"\\r\")\n",
    "\n",
    "    print()\n",
    "\n",
    "    return OrderedDict([('train_loss', epoch_loss.data.item()), (\"train_accuracy\", epoch_accuracy.data.item())])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def freeze_all_blocks(model):\n",
    "    frozen_blocks = 12\n",
    "    for block in model.model.blocks[:frozen_blocks]:\n",
    "        for param in block.parameters():\n",
    "            param.requires_grad=False\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def unfreeze_blocks(model, amount= 1):\n",
    "    \n",
    "    for block in model.model.blocks[11-amount:]:\n",
    "        for param in block.parameters():\n",
    "            param.requires_grad=True\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training Loop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LATransformer(\n",
      "  (model): VisionTransformer(\n",
      "    (patch_embed): PatchEmbed(\n",
      "      (proj): Conv2d(3, 768, kernel_size=(16, 16), stride=(16, 16))\n",
      "    )\n",
      "    (pos_drop): Dropout(p=0.0, inplace=False)\n",
      "    (blocks): ModuleList(\n",
      "      (0): Block(\n",
      "        (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (attn): Attention(\n",
      "          (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "          (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "        (drop_path): Identity()\n",
      "        (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (mlp): Mlp(\n",
      "          (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "          (act): GELU()\n",
      "          (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "          (drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (1): Block(\n",
      "        (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (attn): Attention(\n",
      "          (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "          (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "        (drop_path): Identity()\n",
      "        (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (mlp): Mlp(\n",
      "          (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "          (act): GELU()\n",
      "          (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "          (drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (2): Block(\n",
      "        (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (attn): Attention(\n",
      "          (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "          (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "        (drop_path): Identity()\n",
      "        (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (mlp): Mlp(\n",
      "          (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "          (act): GELU()\n",
      "          (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "          (drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (3): Block(\n",
      "        (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (attn): Attention(\n",
      "          (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "          (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "        (drop_path): Identity()\n",
      "        (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (mlp): Mlp(\n",
      "          (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "          (act): GELU()\n",
      "          (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "          (drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (4): Block(\n",
      "        (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (attn): Attention(\n",
      "          (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "          (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "        (drop_path): Identity()\n",
      "        (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (mlp): Mlp(\n",
      "          (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "          (act): GELU()\n",
      "          (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "          (drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (5): Block(\n",
      "        (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (attn): Attention(\n",
      "          (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "          (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "        (drop_path): Identity()\n",
      "        (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (mlp): Mlp(\n",
      "          (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "          (act): GELU()\n",
      "          (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "          (drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (6): Block(\n",
      "        (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (attn): Attention(\n",
      "          (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "          (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "        (drop_path): Identity()\n",
      "        (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (mlp): Mlp(\n",
      "          (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "          (act): GELU()\n",
      "          (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "          (drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (7): Block(\n",
      "        (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (attn): Attention(\n",
      "          (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "          (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "        (drop_path): Identity()\n",
      "        (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (mlp): Mlp(\n",
      "          (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "          (act): GELU()\n",
      "          (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "          (drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (8): Block(\n",
      "        (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (attn): Attention(\n",
      "          (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "          (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "        (drop_path): Identity()\n",
      "        (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (mlp): Mlp(\n",
      "          (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "          (act): GELU()\n",
      "          (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "          (drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (9): Block(\n",
      "        (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (attn): Attention(\n",
      "          (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "          (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "        (drop_path): Identity()\n",
      "        (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (mlp): Mlp(\n",
      "          (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "          (act): GELU()\n",
      "          (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "          (drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (10): Block(\n",
      "        (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (attn): Attention(\n",
      "          (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "          (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "        (drop_path): Identity()\n",
      "        (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (mlp): Mlp(\n",
      "          (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "          (act): GELU()\n",
      "          (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "          (drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "      (11): Block(\n",
      "        (norm1): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (attn): Attention(\n",
      "          (qkv): Linear(in_features=768, out_features=2304, bias=True)\n",
      "          (attn_drop): Dropout(p=0.0, inplace=False)\n",
      "          (proj): Linear(in_features=768, out_features=768, bias=True)\n",
      "          (proj_drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "        (drop_path): Identity()\n",
      "        (norm2): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "        (mlp): Mlp(\n",
      "          (fc1): Linear(in_features=768, out_features=3072, bias=True)\n",
      "          (act): GELU()\n",
      "          (fc2): Linear(in_features=3072, out_features=768, bias=True)\n",
      "          (drop): Dropout(p=0.0, inplace=False)\n",
      "        )\n",
      "      )\n",
      "    )\n",
      "    (norm): LayerNorm((768,), eps=1e-06, elementwise_affine=True)\n",
      "    (head): Linear(in_features=768, out_features=751, bias=True)\n",
      "  )\n",
      "  (avgpool): AdaptiveAvgPool2d(output_size=(14, 768))\n",
      "  (dropout): Dropout(p=0.5, inplace=False)\n",
      "  (classifier0): ClassBlock(\n",
      "    (add_block): Sequential(\n",
      "      (0): Linear(in_features=768, out_features=256, bias=True)\n",
      "      (1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): Dropout(p=0.5, inplace=False)\n",
      "    )\n",
      "    (classifier): Sequential(\n",
      "      (0): Linear(in_features=256, out_features=751, bias=True)\n",
      "    )\n",
      "  )\n",
      "  (classifier1): ClassBlock(\n",
      "    (add_block): Sequential(\n",
      "      (0): Linear(in_features=768, out_features=256, bias=True)\n",
      "      (1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): Dropout(p=0.5, inplace=False)\n",
      "    )\n",
      "    (classifier): Sequential(\n",
      "      (0): Linear(in_features=256, out_features=751, bias=True)\n",
      "    )\n",
      "  )\n",
      "  (classifier2): ClassBlock(\n",
      "    (add_block): Sequential(\n",
      "      (0): Linear(in_features=768, out_features=256, bias=True)\n",
      "      (1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): Dropout(p=0.5, inplace=False)\n",
      "    )\n",
      "    (classifier): Sequential(\n",
      "      (0): Linear(in_features=256, out_features=751, bias=True)\n",
      "    )\n",
      "  )\n",
      "  (classifier3): ClassBlock(\n",
      "    (add_block): Sequential(\n",
      "      (0): Linear(in_features=768, out_features=256, bias=True)\n",
      "      (1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): Dropout(p=0.5, inplace=False)\n",
      "    )\n",
      "    (classifier): Sequential(\n",
      "      (0): Linear(in_features=256, out_features=751, bias=True)\n",
      "    )\n",
      "  )\n",
      "  (classifier4): ClassBlock(\n",
      "    (add_block): Sequential(\n",
      "      (0): Linear(in_features=768, out_features=256, bias=True)\n",
      "      (1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): Dropout(p=0.5, inplace=False)\n",
      "    )\n",
      "    (classifier): Sequential(\n",
      "      (0): Linear(in_features=256, out_features=751, bias=True)\n",
      "    )\n",
      "  )\n",
      "  (classifier5): ClassBlock(\n",
      "    (add_block): Sequential(\n",
      "      (0): Linear(in_features=768, out_features=256, bias=True)\n",
      "      (1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): Dropout(p=0.5, inplace=False)\n",
      "    )\n",
      "    (classifier): Sequential(\n",
      "      (0): Linear(in_features=256, out_features=751, bias=True)\n",
      "    )\n",
      "  )\n",
      "  (classifier6): ClassBlock(\n",
      "    (add_block): Sequential(\n",
      "      (0): Linear(in_features=768, out_features=256, bias=True)\n",
      "      (1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): Dropout(p=0.5, inplace=False)\n",
      "    )\n",
      "    (classifier): Sequential(\n",
      "      (0): Linear(in_features=256, out_features=751, bias=True)\n",
      "    )\n",
      "  )\n",
      "  (classifier7): ClassBlock(\n",
      "    (add_block): Sequential(\n",
      "      (0): Linear(in_features=768, out_features=256, bias=True)\n",
      "      (1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): Dropout(p=0.5, inplace=False)\n",
      "    )\n",
      "    (classifier): Sequential(\n",
      "      (0): Linear(in_features=256, out_features=751, bias=True)\n",
      "    )\n",
      "  )\n",
      "  (classifier8): ClassBlock(\n",
      "    (add_block): Sequential(\n",
      "      (0): Linear(in_features=768, out_features=256, bias=True)\n",
      "      (1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): Dropout(p=0.5, inplace=False)\n",
      "    )\n",
      "    (classifier): Sequential(\n",
      "      (0): Linear(in_features=256, out_features=751, bias=True)\n",
      "    )\n",
      "  )\n",
      "  (classifier9): ClassBlock(\n",
      "    (add_block): Sequential(\n",
      "      (0): Linear(in_features=768, out_features=256, bias=True)\n",
      "      (1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): Dropout(p=0.5, inplace=False)\n",
      "    )\n",
      "    (classifier): Sequential(\n",
      "      (0): Linear(in_features=256, out_features=751, bias=True)\n",
      "    )\n",
      "  )\n",
      "  (classifier10): ClassBlock(\n",
      "    (add_block): Sequential(\n",
      "      (0): Linear(in_features=768, out_features=256, bias=True)\n",
      "      (1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): Dropout(p=0.5, inplace=False)\n",
      "    )\n",
      "    (classifier): Sequential(\n",
      "      (0): Linear(in_features=256, out_features=751, bias=True)\n",
      "    )\n",
      "  )\n",
      "  (classifier11): ClassBlock(\n",
      "    (add_block): Sequential(\n",
      "      (0): Linear(in_features=768, out_features=256, bias=True)\n",
      "      (1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): Dropout(p=0.5, inplace=False)\n",
      "    )\n",
      "    (classifier): Sequential(\n",
      "      (0): Linear(in_features=256, out_features=751, bias=True)\n",
      "    )\n",
      "  )\n",
      "  (classifier12): ClassBlock(\n",
      "    (add_block): Sequential(\n",
      "      (0): Linear(in_features=768, out_features=256, bias=True)\n",
      "      (1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): Dropout(p=0.5, inplace=False)\n",
      "    )\n",
      "    (classifier): Sequential(\n",
      "      (0): Linear(in_features=256, out_features=751, bias=True)\n",
      "    )\n",
      "  )\n",
      "  (classifier13): ClassBlock(\n",
      "    (add_block): Sequential(\n",
      "      (0): Linear(in_features=768, out_features=256, bias=True)\n",
      "      (1): BatchNorm1d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
      "      (2): Dropout(p=0.5, inplace=False)\n",
      "    )\n",
      "    (classifier): Sequential(\n",
      "      (0): Linear(in_features=256, out_features=751, bias=True)\n",
      "    )\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "# Create LA Transformer\n",
    "model = LATransformer(vit_base, lmbd).to(device)\n",
    "print(model.eval())\n",
    "\n",
    "# loss function\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# optimizer\n",
    "optimizer = optim.Adam(model.parameters(),weight_decay=5e-4, lr=lr)\n",
    "\n",
    "# scheduler\n",
    "scheduler = StepLR(optimizer, step_size=1, gamma=gamma)\n",
    "freeze_all_blocks(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "training...\n",
      "Unfrozen Blocks: 1, Current lr: 0.00023999999999999998, Trainable Params: 20962817\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f767a92c4eac4258b98b3c9676782209",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 1 - loss : 82.7351 - acc: 0.0880\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1baf1a35386941478d54f8ee048653bb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 1 - val_loss : 77.1901 - val_acc: 0.0497\n",
      "\n",
      "SAVED!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c349d24f58ee4194b48328d96f9cb2b9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 2 - loss : 59.0334 - acc: 0.2364\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "030f496ac22f4d08b2fe5b2f13df306c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 2 - val_loss : 58.8111 - val_acc: 0.1918\n",
      "\n",
      "SAVED!\n",
      "Unfrozen Blocks: 2, Current lr: 0.000192, Trainable Params: 28050689\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c7284d3fd841487e996202add386669c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 3 - loss : 41.1694 - acc: 0.4632\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d77e806e33104a8bbfb85a04483c94f4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 3 - val_loss : 47.2650 - val_acc: 0.3353\n",
      "\n",
      "SAVED!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e082fac2ed5e4952938c6fd3fb5d4f1a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 4 - loss : 28.3517 - acc: 0.6674\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "57552c299cc54398aa4416c3689fb7b5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 4 - val_loss : 33.9487 - val_acc: 0.5391\n",
      "\n",
      "SAVED!\n",
      "Unfrozen Blocks: 3, Current lr: 0.00015360000000000002, Trainable Params: 35138561\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "66d3571123f849ccb1d30a186c4fee9d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 5 - loss : 18.7140 - acc: 0.8141\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df0b76c5867f4efea8c5f27c4b6b416a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 5 - val_loss : 25.3060 - val_acc: 0.6617\n",
      "\n",
      "SAVED!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "32b7d1fa8e024640849763fbbbb5df8d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 6 - loss : 12.2253 - acc: 0.9050\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7530e23081ff40838c5c340324fb0bee",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 6 - val_loss : 19.0367 - val_acc: 0.7506\n",
      "\n",
      "SAVED!\n",
      "Unfrozen Blocks: 4, Current lr: 0.00012288000000000002, Trainable Params: 42226433\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5d2820792b3d4c078a4687f08bd4f92d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 7 - loss : 8.0031 - acc: 0.9542\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "166716159015465d8430b22cbb0a937f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 7 - val_loss : 14.0309 - val_acc: 0.8325\n",
      "\n",
      "SAVED!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e43fe49292c04b0f83dacbc610d49e58",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 8 - loss : 5.4122 - acc: 0.9771\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af46f3dff0a548f98b97b3868ce2a783",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 8 - val_loss : 11.0224 - val_acc: 0.8602\n",
      "\n",
      "SAVED!\n",
      "Unfrozen Blocks: 5, Current lr: 9.830400000000001e-05, Trainable Params: 49314305\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af2d255d73484d968d636523528a596c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 9 - loss : 3.7149 - acc: 0.9906\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1b1001230f5842bfae334384b74040ae",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 9 - val_loss : 8.5832 - val_acc: 0.8944\n",
      "\n",
      "SAVED!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e7691744d4142ee940783af59c753e5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 10 - loss : 2.7142 - acc: 0.9950\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ceb48891601843519efddf0984ef2e18",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 10 - val_loss : 7.6481 - val_acc: 0.9033\n",
      "\n",
      "SAVED!\n",
      "Unfrozen Blocks: 6, Current lr: 7.864320000000001e-05, Trainable Params: 56402177\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a5eff7b69b8422b9da1bfc8aa601193",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 11 - loss : 2.0092 - acc: 0.9965\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a691c2bbe71f434390741bb0071a3e42",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 11 - val_loss : 6.7372 - val_acc: 0.9137\n",
      "\n",
      "SAVED!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6b239464e5bf46908b445345dc0eb523",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 12 - loss : 1.5912 - acc: 0.9977\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9828497a70f455188333f870c7eb5ff",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 12 - val_loss : 6.0404 - val_acc: 0.9189\n",
      "\n",
      "SAVED!\n",
      "Unfrozen Blocks: 7, Current lr: 6.291456000000001e-05, Trainable Params: 63490049\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e46c51355950451a98a10a10d833c441",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 13 - loss : 1.3100 - acc: 0.9984\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ca1050b552cc41289c74862a47398dab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 13 - val_loss : 5.8097 - val_acc: 0.9230\n",
      "\n",
      "SAVED!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "86c1b41c6a4c4f1da8e89f803c675709",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 14 - loss : 1.0894 - acc: 0.9991\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0b727d9790b64dab897630c639b05a6a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 14 - val_loss : 5.1302 - val_acc: 0.9321\n",
      "\n",
      "SAVED!\n",
      "Unfrozen Blocks: 8, Current lr: 5.0331648000000016e-05, Trainable Params: 70577921\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ddf2a4c6d42d4298b84bca0cda2f78df",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 15 - loss : 0.9347 - acc: 0.9992\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c69748f472b4d1d9c8c0c2ac0b893d5",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 15 - val_loss : 5.5233 - val_acc: 0.9217\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be86fea29a08418fa1fed8cb91aee99c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 16 - loss : 0.9086 - acc: 0.9996\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1616dcc149cd46ac904968b474f6cfe3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 16 - val_loss : 4.4655 - val_acc: 0.9362\n",
      "\n",
      "SAVED!\n",
      "Unfrozen Blocks: 9, Current lr: 4.026531840000002e-05, Trainable Params: 77665793\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3fb8fcd15c544ca4a23474ee01c35e91",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 17 - loss : 0.7159 - acc: 0.9999\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1af37990febc45fc9a8cb3e6921de68b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 17 - val_loss : 4.2927 - val_acc: 0.9414\n",
      "\n",
      "SAVED!\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "277af575763647ddb9be0c49cbd7fb4f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 18 - loss : 0.6362 - acc: 0.9998\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "01e3ff670c0c4613b93381f273ee20a0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 18 - val_loss : 4.2925 - val_acc: 0.9453\n",
      "\n",
      "SAVED!\n",
      "Unfrozen Blocks: 10, Current lr: 3.221225472000002e-05, Trainable Params: 84753665\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b98ea55bef3a4cf9acb4b5dee8d2d911",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 19 - loss : 0.6389 - acc: 0.9997\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f7ec952d59a84c618b9fd97cc14a6923",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 19 - val_loss : 4.5622 - val_acc: 0.9319\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "44f3727d7f5b43b9ae6fd96fd1f66032",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 20 - loss : 0.5667 - acc: 0.9998\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30bea7568e694900876835ca696d7753",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 20 - val_loss : 4.6590 - val_acc: 0.9254\n",
      "\n",
      "Unfrozen Blocks: 11, Current lr: 2.5769803776000016e-05, Trainable Params: 91841537\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "178dec4505884c3f9dd0716e2e22bef2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 21 - loss : 0.5401 - acc: 0.9998\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d43c07645e94afaa6e63d6142ddf2f1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 21 - val_loss : 3.8805 - val_acc: 0.9401\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "47d5ad2ffad44135866f5d204f692c65",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 22 - loss : 0.6303 - acc: 0.9991\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d353c31d9edc454083f96fdca3e3baa4",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 22 - val_loss : 4.4941 - val_acc: 0.9375\n",
      "\n",
      "Unfrozen Blocks: 12, Current lr: 2.0615843020800013e-05, Trainable Params: 91841537\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a68e3064ab547538cc3d505c0122c3a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 23 - loss : 0.5186 - acc: 0.9997\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "83722d7b0b7d406ca2bab64af3881225",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 23 - val_loss : 4.0348 - val_acc: 0.9435\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b535b57cc6443ef80194950ffc73705",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 24 - loss : 0.4421 - acc: 0.9999\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "15720d615707427b979d560257ae9c13",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 24 - val_loss : 3.6783 - val_acc: 0.9464\n",
      "\n",
      "SAVED!\n",
      "Unfrozen Blocks: 13, Current lr: 1.649267441664001e-05, Trainable Params: 91841537\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d647ef490d5e4397885957e56c46c4db",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 25 - loss : 0.4184 - acc: 1.0000\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c5c37e9cea84f249edea413cad517b3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 25 - val_loss : 3.9668 - val_acc: 0.9425\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa957165fe3d460c85a58fe67701842e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 26 - loss : 0.4113 - acc: 1.0000\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df1f36d9d95749a0bc03bba147d001ad",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 26 - val_loss : 3.9590 - val_acc: 0.9398\n",
      "\n",
      "Unfrozen Blocks: 14, Current lr: 1.319413953331201e-05, Trainable Params: 91841537\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b7a42e21863d4061a7616e79d0d62aeb",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 27 - loss : 0.3976 - acc: 1.0000\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "27e232c16fbd46fe9c6d093b9f2683bc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 27 - val_loss : 3.8370 - val_acc: 0.9414\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "36e13b0ec23445d0ad79294a9772e49e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 28 - loss : 0.3917 - acc: 1.0000\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ae58768db8a4bc98c8d2586bb563499",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 28 - val_loss : 3.8097 - val_acc: 0.9422\n",
      "\n",
      "Unfrozen Blocks: 15, Current lr: 1.0555311626649608e-05, Trainable Params: 91841537\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ead97d19cb7640fa878a0f98ecc59db2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=381.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 29 - loss : 0.3875 - acc: 1.0000\n",
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "64008a2d743a4aa88363b78b330385ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(HTML(value=''), FloatProgress(value=0.0, max=24.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch : 29 - val_loss : 0.5044 - val_acc: 0.1576\r"
     ]
    }
   ],
   "source": [
    "best_acc = 0.0\n",
    "y_loss = {} # loss history\n",
    "y_loss['train'] = []\n",
    "y_loss['val'] = []\n",
    "y_err = {}\n",
    "y_err['train'] = []\n",
    "y_err['val'] = []\n",
    "print(\"training...\")\n",
    "output_dir = \"\"\n",
    "best_acc = 0\n",
    "name = \"la_with_lmbd_{}\".format(lmbd)\n",
    "\n",
    "try:\n",
    "    os.mkdir(\"model/\" + name)\n",
    "\n",
    "except:\n",
    "    pass\n",
    "output_dir = \"model/\" + name\n",
    "unfrozen_blocks = 0\n",
    "\n",
    "for epoch in range(num_epochs):\n",
    "\n",
    "    if epoch%unfreeze_after==0:\n",
    "        unfrozen_blocks += 1\n",
    "        model = unfreeze_blocks(model, unfrozen_blocks)\n",
    "        optimizer.param_groups[0]['lr'] *= lr_decay \n",
    "        trainable_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "        print(\"Unfrozen Blocks: {}, Current lr: {}, Trainable Params: {}\".format(unfrozen_blocks, \n",
    "                                                                             optimizer.param_groups[0]['lr'], \n",
    "                                                                             trainable_params))\n",
    "\n",
    "    train_metrics = train_one_epoch(\n",
    "        epoch, model, train_loader, optimizer, criterion,\n",
    "        lr_scheduler=None, saver=None)\n",
    "\n",
    "    eval_metrics = validate(model, valid_loader, criterion)\n",
    "\n",
    "\n",
    "    # update summary\n",
    "    update_summary(epoch, train_metrics, eval_metrics, os.path.join(output_dir, 'summary.csv'),\n",
    "                   write_header=True)\n",
    "\n",
    "    # deep copy the model\n",
    "    last_model_wts = model.state_dict()\n",
    "    if eval_metrics['val_accuracy'] > best_acc:\n",
    "        best_acc = eval_metrics['val_accuracy']\n",
    "        save_network(model, epoch,name)\n",
    "        print(\"SAVED!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}