train_pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.2.2'"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import matplotlib\n",
    "from matplotlib import pyplot as plt\n",
    "matplotlib.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('1.0.0', '0.2.1')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.__version__, torchvision.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "code_folding": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean': [0.485, 0.456, 0.406], 'std': [0.229, 0.224, 0.225]}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imagenet_stats = {\n",
    "    'mean': [0.485, 0.456, 0.406],\n",
    "    'std': [0.229, 0.224, 0.225]\n",
    "}\n",
    "imagenet_stats"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import os\n",
    "import copy\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "import torch\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torchvision import models, transforms\n",
    "import torch.optim as optim\n",
    "import torch.nn as nn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "with open('cat_to_name.json') as f:\n",
    "    cat_to_name = json.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_path = Path('flower_data')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "train_transform = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    transforms.RandomAffine(degrees=45, shear=(10, 10), translate=(.0, .10), scale=(1.0, 1.2)),\n",
    "#         transforms.ColorJitter(brightness=0.3),\n",
    "#         transforms.ColorJitter(brightness=0.1, contrast=0.1, saturation=0.1, hue=0.1),\n",
    "    transforms.Resize(256),\n",
    "    transforms.RandomCrop(224),\n",
    "#         transforms.RandomResizedCrop(),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(**imagenet_stats)\n",
    "])\n",
    "\n",
    "train_transform_basic = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    transforms.Resize(256),\n",
    "    transforms.RandomCrop(224),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(**imagenet_stats)\n",
    "])\n",
    "\n",
    "valid_transform = transforms.Compose([\n",
    "    transforms.Resize(256),\n",
    "    transforms.CenterCrop(224),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize(**imagenet_stats)\n",
    "])\n",
    "\n",
    "valid_transform_tta = transforms.Compose([\n",
    "    transforms.Resize(256),\n",
    "    transforms.TenCrop(224),\n",
    "    transforms.Lambda(lambda crops: torch.stack([\n",
    "        transforms.Normalize(**imagenet_stats)(transforms.ToTensor()(crop))\n",
    "        for crop in crops]))\n",
    "])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fastai tricks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "from flowers.fastai_tricks import *\n",
    "from flowers.schedules import *\n",
    "from flowers.adamw import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Create, Save, Load Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from flowers.model import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "def train_loop(dataloaders, model, criterion, optimizer, scheduler, epochs):\n",
    "    since = time.time()\n",
    "\n",
    "    best_model_wts = None\n",
    "    best_acc = 0.0\n",
    "    best_loss = 100\n",
    "\n",
    "    for epoch in range(epochs):\n",
    "        print(f'Epoch {epoch+1}/{epochs}')\n",
    "        print('-' * 10)\n",
    "\n",
    "        for phase in dataloaders.keys():\n",
    "            if phase == 'train':\n",
    "                model.train()\n",
    "                if hasattr(scheduler, 'step'):\n",
    "                    scheduler.step()\n",
    "            else:\n",
    "                model.eval()\n",
    "            running_loss, running_corrects = 0.0, 0.0\n",
    "\n",
    "            for inputs, labels in dataloaders[phase]:\n",
    "                print('.', end='')\n",
    "                inputs = inputs.to(device)\n",
    "                labels = labels.to(device)\n",
    "\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    if phase == 'train':\n",
    "                        outputs = model(inputs)\n",
    "                        _, preds = torch.max(outputs, 1)\n",
    "                        loss = criterion(outputs, labels)\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "                    else:\n",
    "                        ns, ncrops, c, h, w = inputs.size()\n",
    "                        outputs = model(inputs.view(-1, c, h, w))\n",
    "                        outputs_avg = outputs.view(ns, ncrops, -1).mean(1)\n",
    "                        _, preds = torch.max(outputs_avg, 1)\n",
    "                        loss = criterion(outputs_avg, labels)\n",
    "\n",
    "                    running_loss += loss.item() * inputs.size(0)\n",
    "                    running_corrects += torch.sum(preds == labels.data).item()\n",
    "\n",
    "                if phase == 'train' and hasattr(scheduler, 'batch_step'):\n",
    "                    scheduler.batch_step()\n",
    "            print()\n",
    "\n",
    "            epoch_loss = running_loss / len(dataloaders[phase].dataset)\n",
    "            epoch_acc = running_corrects / len(dataloaders[phase].dataset)\n",
    "            print(f'{phase} Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}')\n",
    "            if phase == 'valid' and epoch_acc > best_acc:\n",
    "                best_acc = epoch_acc\n",
    "                best_loss = epoch_loss\n",
    "                best_model_wts = copy.deepcopy(model.state_dict())\n",
    "\n",
    "        print()\n",
    "\n",
    "    time_elapsed = time.time() - since\n",
    "    print('Training complete in {:.0f}m {:.0f}s'.format(\n",
    "        time_elapsed // 60, time_elapsed % 60))\n",
    "    print('Best val Loss: {:4f}, Best val Acc: {:4f}'.format(best_loss, best_acc))\n",
    "\n",
    "    if best_model_wts:\n",
    "        model.load_state_dict(best_model_wts)\n",
    "    return model, best_loss, best_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "datasets = {\n",
    "    'train': torchvision.datasets.ImageFolder(root=str(data_path/'train'), transform=train_transform),\n",
    "    'valid': torchvision.datasets.ImageFolder(root=str(data_path/'valid'), transform=valid_transform_tta)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 48\n",
    "num_workers = 4\n",
    "dataloaders = {\n",
    "    'train': torch.utils.data.DataLoader(datasets['train'], batch_size=batch_size,\n",
    "                                         shuffle=True, num_workers=num_workers),\n",
    "    'valid': torch.utils.data.DataLoader(datasets['valid'], batch_size=batch_size,\n",
    "                                         shuffle=False, num_workers=num_workers)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# input_iter = iter(dataloaders['train'])\n",
    "# plot_inputs(3, 5, input_iter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# model = load_model(arch, 'resnet34-stage2.pth')\n",
    "# freeze(model[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "arch = models.resnet34"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 3\n",
    "lr_max = 3e-3\n",
    "weight_decay = 5e-2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "# optimizer = optim.SGD(model.parameters(), lr=0.1, momentum=0.9)\n",
    "# scheduler = CyclicLR(optimizer, base_lr=1e-3, max_lr=1e-1,\n",
    "#                      step_size=step_size, mode='triangular2', debug=False)\n",
    "# scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.1)\n",
    "# scheduler = WarmRestartsLR(optimizer, lr_min=1e-4, lr_max=1e-2, T=50, T_mult=2)\n",
    "model = create_model(arch, device)\n",
    "optimizer = AdamW(filter(lambda p: p.requires_grad, model.parameters()),\n",
    "                  lr=0.01, betas=(0.9, 0.99), weight_decay=weight_decay)\n",
    "scheduler = OneCycleScheduler(optimizer, lr_max=lr_max,\n",
    "                              batches=len(dataloaders['train']), epochs=epochs, debug=False)\n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/3\n",
      "----------\n",
      ".........................................................................................................................................\n",
      "train Loss: 3.1069 Acc: 0.3385\n",
      "..................\n",
      "valid Loss: 0.6397 Acc: 0.8778\n",
      "\n",
      "Epoch 2/3\n",
      "----------\n",
      ".........................................................................................................................................\n",
      "train Loss: 0.8014 Acc: 0.7908\n",
      "..................\n",
      "valid Loss: 0.3073 Acc: 0.9364\n",
      "\n",
      "Epoch 3/3\n",
      "----------\n",
      ".........................................................................................................................................\n",
      "train Loss: 0.5096 Acc: 0.8675\n",
      "..................\n",
      "valid Loss: 0.2626 Acc: 0.9499\n",
      "\n",
      "Training complete in 1m 33s\n",
      "Best val Loss: 0.262621, Best val Acc: 0.949878\n"
     ]
    }
   ],
   "source": [
    "model, best_loss, best_acc = train_loop(dataloaders, model, criterion, optimizer, scheduler,\n",
    "                                        epochs=epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_model(model, 'resnet-stage1.pth')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fine-tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 64\n",
    "num_workers = 4\n",
    "dataloaders = {\n",
    "    'train': torch.utils.data.DataLoader(datasets['train'], batch_size=batch_size,\n",
    "                                         shuffle=True, num_workers=num_workers),\n",
    "    'valid': torch.utils.data.DataLoader(datasets['valid'], batch_size=batch_size,\n",
    "                                         shuffle=False, num_workers=num_workers)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 5\n",
    "weight_decay = 2e-4\n",
    "lr_max = 7e-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = load_model(arch, device, 'resnet-stage1.pth')\n",
    "unfreeze(model)\n",
    "optimizer = AdamW(filter(lambda p: p.requires_grad, model.parameters()),\n",
    "                  lr=0.01, betas=(0.9, 0.99), weight_decay=weight_decay)\n",
    "scheduler = OneCycleScheduler(optimizer, lr_max=lr_max,\n",
    "                              batches=len(dataloaders['train']), epochs=epochs, debug=False)\n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/5\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.3996 Acc: 0.8985\n",
      ".............\n",
      "valid Loss: 0.1891 Acc: 0.9621\n",
      "\n",
      "Epoch 2/5\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.2487 Acc: 0.9380\n",
      ".............\n",
      "valid Loss: 0.1565 Acc: 0.9633\n",
      "\n",
      "Epoch 3/5\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.1679 Acc: 0.9582\n",
      ".............\n",
      "valid Loss: 0.1141 Acc: 0.9817\n",
      "\n",
      "Epoch 4/5\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.1239 Acc: 0.9722\n",
      ".............\n",
      "valid Loss: 0.0942 Acc: 0.9853\n",
      "\n",
      "Epoch 5/5\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.0929 Acc: 0.9818\n",
      ".............\n",
      "valid Loss: 0.0908 Acc: 0.9890\n",
      "\n",
      "Training complete in 2m 21s\n",
      "Best val Loss: 0.090757, Best val Acc: 0.988998\n"
     ]
    }
   ],
   "source": [
    "model, best_loss, best_acc = train_loop(dataloaders, model, criterion, optimizer, scheduler,\n",
    "                                        epochs=epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "save_model(model, 'resnet-stage2.pth', class_to_idx=datasets['train'].class_to_idx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \"Polishing\" with simple augs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = {\n",
    "    'train': torchvision.datasets.ImageFolder(root=str(data_path/'train'), transform=train_transform_basic),\n",
    "    'valid': torchvision.datasets.ImageFolder(root=str(data_path/'valid'), transform=valid_transform_tta)\n",
    "}\n",
    "batch_size = 64\n",
    "num_workers = 4\n",
    "dataloaders = {\n",
    "    'train': torch.utils.data.DataLoader(datasets['train'], batch_size=batch_size,\n",
    "                                         shuffle=True, num_workers=num_workers),\n",
    "    'valid': torch.utils.data.DataLoader(datasets['valid'], batch_size=batch_size,\n",
    "                                         shuffle=False, num_workers=num_workers)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "epochs = 15\n",
    "weight_decay = 1e-3\n",
    "lr_max = 1e-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = load_model(models.resnet101, 'resnet-stage2.pth')\n",
    "unfreeze(model)\n",
    "# optimizer = AdamW(filter(lambda p: p.requires_grad, model.parameters()),\n",
    "#                   lr=0.01, betas=(0.9, 0.99), weight_decay=weight_decay)\n",
    "optimizer = optim.SGD(model.parameters(), lr=lr_max, momentum=0.9, weight_decay=weight_decay)\n",
    "scheduler = optim.lr_scheduler.StepLR(optimizer, step_size=5, gamma=0.3)\n",
    "# scheduler = OneCycleScheduler(optimizer, lr_max=lr_max,\n",
    "#                               batches=len(dataloaders['train']), epochs=epochs, debug=False)\n",
    "criterion = nn.CrossEntropyLoss()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/15\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.0071 Acc: 0.9988\n",
      ".............\n",
      "valid Loss: 0.0534 Acc: 0.9914\n",
      "\n",
      "Epoch 2/15\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.0075 Acc: 0.9980\n",
      ".............\n",
      "valid Loss: 0.0566 Acc: 0.9890\n",
      "\n",
      "Epoch 3/15\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.0084 Acc: 0.9979\n",
      ".............\n",
      "valid Loss: 0.0572 Acc: 0.9890\n",
      "\n",
      "Epoch 4/15\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.0062 Acc: 0.9989\n",
      ".............\n",
      "valid Loss: 0.0549 Acc: 0.9914\n",
      "\n",
      "Epoch 5/15\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.0070 Acc: 0.9986\n",
      ".............\n",
      "valid Loss: 0.0555 Acc: 0.9902\n",
      "\n",
      "Epoch 6/15\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.0058 Acc: 0.9992\n",
      ".............\n",
      "valid Loss: 0.0541 Acc: 0.9914\n",
      "\n",
      "Epoch 7/15\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.0090 Acc: 0.9983\n",
      ".............\n",
      "valid Loss: 0.0535 Acc: 0.9914\n",
      "\n",
      "Epoch 8/15\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.0064 Acc: 0.9989\n",
      ".............\n",
      "valid Loss: 0.0561 Acc: 0.9902\n",
      "\n",
      "Epoch 9/15\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.0065 Acc: 0.9986\n",
      ".............\n",
      "valid Loss: 0.0551 Acc: 0.9902\n",
      "\n",
      "Epoch 10/15\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.0083 Acc: 0.9985\n",
      ".............\n",
      "valid Loss: 0.0567 Acc: 0.9914\n",
      "\n",
      "Epoch 11/15\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.0063 Acc: 0.9991\n",
      ".............\n",
      "valid Loss: 0.0553 Acc: 0.9890\n",
      "\n",
      "Epoch 12/15\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.0074 Acc: 0.9983\n",
      ".............\n",
      "valid Loss: 0.0550 Acc: 0.9914\n",
      "\n",
      "Epoch 13/15\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.0074 Acc: 0.9991\n",
      ".............\n",
      "valid Loss: 0.0561 Acc: 0.9902\n",
      "\n",
      "Epoch 14/15\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.0064 Acc: 0.9986\n",
      ".............\n",
      "valid Loss: 0.0554 Acc: 0.9902\n",
      "\n",
      "Epoch 15/15\n",
      "----------\n",
      ".......................................................................................................\n",
      "train Loss: 0.0059 Acc: 0.9989\n",
      ".............\n",
      "valid Loss: 0.0546 Acc: 0.9914\n",
      "\n",
      "Training complete in 12m 27s\n",
      "Best val Loss: 0.053407, Best val Acc: 0.991443\n"
     ]
    }
   ],
   "source": [
    "model, best_loss, best_acc = train_loop(dataloaders, model, criterion, optimizer, scheduler,\n",
    "                                        epochs=epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Final Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "datasets = {\n",
    "    'valid': torchvision.datasets.ImageFolder(root=str(data_path/'valid'), transform=valid_transform)\n",
    "}\n",
    "batch_size = 64\n",
    "num_workers = 4\n",
    "dataloaders = {\n",
    "    'valid': torch.utils.data.DataLoader(datasets['valid'], batch_size=batch_size,\n",
    "                                         shuffle=False, num_workers=num_workers)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      ".............\n",
      "Valid Loss: 0.0711 Acc: 0.9866\n"
     ]
    }
   ],
   "source": [
    "running_loss, running_corrects = 0.0, 0.0\n",
    "for inputs, labels in dataloaders['valid']:\n",
    "    print('.', end='')\n",
    "    inputs = inputs.to(device)\n",
    "    labels = labels.to(device)\n",
    "\n",
    "    optimizer.zero_grad()\n",
    "\n",
    "    with torch.set_grad_enabled(False):\n",
    "        if len(inputs.size()) > 4:\n",
    "            ns, ncrops, c, h, w = inputs.size()\n",
    "            outputs = model(inputs.view(-1, c, h, w))\n",
    "            outputs = outputs.view(ns, ncrops, -1).mean(1)\n",
    "            _, preds = torch.max(outputs, 1)\n",
    "        else:\n",
    "            outputs = model(inputs)\n",
    "            _, preds = torch.max(outputs, 1)\n",
    "        loss = criterion(outputs, labels)\n",
    "\n",
    "        running_loss += loss.item() * inputs.size(0)\n",
    "        running_corrects += torch.sum(preds == labels.data).item()\n",
    "\n",
    "epoch_loss = running_loss / len(dataloaders['valid'].dataset)\n",
    "epoch_acc = running_corrects / len(dataloaders['valid'].dataset)\n",
    "print(f'\\nValid Loss: {epoch_loss:.4f} Acc: {epoch_acc:.4f}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Hyperopt Experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import time\n",
    "import random\n",
    "from hyperopt import fmin, tpe, hp, STATUS_OK, STATUS_FAIL, Trials"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "datasets = {\n",
    "    'train': torchvision.datasets.ImageFolder(root=str(data_path/'train'), transform=train_transform),\n",
    "    'valid': torchvision.datasets.ImageFolder(root=str(data_path/'valid'), transform=valid_transform_tta)\n",
    "}\n",
    "\n",
    "def objective(params):\n",
    "    try:\n",
    "        print(params)\n",
    "        start = time.time()\n",
    "        batch_size = int(params['batch_size'])\n",
    "        dataloaders = {\n",
    "            'train': torch.utils.data.DataLoader(datasets['train'], batch_size=batch_size,\n",
    "                                                 shuffle=True, num_workers=4),\n",
    "            'valid': torch.utils.data.DataLoader(datasets['valid'], batch_size=batch_size,\n",
    "                                                 shuffle=False, num_workers=4)\n",
    "        }\n",
    "        epochs = 15\n",
    "#         model = create_model(params['arch'])\n",
    "        model = load_model(arch, 'resnet-stage1.pth')\n",
    "        unfreeze(model)\n",
    "        optimizer = AdamW(filter(lambda p: p.requires_grad, model.parameters()),\n",
    "                          lr=0.01, betas=(0.9, 0.99), weight_decay=params['weight_decay'])\n",
    "        scheduler = OneCycleScheduler(optimizer, lr_max=params['lr_max'],\n",
    "                                      batches=len(dataloaders['train']), epochs=epochs, debug=False)\n",
    "        criterion = nn.CrossEntropyLoss()\n",
    "        model, best_loss, best_acc = train_loop(dataloaders, model, criterion, optimizer, scheduler,\n",
    "                                                epochs=epochs)\n",
    "        res = {\n",
    "            'loss': best_loss,\n",
    "            'acc': best_acc,\n",
    "            'status': STATUS_OK,\n",
    "            'time': time.time() - start\n",
    "        }\n",
    "    except KeyboardInterrupt as e:\n",
    "        raise\n",
    "    except BaseException as e:\n",
    "        print(e)\n",
    "        res = {\n",
    "            'loss': 100,\n",
    "            'acc': 0,\n",
    "            'status': STATUS_FAIL,\n",
    "            'time': time.time() - start\n",
    "        }\n",
    "    finally:\n",
    "        if model:\n",
    "            model.to('cpu')\n",
    "        torch.cuda.empty_cache()\n",
    "        print(res)\n",
    "        params.update(res)\n",
    "        return params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "space = {\n",
    "    'batch_size': hp.choice('batch_size', np.arange(16, 256, 16, dtype=int)),\n",
    "    'lr_max': hp.loguniform('lr_max', np.log(1e-5), np.log(1e-2)),\n",
    "    'weight_decay': hp.loguniform('weight_decay', np.log(1e-6), np.log(1)),\n",
    "#     'arch': hp.choice('arch', [models.resnet34, models.resnet50, models.resnet101]),\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "trials = Trials()\n",
    "best = fmin(objective,\n",
    "            space=space,\n",
    "            algo=tpe.suggest,\n",
    "            max_evals=50,\n",
    "            trials=trials)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "trials_df = pd.DataFrame(trials.results)\n",
    "trials_df.to_csv('trials-stage2.csv', sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "# trials_df = pd.read_csv('trials-stage1.csv', sep='\\t')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# trials_df.sort_values(by='status')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acc</th>\n",
       "      <th>batch_size</th>\n",
       "      <th>loss</th>\n",
       "      <th>lr_max</th>\n",
       "      <th>status</th>\n",
       "      <th>time</th>\n",
       "      <th>weight_decay</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.995110</td>\n",
       "      <td>48</td>\n",
       "      <td>0.056657</td>\n",
       "      <td>0.000038</td>\n",
       "      <td>ok</td>\n",
       "      <td>804.644610</td>\n",
       "      <td>0.000001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>0.995110</td>\n",
       "      <td>64</td>\n",
       "      <td>0.050929</td>\n",
       "      <td>0.000073</td>\n",
       "      <td>ok</td>\n",
       "      <td>785.925107</td>\n",
       "      <td>0.000272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.993888</td>\n",
       "      <td>64</td>\n",
       "      <td>0.057410</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>ok</td>\n",
       "      <td>783.253139</td>\n",
       "      <td>0.459626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.992665</td>\n",
       "      <td>48</td>\n",
       "      <td>0.058264</td>\n",
       "      <td>0.000044</td>\n",
       "      <td>ok</td>\n",
       "      <td>803.678368</td>\n",
       "      <td>0.000044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.992665</td>\n",
       "      <td>64</td>\n",
       "      <td>0.054839</td>\n",
       "      <td>0.000074</td>\n",
       "      <td>ok</td>\n",
       "      <td>785.247967</td>\n",
       "      <td>0.000212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>0.992665</td>\n",
       "      <td>32</td>\n",
       "      <td>0.055512</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>ok</td>\n",
       "      <td>853.277575</td>\n",
       "      <td>0.000018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>0.991443</td>\n",
       "      <td>48</td>\n",
       "      <td>0.062161</td>\n",
       "      <td>0.000136</td>\n",
       "      <td>ok</td>\n",
       "      <td>803.593891</td>\n",
       "      <td>0.000194</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.991443</td>\n",
       "      <td>48</td>\n",
       "      <td>0.055415</td>\n",
       "      <td>0.000041</td>\n",
       "      <td>ok</td>\n",
       "      <td>807.265188</td>\n",
       "      <td>0.972279</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>0.990220</td>\n",
       "      <td>64</td>\n",
       "      <td>0.057395</td>\n",
       "      <td>0.000078</td>\n",
       "      <td>ok</td>\n",
       "      <td>785.528910</td>\n",
       "      <td>0.003478</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.988998</td>\n",
       "      <td>48</td>\n",
       "      <td>0.059515</td>\n",
       "      <td>0.000014</td>\n",
       "      <td>ok</td>\n",
       "      <td>803.917024</td>\n",
       "      <td>0.148601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>0.988998</td>\n",
       "      <td>48</td>\n",
       "      <td>0.076942</td>\n",
       "      <td>0.000418</td>\n",
       "      <td>ok</td>\n",
       "      <td>803.142842</td>\n",
       "      <td>0.008809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>0.988998</td>\n",
       "      <td>64</td>\n",
       "      <td>0.067474</td>\n",
       "      <td>0.000246</td>\n",
       "      <td>ok</td>\n",
       "      <td>783.014548</td>\n",
       "      <td>0.000315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.988998</td>\n",
       "      <td>16</td>\n",
       "      <td>0.084892</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>ok</td>\n",
       "      <td>995.561854</td>\n",
       "      <td>0.000007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>0.987775</td>\n",
       "      <td>32</td>\n",
       "      <td>0.055611</td>\n",
       "      <td>0.000018</td>\n",
       "      <td>ok</td>\n",
       "      <td>855.304644</td>\n",
       "      <td>0.000061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.987775</td>\n",
       "      <td>48</td>\n",
       "      <td>0.065198</td>\n",
       "      <td>0.000044</td>\n",
       "      <td>ok</td>\n",
       "      <td>804.524059</td>\n",
       "      <td>0.000023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>0.987775</td>\n",
       "      <td>32</td>\n",
       "      <td>0.064593</td>\n",
       "      <td>0.000162</td>\n",
       "      <td>ok</td>\n",
       "      <td>854.470828</td>\n",
       "      <td>0.000016</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>0.987775</td>\n",
       "      <td>16</td>\n",
       "      <td>0.074814</td>\n",
       "      <td>0.000221</td>\n",
       "      <td>ok</td>\n",
       "      <td>995.380198</td>\n",
       "      <td>0.004952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>0.986553</td>\n",
       "      <td>64</td>\n",
       "      <td>0.094182</td>\n",
       "      <td>0.000597</td>\n",
       "      <td>ok</td>\n",
       "      <td>782.951058</td>\n",
       "      <td>0.000977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>0.986553</td>\n",
       "      <td>64</td>\n",
       "      <td>0.065511</td>\n",
       "      <td>0.000119</td>\n",
       "      <td>ok</td>\n",
       "      <td>784.878294</td>\n",
       "      <td>0.000119</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.986553</td>\n",
       "      <td>32</td>\n",
       "      <td>0.079202</td>\n",
       "      <td>0.000363</td>\n",
       "      <td>ok</td>\n",
       "      <td>854.213687</td>\n",
       "      <td>0.900786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>0.985330</td>\n",
       "      <td>32</td>\n",
       "      <td>0.095480</td>\n",
       "      <td>0.000822</td>\n",
       "      <td>ok</td>\n",
       "      <td>854.505060</td>\n",
       "      <td>0.000003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.977995</td>\n",
       "      <td>48</td>\n",
       "      <td>0.126334</td>\n",
       "      <td>0.003020</td>\n",
       "      <td>ok</td>\n",
       "      <td>804.406476</td>\n",
       "      <td>0.000001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.834963</td>\n",
       "      <td>16</td>\n",
       "      <td>0.633972</td>\n",
       "      <td>0.008461</td>\n",
       "      <td>ok</td>\n",
       "      <td>992.794618</td>\n",
       "      <td>0.012741</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>240</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000068</td>\n",
       "      <td>fail</td>\n",
       "      <td>7.866671</td>\n",
       "      <td>0.000604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>208</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000070</td>\n",
       "      <td>fail</td>\n",
       "      <td>7.131501</td>\n",
       "      <td>0.716590</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>176</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000010</td>\n",
       "      <td>fail</td>\n",
       "      <td>6.850298</td>\n",
       "      <td>0.000003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>46</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>80</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.001228</td>\n",
       "      <td>fail</td>\n",
       "      <td>43.893693</td>\n",
       "      <td>0.000242</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>128</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000191</td>\n",
       "      <td>fail</td>\n",
       "      <td>5.403538</td>\n",
       "      <td>0.001101</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>47</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>64</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.001651</td>\n",
       "      <td>fail</td>\n",
       "      <td>44.035583</td>\n",
       "      <td>0.000058</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>240</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.001122</td>\n",
       "      <td>fail</td>\n",
       "      <td>7.783724</td>\n",
       "      <td>0.034320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>208</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000050</td>\n",
       "      <td>fail</td>\n",
       "      <td>7.902718</td>\n",
       "      <td>0.008249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>112</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000554</td>\n",
       "      <td>fail</td>\n",
       "      <td>5.101038</td>\n",
       "      <td>0.362556</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>80</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000152</td>\n",
       "      <td>fail</td>\n",
       "      <td>43.764013</td>\n",
       "      <td>0.000003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>128</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.003554</td>\n",
       "      <td>fail</td>\n",
       "      <td>5.278311</td>\n",
       "      <td>0.000006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>192</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000010</td>\n",
       "      <td>fail</td>\n",
       "      <td>6.627046</td>\n",
       "      <td>0.002517</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>176</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.001376</td>\n",
       "      <td>fail</td>\n",
       "      <td>6.419624</td>\n",
       "      <td>0.000096</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>224</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.001027</td>\n",
       "      <td>fail</td>\n",
       "      <td>7.632890</td>\n",
       "      <td>0.000129</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>160</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.008257</td>\n",
       "      <td>fail</td>\n",
       "      <td>6.762618</td>\n",
       "      <td>0.002875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>160</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.006028</td>\n",
       "      <td>fail</td>\n",
       "      <td>6.529726</td>\n",
       "      <td>0.000007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>144</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.002154</td>\n",
       "      <td>fail</td>\n",
       "      <td>6.290627</td>\n",
       "      <td>0.032825</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>80</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.002671</td>\n",
       "      <td>fail</td>\n",
       "      <td>43.701720</td>\n",
       "      <td>0.000378</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>224</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000020</td>\n",
       "      <td>fail</td>\n",
       "      <td>7.364373</td>\n",
       "      <td>0.036031</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>224</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000101</td>\n",
       "      <td>fail</td>\n",
       "      <td>8.485760</td>\n",
       "      <td>0.000001</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>192</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.004112</td>\n",
       "      <td>fail</td>\n",
       "      <td>7.348812</td>\n",
       "      <td>0.131728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>144</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000247</td>\n",
       "      <td>fail</td>\n",
       "      <td>5.717525</td>\n",
       "      <td>0.056678</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>224</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000014</td>\n",
       "      <td>fail</td>\n",
       "      <td>7.456245</td>\n",
       "      <td>0.031626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>192</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.007551</td>\n",
       "      <td>fail</td>\n",
       "      <td>7.672545</td>\n",
       "      <td>0.002126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>96</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000030</td>\n",
       "      <td>fail</td>\n",
       "      <td>42.985958</td>\n",
       "      <td>0.000022</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>192</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000097</td>\n",
       "      <td>fail</td>\n",
       "      <td>7.043807</td>\n",
       "      <td>0.000436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>49</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>160</td>\n",
       "      <td>100.000000</td>\n",
       "      <td>0.000061</td>\n",
       "      <td>fail</td>\n",
       "      <td>6.613679</td>\n",
       "      <td>0.001203</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         acc  batch_size        loss    lr_max status        time  \\\n",
       "21  0.995110          48    0.056657  0.000038     ok  804.644610   \n",
       "41  0.995110          64    0.050929  0.000073     ok  785.925107   \n",
       "26  0.993888          64    0.057410  0.000025     ok  783.253139   \n",
       "23  0.992665          48    0.058264  0.000044     ok  803.678368   \n",
       "40  0.992665          64    0.054839  0.000074     ok  785.247967   \n",
       "36  0.992665          32    0.055512  0.000026     ok  853.277575   \n",
       "31  0.991443          48    0.062161  0.000136     ok  803.593891   \n",
       "24  0.991443          48    0.055415  0.000041     ok  807.265188   \n",
       "44  0.990220          64    0.057395  0.000078     ok  785.528910   \n",
       "30  0.988998          48    0.059515  0.000014     ok  803.917024   \n",
       "35  0.988998          48    0.076942  0.000418     ok  803.142842   \n",
       "42  0.988998          64    0.067474  0.000246     ok  783.014548   \n",
       "11  0.988998          16    0.084892  0.000103     ok  995.561854   \n",
       "37  0.987775          32    0.055611  0.000018     ok  855.304644   \n",
       "22  0.987775          48    0.065198  0.000044     ok  804.524059   \n",
       "39  0.987775          32    0.064593  0.000162     ok  854.470828   \n",
       "48  0.987775          16    0.074814  0.000221     ok  995.380198   \n",
       "43  0.986553          64    0.094182  0.000597     ok  782.951058   \n",
       "45  0.986553          64    0.065511  0.000119     ok  784.878294   \n",
       "25  0.986553          32    0.079202  0.000363     ok  854.213687   \n",
       "38  0.985330          32    0.095480  0.000822     ok  854.505060   \n",
       "6   0.977995          48    0.126334  0.003020     ok  804.406476   \n",
       "7   0.834963          16    0.633972  0.008461     ok  992.794618   \n",
       "29  0.000000         240  100.000000  0.000068   fail    7.866671   \n",
       "34  0.000000         208  100.000000  0.000070   fail    7.131501   \n",
       "33  0.000000         176  100.000000  0.000010   fail    6.850298   \n",
       "46  0.000000          80  100.000000  0.001228   fail   43.893693   \n",
       "32  0.000000         128  100.000000  0.000191   fail    5.403538   \n",
       "47  0.000000          64  100.000000  0.001651   fail   44.035583   \n",
       "0   0.000000         240  100.000000  0.001122   fail    7.783724   \n",
       "28  0.000000         208  100.000000  0.000050   fail    7.902718   \n",
       "27  0.000000         112  100.000000  0.000554   fail    5.101038   \n",
       "2   0.000000          80  100.000000  0.000152   fail   43.764013   \n",
       "3   0.000000         128  100.000000  0.003554   fail    5.278311   \n",
       "4   0.000000         192  100.000000  0.000010   fail    6.627046   \n",
       "5   0.000000         176  100.000000  0.001376   fail    6.419624   \n",
       "8   0.000000         224  100.000000  0.001027   fail    7.632890   \n",
       "9   0.000000         160  100.000000  0.008257   fail    6.762618   \n",
       "10  0.000000         160  100.000000  0.006028   fail    6.529726   \n",
       "12  0.000000         144  100.000000  0.002154   fail    6.290627   \n",
       "13  0.000000          80  100.000000  0.002671   fail   43.701720   \n",
       "14  0.000000         224  100.000000  0.000020   fail    7.364373   \n",
       "15  0.000000         224  100.000000  0.000101   fail    8.485760   \n",
       "16  0.000000         192  100.000000  0.004112   fail    7.348812   \n",
       "17  0.000000         144  100.000000  0.000247   fail    5.717525   \n",
       "18  0.000000         224  100.000000  0.000014   fail    7.456245   \n",
       "19  0.000000         192  100.000000  0.007551   fail    7.672545   \n",
       "20  0.000000          96  100.000000  0.000030   fail   42.985958   \n",
       "1   0.000000         192  100.000000  0.000097   fail    7.043807   \n",
       "49  0.000000         160  100.000000  0.000061   fail    6.613679   \n",
       "\n",
       "    weight_decay  \n",
       "21      0.000001  \n",
       "41      0.000272  \n",
       "26      0.459626  \n",
       "23      0.000044  \n",
       "40      0.000212  \n",
       "36      0.000018  \n",
       "31      0.000194  \n",
       "24      0.972279  \n",
       "44      0.003478  \n",
       "30      0.148601  \n",
       "35      0.008809  \n",
       "42      0.000315  \n",
       "11      0.000007  \n",
       "37      0.000061  \n",
       "22      0.000023  \n",
       "39      0.000016  \n",
       "48      0.004952  \n",
       "43      0.000977  \n",
       "45      0.000119  \n",
       "25      0.900786  \n",
       "38      0.000003  \n",
       "6       0.000001  \n",
       "7       0.012741  \n",
       "29      0.000604  \n",
       "34      0.716590  \n",
       "33      0.000003  \n",
       "46      0.000242  \n",
       "32      0.001101  \n",
       "47      0.000058  \n",
       "0       0.034320  \n",
       "28      0.008249  \n",
       "27      0.362556  \n",
       "2       0.000003  \n",
       "3       0.000006  \n",
       "4       0.002517  \n",
       "5       0.000096  \n",
       "8       0.000129  \n",
       "9       0.002875  \n",
       "10      0.000007  \n",
       "12      0.032825  \n",
       "13      0.000378  \n",
       "14      0.036031  \n",
       "15      0.000001  \n",
       "16      0.131728  \n",
       "17      0.056678  \n",
       "18      0.031626  \n",
       "19      0.002126  \n",
       "20      0.000022  \n",
       "1       0.000436  \n",
       "49      0.001203  "
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trials_df.sort_values(by='acc', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "# trials_df['arch_name'] = trials_df.arch.map(lambda a: a.__name__)\n",
    "# trials_df.to_csv('trials.csv', sep='\\t', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'0.9.0'"
      ]
     },
     "execution_count": 122,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "sns.set(style=\"whitegrid\")\n",
    "sns.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = trials_df[trials_df.acc > 0.85]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7f7bf0bac048>"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEBCAYAAACXArmGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAGDVJREFUeJzt3X9M1fe9x/EXHJHZXjo9trJDsPNOq56I9g9MDUkRJyrQQo9zahedtjXF2PbGZF22scWhZEpkcVtmWmvmct2cW5qwpTooUGMagybM3p6YUoZI1rGK4yjIj9sWLT8O3/sH4XNLUc45nMM5nMPzkZjA+X7O9/v5vIPndc7n8z3fb5xlWZYAAJAUH+kOAACmDkIBAGAQCgAAg1AAABiEAgDAIBQAAAahAAAwCAUAgEEoAAAMQgEAYBAKAABjRqQ74MvQ0JB6e3uVkJCguLi4SHcHAKKCZVkaGBjQgw8+qPh4/9//T/lQ6O3tVXNzc6S7AQBRafHixUpKSvK7vc9QKCsr0zvvvKN///vfqqio0OLFi8e08Xq9OnjwoC5evKi4uDjt3r1bW7ZskSRdunRJv/zlL9Xc3KwdO3boRz/6UQDDkRISEiQND2zmzJkBPTeWNDQ0KC0tLdLdiChqMIw6UAPJdw36+/vV3NxsXkP95TMUsrOztXPnTm3fvv2+bSoqKnT9+nWdO3dOPT092rhxozIyMpSamqr58+fr0KFDqqmpUX9/f0Cdk2SmjGbOnKnExMSAnx9Lpvv4JWowgjpQA8m/GgQ67e5zomnlypVyOBzjtqmqqtKWLVsUHx8vu92udevWqaamRpL09a9/XU6nUzNmTPmZKgCY9kJy9pHH41FKSor53eFw6ObNm6HYNQAgjKLm7XtDQ0OkuxBxbrc70l2IOGowjDpQA2lyahCSUHA4HGpra9OKFSskjf3kEAppaWnTeg7R7XYrPT090t2IKGowjDpQA8l3Dfr6+ib0Zjok00e5ubkqLy/X0NCQurq6dP78eeXk5IRi1wCAMPIZCgcPHtTq1at18+ZNvfDCC3r66aclSYWFhfrwww8lSS6XS6mpqdqwYYO2bt2qV155RfPnz5ckvf/++1q9erVOnjypN998U6tXr9bFixcncUgAgInyOX20b98+7du3b8zjJ06cMD/bbDaVlJTc8/krV65UbW1tEF0EAIQL1z4CABiEAgDAIBQAAAahAAAwCAUAgEEoAAAMQgEAYBAKAACDUAAAGIQCAMAgFAAABqEAADAIBQCAQSgAAAxCAQBgEAoAAINQAAAYhAIAwCAUAAAGoQAAMAgFAIBBKAAADEIBAGAQCgAAg1AAABiEAgDAIBQAAAahAAAwCAUAgEEoAAAMQgEAYBAKAACDUAAAGIQCAMAgFAAAhs9QKCsr09q1a7VkyRI1Nzffs43X61VJSYnWrVun9evXq7y83K9tAICpZYavBtnZ2dq5c6e2b99+3zYVFRW6fv26zp07p56eHm3cuFEZGRlKTU0ddxsAYGrx+Ulh5cqVcjgc47apqqrSli1bFB8fL7vdrnXr1qmmpsbnNgDA1BKSNQWPx6OUlBTzu8Ph0M2bN31uAwBMLT6nj6aKhoaGSHch4txud6S7EHHUYBh1oAbS5NQgJKHgcDjU1tamFStWSBr96WC8bYFIS0tTYmJiKLobldxut9LT0yPdjYiiBsOoAzWQfNegr69vQm+mQzJ9lJubq/Lycg0NDamrq0vnz59XTk6Oz20AgKnF5yeFgwcP6ty5c7p9+7ZeeOEFzZ49W2+//bYKCwu1d+9eLV++XC6XSx988IE2bNggSXrllVc0f/58SRp3GwBgavEZCvv27dO+ffvGPH7ixAnzs81mU0lJyT2fP942AMDUwjeaAQAGoQAAMAgFAIBBKAAADEIBAGAQCgAAg1AAABiEAgDAIBQAAAahAAAwCAUAgEEoAAAMQgEAYBAKAAAjam7HCQCBuOBu1anqq7rdfVcPz5mlnXlOrUkP/F4uodpPtCAUAMScC+5WvVb+gfoGvJKkju67eq38A0kK6AU9VPuJJkwfAYg5p6qvmhfyEX0DXp2qvhqR/UQTQgFAzLndfTegxyd7P9GEUAAQcx6eMyugxyd7P9GEUAAQMy64W7Xr4Dl13OOdfGKCTTvznAHtb2eeU4kJtqD3E01YaAYQE768KPxFj0zwrKGR9px9BABR5l6LwtJwIPz3vg0T3u+a9PkxHQJfxvQRgJgwHReFJwOhACAmTMdF4clAKACICdNxUXgysKYAIOqNXIqib8Cr+Pg4DQ1ZoxaXv3ipiv94IEGS9NmdgWmxcBwoQgFAVPvyWUdDQ5b5hDASCF/c/umdAfPc6XDZikAxfQQgqvm6FMX9zkq6V1sQCgCinK+zjvw5+4gzlP4foQAgqvk668ifs484Q+n/EQoAppSRS1U88/2z2nXwnC64W8dt7+uso3ttv19bsNAMYAqZyP0LfF2KYk36fF1t6VRV3cdjnpv0QIJ2b1zOIvMXEAoApozxFo3He+H2dSmK/2lqv+fjX0mcQSB8CdNHAKaMybpUBZfA8B+hAGDKmKxLVXAJDP8RCgCmjGAvVXG/RWougeE/v9YUWlpaVFRUpJ6eHs2ePVtlZWVasGDBqDYdHR0qLi7WjRs3NDg4qD179sjlcvncBgAjgrl/gT+L1NPpvggT5Vco7N+/X9u2bZPL5dLZs2dVXFysU6dOjWpz+PBhpaWl6Y033lBXV5c2bdqkJ554Qg6HY9xtAPBFE71/ga9F6ul2X4SJ8jl91NnZqcbGRuXn50uS8vPz1djYqK6urlHtmpqalJmZKUmy2+1aunSpqqurfW4DgFBgMTk0fH5S8Hg8Sk5Ols02PB9ns9k0b948eTwe2e12027ZsmWqqqrS8uXLdePGDV25ckWpqak+t/mroaEhoPaxyO12R7oLEUcNhlGHsTV46AGb/vfO2GscPfSALWbrNRnjCtn3FIqKilRaWiqXy6WUlBRlZGSYIBlvm7/S0tKUmJgYqu5GHbfbrfT09Eh3I6KowTDqcO8avKix92hOTLDpxY2PKz0Gp418/R309fVN6M20z1BwOBy6deuWvF6vbDabvF6v2tvbx6wH2O12HTlyxPxeWFioRYsW+dwGTBdfvKb/ZC50hus4kTruBXerfnvGo0/+dHbUcVhMDg2foTB37lw5nU5VVlbK5XKpsrJSTqdz1NSRJHV3dyspKUkzZsxQXV2dmpubdfToUZ/bgOlgIpdvmMrHidRxfR2HxeTg+fU9hQMHDuj06dPKycnR6dOnVVJSImn4Hf+HH34oSaqvr9dTTz2l3NxcHT16VMePH9esWbN8bgOmA1/X/I+240TquJEa33Ti15rCwoULVV5ePubxEydOmJ+zsrKUlZV1z+ePtw2YDsJ1ZkykzsCJ9fFNJ3yjGQiDcF1mIVKXc4j18U0nXCUVCIOdec57nhkT6sss+DrOZC0G+zO+kWN3dN9VfHychoYsPRJgHyZax0gtvkcjQgEIg3CdGTPecSZzMdjX+L587KEha0J9GGnz2zMf6JM7Xr/qGKnF92hFKABhEq4zY+53nIneqyDY497v2BPtw5r0+UpSu9/f1Zjsccca1hSAaSKSi7S+jjGZfWBxOjCEAjBNRHKR1tcxJrMPLE4HhlAApolI3lPgXscOVx+4l0JgWFPAtBILZ6HUt/Tq9epzAY8h0peBmJkQP2ZuP9CzjyYi0uOONoQCpo1YOAvlgrtVFe/1aMA78bN3wj3WL9ddGn6n/l9bHg9bX7j8hf+YPsK0EQuXSDhVfdUEwoipPoZYqPt0Qihg2oiFs1CicQzR2OfpjFDAtBELZ6FE4xiisc/TGaGAmHbB3apdB8/pme+f1ed9g5phixu1PdrOQtmZ51RClI2Bs3+iCwvNiFlfXuD89M6AbPFxSnogQZ/dGYjKs1DWpM9XS0uLLjZ9HjVn0nD2T3QhFBCz7rXA6R2y9JXEGfrTz56KUK+Ct+I/H9QLm1dHuhsB4eyf6MH0EWIWC5xA4AgFxCwWOIHAEQqIWSxwAoFjTQExiwVOIHCEAmIaC5xAYJg+AgAYhAIAwCAUAAAGoQAAMAgFAIBBKAAADEIBAGAQCgAAg1AAABiEAgDAIBQAAAahAAAwCAUAgEEoAAAMQgEAYPh1P4WWlhYVFRWpp6dHs2fPVllZmRYsWDCqTUdHh4qLi3Xjxg0NDg5qz549crlckqTOzk79+Mc/lsfj0eDgoFatWqV9+/Zpxgxu5wAAU4lfnxT279+vbdu26Z133tG2bdtUXFw8ps3hw4eVlpamiooK/fGPf9SvfvUreTweSdLx48e1cOFCVVRU6K9//av+/ve/69y5c6EdCQAgaD5DobOzU42NjcrPz5ck5efnq7GxUV1dXaPaNTU1KTMzU5Jkt9u1dOlSVVdXS5Li4uLU29uroaEh9ff3a2BgQMnJyaEeCwAgSD5DwePxKDk5WTbb8A3QbTab5s2bZz4FjFi2bJmqqqpkWZZaW1t15coVtbW1SZJefvlltbS06MknnzT/0tPTJ2E4AIBghGxSv6ioSKWlpXK5XEpJSVFGRoYJkpqaGi1ZskS///3v1dvbq8LCQtXU1Cg3N9fv/Tc0NISqq1HL7XZHugsRRw2GUQdqIE1ODXyGgsPh0K1bt+T1emWz2eT1etXe3i6HwzGqnd1u15EjR8zvhYWFWrRokSTp9OnTKi0tVXx8vJKSkrR27Vpdvnw5oFBIS0tTYmKi3+1jjdvtnvafrqjBMOpADSTfNejr65vQm2mf00dz586V0+lUZWWlJKmyslJOp1N2u31Uu+7ubg0ODkqS6urq1NzcbNYhUlNTVVtbK0nq7+9XXV2dHnvssYA7CwCYXH5NHx04cEBFRUU6duyYHnroIZWVlUka/jSwd+9eLV++XPX19Tp06JDi4+M1Z84cHT9+XLNmzZIk/eQnP9H+/ftVUFAgr9erVatWaevWrZM3KgDAhPgVCgsXLlR5efmYx0+cOGF+zsrKUlZW1j2f/+ijj+rkyZMT7CIAIFz49liIXXC36lT1Vd3uvquH58zSzjyn1qTPj3S3AMAvhEIIXXC36rXyD9Q34JUkdXTf1WvlH0gSwQAgKnDtoxA6VX3VBMKIvgGvTlVfjVCPACAwhEII3e6+G9DjADDVEAoh9PCcWQE9DgBTDaEQQjvznEpMsI16LDHBpp15zgj1CAACw0JzCI0sJnP2EYBoRSiE2Jr0+YQAgKjF9BEAwCAUAAAGoQAAMAgFAIBBKAAADEIBAGAQCgAAg1AAABiEAgDAIBQAAAahAAAwCAUAgEEoAAAMQgEAYBAKAACDUAAAGIQCAMAgFAAABqEAADAIBQCAQSgAAAxCAQBgEAoAAINQAAAYhAIAwCAUAAAGoQAAMGb406ilpUVFRUXq6enR7NmzVVZWpgULFoxq09HRoeLiYt24cUODg4Pas2ePXC6XJOmHP/yhrl27Ztpeu3ZNr7/+urKzs0M3EgBA0PwKhf3792vbtm1yuVw6e/asiouLderUqVFtDh8+rLS0NL3xxhvq6urSpk2b9MQTT8jhcOjnP/+5adfU1KTnnntOmZmZoR0JACBoPqePOjs71djYqPz8fElSfn6+Ghsb1dXVNapdU1OTeaG32+1aunSpqqurx+zvz3/+swoKCjRz5sxQ9B8AEEI+Q8Hj8Sg5OVk2m02SZLPZNG/ePHk8nlHtli1bpqqqKlmWpdbWVl25ckVtbW2j2vT396uiokLf/va3QzgEAECo+DV95I+ioiKVlpbK5XIpJSVFGRkZJkhGnD9/XikpKXI6nQHvv6GhIVRdjVputzvSXYg4ajCMOlADaXJq4DMUHA6Hbt26Ja/XK5vNJq/Xq/b2djkcjlHt7Ha7jhw5Yn4vLCzUokWLRrX5y1/+MuFPCWlpaUpMTJzQc2OB2+1Wenp6pLsRUdRgGHWgBpLvGvT19U3ozbTP6aO5c+fK6XSqsrJSklRZWSmn0ym73T6qXXd3twYHByVJdXV1am5uNusQknTz5k253W4VFBQE3EkAQHj4NX104MABFRUV6dixY3rooYdUVlYmafjTwN69e7V8+XLV19fr0KFDio+P15w5c3T8+HHNmjXL7OOtt97SN7/5TX31q1+dnJEAAILmVygsXLhQ5eXlYx4/ceKE+TkrK0tZWVn33cdLL700ge4BAMKJbzQDAAxCAQBgEAoAAINQAAAYhAIAwCAUAAAGoQAAMAgFAIBBKAAADEIBAGAQCgAAg1AAABiEAgDAIBQAAAahAAAwCAUAgEEoAAAMQgEAYBAKAACDUAAAGIQCAMAgFAAABqEAADAIBQCAQSgAAAxCAQBgEAoAAINQAAAYhAIAwJgR6Q74YlmWJKm/vz/CPYm8vr6+SHch4qjBMOpADaTxazDymjnyGuqvOCvQZ4TZp59+qubm5kh3AwCi0uLFi5WUlOR3+ykfCkNDQ+rt7VVCQoLi4uIi3R0AiAqWZWlgYEAPPvig4uP9XymY8qEAAAgfFpoBAAahAAAwCAUAgEEoAAAMQgEAYBAKAACDUAAAGBENhZaWFj377LPKycnRs88+q3/9619j2nR2dmr37t0qKChQXl6eDhw4oMHBQbO9qqpKBQUFys/PV0FBgW7fvh3GEQQv2Br4qk808KcGHR0deumll8w4z549a7Z5vV6VlJRo3bp1Wr9+vcrLy8PY+9AItgavv/66nn76aRUUFGjTpk26ePFiGHsfGsHWYMQ///lPPf744yorKwtDr0MvFHUI6nXRiqAdO3ZYZ86csSzLss6cOWPt2LFjTJuDBw9ahw8ftizLsvr7+63Nmzdbb7/9tmVZllVfX2/l5eVZ7e3tlmVZ1ieffGJ9/vnnYep9aARbg/G2RQt/avDqq69ar732mmVZltXZ2WllZWVZbW1tlmVZ1ltvvWXt2rXL8nq9Vmdnp5WZmWm1traGbwAhEGwNamtrrTt37liWZVlXr1610tPTrbt374ap96ERbA0sy7IGBwet7373u9arr75q/l9Em2DrEOzrYsQ+KXR2dqqxsVH5+fmSpPz8fDU2Nqqrq2tUu7i4OPX29mpoaEj9/f0aGBhQcnKyJOl3v/uddu3apUceeUSSlJSUpMTExPAOJAihqMF426KBvzVoampSZmamJMlut2vp0qWqrq6WNPyuaMuWLYqPj5fdbte6detUU1MT3oEEIRQ1yMzM1KxZsyRJS5YskWVZ6unpCeMoghOKGkjSb37zG61Zs0YLFiwIW99DKRR1CPZ1MWKh4PF4lJycLJvNJkmy2WyaN2+ePB7PqHYvv/yyWlpa9OSTT5p/6enpkqSPPvpIra2t2r59u771rW/p2LFjAV8RMJJCUYPxtkUDf2uwbNkyVVVVybIstba26sqVK2prazP7SElJMW0dDodu3rwZvkEEKRQ1+KIzZ87o0Ucf1de+9rWw9D8UQlGDpqYmXbp0Sc8//3y4ux8yoahDsK+LU36huaamRkuWLNGlS5dUW1ur999/37wL9Hq9unbtmk6ePKk//OEPqq2tveccY7QbrwbjbYslRUVFun37tlwulw4dOqSMjAzzH2e68KcG7733nn7961/rF7/4RYR6ObnuV4OBgQH99Kc/VUlJybT4uxjvbyHY18WI3U/B4XDo1q1b8nq9stls8nq9am9vl8PhGNXu9OnTKi0tVXx8vJKSkrR27VpdvnxZubm5SklJUW5urmbOnKmZM2cqOztb9fX12rhxY4RGFZhQ1GC8bdHA3xrY7XYdOXLE/F5YWKhFixaZfbS1tWnFihWSxn5ymOpCUQNJunLlin7wgx/o2LFj+sY3vhG2/odCsDXo6OjQ9evXtXv3bknSJ598Isuy9Nlnn+lnP/tZWMcSjFD8LQT7uhixTwpz586V0+lUZWWlJKmyslJOp1N2u31Uu9TUVNXW1koavmlEXV2dHnvsMUnD822XLl0yl4j929/+pqVLl4Z3IEEIRQ3G2xYN/K1Bd3e3Oauqrq5Ozc3NZt41NzdX5eXlGhoaUldXl86fP6+cnJzwDiQIoahBfX29vve97+no0aNatmxZeAcQAsHWICUlRZcvX9a7776rd999V88995y2bt0aVYEgheZvIejXxYmtj4fGP/7xD2vz5s3Whg0brM2bN1sfffSRZVmW9eKLL1r19fWWZVnWxx9/bD3//PNWfn6+lZeXZx04cMAaGBiwLMuyvF6vVVpaauXm5lpPPfWUVVpaanm93oiNZyKCrcF426KFPzW4cOGCtX79eisnJ8f6zne+YzU2NprnDw4OWsXFxVZ2draVnZ1tvfnmmxEZRzCCrcGmTZusVatWWc8884z519TUFJGxTFSwNfiio0ePRu3ZR8HWIdjXRe6nAAAwpvxCMwAgfAgFAIBBKAAADEIBAGAQCgAAg1AAABiEAgDAIBQAAMb/AdC0lyhNoGdhAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(1 - df.loss.values, df.acc.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYEAAAEBCAYAAACe6Rn8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAD1FJREFUeJzt3V9oU/f/x/HXiTPZdEoaYSOtsjKxpSwXzgS8chepWx3obr4MS3GDDWUMLAOpQxBaqDpJLV74paPuZlehvRqTqjOO7xi7GzQiLAjq/uhgDRva1EmlUZPzu9jP8PVr/+SkqYl9Px+Qi57zOSef0EOePSdN4riu6woAYJKv1hMAANQOEQAAw4gAABhGBADAMCIAAIYRAQAwjAgAgGFEAAAMIwIAYNhzCw3I5XL69NNP9fvvv8vv9+uVV15Rf3+/QqGQWltb1dLSIp/vn5YMDAyotbVVkvTdd99pYGBAhUJBr732mo4fP64XXnhhaR8NAMATZ6GPjZiamtLVq1e1detWSVIikdCdO3f02WefqbW1VZcuXdLq1asf22Z6elpvvfWWksmkmpubdfjwYYXDYe3fv3/pHgkAwLMFzwSCwWApAJK0efNmjYyMzLvNDz/8oEgkoubmZklSZ2enDh06VHYEisWipqentXLlSjmOU9Y2AGCd67p68OCBVq9eXbpCs5AFI/DfisWiRkZGFI/HS8vee+89FQoFvfHGG+ru7pbf71c2m1VjY2NpTGNjo7LZbNn3Mz09rWvXrnmZGgDg/7W0tGjNmjVljfX0wvCRI0e0atUq7dmzR5L0/fff66uvvlIymdTPP/+soaEh77OdxcqVK6uyHwCwyMtzaNlnAolEQjdv3tTw8HDpNCMcDkuSXnzxRb377rv68ssvS8t//PHH0rYTExOlseV4dAkoEokoEAiUvR1ml06nFY1Gaz0NYFYcn9WTz+eVyWQ8XUYv60zg5MmTymQyGhoakt/vlyTduXNHMzMzkqSHDx8qlUqpra1NkrRt2zb99NNPunHjhiRpdHRUb7/9tpfHAgB4ChY8E7h+/bpOnz6t5uZmdXZ2SpLWr1+vvXv3qre3V47j6OHDh3r99df1ySefSPrnzKC/v18fffSRisWi2tradPjw4aV9JAAAzxaMwKZNm3T16tVZ142Njc253fbt27V9+/bKZwYAWHK8YxgADCMCAGAYEQAAw4gAABhGBJaRSCQix3GeuMVisVmXO46jSCRS62kDqCFPHxuB+pbJZGZd7jiOFvicQABGcSYAAIYRAQAwjAgAgGFEAAAMIwIAYBgRAADDiAAAGEYEAMAwIgAAhhEBADCMCACAYUQAAAwjAgBgGBEAAMOIAAAYRgQAwDAiAACGEQEAMIwIAIBhRAAADCMCAGAYEQAAw4gAABhGBADAMCIAAIYRAQAwjAg8g0KhkBzHKfsmydN4x3EUCoVq/CgBPA1E4BmUy+Xkum7Zt/HxcU/jXddVLper9cME8BQQAQAwbMEI5HI57du3Tx0dHdq1a5f279+vyclJSdLly5f1zjvvqKOjQx9++KFu375d2m6+dQCA+rBgBBzH0d69e5VKpTQ2NqYNGzZocHBQxWJRBw8eVG9vr1KplGKxmAYHByVp3nUAgPqxYASCwaC2bt1a+nnz5s2amJhQJpNRIBBQLBaTJHV2durChQuSNO86AED98PSaQLFY1MjIiOLxuLLZrBobG0vrQqGQisWipqam5l0HAKgfz3kZfOTIEa1atUp79uzRt99+u1RzKslkMkt+H8+qdDq9pOMr3QaoBMda7ZQdgUQioZs3b2p4eFg+n0/hcFgTExOl9ZOTk/L5fAoGg/Ou8yISiSgQCHjaxopoNFr22HQ67Wl8JfcBVKrS4xNPyufznv94Luty0MmTJ5XJZDQ0NCS/3y/pnyfomZkZjY+PS5JGR0e1Y8eOBdcBAOrHgmcC169f1+nTp9Xc3KzOzk5J0vr16zU0NKSBgQH19fUpn8+rqalJJ06ckCT5fL451wEA6seCEdi0aZOuXr0667otW7ZobGzM8zoAQH3gHcMAYBgRAADDiAAAGEYEAMAwIgAAhhEBADCMCACAYUQAAAwjAgBgGBEAAMOIAAAYRgQAwDAiAACGEQEAMIwIAIBhRAAADPP0RfOoD990t+vXY/8qe3yDpF8veL8PAMsfEXgGvf3v/8h13bLHV/JF3hsdR+4przMD8KzhchAAGEYEAMAwIgAAhhEBADCMCACAYUQAAAwjAgBgGBEAAMOIAAAYRgQAwDAiAACGEQEAMIwIAIBhRAAADCMCAGAYEQAAw4gAABhGBADAsLK+XjKRSCiVSumPP/7Q2NiYWlpaJEnxeFx+v1+BQECS1NPTo23btkmSLl++rN7eXuXzeTU1NenEiRNat27dEj0MAEAlyjoTaG9vVzKZVFNT0xPrTp06pTNnzujMmTOlABSLRR08eFC9vb1KpVKKxWIaHBys7swBAItWVgRisZjC4XDZO81kMgoEAorFYpKkzs5OXbhwobIZAgCWTFmXg+bT09Mj13UVjUZ14MABrV27VtlsVo2NjaUxoVBIxWJRU1NTCgaDZe87k8ksdnrLVjqdXtLxlW4DVIJjrXYWFYFkMqlwOKz79+/r2LFj6u/vr+pln0gkUnq9AY+LRqNlj02n057GV3IfQKUqPT7xpHw+7/mP50X9d9CjS0R+v19dXV26dOlSafnExERp3OTkpHw+n6ezAADA0qs4Avfu3dPdu3clSa7r6vz582pra5P0z1/wMzMzGh8flySNjo5qx44dVZguAKCayrocdPToUV28eFG3bt3SBx98oGAwqOHhYXV3d6tQKKhYLGrjxo3q6+uTJPl8Pg0MDKivr++xfxEFANQXx3Vdt9aT+F+PrmvxmsDsHMeRl19bJddcvd4HUCleE6ieSp47eccwABhGBADAMCIAAIYRAQAwjAgAgGFEAAAMIwIAYBgRAADDiAAAGEYEAMAwIgAAhhEBADCMCACAYUQAAAxb9HcMozYcx1nS/Tc0NCzp/gHUByLwDPL6Of98NwCAuXA5CAAMIwIAYBgRAADDiAAAGEYEAMAwIgAAhhEBADCMCACAYUQAAAwjAgBgGBEAAMOIAAAYRgQAwDAiAACGEQEAMIwIAIBhRAAADCMCAGAYEQAAwxaMQCKRUDweV2trq65du1Za/ttvv2n37t3q6OjQ7t27dePGjbLWAQDqx4IRaG9vVzKZVFNT02PL+/r61NXVpVQqpa6uLvX29pa1DgBQPxaMQCwWUzgcfmzZ7du3deXKFe3cuVOStHPnTl25ckWTk5PzrgMA1JfnKtkom83q5Zdf1ooVKyRJK1as0EsvvaRsNivXdedcFwqFPN1PJpOpZHqYRTqdrvUUgDlxfNZORRF4WiKRiAKBQK2nsSxEo9FaTwGYVTqd5visknw+7/mP54oiEA6H9eeff6pQKGjFihUqFAr666+/FA6H5brunOsAAPWlon8RXbdundra2nT27FlJ0tmzZ9XW1qZQKDTvOgBAfXFc13XnG3D06FFdvHhRt27dUkNDg4LBoM6dO6dffvlFhw4d0t9//621a9cqkUjo1VdflaR515Xj0SkNl4Oqw3EcLfBrBmqGy0HVU8lz54IRqAUiUF1EAPWMCFRPJc+dvGMYAAwjAgBgGBEAAMOIAAAYRgQAwDAiAACGEQEAMIwIAIBhRAAADCMCAGAYEQAAw4gAABhGBADAMCIAAIYRAQAwjAgAgGFEAAAMIwIAYBgRAADDiAAAGEYEAMAwIgAAhhEBADCMCACAYUQAAAwjAgBgGBFYRiKRiBzHeeImadbljuMoEonUeNZYTkKh0JzH2ly3WCzmaXwoFKr1w1xWiMAykslk5LruE7fx8fFZl7uuq0wmU+tpYxnJ5XJzHmtz3eY7Pme75XK5Wj/MZYUIAIBhRAAADCMCAGAYEQAAw4gAABhGBADAMCIAAIY9t9gdxONx+f1+BQIBSVJPT4+2bdumy5cvq7e3V/l8Xk1NTTpx4oTWrVu36AkDAKpn0RGQpFOnTqmlpaX0c7FY1MGDB3X8+HHFYjF9/vnnGhwc1PHjx6txdwCAKlmSy0GZTEaBQECxWEyS1NnZqQsXLizFXQEAFqEqZwI9PT1yXVfRaFQHDhxQNptVY2NjaX0oFFKxWNTU1JSCwWDZ++UjDaonnU7XegowopJjzes2HM/Vs+gIJJNJhcNh3b9/X8eOHVN/f7/efPPNasxNkUik9FoDKpdOpxWNRms9DRjh9Vir5PjkeJ5dPp/3/MfzoiMQDoclSX6/X11dXfr444/1/vvva2JiojRmcnJSPp/P01kAgGfPN93t+vXYvzxt0yDpVw9Xi7/pbvc2KcxrURG4d++eCoWC1qxZI9d1df78ebW1tSkSiWhmZkbj4+OKxWIaHR3Vjh07qjVnAHXq7X//R67retrG65nARseRe8rrzDCXRUXg9u3b6u7uVqFQULFY1MaNG9XX1yefz6eBgQH19fU99i+iAID6sqgIbNiwQV9//fWs67Zs2aKxsbHF7B4AsMR4xzAAGEYEAMAwIgAAhhEBADCMCACAYUQAAAwjAgBgGBEAAMOIAAAYRgQAwLCqfJ8AADziOM6S7r+hoWFJ928NEQBQNV4/QVT6JxqVbIfq4HIQABhGBADAMCIAAIYRAQAwjAgAgGFEAAAMIwIAYBgRAADDiAAAGEYEAMAwIgAAhhEBADCMCAB4KiKRiBzHeeImadblkUikxjO2gU8RBfBUZDKZWZen02lFo9GnPBs8wpkAABhGBADAMCIAAIYRAQAwjAgAgGFEAAAMIwIAYFhdvk/AdV1J0v3792s8k+Ujn8/XegrAnDg+q+PRc+aj59ByOK6X0U/J3bt3de3atVpPAwCeSS0tLVqzZk1ZY+syAsViUdPT01q5cmXpbeUAgPm5rqsHDx5o9erV8vnKu9pflxEAADwdvDAMAIYRAQAwjAgAgGFEAAAMIwIAYBgRAADDiAAAGEYElrFEIqF4PK7W1lbegY26ksvltG/fPnV0dGjXrl3av3+/Jicnaz0tk4jAMtbe3q5kMqmmpqZaTwV4jOM42rt3r1KplMbGxrRhwwYNDg7WelomEYFlLBaLKRwO13oawBOCwaC2bt1a+nnz5s2amJio4YzsIgIAaqpYLGpkZETxeLzWUzGJCACoqSNHjmjVqlXas2dPradiUl1+nwAAGxKJhG7evKnh4eGyP/US1UUEANTEyZMnlclk9MUXX8jv99d6OmbxUdLL2NGjR3Xx4kXdunVLDQ0NCgaDOnfuXK2nBej69evauXOnmpub9fzzz0uS1q9fr6GhoRrPzB4iAACGcREOAAwjAgBgGBEAAMOIAAAYRgQAwDAiAACGEQEAMIwIAIBh/wcIHg1iQCK1tAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "data = [trials_df[trials_df.status == 'fail'].batch_size.values,\n",
    "        trials_df[trials_df.status == 'ok'].batch_size.values]\n",
    "ax.boxplot(data);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7f7bf0aa5710>"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEBCAYAAACXArmGAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAF3tJREFUeJzt3X9s1PXhx/FX76AVGYUeruy6okxY4UarLiWSJoMqMIGtrMSJLjV0MVoj+AW3hcwb6wooJZSRLcv4lbDFjIzEpOrAdm1hjBDFNEROhHUFGk0HRY5f/TGlYkvvPt8/WN9bob/uuOPTT30+EpLe5z5tXvfJh3vd5/3+3OeTYFmWJQAAJLnsDgAAGDooBQCAQSkAAAxKAQBgUAoAAINSAAAYlAIAwKAUAAAGpQAAMCgFAIBBKQAAjBF2BxhIOBxWe3u7Ro4cqYSEBLvjAIAjWJal69eva/To0XK5Bv/5f8iXQnt7uxoaGuyOAQCOlJGRoTFjxgx6/QFLoaysTPv27dMnn3yiiooKZWRk3LJOKBTS+vXr9e677yohIUHPP/+8lixZIkk6fPiwfvOb36ihoUFLly7Vyy+/HMHLkUaOHCnpxgtLTEyM6HftUldXp8zMTLtjRMXJ2SVn53dydon8duote2dnpxoaGsx76GANWApz585VYWGhnn766T7Xqaio0NmzZ7V//361tbVp8eLFysnJUXp6uiZOnKjS0lLV1NSos7MzonCSzJBRYmKikpKSIv59uzgp682cnF1ydn4nZ5fIb6e+skc67D7gQNOMGTPk9Xr7XaeqqkpLliyRy+WSx+PRvHnzVFNTI0m677775PP5NGLEkB+pAoAvvZicfRQMBpWWlmYee71eXbhwIRZ/GgBwBznm43tdXZ3dESISCATsjhA1J2eXnJ3fydkl8tspVtljUgper1fnz5/XAw88IOnWI4dYyMzMdMx4XyAQUHZ2tt0xouLk7JKz8zs5u0R+O/WWvaOjI6oP0zEZPlqwYIHKy8sVDofV0tKiAwcOaP78+bH40wCAO2jAUli/fr1mz56tCxcu6JlnntH3v/99SVJRUZH+8Y9/SJLy8/OVnp6uxx57TE8++aRefPFFTZw4UZJ09OhRzZ49W6+99ppef/11zZ49W++++24cXxIAIFoDDh8VFxeruLj4luU7d+40P7vdbq1bt67X358xY4beeeed24gIALhTuPYRAMCgFAAABqUAADAoBQCAQSkAAAxKAQBgUAoAAINSAAAYlAIAwKAUAAAGpQAAMCgFAIBBKQAADEoBAGBQCgAAg1IAABiUAgDAoBQAAAalAAAwKAUAgEEpAAAMSgEAYFAKAACDUgAAGJQCAMCgFAAABqUAADAoBQCAQSkAAAxKAQBgUAoAAINSAAAYlAIAwKAUAAAGpQAAMAYshbKyMs2ZM0dTp05VQ0NDr+uEQiGtW7dO8+bN03e/+12Vl5cP6jkAwNAyYqAV5s6dq8LCQj399NN9rlNRUaGzZ89q//79amtr0+LFi5WTk6P09PR+nwMADC0DHinMmDFDXq+333Wqqqq0ZMkSuVwueTwezZs3TzU1NQM+BwAYWmIypxAMBpWWlmYee71eXbhwYcDnAABDy4DDR0NFXV2d3REiEggE7I4QNSdnl5yd38nZJfLbKVbZY1IKXq9X58+f1wMPPCCp59FBf89FIjMzU0lJSbGIG3eBQEDZ2dl2x4iKk7NLzs7v5OwS+e3UW/aOjo6oPkzHZPhowYIFKi8vVzgcVktLiw4cOKD58+cP+BwAYGgZ8Ehh/fr12r9/v65cuaJnnnlG48aN01//+lcVFRVp5cqVysrKUn5+vo4fP67HHntMkvTiiy9q4sSJktTvcwCAoWXAUiguLlZxcfEty3fu3Gl+drvdWrduXa+/399zAIChhW80AwAMSgEAYFAKAACDUgAAGJQCAMCgFAAABqUAADAoBQCAQSkAAAxKAQBgUAoAAINSAAAYlAIAwKAUAACGY27HCQwlhwJN2lV9Uldar+melFEqXOjTI9ncJwSxY9c+RikAEToUaNKW8uPquB6SJF1uvaYt5ccliWJATNi5jzF8BERoV/VJ85+1W8f1kHZVn7QpEYYbO/cxSgGI0JXWaxEtByJl5z5GKQARuidlVETLgUjZuY9RCkCEChf6lDTS3WNZ0ki3Chf6bEqE4aZwoU8j3Ak9lo1wJ9yRfYyJZiBC3RN9nH2EeLKs/h/HC6UAROGR7ImUAOJmV/VJhcI9WyAUtrSr+iRnHwHAlw0TzQAAg4lmAIBh58kMw3ZOgcsQIJ7YvxBPj2RP1MnGZtUcOatw2JLLlaC5M9LvyD42LI8Uur8ifrn1miz99yvihwJNdkfDMMD+hXg7FGjS34+eU/g/k83hsKW/Hz13R/axYVkKXIYA8cT+hXjjMhcxxmUIEE/sX4g3zj6KMS5DgHhi/0K8cfZRjHEZAsRT4UKfEm5alvCf5UAs2PkeNixL4ZHsifq/JQ/qqymjlCDpqymj9H9LHuTsEMTEycZm3XzFAes/y4FYsPM9bNieksplCBAvNUfO9rl82RMP3eE0GK7seg8blkcKQDyFw71fmayv5YCTUApAhFyum2cU+l8OOMmwHT4C4mXBzHtVVXum1+VArNj1rflBHSk0Njbqqaee0vz58/XUU0/pX//61y3rXL58WcuWLdOiRYu0cOFC7d27d1DPAU6z7ImH9L2c+8yRgcuVoO/l3Md8AmLGzm/ND+pIYc2aNSooKFB+fr727t2rkpIS7dq1q8c6GzduVGZmprZv366WlhY9/vjjevjhh+X1evt9DnCiZU88RAkgbvr7RrPt91Nobm5WfX298vLyJEl5eXmqr69XS0tLj/VOnTqlWbNmSZI8Ho+mTZum6urqAZ8DAPRk5zeaBzxSCAaDmjBhgtzuG1+kcLvdSk1NVTAYlMfjMetNnz5dVVVVysrK0rlz53Ts2DGlp6cP+Nxg1dXVRbS+3QKBgN0Roubk7JKz8zs5u0T+WEm+261/fx7qdXlfGWOVPWYTzX6/Xxs2bFB+fr7S0tKUk5NjiqS/5wYrMzNTSUlJsYobV4FAQNnZ2XbHiIqTs0vOzu/k7BL5Y+k53ZhT+N8hpKSRbj23+EFl9zJ81Fv2jo6OqD5MD1gKXq9XFy9eVCgUktvtVigU0qVLl26ZD/B4PNq8ebN5XFRUpClTpgz4HOBE29/4sMe17hfMvJc5BsTMkL6fwvjx4+Xz+VRZWSlJqqyslM/n6zF0JEmtra3q6uqSJNXW1qqhocHMQ/T3HOA029/4UFW1Z3pc676q9oy2v/GhzckwXNh5P4VBDR+tXbtWfr9f27ZtU3JyssrKyiTd+MS/cuVKZWVl6cSJEyotLZXL5VJKSop27NihUaNuXNGvv+cAp+EyF4g3O88+GlQpTJ48WeXl5bcs37lzp/k5NzdXubm5vf5+f88BTsNlLhBv3E8BcBAuc4F4s/N+CsP2MhdMBCJeuMwF4q1woa/Xs4+4n0KUmAhEPPm+Mb7Xm+z4vjHejjgYhrifQowxEYh42lV9steb7NyJSUB8eXA/hRhiIhDxZOckIBBvw7IUmAhEPNk5CQjE27Ashb4m/JgIRCzYeVN1IN6G5ZzCsice0ieXr+r4R/+9kfqDU8Yzn4CY6B7nteMGKPjysOsMymFZCocCTTp1pq3HslNn2nQo0MR/XMSEXZOA+HLoPoOyW/cZlJLiXgzDcviov6+IA8BQ198ZlPE2LEuBs0MAOJmdZ1AOy1Lg7BAATmbnGZTDshQKF/rkvmnjuV0JnB0CwBHsPINyWE40S1JCQv+PAWCo6p5M5uyjGNlVfVJdoZ5jb10hi8sQAHCMZU88ZMtp9MNy+IiJZgCIzrAsBSaaASA6w7IUuAwBAERnWM4pcBkCAIjOsCwFicsQAEA0huXwEQAgOpQCAMCgFAAABqUAADAoBQCAQSkAAAxKAQBgUAoAAINSAAAYlAIAwKAUAAAGpQAAMCgFAIBBKQAADEoBAGAM6n4KjY2N8vv9amtr07hx41RWVqZJkyb1WOfy5csqKSnRuXPn1NXVpRdeeEH5+fmSpObmZv3iF79QMBhUV1eXZs6cqeLiYo0YMWxv5wAAjjSoI4U1a9aooKBA+/btU0FBgUpKSm5ZZ+PGjcrMzFRFRYV2796t3/72twoGg5KkHTt2aPLkyaqoqNDbb7+tf/7zn9q/f39sXwkA4LYNWArNzc2qr69XXl6eJCkvL0/19fVqaWnpsd6pU6c0a9YsSZLH49G0adNUXV0tSUpISFB7e7vC4bA6Ozt1/fp1TZgwIdavBQBwmwYshWAwqAkTJsjtdkuS3G63UlNTzVFAt+nTp6uqqkqWZampqUnHjh3T+fPnJUnLly9XY2OjvvOd75h/2dnZcXg5AIDbEbNBfb/frw0bNig/P19paWnKyckxRVJTU6OpU6fqT3/6k9rb21VUVKSamhotWLBg0H+/rq4uVlHviEAgYHeEqDk5u+Ts/E7OLpHfTrHKPmApeL1eXbx4UaFQSG63W6FQSJcuXZLX6+2xnsfj0ebNm83joqIiTZkyRZL05z//WRs2bJDL5dKYMWM0Z84cHTlyJKJSyMzMVFJS0qDXt1MgEHDskZCTs0vOzu/k7BL57dRb9o6Ojqg+TA84fDR+/Hj5fD5VVlZKkiorK+Xz+eTxeHqs19raqq6uLklSbW2tGhoazDxEenq63nnnHUlSZ2enamtr9c1vfjPisACA+BrU8NHatWvl9/u1bds2JScnq6ysTNKNo4GVK1cqKytLJ06cUGlpqVwul1JSUrRjxw6NGjVKkrR69WqtWbNGixYtUigU0syZM/Xkk0/G71UBAKIyqFKYPHmyysvLb1m+c+dO83Nubq5yc3N7/f17771Xr732WpQRAQB3Ct8eA6JwKNCkXdUndaX1mu5JGaXChT49kj3R7ljAbaMUgAgdCjRpS/lxdVwPSZIut17TlvLjkkQxwPG49hEQoV3VJ00hdOu4HtKu6pM2JQJih1IAInSl9VpEywEnoRSACN2TMiqi5YCTUApAhAoX+pQ00t1jWdJItwoX+mxKBMQOE81AhLonkzn7CMMRpQBE4ZHsiZQAhiWGjwAABqUAADAoBQCAQSkAAAxKAQBgUAoAAINSAAAYlAIAwKAUAAAGpQAAMCgFAIBBKQAADEoBAGBQCgAAg1IAABiUAgDAoBQAAAalAAAwKAUAgEEpAAAMSgEAYFAKAACDUgAAGJQCAMCgFAAABqUAADAoBQCAMWIwKzU2Nsrv96utrU3jxo1TWVmZJk2a1GOdy5cvq6SkROfOnVNXV5deeOEF5efnS5J+/vOf6/Tp02bd06dPa+vWrZo7d27sXgkA4LYNqhTWrFmjgoIC5efna+/evSopKdGuXbt6rLNx40ZlZmZq+/btamlp0eOPP66HH35YXq9XmzZtMuudOnVKP/7xjzVr1qzYvhIAwG0bcPioublZ9fX1ysvLkyTl5eWpvr5eLS0tPdY7deqUeaP3eDyaNm2aqqurb/l7b7zxhhYtWqTExMRY5AcAxNCApRAMBjVhwgS53W5JktvtVmpqqoLBYI/1pk+frqqqKlmWpaamJh07dkznz5/vsU5nZ6cqKir0wx/+MIYvAQAQK4MaPhoMv9+vDRs2KD8/X2lpacrJyTFF0u3AgQNKS0uTz+eL+O/X1dXFKuodEQgE7I4QNSdnl5yd38nZJfLbKVbZBywFr9erixcvKhQKye12KxQK6dKlS/J6vT3W83g82rx5s3lcVFSkKVOm9FjnzTffjPooITMzU0lJSVH97p0WCASUnZ1td4yoODm75Oz8Ts4ukd9OvWXv6OiI6sP0gMNH48ePl8/nU2VlpSSpsrJSPp9PHo+nx3qtra3q6uqSJNXW1qqhocHMQ0jShQsXFAgEtGjRoohDAgDujEENH61du1Z+v1/btm1TcnKyysrKJN04Gli5cqWysrJ04sQJlZaWyuVyKSUlRTt27NCoUaPM3/jLX/6iRx99VGPHjo3PKwEA3LZBlcLkyZNVXl5+y/KdO3ean3Nzc5Wbm9vn31i2bFkU8QAAdxLfaAYAGJQCAMCgFAAABqUAADAoBQCAQSkAAAxKAQBgUAoAAINSAAAYlAIAwKAUAAAGpQAAMCgFAIBBKQAADEoBAGBQCgAAg1IAABiUAgDAoBQAAAalAAAwKAUAgEEpAAAMSgEAYFAKAACDUgAAGJQCAMCgFAAABqUAADAoBQCAMcLuAAOxLEuS1NnZaXOSyHR0dNgdIWpOzi45O7+Ts0vkt9PN2bvfM7vfQwcrwYr0N+6wzz77TA0NDXbHAABHysjI0JgxYwa9/pAvhXA4rPb2do0cOVIJCQl2xwEAR7AsS9evX9fo0aPlcg1+pmDIlwIA4M5hohkAYFAKAACDUgAAGJQCAMCgFAAABqUAADAoBQCAMeQvczHULV++XOfOnZPL5dLdd9+tX/3qV/L5fJozZ44SExOVlJQkSVq1apVmzZplc9rebdmyRb///e9VUVGhjIwMffjhhyopKVFHR4e+/vWv69e//rXGjx9vd8w+3Zx/6tSpysjIMF/Y2bRpk6ZOnWpzyp762j+csu37yu+Ebd/R0aENGzaotrZWSUlJeuihh/Tqq6+qsbFRfr9fbW1tGjdunMrKyjRp0iS7496ir/wxe8+xcFs+/fRT8/Pf/vY3a/HixZZlWdajjz5qnT592q5Yg1ZXV2c9++yzJm8oFLLmzZtnvf/++5ZlWdbWrVstv99vc8q+3ZzfsiwrIyPDunr1qs3J+tfb/uGkbd/X/u2Ebf/qq69apaWlVjgctizLsi5fvmxZlmUtXbrU2rNnj2VZlrVnzx5r6dKltmXsT1/5Y/Wew/DRbfrfa4pcvXrVUZfi6Ozs1CuvvKK1a9eaZXV1dUpKStKMGTMkST/60Y9UU1NjU8L+9ZbfyZy07Z2qvb1de/bs0UsvvWT+r95zzz1qbm5WfX298vLyJEl5eXmqr69XS0uLnXFv0Vf+WGL4KAZ++ctf6r333pNlWfrDH/5glq9atUqWZSk7O1s/+9nPlJycbGPKW/3ud7/TD37wA6Wnp5tlwWBQaWlp5rHH41E4HDaH1ENJb/m7LV26VKFQSLNnz9aKFSuUmJhoQ8L+3bx/OGnbS33v30N52zc1NWncuHHasmWLjhw5otGjR+ull17SXXfdpQkTJsjtdkuS3G63UlNTFQwG5fF4bE79X33l7/4gEYv3HI4UYqC0tFSHDh3ST3/6U23atEmStHv3br399tt68803ZVmWXnnlFZtT9nTs2DHV1dWpoKDA7ihR6S//oUOH9NZbb2n37t366KOPtHXrVhsS9m+o7x8D6Sv/UN/2oVBITU1N+ta3vqW33npLq1at0ooVK/T555/bHW1Q+sp/9erVmO1TlEIMLV68WEeOHFFra6u8Xq8kKTExUQUFBfrggw9sTtfT+++/r48//lhz587VnDlzdOHCBT377LM6c+aMzp8/b9ZraWmRy+Uacp9U+8p/+PBhs+2/8pWvaMmSJUNu20vqdf/wer2O2PZS7/n/d/lQ3fZer1cjRowww0QPPvigUlJSdNddd+nixYsKhUKSbrz5Xrp0ybyeoaKv/I2NjTF7z6EUbkN7e7uCwaB5fPDgQY0dO1ZJSUn67LPPJN24fG1VVZV8Pp9dMXv1/PPP6/Dhwzp48KAOHjyor33ta/rjH/+o5557Tl988YWOHj0qSXr99de1YMECm9Peqq/8WVlZ+uKLLyRJXV1d2rdv35Db9p9//nmv+0dmZqYjtn1f+f/9738P+W3v8Xg0c+ZMvffee5KkxsZGNTc3a9KkSfL5fKqsrJQkVVZWyufzDamhI6nv/KmpqTF7z+HS2bfhypUrWr58ua5duyaXy6WxY8fq5ZdfVnJyslasWKFQKKRwOKzJkyeruLhYqampdkfu05w5c7Rjxw5lZGTogw8+0Jo1a3qcFhnryaxY687f3t6ukpISJSQkqKurS9/+9re1evVqjR492u6IRlNTU5/7hxO2fV/5P/nkkyG/7aUb+VevXq22tjaNGDFCP/nJT5Sbm6uPP/5Yfr9fn376qZKTk1VWVqb777/f7ri36C3//fffH7P3HEoBAGAwfAQAMCgFAIBBKQAADEoBAGBQCgAAg1IAABiUAgDAoBQAAMb/A9qhlVJ9tEhgAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fig, ax = plt.subplots()\n",
    "df = trials_df\n",
    "df = df[(df.status == 'ok')&(df.batch_size < 100)&(df.batch_size > 20)]\n",
    "ax.scatter(df.batch_size.values, df.acc.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = trials_df.copy()\n",
    "df['arch_name'] = df.arch.map(lambda a: a.__name__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 217,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYgAAAEHCAYAAAC0pdErAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAH0VJREFUeJzt3X9Q1HX+B/Dnsrj+IjKt+UATUnOgaKxX51Y0QBZEOAEHHDIV5XARkpfdnXUdFt8GGtQMCC6SmuJKb8rDy6k4MqZajrzsPA2BJpZx7lIuEknWZBFlFRZ2398/yL02PiJe++HDh30+ZpzZ3c97P/vafcs+P+/PZz/vj04IIUBERPQDfmoXQEREUxMDgoiIZDEgiIhIFgOCiIhkMSCIiEiWv9oFeEtLS4vaJRARadLy5ctlH582AQFc+E0SEZG88TauuYuJiIhkMSCIiEgWA4KIiGQxIIiISBYDgoiIZDEgiIhIFgOCiIhkKXoexN69e7F582a4XC5kZmYiLy/PY3l3dzcKCgpgs9kwb948lJWVISgoCABQWlqKTz75BC6XC9HR0fi///s/6HQ6JcslIo3atm0b9u3b5/X1DgwMAAACAgK8vm4AiI6ORk5OjiLr9gbFRhBOpxPFxcV47bXXUF9fj/fffx9HjhzxaFNSUoK0tDTs3r0bjzzyCMrLywEAra2taG1txXvvvYf3338fFosFTU1NSpVKRCRrcHAQg4ODapehGsVGEG1tbQgNDUVISAgAICkpCY2NjQgLC3O36ejowFNPPQUAiIqKwrp16wAAOp0ODocDw8PDEEJgeHgYV155pVKlEpHG5eTkKLIl/tBDDwEAXn/9da+vWwsUCwir1ereXQQAkiShra3No01ERATMZjOys7PR0NAAu92Ovr4+3HjjjbjlllsQExMDIQQeeOAB/OQnP7noa3I+JiLypqGhIQC++92i6lxM+fn52LhxI2pra2EymSBJEvR6Pb7++mt0dHTgk08+ATC6ddDc3AyTyTTu+jgXExF508yZMwFM7++W8cJPsYCQJAk9PT3u+1arFZIkjWlTVVUFALDb7TCbzQgMDMSuXbvw05/+FHPnzgUAxMbG4vPPP79oQBARkfcodpDaaDSis7MTXV1dcDgcqK+vR1xcnEcbm80Gl8sFAKiurkZGRgYA4Oqrr8bBgwcxMjKC4eFhHDx4cEK7mIiIyHsUG0H4+/ujsLAQubm5cDqdyMjIQHh4OCorKxEZGYn4+Hg0NTWhoqICOp0OJpMJRUVFAIDExEQcOHAAKSkp0Ol0iI2NHRMuRESkLJ0QQqhdhDe0tLRM6/2ERDT5fOFXTON9d/JMaiIiksWAICIiWQwIIiKSxYAgIiJZDAgiIpKl6pnURORb8vPz0dvbq3YZE3by5EkA//01k1YsWLAApaWlP3o9DAgimjS9vb349sQJBPhpY+eF/rsTec99FxRaMPBdzd7AgCCiSRXg54cHLp+vdhnT1o5+m9fWpY0YJyKiSceAICIiWQwIIiKSxYAgIiJZDAgiIpLFgCAiIln8mSsRTZqBgQGcc7m8+lNM8jTgcsE5MOCVdXEEoQEWiwUWi0XtMojIx3AEoQE1NTUAgC1btqhcCdGPExAQAP3gIE+UU9COfhtmBwR4ZV0cQUxxFosF7e3taG9v5yiCiCYVRxBT3PnRw/nbHEUoZ9u2bdi3b5/X1zvw3f7gAC9t1X1fdHQ0cnJyvL5eJQ1o6BjE4HfzGs3SyNxRwOjnO9tL62JAEClscHAQgDIBoTULFixQu4RLYv9ukr7ZV16pciUTNxve+5x1QgjhlTWpbLwLb2uZxWJBQUEBAODZZ5+F0WhUuSK6VL5w4fvpyhf6brzvTo4gpjij0YjIyEj3bSKiycKA0ICsrCy1SyAiH8SA0ACOHIjGp9QPDJS+otxU/5EBA4KI6AJmzZqldgmqYkAQkebl5ORM6S1xrdLOj3uJiGhSKRoQe/fuRWJiIhISElBdXT1meXd3N7Kzs5GSkoLVq1ejp6cHAHDgwAGkpqa6/xmNRvztb39TslQiIvoBxXYxOZ1OFBcXY/v27ZAkCatWrUJcXBzCwsLcbUpKSpCWlob09HTs378f5eXlKCsrQ1RUFOrq6gAAp06dwl133YXo6GilSiUiIhmKjSDa2toQGhqKkJAQGAwGJCUlobGx0aNNR0cHoqKiAABRUVFjlgPARx99hNjYWMye7a2Tx4mIaCIUCwir1YqgoCD3fUmSYLVaPdpERETAbDYDABoaGmC329HX1+fRpr6+HsnJyUqVSUREF6Dqr5jy8/OxceNG1NbWwmQyQZIk6PV69/ITJ07gyy+/RExMzITW19LSolSpRP+zoaEhAPz/SdqjWEBIkuQ+6AyMjigkSRrTpqqqCgBgt9thNpsRGBjoXv7BBx8gISEBM2bMmNBrTse5mMhTfn4+ent71S7jkpw5cwYA8PLLL6tcycQtWLAApaWlapdBk2C8DRfFAsJoNKKzsxNdXV2QJAn19fUoLy/3aGOz2TBv3jz4+fmhuroaGRkZHsvr6+vx+OOPK1UiaVBvby9OfHsCfrO1cwqPy290PsyTA9qY4tp1bkTtEmiKUOyvzN/fH4WFhcjNzYXT6URGRgbCw8NRWVmJyMhIxMfHo6mpCRUVFdDpdDCZTCgqKnI//9ixYzh+/DhuvvlmpUokjfKb7Y8rVi5Uu4xpq+/Do2qXQFOEopthK1aswIoVKzwe++1vf+u+vXLlSqxcuVL2uddccw0+/fRTJcsjIqJx8ExqIiKSxYAgIiJZDAgiIpLFgCAiIlkMCCIiksWAICIiWQwIIiKSxYAgIiJZDAgiIpLFgCAiIlkMCCIiksWAICIiWQwIIiKSxYAgIiJZ2rnqigZs27YN+/bt8/p6BwYGAAABAQFeXzcAREdHIycnR5F1e9vAwABc50Z4zQIFuc6NYAADapdBUwBHEBowODiIwcFBtcsgIh/DEYQX5eTkKLIl/tBDDwEAXn/9da+vW2sCAgIwCAevKKegvg+PKjZaJW3hCIKIiGT55AgiPz8fvb29apcxYSdPngTw35GEVixYsAClpaVql0FE/yOfDIje3l6cOPEtdDNmq13KhIjvBnrf9mnnwKEYPqd2CUT0I/lkQACAbsZsBIT9XO0ypq2BI++pXQLRj2axWAAARqNR5UrUwWMQREQXUFNTg5qaGrXLUA0DgohIhsViQXt7O9rb290jCV/DgCAikvH9kYOvjiIYEEREJMsnD1IPDAxADJ/jgVQFieFzGNDOj66IxsjKykJBQYH7ti/yyYAgIroYo9GIyMhI921f5JMBERAQgHPD4M9cFTRw5D3FpmtQarI+l8MJOIXX16sovQ5+Br1XV+k6NwJwpg0AvjtyOE/RgNi7dy82b94Ml8uFzMxM5OXleSzv7u5GQUEBbDYb5s2bh7KyMgQFBQEAvvnmGzz99NM4fvw4dDodqqurcc0113itNi3tYhJOBwBApzeoXMnEjZ4o5/1vmQULFnh9necNDAxoblLEWYZZ3g/iAGU/Zy3x1ZHDeYoFhNPpRHFxMbZv3w5JkrBq1SrExcUhLCzM3aakpARpaWlIT0/H/v37UV5ejrKyMgDAhg0bsHbtWkRHR8Nut8PPz3vH07X2n//8VBtXXqGlzboART5nTt1BNHkUC4i2tjaEhoYiJCQEAJCUlITGxkaPgOjo6MBTTz0FAIiKisK6desAAEeOHMHIyAiio6MBAHPnzvVqbVr7kuFsrkSkBsUCwmq1uncXAYAkSWhra/NoExERAbPZjOzsbDQ0NMBut6Ovrw+dnZ0IDAzEo48+imPHjuHWW2/FE088Ab1+/H2tLS0tirwXtQ0NDQGYvu+PiKYmVQ9S5+fnY+PGjaitrYXJZIIkSdDr9RgZGUFzczP++te/Ijg4GI899hjeffddZGZmjru+5cuXT1Ll8pS6otyZM2cAAC+//LLX1w1o64pyRORd4214KhYQkiShp6fHfd9qtUKSpDFtqqqqAAB2ux1msxmBgYEICgrCkiVL3Lun4uPj8cUXXyhV6pQ3a9YstUsgIh+kWEAYjUZ0dnaiq6sLkiShvr4e5eXlHm3O/3rJz88P1dXVyMjIcD/39OnTsNlsmD9/Pj777DP375GnMqWuKEdEpAbFAsLf3x+FhYXIzc2F0+lERkYGwsPDUVlZicjISMTHx6OpqQkVFRXQ6XQwmUwoKioCAOj1emzYsAHZ2dkAgOuvv/6iu5eIiMi7dEIIjZ0ZJK+lpUX1YxBERFoz3ncnJ+sjIiJZDAgiIpLFgCAiIlkMCCIiksWAICIiWQwIIiKSxYAgIiJZDAgiIpLFgCAiIlkMCCIiksWAICIiWRMKiC+//BJnz55137fb7Th8+LBiRRERkfomFBBPPvkkZsyY4b4/Y8YMbNiwQbGiiIhIfRMKCKfT6REQBoMBTqdTsaKIiEh9EwoIf39/dHV1ue8fPXr0oteHJiLSOovFAovFonYZqpnQBYMeffRR3HfffVixYgUA4JNPPsGmTZsULYyISG01NTUAgC1btqhciTomFBB33HEHduzYgX379gEA8vLyEBoaqmhhRERqslgsaG9vd982Go0qVzT5JrSLyWaz4eqrr8b999+P+++/H8HBwbDZbErXRkSkmvOjhx/e9iUTCoiHH37Y46D0yMgI1q5dq1hRRESkvgkFhMPhwOzZs93358yZg6GhIcWKIiJSW1ZWluxtXzKhYxDA6G6m+fPnAwB6e3vhcrkUK4qISG1GoxGRkZHu275oQgGxevVq3HfffUhNTYUQAu+99x7WrFmjdG1ERKry1ZHDeRMKiFWrVmHhwoX4+9//Dp1Oh02bNuGmm25SujYiIlX56sjhvAkFxJkzZ/Dpp5/i8OHDGBwcdJ848sYbbyhaHBERqWdCB6kLCgqg1+vR2dmJe+65B3q9HsuWLVO6NiIiUtGEAuLrr7/G+vXrMWvWLCQnJ+PVV19Fc3Oz0rUREZGKJhQQBoMBwOgsrqdOncKMGTN4ohwR0TQ3oWMQ1157LU6dOoWUlBTcc889uOyyy3D99dcrXRsREaloQgHx/PPPAwAefPBBGI1GnDlzBrGxsRd93t69e7F582a4XC5kZmYiLy/PY3l3dzcKCgpgs9kwb948lJWVISgoCACwZMkSLFq0CAAQHByMV1555ZLeGBER/TgTPlHuPJPJNKF2TqcTxcXF2L59OyRJwqpVqxAXF4ewsDB3m5KSEqSlpSE9PR379+9HeXk5ysrKAACzZs1CXV3dpZZHREReotg1qdva2hAaGoqQkBAYDAYkJSWhsbHRo01HRweioqIAAFFRUWOWExGRei55BDFRVqvVvbsIACRJQltbm0ebiIgImM1mZGdno6GhAXa7HX19fbjiiiswNDSEX/ziF/D390deXh7uvPPOi75mS0uL198HEZGvUiwgJiI/Px8bN25EbW0tTCYTJElyX6luz549kCQJXV1dyM7OxqJFi7Bw4cJx17d8+fLJKJuIaNoYb8NasYCQJAk9PT3u+1arFZIkjWlTVVUFALDb7TCbzQgMDHQvA4CQkBDcfPPNOHTo0EUDgoiIvEexYxBGoxGdnZ3o6uqCw+FAfX094uLiPNrYbDb3rLDV1dXIyMgAAPT398PhcLjbtLa2ehzcJiIi5Sk2gvD390dhYSFyc3PhdDqRkZGB8PBwVFZWIjIyEvHx8WhqakJFRQV0Oh1MJhOKiooAjB68Lioqgk6ngxACa9asYUAQEU0ynRBCqF2EN7S0tPAYBBHRJRrvu1OxXUxENMpisbhnQCbSEgYEkcJqamp89qL3pG0MCCIFWSwWtLe3o729naMI0hwGBJGCvj9y4CiCtIYBQUREshgQRAr6/kXvv3+bSAtUnWqDaLozGo2IjIx03ybSEgYEkcI4ciCtYkAQKYwjB9IqHoMgUhhPlCOtYkAQKYwnypFWMSCIFMQT5UjLGBBECuKJcqRlDAgiIpLFgCBSEE+UIy3jz1yJFMQT5UjLGBBECuPIgbSKAUGkMI4cSKt4DIKIiGQxIIiISBYDgoiIZDEgiIhIFgOCiIhkMSCIiEgWA4KIiGQxIIiISBYDgoiIZDEgNIBXJNM29h9plaIBsXfvXiQmJiIhIQHV1dVjlnd3dyM7OxspKSlYvXo1enp6PJYPDAzgtttuQ3FxsZJlTnm8Ipm2sf9IqxQLCKfTieLiYrz22muor6/H+++/jyNHjni0KSkpQVpaGnbv3o1HHnkE5eXlHstfeOEF3HTTTUqVqAm8Ipm2sf9IyxQLiLa2NoSGhiIkJAQGgwFJSUlobGz0aNPR0YGoqCgAQFRUlMfy9vZ29Pb2Ijo6WqkSNYFXJNM29h9pmWKzuVqtVgQFBbnvS5KEtrY2jzYREREwm83Izs5GQ0MD7HY7+vr6cPnll6OkpARlZWX45z//OeHXbGlp8Vr9U8WZM2c8bk/H9zidsf9Iy1Sd7js/Px8bN25EbW0tTCYTJEmCXq9HTU0NbrvtNo+AmYjly5crVKl6DAYDCgoKAAAPP/wwp47WGPYfTXXjbbQoFhCSJHkcdLZarZAkaUybqqoqAIDdbofZbEZgYCA+//xztLS0YOfOnbDb7RgeHsacOXPwxBNPKFXulMUrkmkb+4+0TLGAMBqN6OzsRFdXFyRJQn19/ZiD0DabDfPmzYOfnx+qq6uRkZEBAB7t3n33XbS3t/tkOJzHK5JpG/uPtEqxgPD390dhYSFyc3PhdDqRkZGB8PBwVFZWIjIyEvHx8WhqakJFRQV0Oh1MJhOKioqUKkfTuOWpbew/0iqdEEKoXYQ3tLS0TMtjEEREShrvu5NnUhMRkSwGBBERyWJAEBGRLAYEERHJYkAQEZEsBgQREcliQBARkSwGBBERyWJAEBGRLAYEERHJYkAQEZEsBgQREcliQBARkSwGBBERyWJAEBGRLAYEERHJYkAQEZEsBgQREcliQBARkSwGBBERyWJAEBGRLAYEERHJYkAQEZEsBgQREcliQBARkSwGBBERyWJAEBGRLEUDYu/evUhMTERCQgKqq6vHLO/u7kZ2djZSUlKwevVq9PT0uB9PT09HamoqkpKSsHPnTiXLJCIiGf5KrdjpdKK4uBjbt2+HJElYtWoV4uLiEBYW5m5TUlKCtLQ0pKenY//+/SgvL0dZWRmuuuoqvPXWWzAYDLDb7UhJSUFcXBwkSVKqXCIi+gHFRhBtbW0IDQ1FSEgIDAYDkpKS0NjY6NGmo6MDUVFRAICoqCj3coPBAIPBAABwOBxwuVxKlUlERBegWEBYrVYEBQW570uSBKvV6tEmIiICZrMZANDQ0AC73Y6+vj4AwPHjx5GSkoLbb78da9as4eiBiGiSKbaLaSLy8/OxceNG1NbWwmQyQZIk6PV6AEBwcDB2794Nq9WKdevWITExEVdeeeW462tpaZmMsomIfIJiASFJkvugMzA6ovjhKECSJFRVVQEA7HY7zGYzAgMDx7QJDw9Hc3MzVq5cOe5rLl++3EvVExH5hvE2rBXbxWQ0GtHZ2Ymuri44HA7U19cjLi7Oo43NZnMfX6iurkZGRgYAoKenB4ODgwCA/v5+tLa24rrrrlOqVCIikqHYCMLf3x+FhYXIzc2F0+lERkYGwsPDUVlZicjISMTHx6OpqQkVFRXQ6XQwmUwoKioCMHrw+rnnnoNOp4MQAjk5OVi8eLFSpRIRkQydEEKoXYQ3tLS0cBcTEdElGu+7k2dSExGRLAYEERHJYkAQEZEsBoQGWCwWWCwWtcsgIh/DgNCAmpoa1NTUqF0GEfkYBsQUZ7FY0N7ejvb2do4iiGhSMSCmuO+PHDiKIKLJxIAgIiJZDIgpLisrS/Y2EZHSVJ3NlS7OaDQiMjLSfZuIaLIwIDSAIwciUgMDQgM4ciAiNfAYBBERyWJAEBGRLAYEERHJYkAQEZEsBgQREcmaVr9iGu/i20REdGmmzSVHiYjIu7iLiYiIZDEgiIhIFgOCiIhkMSCIiEgWA4KIiGQxIIiISBYDQuNOnz6NP//5z+773d3dSE9PR2pqKpKSkrBz584xz1m7di2Sk5Mns0yS8cO+A4AlS5YgNTUVqampWLt2rfvxrq4uZGZmIiEhAevXr4fD4ZjscgnyffbQQw/BZDLh4Ycf9nj8Qn128OBBpKenY+nSpfjwww8nrfb/BQNCYUIIuFwuxdZ/+vRpjxC46qqr8NZbb6Gurg67du3CH//4R1itVvdys9mMuXPnKlbPdDLZfQcAs2bNQl1dHerq6vDKK6+4H3/++efxy1/+Eg0NDQgMDMTbb7+tWF1apkaf5ebmorS0dEzbC/VZcHAwtmzZoomNNAaEAo4dO4bExETk5+cjOTkZdXV1uOeee5Ceno7f/OY3sNvtAEb/A919991ISUlBSUkJAODJJ5/Epk2bcO+99yI+Pt5jC+O1115DRkYGUlJS8OKLLwIAysvLcfToUaSmpqKkpAQGgwEGgwEA4HA4PP5Y7HY7tm/fjl/96leT9VFojpp9dyFCCBw4cACJiYkAgPT0dDQ2Nir1EWiO2n126623jtnoGq/PrrnmGkRERMDPTwNfv4K8rqurSyxevFh8/vnnore3V2RlZQm73S6EEOLVV18VW7duFTabTdx1113C5XIJIYTo7+8XQgixYcMG8etf/1o4nU5x+PBhceeddwohhPj000/F008/LVwul3A6nSIvL080NTWJrq4ukZSU5PH633zzjUhOThbLli0TO3bscD++efNmYTabZZ9Do9TuuyVLloj09HSRmZkpGhoahBBC9Pb2utclxGj/sv/+S+0+E0KIAwcOiLy8PPf9ifTZhg0bxAcffODdD8PLptVcTFPJ1VdfjRtuuAF79uzBkSNHcN999wEAhoeHccMNN+Cyyy7DzJkzUVBQgDvuuAO33367+7l33nkn/Pz8EBYWhpMnTwIA9u3bh3379iEtLQ0AcPbsWXR2diI4OHjMawcHB2P37t2wWq1Yt24dEhMT8e233+Lo0aMoKCjAsWPHlP8ANEzNvtuzZw8kSUJXVxeys7OxaNEiBAQEKP+mNU7NPpvOGBAKmTNnDoDRoWZ0dDQqKirGtHn77bexf/9+fPjhh9ixYwfeeOMNAHDvIvo+IQTy8vJw7733ejw+3pe9JEkIDw9Hc3MzbDYb2tvbERcXh5GREdhsNqxevRpvvvnmj3mb05KafSdJEgAgJCQEN998Mw4dOoTExEScPn0aIyMj8Pf3R09Pj7sdjZoKf2/fd8UVV0yLPtPATjBtu+GGG9Da2oqvv/4awOiWyFdffQW73Y4zZ85gxYoVKCgowL///e9x1xMTE4N33nnHvT/VarWit7cXc+fOdT8GAD09PRgcHAQA9Pf3o7W1Fddddx2ysrLwj3/8Ax9//DFqampw7bXXMhwuYrL7rr+/3/1LF5vNhtbWVoSFhUGn0+GWW27BRx99BACora1FXFycEm9Z8ya7zy5kuvQZRxAKmz9/PrZs2YLHH3/c/ce/fv16zJ07F4888giGhoYAjB4sG09MTAw6OjrcWzRz5sxBWVkZFi5ciJ/97GdITk5GbGwsYmJi8Nxzz0Gn00EIgZycHCxevFjZNzlNTXbfJSQkoKioyN13a9asQVhYGADg97//PR577DG88MILWLJkCTIzMxV859o12X22YcMGZGVl4T//+Q/Onj2L2267DZs3b0ZsbOwF+6ytrQ2PPvooTp8+jT179mDr1q2or69X8FP533G6byIiksVdTEREJIsBQUREshgQREQkiwFBRESyGBBERCSLAUFERLIYEESXaOvWreNOrkc0XTAgiGQ4nU61SyBSHc+kJp/zu9/9Dl999RWGh4excOFCPPvss/jXv/6FTZs2ITIyEocOHcL69ethMpnw7LPPor29HTqdDiaTCYWFhQBGp15Ys2YNurq6sHDhQlRWVmL27NkXfM3FixfjscceQ0NDA06dOoX8/Hz3VNBy9Vx++eX47LPPsHnzZixbtgxffPEF/P39UVpaiqqqKhw+fBjBwcHYunUr5syZA4fDgT/84Q84ePAgHA4HFi9ejGeeeYbX/qAfR61pZInU0tvb675dUVEhysrKxIEDB0RERIRobW11L3vyySdFcXGxcDqdHs978cUXRUJCgujv7xcul0s8+OCD4q233hr3NRctWiTefPNNIYQQzc3NIiYmZtx6hBidQnrp0qXi0KFDQgghnnnmGREbGyuOHz8uhBAiNzdX7Nq1SwghxEsvvSReeukl93pKS0tFRUXFJX4yRJ44giCfU1dXh927d2N4eBhnz57Ftddei9jYWISGhuLGG290t9uzZw/effdd94Vd5s+f714WExODwMBAAMCyZctw9OjRi77u3XffDWB0QrkTJ05gaGgIM2fOlK3nvOuuuw5LliwBACxduhTffPMNgoKCAADXX3+9e1K6jz/+GAMDA+7J4RwOByIiIv7Xj4gIAHcxkY9pbm7Gzp078Ze//AXz58/H7t27sWvXLgD/nTJ6ImbOnOm+rdfr3ZPATeQ5er0eADAyMgKLxXLBegDPqaj1ev0FX1cIgaKiItx6660Tfg9EF8OD1ORTTp8+jYCAAMybNw8OhwPvvPPOBdvecccdeP311yG+m8/SZrOpWs944uLi8Kc//ck91fvAwAA6Ojq8WSr5IAYE+ZTY2FgsXLgQiYmJeOCBB7B06dILtn3qqadgt9uRnJyMn//853j55ZdVrWc8eXl5iIiIwKpVq5CSkoKsrCwGBP1onO6biIhkcQRBRESyeJCayEuqqqrQ0NAw5vFt27ZhwYIFKlRE9ONwFxMREcniLiYiIpLFgCAiIlkMCCIiksWAICIiWf8P9Cit+Q3shYMAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df = df[df.acc > 0.8]\n",
    "ax = sns.boxplot(x=df.arch_name, y=df.acc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>acc</th>\n",
       "      <th>batch_size</th>\n",
       "      <th>loss</th>\n",
       "      <th>lr_max</th>\n",
       "      <th>status</th>\n",
       "      <th>time</th>\n",
       "      <th>weight_decay</th>\n",
       "      <th>acc_loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>41</th>\n",
       "      <td>0.995110</td>\n",
       "      <td>64</td>\n",
       "      <td>0.050929</td>\n",
       "      <td>0.000073</td>\n",
       "      <td>ok</td>\n",
       "      <td>785.925107</td>\n",
       "      <td>0.000272</td>\n",
       "      <td>0.944430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>0.995110</td>\n",
       "      <td>48</td>\n",
       "      <td>0.056657</td>\n",
       "      <td>0.000038</td>\n",
       "      <td>ok</td>\n",
       "      <td>804.644610</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.938730</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40</th>\n",
       "      <td>0.992665</td>\n",
       "      <td>64</td>\n",
       "      <td>0.054839</td>\n",
       "      <td>0.000074</td>\n",
       "      <td>ok</td>\n",
       "      <td>785.247967</td>\n",
       "      <td>0.000212</td>\n",
       "      <td>0.938229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>0.992665</td>\n",
       "      <td>32</td>\n",
       "      <td>0.055512</td>\n",
       "      <td>0.000026</td>\n",
       "      <td>ok</td>\n",
       "      <td>853.277575</td>\n",
       "      <td>0.000018</td>\n",
       "      <td>0.937560</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>0.993888</td>\n",
       "      <td>64</td>\n",
       "      <td>0.057410</td>\n",
       "      <td>0.000025</td>\n",
       "      <td>ok</td>\n",
       "      <td>783.253139</td>\n",
       "      <td>0.459626</td>\n",
       "      <td>0.936829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>0.991443</td>\n",
       "      <td>48</td>\n",
       "      <td>0.055415</td>\n",
       "      <td>0.000041</td>\n",
       "      <td>ok</td>\n",
       "      <td>807.265188</td>\n",
       "      <td>0.972279</td>\n",
       "      <td>0.936502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>0.992665</td>\n",
       "      <td>48</td>\n",
       "      <td>0.058264</td>\n",
       "      <td>0.000044</td>\n",
       "      <td>ok</td>\n",
       "      <td>803.678368</td>\n",
       "      <td>0.000044</td>\n",
       "      <td>0.934829</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>44</th>\n",
       "      <td>0.990220</td>\n",
       "      <td>64</td>\n",
       "      <td>0.057395</td>\n",
       "      <td>0.000078</td>\n",
       "      <td>ok</td>\n",
       "      <td>785.528910</td>\n",
       "      <td>0.003478</td>\n",
       "      <td>0.933386</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>37</th>\n",
       "      <td>0.987775</td>\n",
       "      <td>32</td>\n",
       "      <td>0.055611</td>\n",
       "      <td>0.000018</td>\n",
       "      <td>ok</td>\n",
       "      <td>855.304644</td>\n",
       "      <td>0.000061</td>\n",
       "      <td>0.932844</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>0.988998</td>\n",
       "      <td>48</td>\n",
       "      <td>0.059515</td>\n",
       "      <td>0.000014</td>\n",
       "      <td>ok</td>\n",
       "      <td>803.917024</td>\n",
       "      <td>0.148601</td>\n",
       "      <td>0.930138</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>0.991443</td>\n",
       "      <td>48</td>\n",
       "      <td>0.062161</td>\n",
       "      <td>0.000136</td>\n",
       "      <td>ok</td>\n",
       "      <td>803.593891</td>\n",
       "      <td>0.000194</td>\n",
       "      <td>0.929813</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>39</th>\n",
       "      <td>0.987775</td>\n",
       "      <td>32</td>\n",
       "      <td>0.064593</td>\n",
       "      <td>0.000162</td>\n",
       "      <td>ok</td>\n",
       "      <td>854.470828</td>\n",
       "      <td>0.000016</td>\n",
       "      <td>0.923972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>0.987775</td>\n",
       "      <td>48</td>\n",
       "      <td>0.065198</td>\n",
       "      <td>0.000044</td>\n",
       "      <td>ok</td>\n",
       "      <td>804.524059</td>\n",
       "      <td>0.000023</td>\n",
       "      <td>0.923374</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42</th>\n",
       "      <td>0.988998</td>\n",
       "      <td>64</td>\n",
       "      <td>0.067474</td>\n",
       "      <td>0.000246</td>\n",
       "      <td>ok</td>\n",
       "      <td>783.014548</td>\n",
       "      <td>0.000315</td>\n",
       "      <td>0.922266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45</th>\n",
       "      <td>0.986553</td>\n",
       "      <td>64</td>\n",
       "      <td>0.065511</td>\n",
       "      <td>0.000119</td>\n",
       "      <td>ok</td>\n",
       "      <td>784.878294</td>\n",
       "      <td>0.000119</td>\n",
       "      <td>0.921923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48</th>\n",
       "      <td>0.987775</td>\n",
       "      <td>16</td>\n",
       "      <td>0.074814</td>\n",
       "      <td>0.000221</td>\n",
       "      <td>ok</td>\n",
       "      <td>995.380198</td>\n",
       "      <td>0.004952</td>\n",
       "      <td>0.913875</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>0.988998</td>\n",
       "      <td>48</td>\n",
       "      <td>0.076942</td>\n",
       "      <td>0.000418</td>\n",
       "      <td>ok</td>\n",
       "      <td>803.142842</td>\n",
       "      <td>0.008809</td>\n",
       "      <td>0.912902</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>0.986553</td>\n",
       "      <td>32</td>\n",
       "      <td>0.079202</td>\n",
       "      <td>0.000363</td>\n",
       "      <td>ok</td>\n",
       "      <td>854.213687</td>\n",
       "      <td>0.900786</td>\n",
       "      <td>0.908416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>0.988998</td>\n",
       "      <td>16</td>\n",
       "      <td>0.084892</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>ok</td>\n",
       "      <td>995.561854</td>\n",
       "      <td>0.000007</td>\n",
       "      <td>0.905039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43</th>\n",
       "      <td>0.986553</td>\n",
       "      <td>64</td>\n",
       "      <td>0.094182</td>\n",
       "      <td>0.000597</td>\n",
       "      <td>ok</td>\n",
       "      <td>782.951058</td>\n",
       "      <td>0.000977</td>\n",
       "      <td>0.893637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>0.985330</td>\n",
       "      <td>32</td>\n",
       "      <td>0.095480</td>\n",
       "      <td>0.000822</td>\n",
       "      <td>ok</td>\n",
       "      <td>854.505060</td>\n",
       "      <td>0.000003</td>\n",
       "      <td>0.891251</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.977995</td>\n",
       "      <td>48</td>\n",
       "      <td>0.126334</td>\n",
       "      <td>0.003020</td>\n",
       "      <td>ok</td>\n",
       "      <td>804.406476</td>\n",
       "      <td>0.000001</td>\n",
       "      <td>0.854441</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         acc  batch_size      loss    lr_max status        time  weight_decay  \\\n",
       "41  0.995110          64  0.050929  0.000073     ok  785.925107      0.000272   \n",
       "21  0.995110          48  0.056657  0.000038     ok  804.644610      0.000001   \n",
       "40  0.992665          64  0.054839  0.000074     ok  785.247967      0.000212   \n",
       "36  0.992665          32  0.055512  0.000026     ok  853.277575      0.000018   \n",
       "26  0.993888          64  0.057410  0.000025     ok  783.253139      0.459626   \n",
       "24  0.991443          48  0.055415  0.000041     ok  807.265188      0.972279   \n",
       "23  0.992665          48  0.058264  0.000044     ok  803.678368      0.000044   \n",
       "44  0.990220          64  0.057395  0.000078     ok  785.528910      0.003478   \n",
       "37  0.987775          32  0.055611  0.000018     ok  855.304644      0.000061   \n",
       "30  0.988998          48  0.059515  0.000014     ok  803.917024      0.148601   \n",
       "31  0.991443          48  0.062161  0.000136     ok  803.593891      0.000194   \n",
       "39  0.987775          32  0.064593  0.000162     ok  854.470828      0.000016   \n",
       "22  0.987775          48  0.065198  0.000044     ok  804.524059      0.000023   \n",
       "42  0.988998          64  0.067474  0.000246     ok  783.014548      0.000315   \n",
       "45  0.986553          64  0.065511  0.000119     ok  784.878294      0.000119   \n",
       "48  0.987775          16  0.074814  0.000221     ok  995.380198      0.004952   \n",
       "35  0.988998          48  0.076942  0.000418     ok  803.142842      0.008809   \n",
       "25  0.986553          32  0.079202  0.000363     ok  854.213687      0.900786   \n",
       "11  0.988998          16  0.084892  0.000103     ok  995.561854      0.000007   \n",
       "43  0.986553          64  0.094182  0.000597     ok  782.951058      0.000977   \n",
       "38  0.985330          32  0.095480  0.000822     ok  854.505060      0.000003   \n",
       "6   0.977995          48  0.126334  0.003020     ok  804.406476      0.000001   \n",
       "\n",
       "    acc_loss  \n",
       "41  0.944430  \n",
       "21  0.938730  \n",
       "40  0.938229  \n",
       "36  0.937560  \n",
       "26  0.936829  \n",
       "24  0.936502  \n",
       "23  0.934829  \n",
       "44  0.933386  \n",
       "37  0.932844  \n",
       "30  0.930138  \n",
       "31  0.929813  \n",
       "39  0.923972  \n",
       "22  0.923374  \n",
       "42  0.922266  \n",
       "45  0.921923  \n",
       "48  0.913875  \n",
       "35  0.912902  \n",
       "25  0.908416  \n",
       "11  0.905039  \n",
       "43  0.893637  \n",
       "38  0.891251  \n",
       "6   0.854441  "
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = trials_df.copy()\n",
    "df['acc_loss'] = df.acc * (1 - df.loss)\n",
    "df = df[df.acc > 0.95]\n",
    "# df = df[df.arch_name == 'resnet101']\n",
    "df.sort_values(by='acc_loss', ascending=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7f7b61da3278>"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEHCAYAAABFroqmAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3XtYVPeBN/DvmTMXkIs46JDxFhQCToIahRhvxAQ3JUbzso1JYWFDNN3axybbdLd9W7rhFVjp+7zk3TbdbUt8261atKmN6ybUIRez2eaiSWMkGsVLCAYSLwMIqAjCXM457x9UKzmjDjBzDjN8P8+T5zFnLueLHOc753fO+R1BURQFRERE1zDoHYCIiEYflgMREamwHIiISIXlQEREKiwHIiJSYTkQEZEKy4GIiFRYDkREpMJyICIiFZYDERGpGPUOMBSyLKO3txcmkwmCIOgdh4goLCiKAq/Xi5iYGBgMge0ThFU59Pb2orGxUe8YRERhKS0tDXFxcQE9N6zKwWQyARj4Ac1ms85pSEsNDQ3IyMjQOwbpjNvB8Hg8HjQ2Nl79DA1EWJXDlaEks9kMi8WicxrSGn/nBHA7GImhDMfzgDQREamwHIiISIXlQEREKrqVw8aNG1FYWIhNmzbpFSGiKYoCr08Cb/RHRMOhywHpI0eOQBRFvPDCC/j2t7+Njo4OTJw4UY8oEUeSZLzw+gk49zWjz+2DNT4KxSscyLlrut7RiCiM6FIOhw8fxsKFCwEAd911F44ePYply5bpESXi/HTHQbx/5CzcXhkA0HmxH9W7DsMrychdmKxvOAq6fo8Pr7//Od45dAbRFhEPLp6BRbPtvEiURmzEw0pVVVXIyclBenr6oAvUmpubkZ+fj9zcXOTn56OlpeXqY93d3YiNjQUAxMTEoLu7e6QxCMC5833Yd/gvxXCF2yuh5pXjkGQOMUUSr0/C93/2LmpePY7GL87j40878NzvPsIvX27QOxpFgBHvOSxfvhzFxcUoKioatLysrAyFhYXIy8tDbW0tNmzYgJqaGgBAfHw8enp6AAxc9Tx9+tCGPBoauPH7c/xUHwyC/wK43O/Bu+99iLhoUeNUwVNfX693hFHl4Ge9ON12CV7pL7/zfo+E1977DDOtl2GNDavLmALG7UAbI956srKyVMs6Oztx7NgxbNmyBQCwatUqbNy4EV1dXbBarZg9ezbq6uqQk5ODDz/8ECtWrBjSOjMyMnghjB8xE7tQ+8F7ACTVY4JgwMIF8xFlDs8PjPr6emRmZuodY1TZc2T/oGK4wmgUIUTbkZkZeceZuB0Mj9vtHvKX6pCcreRyuZCUlARRHPiWKooibDYbXC4XAGDOnDnweDwoLCzErFmzeDA6SNKnT0BCrAVfHm42GQ1YPMcetsVA/o2Ps8Dg59iCIAiIGxf4NAlE/uj2aVFWVqbXqiOWIAioWLcIz2zah94+H2RFARQgZep4fGv1XL3jUZCtWJSMNz/8Ah7v4L0Hoyhg/qwknVJRpAhJOdjtdrS1tUGSJIiiCEmS0N7eDrvdHorV0TUmT4rFvz/zFRz+9Bw6LvRhxuTxSJ2WoHcsCoEZk8fjydVzUb3rMERRgKIA0RYjyv5uIUxGXt9KIxOSckhMTITD4YDT6UReXh6cTiccDgesVmsoVkdfIhoEzEu36R2DNJBz13QsnjsZJ1q6EGU2Im36BBgMPI2VRm7E5VBZWYk9e/ago6MDa9euRUJCAurq6lBeXo6SkhJUV1cjPj4eVVVVwchLRF8SZTbizjR+GaDgGnE5lJaWorS0VLU8JSUFO3fuHOnbR4xLlz348Fgr3F4Zd942CfaJMXpHIiK6Lp6+ooE39n+OTbsOw2AYGBdWFAX3Zk7Dk4/M5RAAEY1KPGoVYs1nL+L//ecReHwy+j0S3F4JHp+Mtz86jdfeb9E7HhGRXyyHEKvb1wyvJKuWu70SXnq7SYdEREQ3x3IIMVdnL+TrzGl04ZJb4zRERIFhOYTYrOkTrnvO+fRb4jROQ0QUGJZDiD24ZAaMovqv2WwSUfSAQ4dEREQ3x3IIscTx0fjf31qCKZNiYTaJiDKLiBtnwt8/OhfzeaEaEY1SPJVVA6lTE/D8D3LQ2nkZHq+EqbZYiH72JoiIRguWg0YEQeCFb0QUNvj1lYiIVFgORESkwnIgIiIVlgMREamwHIiISIXlQEREKmOqHDxeCecv9V93riMiIhowJq5z8Hgl/PLlI/jjgVNQMHCf3cdX3o6v3H2r3tGIiEalMVEOP/5tPQ4cb4PHNzB1ttfnwS9fOoJoixHZd07ROR0R0egT8cNKHRf68OE1xXCF2yth+6vHdUpFRDS6RXw5nDnXc90ps9vPX9Y4DRFReIj4crAnxsDrU9+JDQAmJkRrnIaIKDzoUg4XLlzAww8/jHnz5oV8XTbrONyZNkm192AxiSjMnRXy9RMRhSNdyiEmJgabN2/G3LlzNVnf9/82C0vmTIbJaIDFJCImauBspfsyp2myfiKicKPL2UomkwkJCQmarS/KYsR3izKxfvUcXLrsReL4KL93ZyMiogEBf0JWVVUhJycH6enpaGxsvLq8ubkZ+fn5yM3NRX5+PlpaWkKRMyjGRZmQZB3HYiAiuomA9xyWL1+O4uJiFBUVDVpeVlaGwsJC5OXloba2Fhs2bEBNTQ0AoKmpCRUVFYOen52djXXr1o0odENDw4heT+Gpvr5e7wg0CnA70EbA5ZCVlaVa1tnZiWPHjmHLli0AgFWrVmHjxo3o6uqC1WpFamoqtm3bFry0f5aRkQGLxRL096XRq76+HpmZmXrHIJ1xOxget9s95C/VIxpfcblcSEpKgiiKAABRFGGz2eByuW762jVr1uD48eNYs2bNoGEqIiLSn27TZ2zdulWvVRMR0U2MaM/Bbrejra0NkiQBACRJQnt7O+x2e1DCERGRPkZUDomJiXA4HHA6nQAAp9MJh8MBq9UalHBERKSPgIeVKisrsWfPHnR0dGDt2rVISEhAXV0dysvLUVJSgurqasTHx6OqqiqUeYmISAMBl0NpaSlKS0tVy1NSUrBz586ghiIiIn3xajAiIlJhORARkQrLgYiIVFgORESkwnIgIiIVlgMREamwHIiISIXlQEREKiwHIiJSYTkQEZEKy4GIiFRYDkREpMJyICIiFZYDERGpsByIiEiF5UBERCosByIiUmE5EBGRCsuBiIhUWA5ERKTCciAiIhVdyuHAgQP42te+hoKCAmzevFmPCEREdAO6lMO0adOwfft27NixA3/84x/R19enRwwiIroOox4rTUpKuvpnURRhMHB0iygULva44dz7GQ41noN1fBT+R3YK7piZqHcsCgMBl0NVVRVef/11nDlzBrt370ZaWhoAoLm5GSUlJbhw4QISEhJQVVWF5OTkgN5z3759mD59OiwWy7DCE9H1dV7sw9M/eQuX+33w+mQIAOqPt+PxlbfjoeyZesejUU5QFEUJ5IkHDhzAlClTUFRUhE2bNl0th+LiYqxevRp5eXmora3Frl27UFNTAwBoampCRUXFoPfJzs7GunXr0Nraiu9///t4/vnnERMTE1BYt9uNhoaGofx8RGPWS+934XDLZXz5X7hRBL771cmINnOPfazJyMgI+Mt4wHsOWVlZqmWdnZ04duwYtmzZAgBYtWoVNm7ciK6uLlitVqSmpmLbtm2q13k8HpSUlKC8vDzgYrjWUH5Aigz19fXIzMzUO0ZY+fHLr6iKAQBMJiOMcVORmWHXPtQIcTsYnuF8sR7RVweXy4WkpCSIoghg4PiBzWaDy+W64et2796NpqYmlJWV4bHHHkNbW9tIYhCRHyaj6P8BBTCbrvMY0Z/pckB69erVWL16tR6rJhoz7l8wHS+91QSPTx603GAQMDtlok6pKFyMaM/Bbrejra0NkiQBACRJQnt7O+z28NtdJYo0j/5VGmZOGY8o88BegsUkIsos4pm1C2Ay8ngD3diI9hwSExPhcDjgdDqRl5cHp9MJh8MBq9UarHxENEwWk4iqp7JxuOkcjjd3ISHOgqV3TkHcOLPe0SgMBFwOlZWV2LNnDzo6OrB27VokJCSgrq4O5eXlKCkpQXV1NeLj41FVVRXKvESjXmtnLw4cb4PJaMDCDDvGx+p38oTBIODONBvuTLPploHCU8Cnso4GV46482ylsSdczlL57avHseutJggCIAgCFEXBt782D8vmT9U7WkQIl+1gtBnOZycHHomC5OhnnXjp7ZPw+mR4vDLcHgker4x/e/Egznf36x2PaEhYDkRBsueDz+HxSn4f23f4rMZpiEaG5UAUJP1uH/yN0UqyArfHf2kQjVYsB9JEv8eH/z7wBXa88Qn+1OCCJMk3f1GYWTp3Cixm9cVlosGArNuT/LyCaPTS5SI4GltaXN34YfVe+Hwy+j0Soi1GJMRZ8OxT2UiIi5wTCxbPseOV95vRdOoC+v+8p2Axi1ieNQ233hKvczqioWE5UEgpioLKzR+g57L36rI+tw8er4Rf7DyEZ564W8d0wSWKBmz85mK8e+gM3v7oNMwmEV+5+1ZkzuJppBR+WA4UUs1nu3Gxx61aLskKDpxog8crRdQ8P0bRgPsyp+G+zGl6RyEaER5zoJDqc/tgMAh+H1MUwOuLvGMPRJGA5UAhlTJ1PCTZ/3WW9okxiIk2aZyIiALBcqCQijIbsXbV7bBcM3QkCAPz/nxr9VwdkxHRjfCYA4XcyiUzYU+MxYtvNqK96zJSpo5Hwf3pSJmaoHc0IroOlgNpYv4sG+bzrB2isMFhJSIiUmE5EBGRCsuBiIhUWA5ERKTCciAiIhWWAxERqbAciIhIheVAREQqLAciIlLRvBwOHz6MgoICFBQU4LnnntN69UQ0QpIk4+TpCzhzrkfvKBRCmk+f4XA4sGPHDgDA448/jp6eHsTGxmodg4iG4YMGF3664yAkWYEsK7BZx6H0iQWYPJH/hiON5nsOJtPAFM2SJMFmsyEqKkrrCEQ0DKfaLuH/bq9HT58XfW4f3F4Jp9sv4Z+q9113WnYKXwGVQ1VVFXJycpCeno7Gxsary5ubm5Gfn4/c3Fzk5+ejpaUloJXu3r0bDz74IOLj42E0cu4/onDwynvN8EqDb86kKMDlfh8+bjynUyoKFUFRlJtW/oEDBzBlyhQUFRVh06ZNSEtLAwAUFxdj9erVyMvLQ21tLXbt2oWamhoAQFNTEyoqKga9T3Z2NtatWwcAkGUZTz/9NJ566imkp6cHFNbtdqOhoWFIPyARBcfv3u7AJ2f6VctNooCVdyXgzpkxOqSiocjIyIDFYgnouQF9bc/KylIt6+zsxLFjx7BlyxYAwKpVq7Bx40Z0dXXBarUiNTUV27ZtU73O4/HAbDbDYDAgJiYm4KDXGsoPSJGhvr4emZmZescY01r7P0PL7mNwe6VBywWDgAfunY8pk0J/3IHbwfAM54v1sI85uFwuJCUlQRQH7vAliiJsNhtcLtcNX/fmm2/iscceQ1FREZKSkpCcnDzcCESkoeVZ0zEh3gKj+JePDYtJxNI5UzQpBtKW5gP+K1aswIoVK7ReLRGNUJTFiOe+swy7/tiEfYfPIsosYuWSGbh/wa16R6MQGHY52O12tLW1QZIkiKIISZLQ3t4Ou90ezHxENIrEjjPj8ZW34/GVt+sdhUJs2MNKiYmJcDgccDqdAACn0wmHwwGr1Rq0cEREpI+A9hwqKyuxZ88edHR0YO3atUhISEBdXR3Ky8tRUlKC6upqxMfHo6qqKtR5iYhIAwGVQ2lpKUpLS1XLU1JSsHPnzqCHIiIifXHiPSIiUmE5EBGRCsuBiIhUWA5ERKTCciDSiKIoON/dj94+r95RiG6KU6ISaeBQYzt+vvNjdHX3Q1EUZKRMxD/8zXxY4zllPY1O3HOgkFAUBZ+duYhDje242OPWO46ums9eROXm/WjrugyvT4ZPUnC4qQM/+Pm7vA8CjVrcc6Cg+6K1G5Vb9uN8dz8MBgFen4ycrGlY//AciOLY+z7yH29+Co9v8EymsqzgYo8H9SfasOD2W3RKRnR9LAcKqj63DyW/2Iuey15c+534rfrTGBdlxBMPZeiWTS8trd3wd9cUj0/CmfYegNMU0Sg09r7GUUi9c/AMvD4ZX/4sdHslvLKvRXUvgLEg+ZZ4CIJ6udlowBQbp7qm0YnlQEHVfPYi+j3+C0AQgPPd6juJRbpHlt8Gs1EctMwgAPExFmTOStIpFdGNsRwoqG5JHAez0f9m5fHJiI8xa5xoQFvXZbx/5CwavziPAO6MG1QzJo/HP61dgEkTomEyGmAUDchImYiqp5ZCNPjZpSAaBXjMgYLqvsxp2Oo85vcxRVHQ1d2PcVEmzfJ4fTJ+/EI99h9thUk0QFYUJI6PRsW6RUiyjtMsx/x0G379zP3o6u6HxWxEbLR2fwdEw8E9Bwqq8bEWiKL/b8MCBo5JaOk3dUfx4bFWeH0yLrt96PdIcHX0oHTTPs33IARBQOL4aBYDhQWWAwWdLPtfrihAv8enWQ5JkvHanz6Hxzs4kKwAF3rcONbcpVkWonDDcqCgm52a6He5xSwiy6HdAdg+jwRJ8t9UAgR0XOjTLAtRuGE5UNB9/aEMRJnFQadvmk0i0m+dgNkpEzXLMc5iRMx1hnAkWUby5HjNshCFG5YDBd2t9nj85DvLsHi2HfExZiRZx6HwK+ko+7tFEPyd8B8iBoOAotxZsJgGn0ZqMhrgSE7ErbewHIiuh2crUUhMS4pDyeMLNFufLCto7eyFxSwicXz01eUPLEqGJCt44fUTcP/5+ot75k3FNx+erVk2onDEcqCwd6ixHc/97iB6+7yQFQXJ9niUFN8Fm3UcBEHAqqUzsWLxDHT3uDEu2qTakyAiNQ4rUVhzdfSicvN+dHX3w+2V4PXJOHn6In5YvXfQjKeiQcCE+CgWA1GAdCuHrVu3Ys2aNXqtniLEK+81Q5K/fKqqgkuXvTj86TmdUhGFP13Kwev14sSJE3qsmiJMa2cvfJL6YjZFUdB5kaeqEg2XLuVQW1uLlStX6rFqijCzUybCYlJvxrKi4LbpE3RIRBQZAiqHqqoq5OTkID09HY2NjVeXNzc3Iz8/H7m5ucjPz0dLS8tN30uWZezduxfZ2dnDDk10xV8tmI64GMugCewsJhGZs5J4qirRCAR0ttLy5ctRXFyMoqKiQcvLyspQWFiIvLw81NbWYsOGDaipqQEANDU1oaKiYtDzs7OzMX36dOTk5IwodENDw4heT+Gpvr7e7/I1OQl4+0g3Tpzug0kUkHVbLO5ON1z3+RTe+HvVhqAMYfaxnJwcbNq0CWlpaejs7ERubi4++OADiKIISZJw9913Y8+ePbBardd9j+effx779++HwWBAQ0MDvve97+HRRx8NaP1utxsNDQ3IyMiAxWIJNDZFgPr6emRmZuodg3TG7WB4hvPZOezrHFwuF5KSkiCKA6cGiqIIm80Gl8t1w3JYv3491q9fDwBYs2ZNwMVARETa0fUiuK1bt+q5+oCdaruE94+4IAjA4jmTMWUSb+1IRJFt2OVgt9vR1tYGSZKuDiu1t7fDbrcHM5/ufvvqcfzn202QJAUQgB1vfIKC+9Px6PI0vaMREYXMsE9lTUxMhMPhgNPpBAA4nU44HI4bDimFm6ZTF/DS2yfh8cqQZAWSpMDjlbHjjU/weWu33vGIiEImoHKorKzEPffcg9bWVqxdu/bqNQrl5eXYvn07cnNzsX37dtXZSeHurY9OweOTVMt9kqL5Hc2IiLQU0LBSaWkpSktLVctTUlKwc+fOoIcaLSRZAfycy6XIynVvIjNaKYqCcxf6YBQNsMZH6R2HiEY5zsp6A4vnTMYb+7+4OtXzFWaTiMVzJuuUaugONbbjZy8ewoUeNxQFmDIpFv/wN/Mxc8p4vaMR0SjFWVlvIGNmIpbMmYwo88DpugIGbnV5X9ZUpIXJ1AwnT19A5eb9aD/fB49Xhtcno8XVjR/+Yi+6uvv1jkdEoxT3HG5AEAR8p2Ae7p0/Fe8cPANBAO7NnKrprS5H6vdvNPo9buKVZLyyrxl/u8KhQyoiGu1YDjchCALmpdswL92md5RhaTp9Af6ugff6ZBxv6dI+EBGFBQ4rRbhJE6L9LjcYBNgnxmichojCBcshwq2+7zZYzOq7nxlFA1YtnalDIiIKByyHCLfgjlvwSM5tMBkNiLaIiLYYYTGJePKRuUi2c0prIvKPxxzGgIL70/HAwmR8/Ok5GEUD5qVPwrgok96xiGgUYzmMEQlxFiybP1XvGEQUJlgOY4CiKNh76Cxeea8ZfR4fsudOwYNLZiDawl8/EfnHT4cx4Oc7P8Y7B0+j/89Xep9qvYT/+vALPPedZYhiQRCRHzwgHeFOtV3CWx+duloMAODxyTh3vg9vfviFjsmIaDRjOWistbMX7x9x4dNT5zGEO7QO28efnvM7eaDbK+H9BlfI109E4YljChpxeyU8W/MhDjWeg9FogCwrmJgQjYpvLILNOi5k642JNsEgCoBv8HIBQNw4c8jWS0ThjXsOGnl+18c41HgOHp+My/0+9HsknD3Xg9JN74V0D+LuO27xu+dgNol4cMmMkK03HDWfvYhntx3Ak8/+N/5l+wF87uINnWjsYjlooM/twzsHz8DjG3wPCFkBLvS4caw5dHMcjYsyYcPXF2KcxYhoixHRFhEmowH596eF1QSCofbxp+fwP//tXez9+Ay+aLuEdw+dwXf/9R00nOzQOxqRLjispIELl9wQDQK8fh5TFAWtnb24Y2ZiyNY/O3UiaioewMFP2tHvkTD3tomYEMcb/lyhKAp+8R8fw+39y0F7WRkYCqzedRjV38/RMR2RPlgOGrCOj/I7MyowMOIz/Za4kGewmEQszLCHfD3hqLffh/bzl/0+duZcD/rdPp7yS2MOh5U0YDGJWLlkBiymwRPgGUUB05PicNu08LhxUKQyGw0QIPh9zCAIMBn5z4TGHm71GileeTtyF94Ks9GAcVFGmIwGzEmdhIp1i/SONuaZTSIWzbbDKA4uCJPRgHvunAJR5D8TGnu4r6wR0SDgG389G0UPzMLZjl4kxkdhQjzH/UeLJx+Zi/auy2hp7YZBECDLClKmjsc3H56tdzQiXWheDqdPn0ZBQQFmzJgBu92OZ599VusIuhoXZULq1AS9Y9CXxESb8C9P34OmUxdw+lwPptlikcLfE41huuw5LFu2DD/60Y/0WDXRDaVOS0DqNJYCkS6DqXv37kVhYSH+8Ic/6LF6IiK6iYDKoaqqCjk5OUhPT0djY+PV5c3NzcjPz0dubi7y8/PR0tJy0/ey2Wx47bXXsHnzZvz+97/H+fPnhx2eiIhCI6BhpeXLl6O4uBhFRUWDlpeVlaGwsBB5eXmora3Fhg0bUFNTAwBoampCRUXFoOdnZ2dj3bp1V/8/KysLp06dwoQJQzuVs6GhYUjPp/Dm8ck4eLIXv3nzNUSbDbgrLQYzkngwf6yqr6/XO8KYEFA5ZGVlqZZ1dnbi2LFj2LJlCwBg1apV2LhxI7q6umC1WpGamopt27apXtfb24uYmBgoioKGhgZV4QQiIyMDFotlyK+j8HO534t//Ok7aO/qwZULmE+2evDVZSkoWuHQNxxprr6+HpmZmXrHCDtut3vIX6qHfczB5XIhKSkJojhwYZcoirDZbHC5bjwN9MGDB/Hwww+joKAAS5Ysgc1mG24EGgNq3zmJ9vOXcc3MFnB7Jex6q+m6VzUT0chpfrbS0qVLsXTpUq1XS2Hq3YNn4P3ShIUAYBCA+hPtWLEoWftQRGPAsPcc7HY72traIEkDX+kkSUJ7ezvsds7fQ8FjvM7UFYIgwCT6n/KCiEZu2OWQmJgIh8MBp9MJAHA6nXA4HLBarUELR5S78FbVnFQAIMsKFtzBLyJEoRJQOVRWVuKee+5Ba2sr1q5di5UrVwIAysvLsX37duTm5mL79u2qs5OIRip3YTLumJkIs3FgL8EoGmA2GfD3X7sT8TG8kx1RqAiKFjcyDpIrR9x5ttLYoigKdr36HnqUBMRGm7Bs3jRMmhCtdyzSAc9WGp7hfHZy4j0a9QRBwIykKGRm3qF3FKIxg3MRExGRCsuBiIhUWA5ERKTCYw5EpIvuXg8ONbbDZBQxf5bN7ynLpB+WAxFp7rX3W/Crl48MugXr//r63ZidMlG/UDQIh5WISFOn2i7hV7VH4PHJ6HP7rv638dcfwH3tJFqkK5YDEWnq7Y9OQ5L8X1710Yl2jdPQ9bAciEhTbq8ESfZfDh7uOYwaLAci0tSi2XZYzOqDzz5Jxrx0TuE/WrAciEhTjmQr7p0/FRazCEEADAYBZpMB38jL4HxZowjPViIiTQmCgCcfmYucrGl474gLFqOIezOnYlpSnN7R6BosByLSnCAIuH1GIm6fkah3FLoODisREZEKy4GIiFRYDkREpMJyICIiFZYDERGp8GylCNDT58U7B0+jtbMXt94Sj6V3TuEMl0Q0IiyHMHf0s05U/PufIMsK3F4JUWYRm3cfxf95cinPGyeiYeOwUhjzeCX886//hD637+pslv0eCZd6Pdi4+QMoiv/5a4iIbkaXPYcdO3bg1VdfhSzL2Lx5M0wmkx4xwt6B423w9/mvADjf3Y/PzlxEytQEzXMRUXCcaOnCi//ViI6LfVg0ezK+uiwFURZtPrY1L4ezZ8+isbERv/nNb7RedcS52OOGJMt+HzMYBFzs8WiciIiC5ePGc/jnzR9cnan2THsP9h9txU++cw8EQQj5+jUfVtq3bx/6+vpQXFyMn/3sZ1qvPqKkTkuAAP8bidcnY8bkeI0TEVGwbK07OmgKc49Pxulzl9BwslOT9QdUDlVVVcjJyUF6ejoaGxuvLm9ubkZ+fj5yc3ORn5+PlpaWm75XZ+fAD1ZTU4PTp0/j+PHjw0tOuG3aBMycMh5GcfCv0WISkT13CibER+mUjIhGqq3rsnqhArR29mqy/oCGlZYvX47i4mIUFRUNWl5WVobCwkLk5eWhtrYWGzZsQE1NDQCgqakJFRUVg56fnZ2NuLg4LFiwAACQlZWF5uZmOByOIYVuaGgY0vMj2V/fFYU/yBacON0H0SCZNJQlAAAEsUlEQVRAVoA5yVFYcpuM+vp6veMFVaT9PDQ8Y2U7sE8Q0XPZi2sPK/okGVKvC/X1HSFff0DlkJWVpVrW2dmJY8eOYcuWLQCAVatWYePGjejq6oLVakVqaiq2bdumet3Ro0fhdDoBAI2NjVi1atWQQ2dkZMBisQz5dZFq8cKBax3Od/djYkI0ojU6YKWl+vp6ZGZm6h2DdDaWtoPJt/biH//1bXi9MtxeCRaziPsXTMcDOXOG/F5ut3vIX6qH/SnicrmQlJQEURy42EoURdhsNrhcLlit1uu+7o477sDLL7+Mxx57DMnJyZg7d+5wI9A1YqNNiI3mWV9EkcI+MQa/+qf78e7B0+jq7se8dBscydf/bA02Xb5iPvPMM3qslogorMRGm7Bi8Qxd1j3ss5Xsdjva2togSQNH0yVJQnt7O+x2e9DCERGRPoZdDomJiXA4HFePHzidTjgcjhsOKRERUXgIaFipsrISe/bsQUdHB9auXYuEhATU1dWhvLwcJSUlqK6uRnx8PKqqqkKdl4iINBBQOZSWlqK0tFS1PCUlBTt37gx6KCIi0hcn3iMiIpWwOiH+yiyjHg/nDBqL3G633hFoFOB2MHRXPjOHMlOzoITRvM6XLl0aNH0HEREFLi0tDXFxgd3nJazKQZZl9Pb2wmQyaTIrIRFRJFAUBV6vFzExMTAYAjuaEFblQERE2uABaSIiUmE5EBGRCsuBiIhUWA5ERKTCciAiIhWWAxERqbAciIhIheVAREQqLAciIlJhOVBE2bp1K9asWaN3DNLJ4cOHUVBQgIKCAjz33HN6xwlrYTUrK9GNeL1enDhxQu8YpCOHw4EdO3YAAB5//HH09PQgNjZW51ThiXsOFDFqa2uxcuVKvWOQjkwmE4CBe9rbbDZERUXpnCh8sRxIN1VVVcjJyUF6evqgqdibm5uRn5+P3Nxc5Ofno6Wl5abvJcsy9u7di+zs7BAmplAI5nYAALt378aDDz6I+Ph4GI0cHBku/s2RbpYvX47i4mIUFRUNWl5WVobCwkLk5eWhtrYWGzZsQE1NDQCgqakJFRUVg56fnZ2N6dOnIycnR7PsFDzB3A7WrVuHhx56CCtXrsTTTz+NTz75BOnp6Zr9LBFFIdLZfffdp3zyySeKoihKR0eHkpmZqfh8PkVRFMXn8ymZmZlKZ2fnDd+jurpaWbNmjfLEE08oCxYsUF588cWQ56bgCsZ24Ha7r/75Bz/4gdLc3ByyvJGOew40qrhcLiQlJUEURQCAKIqw2WxwuVywWq3Xfd369euxfv16AMCaNWvw6KOPapKXQmO428Gbb76JF154AbIsIysrC8nJyRoljjwsB4o4W7du1TsC6WTFihVYsWKF3jEiAg9I06hit9vR1tYGSZIADJx10t7eDrvdrnMy0hK3A/2xHGhUSUxMhMPhgNPpBAA4nU44HI4bDiVQ5OF2oD/eQ5p0U1lZiT179qCjowMTJkxAQkIC6urqcPLkSZSUlKC7uxvx8fGoqqrCzJkz9Y5LIcLtYHRiORARkQqHlYiISIXlQEREKiwHIiJSYTkQEZEKy4GIiFRYDkREpMJyICIiFZYDERGpsByIiEjl/wNdOXyIMGMvswAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.subplot()\n",
    "plt.xscale('log')\n",
    "plt.yscale('log')\n",
    "plt.minorticks_on()\n",
    "plt.scatter(x=df.lr_max.values, y=df.weight_decay.values, s=(df.acc.values - 0.95)*1000)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TTA Experiments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "............."
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.0782444745229334, 0.9865525672371638)"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tta_probs_all = []\n",
    "tta_labels_all = []\n",
    "\n",
    "model.eval()\n",
    "running_loss, running_corrects = 0.0, 0.0\n",
    "dataloaders = load_data(batch_size=64, tta=True)\n",
    "for inputs, labels in dataloaders['valid']:\n",
    "    print('.', end='')\n",
    "    inputs = inputs.to(device)\n",
    "    labels = labels.to(device)\n",
    "    \n",
    "    with torch.set_grad_enabled(False):\n",
    "        ns, ncrops, c, h, w = inputs.size()\n",
    "        outputs = model(inputs.view(-1, c, h, w))\n",
    "        outputs_avg = outputs.view(ns, ncrops, -1).mean(1)\n",
    "        _, preds = torch.max(outputs_avg, 1)\n",
    "        loss = criterion(outputs_avg, labels)\n",
    "        \n",
    "        tta_probs_all.append(outputs_avg.cpu().numpy())\n",
    "        tta_labels_all.append(labels.cpu().numpy())\n",
    "\n",
    "        running_loss += loss.item() * ns\n",
    "        running_corrects += torch.sum(preds == labels.data).cpu().numpy()\n",
    "\n",
    "tta_probs_all = np.concatenate(tta_probs_all)\n",
    "tta_labels_all = np.concatenate(tta_labels_all)\n",
    "        \n",
    "epoch_loss = running_loss / len(dataloaders['valid'].dataset)\n",
    "epoch_acc = running_corrects / len(dataloaders['valid'].dataset)\n",
    "epoch_loss, epoch_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "# np.concatenate(avg_preds_all),\n",
    "# np.concatenate(labels_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "............."
     ]
    },
    {
     "data": {
      "text/plain": [
       "(0.07271976198791583, 0.9816625916870416)"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "probs_all = []\n",
    "labels_all = []\n",
    "\n",
    "model.eval()\n",
    "running_loss, running_corrects = 0.0, 0.0\n",
    "dataloaders = load_data(batch_size=64, tta=False)\n",
    "for inputs, labels in dataloaders['valid']:\n",
    "    print('.', end='')\n",
    "    inputs = inputs.to(device)\n",
    "    labels = labels.to(device)\n",
    "    \n",
    "    with torch.set_grad_enabled(False):\n",
    "        outputs = model(inputs.view(-1, c, h, w))\n",
    "        _, preds = torch.max(outputs, 1)\n",
    "        loss = criterion(outputs, labels)\n",
    "        \n",
    "        probs_all.append(outputs)\n",
    "        labels_all.append(labels)\n",
    "\n",
    "        running_loss += loss.item() * ns\n",
    "        running_corrects += torch.sum(preds == labels.data).cpu().numpy()\n",
    "\n",
    "probs_all = np.concatenate(probs_all)\n",
    "labels_all = np.concatenate(labels_all)\n",
    "        \n",
    "epoch_loss = running_loss / len(dataloaders['valid'].dataset)\n",
    "epoch_acc = running_corrects / len(dataloaders['valid'].dataset)\n",
    "epoch_loss, epoch_acc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.0 803 0.9816625916870416\n",
      "0.1 804 0.9828850855745721\n",
      "0.2 806 0.9853300733496333\n",
      "0.30000000000000004 806 0.9853300733496333\n",
      "0.4 805 0.9841075794621027\n",
      "0.5 805 0.9841075794621027\n",
      "0.6000000000000001 806 0.9853300733496333\n",
      "0.7000000000000001 806 0.9853300733496333\n",
      "0.8 807 0.9865525672371638\n",
      "0.9 807 0.9865525672371638\n",
      "1.0 807 0.9865525672371638\n"
     ]
    }
   ],
   "source": [
    "for beta in np.linspace(0, 1, 11):\n",
    "    comb_probs = beta * tta_probs_all + (1 - beta) * probs_all\n",
    "    corrects = (comb_probs.argmax(axis=1) == labels_all)\n",
    "    print(beta, corrects.sum(), corrects.mean())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "a = np.asarray(list(zip(tta_probs_all.argmax(axis=1), probs_all.argmax(axis=1), labels_all)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 66,  66,  13],\n",
       "       [ 28,  28,  26],\n",
       "       [ 13,  25,  36],\n",
       "       [ 64,  64,  47],\n",
       "       [ 52,  47,  47],\n",
       "       [ 28,  28,  51],\n",
       "       [ 75,  75,  69],\n",
       "       [ 28,  28,  76],\n",
       "       [ 96,  96,  95],\n",
       "       [ 99,  99,  98],\n",
       "       [ 24,  24, 101]])"
      ]
     },
     "execution_count": 131,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a[~((a[:,0] == a[:,2]) | (a[:,0] == a[:,2]))]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Cross Validation Model Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "code_folding": []
   },
   "outputs": [],
   "source": [
    "from flowers.dataset import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_valid_ds = TrainValidListDataset(data_path, 'train', 'valid')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "np.random.seed(13)\n",
    "\n",
    "arch = models.resnet101\n",
    "\n",
    "cv_results = []\n",
    "cv = KFold(n_splits=5, shuffle=True)\n",
    "for i, (train_inds, valid_inds) in enumerate(cv.split(train_valid_ds)):\n",
    "    datasets = {\n",
    "        'train': SubsetDataset(train_valid_ds, train_inds, train_transform),\n",
    "        'valid': SubsetDataset(train_valid_ds, valid_inds, valid_transform_tta)\n",
    "    }\n",
    "    print(f'Fold: {i}', len(datasets['train']), len(datasets['valid']))\n",
    "    batch_size = 48\n",
    "    num_workers = 4\n",
    "    dataloaders = {\n",
    "        'train': torch.utils.data.DataLoader(datasets['train'], batch_size=batch_size,\n",
    "                                             shuffle=True, num_workers=num_workers),\n",
    "        'valid': torch.utils.data.DataLoader(datasets['valid'], batch_size=batch_size,\n",
    "                                             shuffle=False, num_workers=num_workers)\n",
    "    }\n",
    "    epochs = 15\n",
    "    lr_max = 3e-3\n",
    "    weight_decay = 5e-2\n",
    "    \n",
    "    model = create_model(arch)\n",
    "    optimizer = AdamW(filter(lambda p: p.requires_grad, model.parameters()),\n",
    "                      lr=0.01, betas=(0.9, 0.99), weight_decay=weight_decay)\n",
    "    scheduler = OneCycleScheduler(optimizer, lr_max=lr_max,\n",
    "                                  batches=len(dataloaders['train']), epochs=epochs, debug=False)                  \n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    model, best_loss_head, best_acc_head = train_loop(dataloaders, model, criterion, optimizer, scheduler,\n",
    "                                                      epochs=epochs)\n",
    "    \n",
    "    batch_size = 64\n",
    "    num_workers = 4\n",
    "    dataloaders = {\n",
    "        'train': torch.utils.data.DataLoader(datasets['train'], batch_size=batch_size,\n",
    "                                             shuffle=True, num_workers=num_workers),\n",
    "        'valid': torch.utils.data.DataLoader(datasets['valid'], batch_size=batch_size,\n",
    "                                             shuffle=False, num_workers=num_workers)\n",
    "    }\n",
    "    epochs = 15\n",
    "    lr_max = 7e-5\n",
    "    weight_decay = 2e-4\n",
    "    \n",
    "    unfreeze(model)\n",
    "    optimizer = AdamW(filter(lambda p: p.requires_grad, model.parameters()),\n",
    "                      lr=0.01, betas=(0.9, 0.99), weight_decay=weight_decay)\n",
    "    scheduler = OneCycleScheduler(optimizer, lr_max=lr_max,\n",
    "                                  batches=len(dataloaders['train']), epochs=epochs, debug=False)\n",
    "    criterion = nn.CrossEntropyLoss()\n",
    "    model, best_loss, best_acc = train_loop(dataloaders, model, criterion, optimizer, scheduler,\n",
    "                                            epochs=epochs)\n",
    "    \n",
    "    model_path = f'resnet-cv-{i}.pth'\n",
    "    save_model(model, model_path)\n",
    "    \n",
    "    d = {\n",
    "        'model_path': model_path,\n",
    "        'best_loss_head': best_loss_head,\n",
    "        'best_acc_head': best_acc_head,\n",
    "        'best_loss': best_loss,\n",
    "        'best_acc': best_acc\n",
    "    }\n",
    "    cv_results.append(d)\n",
    "    print(f'Results: {d}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>best_acc</th>\n",
       "      <th>best_acc_head</th>\n",
       "      <th>best_loss</th>\n",
       "      <th>best_loss_head</th>\n",
       "      <th>model_path</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.993216</td>\n",
       "      <td>0.969471</td>\n",
       "      <td>0.038495</td>\n",
       "      <td>0.112781</td>\n",
       "      <td>resnet-cv-0.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.993894</td>\n",
       "      <td>0.971506</td>\n",
       "      <td>0.043253</td>\n",
       "      <td>0.123707</td>\n",
       "      <td>resnet-cv-1.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.981004</td>\n",
       "      <td>0.965400</td>\n",
       "      <td>0.082509</td>\n",
       "      <td>0.146316</td>\n",
       "      <td>resnet-cv-2.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.986431</td>\n",
       "      <td>0.964043</td>\n",
       "      <td>0.073682</td>\n",
       "      <td>0.140441</td>\n",
       "      <td>resnet-cv-3.pth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.987788</td>\n",
       "      <td>0.976255</td>\n",
       "      <td>0.059606</td>\n",
       "      <td>0.115909</td>\n",
       "      <td>resnet-cv-4.pth</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   best_acc  best_acc_head  best_loss  best_loss_head       model_path\n",
       "0  0.993216       0.969471   0.038495        0.112781  resnet-cv-0.pth\n",
       "1  0.993894       0.971506   0.043253        0.123707  resnet-cv-1.pth\n",
       "2  0.981004       0.965400   0.082509        0.146316  resnet-cv-2.pth\n",
       "3  0.986431       0.964043   0.073682        0.140441  resnet-cv-3.pth\n",
       "4  0.987788       0.976255   0.059606        0.115909  resnet-cv-4.pth"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "cv_results_df = pd.DataFrame(cv_results)\n",
    "cv_results_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# test_ds = torchvision.datasets.ImageFolder(root=str(data_path / 'test'), transform=valid_transform_tta)\n",
    "# test_dl = torch.utils.data.DataLoader(test_ds, batch_size=64, shuffle=False, num_workers=4)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Predict and Submit Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 1.0\n",
      "1 0.9987775061124694\n",
      "2 0.9975550122249389\n",
      "3 0.9963325183374083\n",
      "4 0.9951100244498777\n",
      "5 0.9938875305623472\n",
      "6 0.9926650366748166\n",
      "7 0.991442542787286\n",
      "8 0.9902200488997555\n",
      "9 0.988997555012225\n",
      "10 0.9877750611246944\n",
      "11 0.9865525672371638\n",
      "12 0.9853300733496333\n",
      "13 0.9841075794621027\n",
      "14 0.9828850855745721\n",
      "15 0.9816625916870416\n",
      "16 0.980440097799511\n",
      "17 0.9792176039119804\n",
      "18 0.9779951100244498\n",
      "19 0.9767726161369193\n"
     ]
    }
   ],
   "source": [
    "n = 818\n",
    "for i in range(20):\n",
    "    print(i, (n - i) / n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "from flowers.test_predict import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "............."
     ]
    }
   ],
   "source": [
    "test_ds = torchvision.datasets.ImageFolder(root=str(data_path / 'test'), transform=valid_transform_tta)\n",
    "test_dl = torch.utils.data.DataLoader(test_ds, batch_size=64, shuffle=False, num_workers=4)\n",
    "test_preds = predict_test(test_dl, model)\n",
    "# test_preds = ensemble_predict_test(models.resnet101, test_dl, [f'resnet-cv-{i}.pth' for i in range(5)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx_to_class = {v: k for k, v in dataloaders['valid'].dataset.class_to_idx.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Saved to file: submission.csv\n"
     ]
    }
   ],
   "source": [
    "create_submission(test_preds, idx_to_class, fn='submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !ls -lah |grep submission"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !kaggle competitions submit oxford-102-flower-pytorch -f submission.csv\\\n",
    "#  -m \"resnet101, 15 epochs, bs 48/64, 1cycle + other fastai tricks, lr=3e-3/7e-5, AdamW wd=5e-2/2e-4, basic affine aug, 45 rot, 10xTTA, retrained\""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Environment (conda_pytorch_p36)",
   "language": "python",
   "name": "conda_pytorch_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}