CNN - Regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Set-up\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pylab as plt\n",
    "import seaborn as sns\n",
    "import warnings\n",
    "from IPython.display import clear_output\n",
    "import nibabel as nib\n",
    "\n",
    "import torch\n",
    "import torchvision.models as models\n",
    "import torchvision.transforms as transforms\n",
    "import torch.optim as optim\n",
    "import torch.nn as nn\n",
    "from torch.optim import lr_scheduler\n",
    "import torch.nn.functional as F\n",
    "import torch.utils.data as utils\n",
    "import time\n",
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([536, 3, 224, 224]) torch.Size([536])\n"
     ]
    }
   ],
   "source": [
    "x = torch.load(\"../../../data/data.pt\")\n",
    "y = torch.load(\"../../../data/ages.pt\")\n",
    "n = x.shape[0]\n",
    "print(x.shape,y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1570, 3, 224, 224]) torch.Size([1570, 1])\n"
     ]
    }
   ],
   "source": [
    "fcp_x = torch.load(\"../../../data/fcp_data.pt\")\n",
    "fcp_y = torch.load(\"../../../data/fcp_ages.pt\")\n",
    "fcp_n = fcp_x.shape[0]\n",
    "combined_x = torch.cat((x,fcp_x))\n",
    "combined_y = torch.cat((y,fcp_y))\n",
    "combined_y = torch.reshape(combined_y,(fcp_n + n,1))\n",
    "N = combined_x.shape[0]\n",
    "print(combined_x.shape,combined_y.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Normalizing the data (according to the stats provided) before feeding into the pretrained model\n",
    "x_normalized = torch.Tensor(x.shape)\n",
    "for i in range(n):\n",
    "    x_normalized[i] = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "                                     std=[0.229, 0.224, 0.225]).__call__(x[i,:,:])\n",
    "#     fcp_x_normalized[i] = transforms.Normalize(mean=[0.485, 0.456, 0.406],\n",
    "#                                      std=[0.229, 0.224, 0.225]).__call__(fcp_x[i,:,:])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "using device: cuda\n"
     ]
    }
   ],
   "source": [
    "USE_GPU = True\n",
    "\n",
    "dtype = torch.float32 # we will be using float throughout this tutorial\n",
    "\n",
    "if USE_GPU and torch.cuda.is_available():\n",
    "    device = torch.device('cuda')\n",
    "else:\n",
    "    device = torch.device('cpu')\n",
    "\n",
    "print('using device:', device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = torch.reshape(y,(n,1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 322 (0.60) training data points, 107 (0.20) validation data points,\u0007nd 107 (0.20) test data points\n"
     ]
    }
   ],
   "source": [
    "# dataloaders = {i: torch.utils.data.DataLoader(dataset, batch_size=4,\n",
    "#                                              shuffle=True, num_workers=4)\n",
    "#               for i in ['train', 'val']}\n",
    "train_indices = [i for i in range(n) if (i % 5 == 0 or i % 5 == 2 or i % 5 == 4)]\n",
    "val_indices = [i for i in range(n) if i % 5 == 3]\n",
    "test_indices = [i for i in range(n) if i not in train_indices and i not in val_indices]\n",
    "\n",
    "print(\"There are %d (%.2f) training data points, %d (%.2f) validation data points,\\and %d (%.2f) test data points\" %(len(train_indices),len(train_indices)/n,len(val_indices),\\\n",
    "                                                                                                                len(val_indices)/n,len(test_indices),len(test_indices)/n))\n",
    "train_dataset = utils.TensorDataset(x_normalized[train_indices],y[train_indices])\n",
    "val_dataset = utils.TensorDataset(x_normalized[val_indices],y[val_indices])      \n",
    "dataloaders = {}\n",
    "dataloaders['train'] = torch.utils.data.DataLoader(train_dataset, batch_size=4,\n",
    "                                             shuffle=True,num_workers=4)\n",
    "dataloaders['val'] = torch.utils.data.DataLoader(val_dataset, batch_size=4,\n",
    "                                             shuffle=True,num_workers=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_model(model, criterion, optimizer, scheduler, num_epochs=25):\n",
    "    since = time.time()\n",
    "\n",
    "    best_model_wts = copy.deepcopy(model.state_dict())\n",
    "    best_offset = float('inf')\n",
    "\n",
    "    for epoch in range(num_epochs):\n",
    "        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n",
    "        print('-' * 10)\n",
    "\n",
    "        # Each epoch has a training and validation phase\n",
    "        for phase in ['train', 'val']:\n",
    "            if phase == 'train':\n",
    "                scheduler.step()\n",
    "                model.train()  # Set model to training mode\n",
    "            else:\n",
    "                model.eval()   # Set model to evaluate mode\n",
    "\n",
    "            running_loss = 0.0\n",
    "            running_offset = 0\n",
    "\n",
    "            # Iterate over data.\n",
    "            for inputs, labels in dataloaders[phase]:\n",
    "                inputs = inputs.to(device)\n",
    "                labels = labels.to(device,dtype = torch.float)\n",
    "\n",
    "                # zero the parameter gradients\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # forward\n",
    "                # track history if only in train\n",
    "                with torch.set_grad_enabled(phase == 'train'):\n",
    "                    outputs = model(inputs)\n",
    "                    loss = criterion(outputs, labels)\n",
    "\n",
    "                    # backward + optimize only if in training phase\n",
    "                    if phase == 'train':\n",
    "                        loss.backward()\n",
    "                        optimizer.step()\n",
    "\n",
    "                # statistics\n",
    "                running_loss += loss.item() * inputs.size(0)\n",
    "                running_offset += torch.sum(torch.abs(outputs - labels.data))\n",
    "                \n",
    "            if phase == 'train':\n",
    "                epoch_loss = running_loss / (n * 0.6)\n",
    "                epoch_offset = running_offset / (n * 0.6)\n",
    "               \n",
    "            else:\n",
    "                epoch_loss = running_loss / (n * 0.2)\n",
    "                epoch_offset = running_offset / (n * 0.2)\n",
    "            \n",
    "            print('{} Loss: {:.4f} Diff: {:.4f}'.format(\n",
    "                phase, epoch_loss, epoch_offset))\n",
    "\n",
    "            # deep copy the model\n",
    "            if phase == 'val' and epoch_offset < best_offset:\n",
    "                best_offset = epoch_offset\n",
    "                best_model_wts = copy.deepcopy(model.state_dict())\n",
    "\n",
    "        print()\n",
    "\n",
    "    time_elapsed = time.time() - since\n",
    "    print('Training complete in {:.0f}m {:.0f}s'.format(\n",
    "        time_elapsed // 60, time_elapsed % 60))\n",
    "    print('Best val Diff: {:4f}'.format(best_offset))\n",
    "\n",
    "    # load best model weights\n",
    "    model.load_state_dict(best_model_wts)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_ft = models.resnet101(pretrained=True)\n",
    "\n",
    "num_ftrs = model_ft.fc.in_features\n",
    "model_ft.fc = nn.Linear(num_ftrs, 1)\n",
    "\n",
    "model_ft = model_ft.to(device)\n",
    "\n",
    "criterion = nn.MSELoss()\n",
    "\n",
    "# Observe that all parameters are being optimized\n",
    "# optimizer_ft = optim.SGD(model_ft.parameters(), lr=0.001, momentum=0.9)\n",
    "optimizer_ft = optim.Adam(model_ft.parameters(), lr=0.001)\n",
    "\n",
    "# Decay LR by a factor of 0.1 every 7 epochs\n",
    "exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 0/49\n",
      "----------\n",
      "train Loss: 125.5311 Diff: 9.2308\n",
      "val Loss: 99.1176 Diff: 8.1358\n",
      "\n",
      "Epoch 1/49\n",
      "----------\n",
      "train Loss: 138.9898 Diff: 9.6144\n",
      "val Loss: 100.4619 Diff: 8.0238\n",
      "\n",
      "Epoch 2/49\n",
      "----------\n",
      "train Loss: 103.7960 Diff: 8.0050\n",
      "val Loss: 97.9270 Diff: 8.0415\n",
      "\n",
      "Epoch 3/49\n",
      "----------\n",
      "train Loss: 110.8339 Diff: 8.4225\n",
      "val Loss: 98.5248 Diff: 8.0560\n",
      "\n",
      "Epoch 4/49\n",
      "----------\n",
      "train Loss: 133.0807 Diff: 8.9952\n",
      "val Loss: 100.5171 Diff: 8.0231\n",
      "\n",
      "Epoch 5/49\n",
      "----------\n",
      "train Loss: 110.7878 Diff: 8.4692\n",
      "val Loss: 101.4549 Diff: 7.9981\n",
      "\n",
      "Epoch 6/49\n",
      "----------\n",
      "train Loss: 121.5263 Diff: 8.8809\n",
      "val Loss: 106.0599 Diff: 8.1464\n",
      "\n",
      "Epoch 7/49\n",
      "----------\n",
      "train Loss: 118.8650 Diff: 8.8063\n",
      "val Loss: 98.4922 Diff: 8.1197\n",
      "\n",
      "Epoch 8/49\n",
      "----------\n",
      "train Loss: 111.3667 Diff: 8.4513\n",
      "val Loss: 99.3333 Diff: 8.1445\n",
      "\n",
      "Epoch 9/49\n",
      "----------\n",
      "train Loss: 128.4246 Diff: 9.1009\n",
      "val Loss: 98.9834 Diff: 8.0222\n",
      "\n",
      "Epoch 10/49\n",
      "----------\n",
      "train Loss: 100.3451 Diff: 7.9407\n",
      "val Loss: 98.7751 Diff: 8.1253\n",
      "\n",
      "Epoch 11/49\n",
      "----------\n",
      "train Loss: 119.3856 Diff: 8.7490\n",
      "val Loss: 106.5119 Diff: 8.1758\n",
      "\n",
      "Epoch 12/49\n",
      "----------\n",
      "train Loss: 120.6016 Diff: 8.9420\n",
      "val Loss: 98.5849 Diff: 8.1120\n",
      "\n",
      "Epoch 13/49\n",
      "----------\n",
      "train Loss: 122.5513 Diff: 8.7885\n",
      "val Loss: 98.5204 Diff: 8.1053\n",
      "\n",
      "Epoch 14/49\n",
      "----------\n",
      "train Loss: 121.7740 Diff: 8.8470\n",
      "val Loss: 99.3697 Diff: 8.1840\n",
      "\n",
      "Epoch 15/49\n",
      "----------\n",
      "train Loss: 119.7769 Diff: 8.9540\n",
      "val Loss: 98.4161 Diff: 8.0631\n",
      "\n",
      "Epoch 16/49\n",
      "----------\n",
      "train Loss: 119.2071 Diff: 9.0271\n",
      "val Loss: 103.2513 Diff: 8.0375\n",
      "\n",
      "Epoch 17/49\n",
      "----------\n",
      "train Loss: 117.5351 Diff: 8.6134\n",
      "val Loss: 98.2633 Diff: 8.0234\n",
      "\n",
      "Epoch 18/49\n",
      "----------\n",
      "train Loss: 95.4565 Diff: 7.9537\n",
      "val Loss: 102.2331 Diff: 8.0271\n",
      "\n",
      "Epoch 19/49\n",
      "----------\n",
      "train Loss: 121.8633 Diff: 8.8290\n",
      "val Loss: 99.9798 Diff: 8.2322\n",
      "\n",
      "Epoch 20/49\n",
      "----------\n",
      "train Loss: 97.7344 Diff: 8.0184\n",
      "val Loss: 99.8539 Diff: 8.2285\n",
      "\n",
      "Epoch 21/49\n",
      "----------\n",
      "train Loss: 97.8960 Diff: 7.7898\n",
      "val Loss: 99.3363 Diff: 8.0616\n",
      "\n",
      "Epoch 22/49\n",
      "----------\n",
      "train Loss: 134.5020 Diff: 9.2729\n",
      "val Loss: 98.7428 Diff: 8.0083\n",
      "\n",
      "Epoch 23/49\n",
      "----------\n",
      "train Loss: 103.7140 Diff: 8.2461\n",
      "val Loss: 98.8619 Diff: 8.0540\n",
      "\n",
      "Epoch 24/49\n",
      "----------\n",
      "train Loss: 106.6366 Diff: 8.3207\n",
      "val Loss: 99.9494 Diff: 7.9492\n",
      "\n",
      "Epoch 25/49\n",
      "----------\n",
      "train Loss: 105.9608 Diff: 8.2715\n",
      "val Loss: 97.8860 Diff: 8.0661\n",
      "\n",
      "Epoch 26/49\n",
      "----------\n",
      "train Loss: 118.2355 Diff: 8.9775\n",
      "val Loss: 106.9444 Diff: 8.2062\n",
      "\n",
      "Epoch 27/49\n",
      "----------\n",
      "train Loss: 122.2781 Diff: 8.8753\n",
      "val Loss: 98.9571 Diff: 8.1476\n",
      "\n",
      "Epoch 28/49\n",
      "----------\n",
      "train Loss: 109.1367 Diff: 8.3444\n",
      "val Loss: 98.5463 Diff: 8.0866\n",
      "\n",
      "Epoch 29/49\n",
      "----------\n",
      "train Loss: 124.0362 Diff: 8.8808\n",
      "val Loss: 101.3152 Diff: 7.9949\n",
      "\n",
      "Epoch 30/49\n",
      "----------\n",
      "train Loss: 111.3280 Diff: 8.4941\n",
      "val Loss: 98.5498 Diff: 8.0479\n",
      "\n",
      "Epoch 31/49\n",
      "----------\n",
      "train Loss: 114.4809 Diff: 8.6184\n",
      "val Loss: 100.7175 Diff: 8.0088\n",
      "\n",
      "Epoch 32/49\n",
      "----------\n",
      "train Loss: 111.1905 Diff: 8.4199\n",
      "val Loss: 99.3562 Diff: 8.0639\n",
      "\n",
      "Epoch 33/49\n",
      "----------\n",
      "train Loss: 108.3079 Diff: 8.6229\n",
      "val Loss: 99.2157 Diff: 8.1604\n",
      "\n",
      "Epoch 34/49\n",
      "----------\n",
      "train Loss: 114.3986 Diff: 8.6871\n",
      "val Loss: 102.1412 Diff: 7.9996\n",
      "\n",
      "Epoch 35/49\n",
      "----------\n",
      "train Loss: 136.4774 Diff: 9.5858\n",
      "val Loss: 107.3876 Diff: 8.2225\n",
      "\n",
      "Epoch 36/49\n",
      "----------\n",
      "train Loss: 121.9563 Diff: 8.7652\n",
      "val Loss: 99.3886 Diff: 8.1730\n",
      "\n",
      "Epoch 37/49\n",
      "----------\n",
      "train Loss: 131.8338 Diff: 9.2103\n",
      "val Loss: 100.2646 Diff: 7.9595\n",
      "\n",
      "Epoch 38/49\n",
      "----------\n",
      "train Loss: 118.2760 Diff: 8.7229\n",
      "val Loss: 97.8598 Diff: 8.0707\n",
      "\n",
      "Epoch 39/49\n",
      "----------\n",
      "train Loss: 105.8709 Diff: 8.1960\n",
      "val Loss: 101.0221 Diff: 8.2664\n",
      "\n",
      "Epoch 40/49\n",
      "----------\n",
      "train Loss: 113.2238 Diff: 8.5045\n",
      "val Loss: 98.0245 Diff: 8.0724\n",
      "\n",
      "Epoch 41/49\n",
      "----------\n",
      "train Loss: 111.7083 Diff: 8.3701\n",
      "val Loss: 101.8436 Diff: 8.0018\n",
      "\n",
      "Epoch 42/49\n",
      "----------\n",
      "train Loss: 111.2107 Diff: 8.4976\n",
      "val Loss: 103.3359 Diff: 8.0497\n",
      "\n",
      "Epoch 43/49\n",
      "----------\n",
      "train Loss: 111.0361 Diff: 8.4723\n",
      "val Loss: 99.5494 Diff: 8.1098\n",
      "\n",
      "Epoch 44/49\n",
      "----------\n",
      "train Loss: 123.0523 Diff: 9.0344\n",
      "val Loss: 99.5922 Diff: 8.1810\n",
      "\n",
      "Epoch 45/49\n",
      "----------\n",
      "train Loss: 119.2200 Diff: 8.6985\n",
      "val Loss: 99.9472 Diff: 8.1761\n",
      "\n",
      "Epoch 46/49\n",
      "----------\n",
      "train Loss: 111.1093 Diff: 8.3446\n",
      "val Loss: 98.4960 Diff: 8.0851\n",
      "\n",
      "Epoch 47/49\n",
      "----------\n",
      "train Loss: 115.6698 Diff: 8.4845\n",
      "val Loss: 100.1791 Diff: 8.0268\n",
      "\n",
      "Epoch 48/49\n",
      "----------\n",
      "train Loss: 112.8370 Diff: 8.6123\n",
      "val Loss: 97.8460 Diff: 8.0010\n",
      "\n",
      "Epoch 49/49\n",
      "----------\n",
      "train Loss: 115.1559 Diff: 8.5197\n",
      "val Loss: 98.2386 Diff: 8.1241\n",
      "\n",
      "Training complete in 36m 29s\n",
      "Best val Diff: 7.949244\n"
     ]
    }
   ],
   "source": [
    "model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,\n",
    "                       num_epochs=50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def check_error(data,labels,model):\n",
    "    print('Checking error on test set')   \n",
    "    total_abs_err = 0.0\n",
    "    num_samples = data.shape[0]\n",
    "    model.eval()  # set model to evaluation mode\n",
    "    with torch.no_grad():\n",
    "        data = data.to(device=device, dtype=dtype)  # move to device, e.g. GPU\n",
    "        labels = labels.to(device=device,dtype=dtype)\n",
    "        scores = model(data)\n",
    "        total_abs_err += torch.abs(labels - scores).sum()\n",
    "        mean_abs_err = float(total_abs_err) / num_samples\n",
    "        print('Got error %.4f' % (mean_abs_err))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Checking error on test set\n",
      "Got error 8.6252\n"
     ]
    }
   ],
   "source": [
    "# print(x_normalized[test_indices,:].shape)\n",
    "check_error(x_normalized[test_indices,:], y[test_indices],model_ft)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(model_ft,\"best_reg_model_res18.pth\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Environment (conda_pytorch_p36)",
   "language": "python",
   "name": "conda_pytorch_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}