bp-mnist-fully-connected.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "5d33c3e3",
   "metadata": {},
   "source": [
    "# (Reference) Backprop on MNIST dataset using Fully Connected Layers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "202f92d9",
   "metadata": {},
   "source": [
    "### Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "c4b9867a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch \n",
    "import torch.nn as nn  \n",
    "from torchvision.datasets import MNIST \n",
    "from torchvision import datasets, transforms\n",
    "from torchvision.transforms import Compose, ToTensor, Normalize, Lambda \n",
    "from torch.utils.data import DataLoader \n",
    "from torch.optim import Adam "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2eb66b72",
   "metadata": {},
   "source": [
    "### Model Architecture, Loss and Optimization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "48b4f28b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define the model architecture.\n",
    "model = nn.Sequential(\n",
    "    nn.Flatten(),\n",
    "    nn.Linear(28*28, 2000),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(2000, 2000),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(2000, 2000),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(2000, 2000),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(2000, 10)\n",
    ")\n",
    "\n",
    "# Loss and optimizer\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = Adam(model.parameters(), lr=0.001)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3a356cc3",
   "metadata": {},
   "source": [
    "### Device Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "29c320ef",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Flatten(start_dim=1, end_dim=-1)\n",
       "  (1): Linear(in_features=784, out_features=2000, bias=True)\n",
       "  (2): ReLU()\n",
       "  (3): Linear(in_features=2000, out_features=2000, bias=True)\n",
       "  (4): ReLU()\n",
       "  (5): Linear(in_features=2000, out_features=2000, bias=True)\n",
       "  (6): ReLU()\n",
       "  (7): Linear(in_features=2000, out_features=2000, bias=True)\n",
       "  (8): ReLU()\n",
       "  (9): Linear(in_features=2000, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "model.to(device) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6d836f91",
   "metadata": {},
   "source": [
    "### Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d7919eba",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Data loading + transform\n",
    "transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.1307,), (0.3081,))\n",
    "])\n",
    "\n",
    "train_loader = DataLoader(MNIST(\"./data/\", train=True, download=True, transform=transform), batch_size=50000) \n",
    "test_loader = DataLoader(MNIST(\"./data/\", train=False, download=True, transform=transform), batch_size=10000) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "db5fc7e2",
   "metadata": {},
   "source": [
    "### Train and Test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "533099a9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1, Train Loss: 2.2661, Train Acc: 15.07%\n",
      "Test Acc: 58.33%\n",
      "\n",
      "Epoch 2, Train Loss: 2.0292, Train Acc: 51.27%\n",
      "Test Acc: 28.86%\n",
      "\n",
      "Epoch 3, Train Loss: 2.2187, Train Acc: 32.84%\n",
      "Test Acc: 51.82%\n",
      "\n",
      "Epoch 4, Train Loss: 1.4772, Train Acc: 49.63%\n",
      "Test Acc: 52.42%\n",
      "\n",
      "Epoch 5, Train Loss: 1.3186, Train Acc: 54.69%\n",
      "Test Acc: 70.87%\n",
      "\n",
      "Epoch 6, Train Loss: 0.9296, Train Acc: 71.08%\n",
      "Test Acc: 82.21%\n",
      "\n",
      "Epoch 7, Train Loss: 0.6182, Train Acc: 81.56%\n",
      "Test Acc: 83.35%\n",
      "\n",
      "Epoch 8, Train Loss: 0.5702, Train Acc: 83.40%\n",
      "Test Acc: 86.98%\n",
      "\n",
      "Epoch 9, Train Loss: 0.4851, Train Acc: 86.80%\n",
      "Test Acc: 89.08%\n",
      "\n",
      "Epoch 10, Train Loss: 0.4100, Train Acc: 88.79%\n",
      "Test Acc: 90.93%\n",
      "\n",
      "Epoch 11, Train Loss: 0.3247, Train Acc: 90.78%\n",
      "Test Acc: 91.22%\n",
      "\n",
      "Epoch 12, Train Loss: 0.2993, Train Acc: 91.35%\n",
      "Test Acc: 92.15%\n",
      "\n",
      "Epoch 13, Train Loss: 0.2607, Train Acc: 92.43%\n",
      "Test Acc: 92.81%\n",
      "\n",
      "Epoch 14, Train Loss: 0.2332, Train Acc: 93.12%\n",
      "Test Acc: 93.61%\n",
      "\n",
      "Epoch 15, Train Loss: 0.2131, Train Acc: 93.73%\n",
      "Test Acc: 94.36%\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Training loop\n",
    "for epoch in range(15):\n",
    "    model.train()\n",
    "    running_loss = 0.0\n",
    "    correct_train = 0\n",
    "    total_train = 0\n",
    "    for inputs, labels in train_loader:\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "        optimizer.zero_grad()\n",
    "\n",
    "        outputs = model(inputs)\n",
    "        loss = criterion(outputs, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        running_loss += loss.item() * inputs.size(0)\n",
    "\n",
    "        _, predicted = torch.max(outputs.data, 1)\n",
    "        total_train += labels.size(0)\n",
    "        correct_train += (predicted == labels).sum().item()\n",
    "\n",
    "    epoch_loss = running_loss / len(train_loader.dataset)\n",
    "    epoch_acc = 100 * correct_train / total_train\n",
    "    print(f'Epoch {epoch+1}, Train Loss: {epoch_loss:.4f}, Train Acc: {epoch_acc:.2f}%')\n",
    "\n",
    "    model.eval()\n",
    "    correct_test = 0\n",
    "    total_test = 0\n",
    "    with torch.no_grad():\n",
    "        for inputs, labels in test_loader:\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            \n",
    "            outputs = model(inputs)\n",
    "            _, predicted = torch.max(outputs.data, 1)\n",
    "            total_test += labels.size(0)\n",
    "            correct_test += (predicted == labels).sum().item()\n",
    "\n",
    "    test_acc = 100 * correct_test / total_test\n",
    "    print(f'Test Acc: {test_acc:.2f}%\\n')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10 (tensorflow)",
   "language": "python",
   "name": "tensorflow"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}