lab-07_1_tips.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Lab 7-1: Tips"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Author: Seungjae Lee (이승재)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<torch._C.Generator at 0x7f0708f8ffb0>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# For reproducibility\n",
    "torch.manual_seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Training and Test Datasets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = torch.FloatTensor([[1, 2, 1],\n",
    "                             [1, 3, 2],\n",
    "                             [1, 3, 4],\n",
    "                             [1, 5, 5],\n",
    "                             [1, 7, 5],\n",
    "                             [1, 2, 5],\n",
    "                             [1, 6, 6],\n",
    "                             [1, 7, 7]\n",
    "                            ])\n",
    "y_train = torch.LongTensor([2, 2, 2, 1, 1, 1, 0, 0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_test = torch.FloatTensor([[2, 1, 1], [3, 1, 2], [3, 3, 4]])\n",
    "y_test = torch.LongTensor([2, 2, 2])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SoftmaxClassifierModel(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.linear = nn.Linear(3, 3)\n",
    "    def forward(self, x):\n",
    "        return self.linear(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SoftmaxClassifierModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# optimizer 설정\n",
    "optimizer = optim.SGD(model.parameters(), lr=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, optimizer, x_train, y_train):\n",
    "    nb_epochs = 20\n",
    "    for epoch in range(nb_epochs):\n",
    "\n",
    "        # H(x) 계산\n",
    "        prediction = model(x_train)\n",
    "\n",
    "        # cost 계산\n",
    "        cost = F.cross_entropy(prediction, y_train)\n",
    "\n",
    "        # cost로 H(x) 개선\n",
    "        optimizer.zero_grad()\n",
    "        cost.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        print('Epoch {:4d}/{} Cost: {:.6f}'.format(\n",
    "            epoch, nb_epochs, cost.item()\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(model, optimizer, x_test, y_test):\n",
    "    prediction = model(x_test)\n",
    "    predicted_classes = prediction.max(1)[1]\n",
    "    correct_count = (predicted_classes == y_test).sum().item()\n",
    "    cost = F.cross_entropy(prediction, y_test)\n",
    "\n",
    "    print('Accuracy: {}% Cost: {:.6f}'.format(\n",
    "         correct_count / len(y_test) * 100, cost.item()\n",
    "    ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    0/20 Cost: 2.203667\n",
      "Epoch    1/20 Cost: 1.199645\n",
      "Epoch    2/20 Cost: 1.142985\n",
      "Epoch    3/20 Cost: 1.117769\n",
      "Epoch    4/20 Cost: 1.100901\n",
      "Epoch    5/20 Cost: 1.089523\n",
      "Epoch    6/20 Cost: 1.079872\n",
      "Epoch    7/20 Cost: 1.071320\n",
      "Epoch    8/20 Cost: 1.063325\n",
      "Epoch    9/20 Cost: 1.055720\n",
      "Epoch   10/20 Cost: 1.048378\n",
      "Epoch   11/20 Cost: 1.041245\n",
      "Epoch   12/20 Cost: 1.034285\n",
      "Epoch   13/20 Cost: 1.027478\n",
      "Epoch   14/20 Cost: 1.020813\n",
      "Epoch   15/20 Cost: 1.014279\n",
      "Epoch   16/20 Cost: 1.007872\n",
      "Epoch   17/20 Cost: 1.001586\n",
      "Epoch   18/20 Cost: 0.995419\n",
      "Epoch   19/20 Cost: 0.989365\n"
     ]
    }
   ],
   "source": [
    "train(model, optimizer, x_train, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.0% Cost: 1.425844\n"
     ]
    }
   ],
   "source": [
    "test(model, optimizer, x_test, y_test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning Rate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Gradient Descent 에서의 $\\alpha$ 값"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "`optimizer = optim.SGD(model.parameters(), lr=0.1)` 에서 `lr=0.1` 이다"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "learning rate이 너무 크면 diverge 하면서 cost 가 점점 늘어난다 (overshooting)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SoftmaxClassifierModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=1e5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    0/20 Cost: 1.280268\n",
      "Epoch    1/20 Cost: 976950.812500\n",
      "Epoch    2/20 Cost: 1279135.125000\n",
      "Epoch    3/20 Cost: 1198379.000000\n",
      "Epoch    4/20 Cost: 1098825.875000\n",
      "Epoch    5/20 Cost: 1968197.625000\n",
      "Epoch    6/20 Cost: 284763.250000\n",
      "Epoch    7/20 Cost: 1532260.125000\n",
      "Epoch    8/20 Cost: 1651504.000000\n",
      "Epoch    9/20 Cost: 521878.500000\n",
      "Epoch   10/20 Cost: 1397263.250000\n",
      "Epoch   11/20 Cost: 750986.250000\n",
      "Epoch   12/20 Cost: 918691.500000\n",
      "Epoch   13/20 Cost: 1487888.250000\n",
      "Epoch   14/20 Cost: 1582260.125000\n",
      "Epoch   15/20 Cost: 685818.062500\n",
      "Epoch   16/20 Cost: 1140048.750000\n",
      "Epoch   17/20 Cost: 940566.500000\n",
      "Epoch   18/20 Cost: 931638.250000\n",
      "Epoch   19/20 Cost: 1971322.625000\n"
     ]
    }
   ],
   "source": [
    "train(model, optimizer, x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "learning rate이 너무 작으면 cost가 거의 줄어들지 않는다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SoftmaxClassifierModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=1e-10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    0/20 Cost: 3.187324\n",
      "Epoch    1/20 Cost: 3.187324\n",
      "Epoch    2/20 Cost: 3.187324\n",
      "Epoch    3/20 Cost: 3.187324\n",
      "Epoch    4/20 Cost: 3.187324\n",
      "Epoch    5/20 Cost: 3.187324\n",
      "Epoch    6/20 Cost: 3.187324\n",
      "Epoch    7/20 Cost: 3.187324\n",
      "Epoch    8/20 Cost: 3.187324\n",
      "Epoch    9/20 Cost: 3.187324\n",
      "Epoch   10/20 Cost: 3.187324\n",
      "Epoch   11/20 Cost: 3.187324\n",
      "Epoch   12/20 Cost: 3.187324\n",
      "Epoch   13/20 Cost: 3.187324\n",
      "Epoch   14/20 Cost: 3.187324\n",
      "Epoch   15/20 Cost: 3.187324\n",
      "Epoch   16/20 Cost: 3.187324\n",
      "Epoch   17/20 Cost: 3.187324\n",
      "Epoch   18/20 Cost: 3.187324\n",
      "Epoch   19/20 Cost: 3.187324\n"
     ]
    }
   ],
   "source": [
    "train(model, optimizer, x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "적절한 숫자로 시작해 발산하면 작게, cost가 줄어들지 않으면 크게 조정하자."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = SoftmaxClassifierModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=1e-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    0/20 Cost: 1.341573\n",
      "Epoch    1/20 Cost: 1.198802\n",
      "Epoch    2/20 Cost: 1.150877\n",
      "Epoch    3/20 Cost: 1.131977\n",
      "Epoch    4/20 Cost: 1.116242\n",
      "Epoch    5/20 Cost: 1.102514\n",
      "Epoch    6/20 Cost: 1.089676\n",
      "Epoch    7/20 Cost: 1.077479\n",
      "Epoch    8/20 Cost: 1.065775\n",
      "Epoch    9/20 Cost: 1.054511\n",
      "Epoch   10/20 Cost: 1.043655\n",
      "Epoch   11/20 Cost: 1.033187\n",
      "Epoch   12/20 Cost: 1.023091\n",
      "Epoch   13/20 Cost: 1.013356\n",
      "Epoch   14/20 Cost: 1.003968\n",
      "Epoch   15/20 Cost: 0.994917\n",
      "Epoch   16/20 Cost: 0.986189\n",
      "Epoch   17/20 Cost: 0.977775\n",
      "Epoch   18/20 Cost: 0.969660\n",
      "Epoch   19/20 Cost: 0.961836\n"
     ]
    }
   ],
   "source": [
    "train(model, optimizer, x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Preprocessing (데이터 전처리)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "데이터를 zero-center하고 normalize하자."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train = torch.FloatTensor([[73, 80, 75],\n",
    "                             [93, 88, 93],\n",
    "                             [89, 91, 90],\n",
    "                             [96, 98, 100],\n",
    "                             [73, 66, 70]])\n",
    "y_train = torch.FloatTensor([[152], [185], [180], [196], [142]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "$$ x'_j = \\frac{x_j - \\mu_j}{\\sigma_j} $$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "여기서 $\\sigma$ 는 standard deviation, $\\mu$ 는 평균값 이다."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "mu = x_train.mean(dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "sigma = x_train.std(dim=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "norm_x_train = (x_train - mu) / sigma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[-1.0674, -0.3758, -0.8398],\n",
      "        [ 0.7418,  0.2778,  0.5863],\n",
      "        [ 0.3799,  0.5229,  0.3486],\n",
      "        [ 1.0132,  1.0948,  1.1409],\n",
      "        [-1.0674, -1.5197, -1.2360]])\n"
     ]
    }
   ],
   "source": [
    "print(norm_x_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Normalize와 zero center한 X로 학습해서 성능을 보자"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultivariateLinearRegressionModel(nn.Module):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.linear = nn.Linear(3, 1)\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.linear(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MultivariateLinearRegressionModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=1e-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(model, optimizer, x_train, y_train):\n",
    "    nb_epochs = 20\n",
    "    for epoch in range(nb_epochs):\n",
    "\n",
    "        # H(x) 계산\n",
    "        prediction = model(x_train)\n",
    "\n",
    "        # cost 계산\n",
    "        cost = F.mse_loss(prediction, y_train)\n",
    "\n",
    "        # cost로 H(x) 개선\n",
    "        optimizer.zero_grad()\n",
    "        cost.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        print('Epoch {:4d}/{} Cost: {:.6f}'.format(\n",
    "            epoch, nb_epochs, cost.item()\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    0/20 Cost: 29785.091797\n",
      "Epoch    1/20 Cost: 18906.164062\n",
      "Epoch    2/20 Cost: 12054.674805\n",
      "Epoch    3/20 Cost: 7702.029297\n",
      "Epoch    4/20 Cost: 4925.733398\n",
      "Epoch    5/20 Cost: 3151.632568\n",
      "Epoch    6/20 Cost: 2016.996094\n",
      "Epoch    7/20 Cost: 1291.051270\n",
      "Epoch    8/20 Cost: 826.505310\n",
      "Epoch    9/20 Cost: 529.207336\n",
      "Epoch   10/20 Cost: 338.934204\n",
      "Epoch   11/20 Cost: 217.153549\n",
      "Epoch   12/20 Cost: 139.206741\n",
      "Epoch   13/20 Cost: 89.313782\n",
      "Epoch   14/20 Cost: 57.375462\n",
      "Epoch   15/20 Cost: 36.928429\n",
      "Epoch   16/20 Cost: 23.835772\n",
      "Epoch   17/20 Cost: 15.450428\n",
      "Epoch   18/20 Cost: 10.077808\n",
      "Epoch   19/20 Cost: 6.633700\n"
     ]
    }
   ],
   "source": [
    "train(model, optimizer, norm_x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Overfitting"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "너무 학습 데이터에 한해 잘 학습해 테스트 데이터에 좋은 성능을 내지 못할 수도 있다."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "이것을 방지하는 방법은 크게 세 가지인데:\n",
    "\n",
    "1. 더 많은 학습 데이터\n",
    "2. 더 적은 양의 feature\n",
    "3. **Regularization**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Regularization: Let's not have too big numbers in the weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_with_regularization(model, optimizer, x_train, y_train):\n",
    "    nb_epochs = 20\n",
    "    for epoch in range(nb_epochs):\n",
    "\n",
    "        # H(x) 계산\n",
    "        prediction = model(x_train)\n",
    "\n",
    "        # cost 계산\n",
    "        cost = F.mse_loss(prediction, y_train)\n",
    "        \n",
    "        # l2 norm 계산\n",
    "        l2_reg = 0\n",
    "        for param in model.parameters():\n",
    "            l2_reg += torch.norm(param)\n",
    "            \n",
    "        cost += l2_reg\n",
    "\n",
    "        # cost로 H(x) 개선\n",
    "        optimizer.zero_grad()\n",
    "        cost.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        print('Epoch {:4d}/{} Cost: {:.6f}'.format(\n",
    "            epoch+1, nb_epochs, cost.item()\n",
    "        ))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = MultivariateLinearRegressionModel()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.SGD(model.parameters(), lr=1e-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch    1/20 Cost: 29477.810547\n",
      "Epoch    2/20 Cost: 18798.513672\n",
      "Epoch    3/20 Cost: 12059.364258\n",
      "Epoch    4/20 Cost: 7773.400879\n",
      "Epoch    5/20 Cost: 5038.263672\n",
      "Epoch    6/20 Cost: 3290.066406\n",
      "Epoch    7/20 Cost: 2171.882568\n",
      "Epoch    8/20 Cost: 1456.434570\n",
      "Epoch    9/20 Cost: 998.598267\n",
      "Epoch   10/20 Cost: 705.595398\n",
      "Epoch   11/20 Cost: 518.073608\n",
      "Epoch   12/20 Cost: 398.057220\n",
      "Epoch   13/20 Cost: 321.242920\n",
      "Epoch   14/20 Cost: 272.078247\n",
      "Epoch   15/20 Cost: 240.609131\n",
      "Epoch   16/20 Cost: 220.465637\n",
      "Epoch   17/20 Cost: 207.570602\n",
      "Epoch   18/20 Cost: 199.314804\n",
      "Epoch   19/20 Cost: 194.028214\n",
      "Epoch   20/20 Cost: 190.642014\n"
     ]
    }
   ],
   "source": [
    "train_with_regularization(model, optimizer, norm_x_train, y_train)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}