week01/intro_to_pytorch.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Introduction to PyTorch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How to create tensors?"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    All of deep learning is computations on tensors, which are generalizations of a matrix that can be indexed in more than 2 dimensions.\n",
    "    So, let's look at how to create tensors, some operations on them."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Populating the interactive namespace from numpy and matplotlib\n"
     ]
    }
   ],
   "source": [
    "%pylab inline\n",
    "\n",
    "import torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    The main way to create a tensor is to initialise it from an existing list or `numpy.ndarray`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([1, 2, 3, 4, 5])\n",
      "tensor([1, 2, 3, 4, 5])\n"
     ]
    }
   ],
   "source": [
    "l = [1, 2, 3, 4, 5]\n",
    "l_tensor = torch.tensor(l)\n",
    "print(l_tensor)\n",
    "\n",
    "l_numpy = np.array(l)\n",
    "l_tensor = torch.from_numpy(l_numpy)\n",
    "print(l_tensor)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "     But in real life you usually create the tensors you need directly in `pytorch` without any extra memory allocation.\n",
    "     There are so many ways to do this and let's look at some of them"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x = torch.arange(10)\n",
    "x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    We can see the type of elements in the tensor, its size, the number of dimensions in it and much more"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "type of elements: torch.int64\n",
      "size (or shape) : torch.Size([10])\n",
      "number of dim: 1\n"
     ]
    }
   ],
   "source": [
    "print(f\"type of elements: {x.dtype}\")\n",
    "print(f\"size (or shape) : {x.size()}\")\n",
    "print(f\"number of dim: {x.dim()}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Apart from `torch.arange` we can create a tensor of all zeros, ones or just random from some distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set the desired tensor size (or shape)\n",
    "desired_shape = (1, 5, 10)\n",
    "\n",
    "ones_tensor = torch.ones(desired_shape)\n",
    "zeros_tensor = torch.zeros(desired_shape)\n",
    "\n",
    "# each element of the tensor is sampled from a normal standard distribution\n",
    "random_tensor = torch.randn(desired_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[ 1.9585e-01, -6.1831e-01, -9.5756e-01,  3.9901e-01,  9.7058e-01,\n",
      "           1.2543e-01, -1.3706e+00, -2.4149e-03,  3.2547e+00, -5.4886e-01],\n",
      "         [ 3.9639e-02,  3.1375e-01,  1.2110e+00, -7.2277e-01,  1.3573e+00,\n",
      "           1.3111e+00, -3.7105e-02, -1.3918e+00,  1.2738e+00, -4.9734e-02],\n",
      "         [ 1.9835e+00, -6.2938e-01, -4.9263e-01, -1.0369e-01, -1.5734e+00,\n",
      "          -1.3818e+00, -7.6623e-03,  8.4458e-02, -6.8961e-01, -7.7442e-01],\n",
      "         [-4.4600e-01,  2.4276e+00, -7.7969e-01, -8.7114e-01, -8.3761e-02,\n",
      "           8.7588e-01, -2.0866e-01, -4.6978e-01,  2.9023e-03, -1.1727e+00],\n",
      "         [ 2.1426e-01,  7.1265e-01,  3.0820e-01,  2.5808e-01, -5.4970e-01,\n",
      "          -2.0470e+00, -1.2749e+00, -5.7189e-02,  3.3236e-01, -8.0916e-01]]])\n",
      "torch.Size([1, 5, 10])\n"
     ]
    }
   ],
   "source": [
    "print(random_tensor)\n",
    "print(random_tensor.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## How to change shape of tensor?\n",
    "\n",
    "    There are several ways to resize the tensors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([100])\n",
      "tensor([ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9, 10, 11, 12, 13, 14, 15, 16, 17,\n",
      "        18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35,\n",
      "        36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53,\n",
      "        54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71,\n",
      "        72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89,\n",
      "        90, 91, 92, 93, 94, 95, 96, 97, 98, 99])\n"
     ]
    }
   ],
   "source": [
    "tensor = torch.arange(100)\n",
    "\n",
    "print(tensor.shape)\n",
    "print(tensor)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Now we would like to create a 10x10 matrix from the `tensor'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([10, 10])\n",
      "tensor([[ 0,  1,  2,  3,  4,  5,  6,  7,  8,  9],\n",
      "        [10, 11, 12, 13, 14, 15, 16, 17, 18, 19],\n",
      "        [20, 21, 22, 23, 24, 25, 26, 27, 28, 29],\n",
      "        [30, 31, 32, 33, 34, 35, 36, 37, 38, 39],\n",
      "        [40, 41, 42, 43, 44, 45, 46, 47, 48, 49],\n",
      "        [50, 51, 52, 53, 54, 55, 56, 57, 58, 59],\n",
      "        [60, 61, 62, 63, 64, 65, 66, 67, 68, 69],\n",
      "        [70, 71, 72, 73, 74, 75, 76, 77, 78, 79],\n",
      "        [80, 81, 82, 83, 84, 85, 86, 87, 88, 89],\n",
      "        [90, 91, 92, 93, 94, 95, 96, 97, 98, 99]])\n"
     ]
    }
   ],
   "source": [
    "matrix = tensor.reshape(10, 10)\n",
    "\n",
    "print(matrix.shape)\n",
    "print(matrix)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    What if we want a tenson of order 3 with dimensions 2x10x5?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([2, 10, 5])\n",
      "tensor([[[ 0,  1,  2,  3,  4],\n",
      "         [ 5,  6,  7,  8,  9],\n",
      "         [10, 11, 12, 13, 14],\n",
      "         [15, 16, 17, 18, 19],\n",
      "         [20, 21, 22, 23, 24],\n",
      "         [25, 26, 27, 28, 29],\n",
      "         [30, 31, 32, 33, 34],\n",
      "         [35, 36, 37, 38, 39],\n",
      "         [40, 41, 42, 43, 44],\n",
      "         [45, 46, 47, 48, 49]],\n",
      "\n",
      "        [[50, 51, 52, 53, 54],\n",
      "         [55, 56, 57, 58, 59],\n",
      "         [60, 61, 62, 63, 64],\n",
      "         [65, 66, 67, 68, 69],\n",
      "         [70, 71, 72, 73, 74],\n",
      "         [75, 76, 77, 78, 79],\n",
      "         [80, 81, 82, 83, 84],\n",
      "         [85, 86, 87, 88, 89],\n",
      "         [90, 91, 92, 93, 94],\n",
      "         [95, 96, 97, 98, 99]]])\n"
     ]
    }
   ],
   "source": [
    "tensor_3d = tensor.reshape(2, 10, 5)\n",
    "\n",
    "print(tensor_3d.shape)\n",
    "print(tensor_3d)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Sometimes we want to swap dimensions, e.g. from 2x10x5 we want to get 5x10x2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([5, 10, 2])\n",
      "torch.Size([5, 10, 2])\n",
      "torch.Size([5, 10, 2])\n"
     ]
    }
   ],
   "source": [
    "tensor_3d_v2 = tensor_3d.transpose(dim0=0, dim1=2)\n",
    "print(tensor_3d_v2.shape)\n",
    "\n",
    "# or this way. it helps not to make a mistake\n",
    "tensor_3d_v3 = tensor_3d.transpose(dim0=0, dim1=-1)\n",
    "print(tensor_3d_v3.shape)\n",
    "\n",
    "# or this way\n",
    "tensor_3d_v4 = tensor_3d.permute(2, 1, 0)\n",
    "print(tensor_3d_v4.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    You also able to add imagine dimension. It is often useful"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 100])\n"
     ]
    }
   ],
   "source": [
    "unsqueezed_tensot = tensor.unsqueeze(0)\n",
    "print(unsqueezed_tensot.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    And in fact, there are many other useful functions that often come in handy.\n",
    "    For example, `.chunk`, `.unfold`, `.flatten`, `.scatter`, `.gather`, ....\n",
    "    \n",
    "    So, feel free to dive into the PyTorch documentation and find out more yourself :)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Operations on tensors\n",
    "\n",
    "    In general, PyTorch has all the same things as NumPy and maybe even a little more\n",
    "    Let's look at the aggregating functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([10, 100, 1000])\n"
     ]
    }
   ],
   "source": [
    "tensor = torch.randn(10, 100, 1000)\n",
    "print(tensor.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(tensor(0.0014), tensor(1377.3700), tensor(4.8508), tensor(-5.1751))"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tensor.mean(), tensor.sum(), tensor.max(), tensor.min()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Note, that almost every aggregation function has argument `dim` and `keepdim`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([100, 1000])\n",
      "torch.Size([1000])\n"
     ]
    }
   ],
   "source": [
    "# take mean over 0 dimension\n",
    "print(tensor.mean(dim=0).shape)\n",
    "\n",
    "# or over 0 and 1 dimensions\n",
    "print(tensor.mean(dim=(0, 1)).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1, 100, 1])\n"
     ]
    }
   ],
   "source": [
    "# take mean over 0 and 2 dimensions and keep dimensions\n",
    "print(tensor.mean(dim=(0, 2), keepdim=True).shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor(1000.4949)\n"
     ]
    }
   ],
   "source": [
    "# Frobenius norm\n",
    "print(tensor.norm(p='fro'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Linear algrebra is also arranged very similarly to NumPy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "A = torch.randn(10, 20)\n",
    "B = torch.randn(20, 30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "C = A.matmul(B)\n",
    "\n",
    "# or\n",
    "C = A @ B\n",
    "\n",
    "# or\n",
    "C = torch.matmul(A, B)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Automatic Differentiation Engine\n",
    "\n",
    "    Gradient automatic computation using backpropagation:\n",
    "        1) Create a tensor/tensors which requires gradient: a = torch.tensor(..., requires_grad=True)\n",
    "        2) Define some differentiable scalar function loss = whatever(a)\n",
    "        3) Request backward pass loss.backward()\n",
    "        4) The gradient is available as a.grad\n",
    "        \n",
    "    Note the loss must be a function of at least one tensor which requires gradient.\n",
    "    For more information see https://pytorch.org/docs/stable/autograd.html\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Let's look on a simple example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x: tensor([-0.9663, -0.0527, -0.4354, -0.0563])\n",
      "y: tensor([ 0.5330, -1.4276, -0.8498,  0.5343])\n",
      "dp / dx: tensor([ 0.5330, -1.4276, -0.8498,  0.5343])\n",
      "dp / dy: tensor([-0.9663, -0.0527, -0.4354, -0.0563])\n"
     ]
    }
   ],
   "source": [
    "x = torch.randn(4, requires_grad=True)\n",
    "y = torch.randn(4, requires_grad=True)\n",
    "z = x * y + 10\n",
    "p = z.sum()\n",
    "p.backward()\n",
    "print('x:', x.data)\n",
    "print('y:', y.data)\n",
    "print('dp / dx:', x.grad)\n",
    "print('dp / dy:', y.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Detaching variables. .detach() returns a copy of a variable, through which backpropagation doesn't pass.\n",
    "    If you average or just store batch losses during the optimization, you'd better detach those losses before averaging or storing.\n",
    "    Otherwise the computational graphs for each batch would be stored in memory as well as batch losses, so you run out of memory very fast.\n",
    "    \n",
    "    Also there is in-place version (.detach_()) which doesn't create detached copy of the variable, but detach variable itself from its computational graph."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x: tensor([-1.4108,  1.2233, -0.3430,  0.8104])\n",
      "y: tensor([-1.1435,  0.8602, -1.2313,  0.0317])\n",
      "dp / dx: tensor([-1.1435,  0.8602, -1.2313,  0.0317])\n",
      "dp / dy: None\n"
     ]
    }
   ],
   "source": [
    "x = torch.randn(4, requires_grad=True)\n",
    "y0 = torch.randn(4, requires_grad=True)\n",
    "y = y0.detach()\n",
    "z = x * y + 10\n",
    "p = z.sum()\n",
    "p.backward()\n",
    "print('x:', x.data)\n",
    "print('y:', y.data)\n",
    "print('dp / dx:', x.grad)\n",
    "print('dp / dy:', y.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    You can also reset the gradient of a specific variable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x: tensor([1., 1., 1., 1.], requires_grad=True)\n",
      "dp / dx: tensor([2., 2., 2., 2.])\n",
      "x: tensor([1., 1., 1., 1.], requires_grad=True)\n",
      "dp / dx: tensor([1., 1., 1., 1.])\n"
     ]
    }
   ],
   "source": [
    "x = torch.tensor([1., 1, 1, 1], requires_grad=True)\n",
    "y = x ** 2\n",
    "p = y.sum()\n",
    "p.backward()\n",
    "print('x:', x)\n",
    "print('dp / dx:', x.grad)\n",
    "y = 1 / x\n",
    "p = y.sum()\n",
    "p.backward()\n",
    "print('x:', x)\n",
    "print('dp / dx:', x.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x: tensor([1., 1., 1., 1.], requires_grad=True)\n",
      "dp / dx: tensor([2., 2., 2., 2.])\n",
      "x: tensor([1., 1., 1., 1.], requires_grad=True)\n",
      "dp / dx: tensor([-1., -1., -1., -1.])\n"
     ]
    }
   ],
   "source": [
    "x = torch.tensor([1, 1, 1, 1.], requires_grad=True)\n",
    "y = x ** 2\n",
    "p = y.sum()\n",
    "p.backward()\n",
    "print('x:', x)\n",
    "print('dp / dx:', x.grad)\n",
    "x.grad.zero_()         # zero gradinents\n",
    "y = 1 / x\n",
    "p = y.sum()\n",
    "p.backward()\n",
    "print('x:', x)\n",
    "print('dp / dx:', x.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    For memory saving, gradients are stored only for so called leaf-variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x: tensor([ 0.5853,  1.0700, -1.2298,  1.1589], requires_grad=True)\n",
      "y: tensor([ 1.5853,  2.0700, -0.2298,  2.1589], grad_fn=<AddBackward0>)\n",
      "p: tensor(5.5844, grad_fn=<SumBackward0>)\n",
      "x.grad: tensor([1., 1., 1., 1.])\n",
      "y.grad: None\n",
      "p.grad: None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/markaa/miniconda3/lib/python3.7/site-packages/ipykernel_launcher.py:11: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the gradient for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations.\n",
      "  # This is added back by InteractiveShellApp.init_path()\n",
      "/Users/markaa/miniconda3/lib/python3.7/site-packages/ipykernel_launcher.py:12: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the gradient for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations.\n",
      "  if sys.path[0] == '':\n"
     ]
    }
   ],
   "source": [
    "# Don't pay attention on UserWarning from Python\n",
    "\n",
    "x = torch.randn(4, requires_grad=True)  # leaf variable\n",
    "y = x + 1                               # not a leaf variable\n",
    "p = y.sum()                             # not a leaf variable\n",
    "p.backward()\n",
    "print('x:', x)\n",
    "print('y:', y)\n",
    "print('p:', p)\n",
    "print('x.grad:', x.grad)\n",
    "print('y.grad:', y.grad)\n",
    "print('p.grad:', p.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x.grad: tensor([1., 1., 1., 1.])\n",
      "y.grad: tensor([1., 1., 1., 1.])\n",
      "z.grad: None\n",
      "p.grad: None\n",
      "x.is_leaf: True\n",
      "y.is_leaf: True\n",
      "z.is_leaf: False\n",
      "p.is_leaf: False\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/markaa/miniconda3/lib/python3.7/site-packages/ipykernel_launcher.py:8: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the gradient for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations.\n",
      "  \n",
      "/Users/markaa/miniconda3/lib/python3.7/site-packages/ipykernel_launcher.py:9: UserWarning: The .grad attribute of a Tensor that is not a leaf Tensor is being accessed. Its .grad attribute won't be populated during autograd.backward(). If you indeed want the gradient for a non-leaf Tensor, use .retain_grad() on the non-leaf Tensor. If you access the non-leaf Tensor by mistake, make sure you access the leaf Tensor instead. See github.com/pytorch/pytorch/pull/30531 for more informations.\n",
      "  if __name__ == '__main__':\n"
     ]
    }
   ],
   "source": [
    "x = torch.randn(4, requires_grad=True)  # leaf variable\n",
    "y = torch.randn(4, requires_grad=True)  # leaf variable\n",
    "z = x + y    # not a leaf variable\n",
    "p = z.sum()  # not a leaf variable\n",
    "p.backward()\n",
    "print('x.grad:', x.grad)\n",
    "print('y.grad:', y.grad)\n",
    "print('z.grad:', z.grad)\n",
    "print('p.grad:', p.grad)\n",
    "print('x.is_leaf:', x.is_leaf)\n",
    "print('y.is_leaf:', y.is_leaf)\n",
    "print('z.is_leaf:', z.is_leaf)\n",
    "print('p.is_leaf:', p.is_leaf)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    One can use .retain_grad() to save gradients for non-leaf variables."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dp / dx: tensor([-3.0129,  1.4726, -1.1907,  1.3036])\n",
      "dp / dz: tensor([ 0.7252,  0.6488, -0.6153, -2.9357])\n",
      "dp / dw: tensor([ 0.3626,  0.3244, -0.3077, -1.4678])\n"
     ]
    }
   ],
   "source": [
    "x = torch.randn(4, requires_grad=True)   # leaf variable\n",
    "z = torch.randn(4, requires_grad=True)   # leaf variable\n",
    "w = z * 2      # not a leaf variable\n",
    "y = x * w + 1  # forward pass before retaining gradient is ok\n",
    "p = y.sum()\n",
    "\n",
    "w.retain_grad()\n",
    "\n",
    "p.backward()\n",
    "print('dp / dx:', x.grad)\n",
    "print('dp / dz:', z.grad)\n",
    "print('dp / dw:', w.grad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Note that even if there are non-leaf varibales which require gradient in the computational graph, .backward() fail.\n",
    "    For correct work one of the leaf variables must require gradient."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "can't retain_grad on Tensor that has requires_grad=False",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-26-a1d88584eebf>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0mp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m \u001b[0mw\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretain_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mretain_grad\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    324\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mhandle_torch_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mTensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mretain_grad\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrelevant_args\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    325\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrequires_grad\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 326\u001b[0;31m             \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"can't retain_grad on Tensor that has requires_grad=False\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    327\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_leaf\u001b[0m\u001b[0;34m:\u001b[0m  \u001b[0;31m# no-op for leaves\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    328\u001b[0m             \u001b[0;32mreturn\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mRuntimeError\u001b[0m: can't retain_grad on Tensor that has requires_grad=False"
     ]
    }
   ],
   "source": [
    "# will not work\n",
    "x = torch.randn(4, requires_grad=False)   # leaf variable\n",
    "z = torch.randn(4, requires_grad=False)   # leaf variable\n",
    "w = z * 2      # not a leaf variable\n",
    "y = x * w + 1  # forward pass before retaining gradient is ok\n",
    "p = y.sum()\n",
    "\n",
    "w.retain_grad()\n",
    "\n",
    "p.backward()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Dataset API\n",
    "    PyTorch provides convenient functionality for working with data during neural network training."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torch.utils.data import DataLoader, Dataset"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    The `Dataset` is a class that initiates our data somehow in the constructor.\n",
    "    It also has two main methods __getitem__ and __len__.\n",
    "    Let's make a simple dataset, where one element is a vector of size 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleDataset(Dataset):\n",
    "    \n",
    "    def __init__(self, data: torch.Tensor):\n",
    "        self.data = data\n",
    "    \n",
    "    def __getitem__(self, index: int):\n",
    "        return self.data[index]\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "example_data = torch.randn(1000, 10)\n",
    "simple_dataset = SimpleDataset(example_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([ 2.5797,  0.5153, -1.8101, -0.2936, -2.2498,  1.5486,  0.1375, -1.7653,\n",
       "         0.7241,  1.1529])"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "simple_dataset[10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(simple_dataset)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Second important class is `Dataloader`.\n",
    "    It takes some objects from the dataset and gather them into a batch."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "simple_dataloader = DataLoader(simple_dataset, batch_size=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Note that `simple_dataloader` is an iterator and has not yet done anything with the data.\n",
    "    Let's take a look at the first batсh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from itertools import islice\n",
    "\n",
    "first_batch = list(islice(simple_dataloader, 0, 1))[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([10, 10])\n",
      "tensor([[ 4.7640e-01, -2.4392e-01, -3.0228e-01,  2.4409e-01,  1.1879e+00,\n",
      "          6.1010e-01, -6.1443e-01,  6.6263e-01,  1.0846e-01, -2.8123e-01],\n",
      "        [ 1.6125e+00, -5.3476e-01, -3.0822e-01,  3.6862e-01, -5.0913e-01,\n",
      "         -5.8651e-01, -2.1725e-01,  1.9400e-01, -6.9170e-01,  1.5447e+00],\n",
      "        [-2.0479e+00,  6.6786e-01,  1.1241e+00,  5.4351e-01, -7.5457e-01,\n",
      "         -1.5541e+00, -1.8343e+00, -5.0052e-01, -5.3663e-02, -1.1224e+00],\n",
      "        [ 5.5327e-01, -4.0333e-01,  1.0972e+00, -5.6430e-01, -1.4030e+00,\n",
      "         -2.0434e+00,  8.4237e-02,  1.3065e+00,  6.5699e-01,  3.9171e-01],\n",
      "        [-1.8678e-01, -2.0547e+00, -1.7454e-01, -3.5228e-01, -8.0769e-02,\n",
      "         -6.1941e-01, -9.3604e-01, -1.0666e+00,  1.9389e+00,  4.4494e-01],\n",
      "        [ 6.2038e-02, -7.4033e-01, -3.9782e-01,  1.7042e+00, -9.0481e-01,\n",
      "          1.2189e+00, -1.3114e+00, -1.7935e+00, -2.7893e-01,  1.0207e+00],\n",
      "        [ 1.8920e-01, -3.2648e-01, -1.4454e+00,  9.2502e-01,  8.5233e-01,\n",
      "         -2.6312e-01,  1.6876e-01, -8.6392e-01,  6.4506e-02,  3.7468e-01],\n",
      "        [-1.3852e+00, -2.6164e-01, -2.0916e-01, -7.7036e-01,  4.1935e-01,\n",
      "         -1.9363e+00, -2.9584e-01, -2.3402e-01, -1.7375e-02,  3.9466e-04],\n",
      "        [-5.0133e-01,  3.4153e-01, -1.0849e+00, -6.8443e-01,  9.0952e-01,\n",
      "          6.5708e-01, -7.9250e-01,  7.1558e-01, -7.9662e-01, -1.4550e-01],\n",
      "        [ 5.3165e-01,  9.4228e-01, -2.0971e-01, -8.4808e-02, -7.7996e-01,\n",
      "         -3.5170e-01, -1.8163e+00, -4.6374e-01,  4.0067e-01,  1.9168e+00]])\n"
     ]
    }
   ],
   "source": [
    "print(first_batch.shape)\n",
    "print(first_batch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    DataLoader has many arguments for detailed configuration. Check the documentation so you can use it in the future"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Optimization"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "    Let's use optimization in PyTorch to find minimum of some simple function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "from IPython import display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "def func(x):\n",
    "    return (x - 10) ** 2 + 14"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAzI0lEQVR4nO3deXhU5dnH8e+dnSxkYQlLQsIq+5YIiILgCqiAVBRFxYpSrRaX2or19cWN1qXWaqtWVFqs1IgoFUVFRIIiguwgJEAStoQQIGwJ2ZPn/WMOviMmkJnJ5EyS+3Ndc2XmOdtvziRz55zznHPEGINSSqmmzc/uAEoppeynxUAppZQWA6WUUloMlFJKocVAKaUUEGB3AHe1bNnSJCYmujXtqVOnCAsLq9tAdUBzuUZzuUZzuaax5lq/fv0RY0yrnw0wxjTIR1JSknHX8uXL3Z7WmzSXazSXazSXaxprLmCdqeY7VXcTKaWU0mKglFJKi4FSSika8AFkpVTjVF5eTnZ2NiUlJbbmiIyMJC0tzdYM1altrpCQEOLi4ggMDKzVfLUYKKV8SnZ2NhERESQmJiIituUoKCggIiLCtuXXpDa5jDHk5+eTnZ1Nx44dazVf3U2klPIpJSUltGjRwtZC0NCJCC1atHBp60qLgVLK52gh8Jyr6/CcxUBE5ojIIRH5wanteRFJF5EtIrJQRKKchj0iIhkiskNErnRqH2W1ZYjIDKf2jiKyxmp/T0SCXHoHLqiqMvxnzT6+P1jhrUUopVSDVJstg38Bo85oWwr0Nsb0BXYCjwCISE9gEtDLmuZVEfEXEX/gFWA00BO40RoX4FngRWNMF+AYMNWjd3QWIvDe2n0s3FWG0fs4KKVqICLccccdP76uqKigVatWXH311QAsWrSIZ5555qzzOHDgANddd51Xc9alcxYDY8zXwNEz2r4wxpz+93o1EGc9HwekGGNKjTG7gQxgkPXIMMZkGWPKgBRgnDi2Yy4BFljTzwXGe/aWaiYiTBmaSO4pw8qMI95ajFKqgQsLCyMtLY3i4mIAli5dSvv27X8cPnbsWGbMmFHT5AC0a9eOBQsWnHUcX1IXvYluB96znrfHURxOy7baAPaf0T4YaAEcdyoszuP/jIhMA6YBxMbGkpqa6nLYiCpDRKDhhUXrqUwKcXl6byosLHTrPXmb5nKN5nLNmbkiIyMpKCio9fQB8+cT/MQTSHY2Ji6O0pkzqbj+eo9zXXbZZSxYsIDx48fz9ttvM2HCBFatWkVBQQHz5s1jw4YNvPDCC9x1111ERESwceNGDh06xJNPPsn48ePZu3cv119/PWvWrGHevHl88sknFBUVkZmZyW9+8xvKy8tJSUkhKCiIBQsWEBMTw5gxY3j66acZOHAg+fn5XHzxxfzwww8/mT4jI4Pp06dXO/2ZSkpKav2Ze1QMRORRoAKY58l8assYMxuYDZCcnGxGjBjh1nxGZn7Bx1nldOoziA4tQuswoWdSU1Nx9z15k+ZyjeZyzZm50tLSat+lc948mD4diooAkP37aTZ9OoSEwOTJHuWaOHEiL7zwAhMnTiQtLY1f/epXfP/990RERBASEkJQUBAREREEBgaSn5/Pd999R3p6OmPHjuWWW24hPDwcPz+/H8dPT09n48aNlJSU0KVLF5599lk2b97MAw88wMKFC7n//vvx9/cnLCyMiIgISktLEZGfTX/kyBH69+9f7fRnCgkJYcCAAbV6v273JhKR24Crgcnm/3fA5wDxTqPFWW01tecDUSIScEa7V42MD8BfhLe/2+PtRSmlvOnRR38sBD8qKnK0e6h3797s2bOHd999lzFjxpx13PHjx+Pn50fPnj3Jy8urdpyRI0cSERFBq1atiIyM5JprrgGgT58+7Nmz55x5Tk/fsmVLt6Y/F7eKgYiMAn4PjDXGOH8Si4BJIhIsIh2BrsD3wFqgq9VzKAjHQeZFVhFZDpw+yjIF+Mi9t1J70SF+jO7TlvfW7edUqfYsUqrB2rfPtXYXjR07loceeogbb7zxrOMFBwf/+LymzinO4/j5+f342s/Pj4oKx/dQQEAAVVVVAD87R6A203uiNl1L3wW+A84TkWwRmQr8HYgAlorIJhH5B4AxZhswH9gOfA7cY4yptI4J3AssAdKA+da4AA8DD4pIBo5jCG95/K5q4bahCRSUVLBwo9c3RJRS3tKhg2vtLrr99tuZOXMmffr0qZP5nUtiYiLr168HqPeDz+c8ZmCMqa4k1viFbYyZBcyqpv1T4NNq2rNw9DaqVwM7RNOnfST/WrWHyYM76EkuSjVEs2bBtGk/3VUUGuporwNxcXFMnz69TuZVGw899BDXX389s2fP5qqrrqq35QJN++Y2C9btNwkPf2K+3nnI7XnVpcZ6Mw1v0VyuaSi5tm/f7toM3nnHmIQEY0QcP995p05ynTx5sk7mU9dcyVXdukRvbvNzV/drS8vwYOas3G13FKWUuyZPhj17oKrK8dPDXkRNVZMuBsEB/tw8pAPLdxwm63Ch3XGUUso2TboYAEwenECQvx//WrXH7ihKKYvRy8V4zNV12OSLQauIYK7p144F67M5UVxudxylmryQkBDy8/O1IHjAWPczCAmp/VUW9OY2wC8vTOSDDdnMX7ufO4d3sjuOUk1aXFwc2dnZHD582NYcJSUlLn2Z1pfa5jp9p7Pa0mIA9G4fyaCOMfxr1R5+eWEiAf5NfoNJKdsEBgbW+u5c3pSamlrrSznUJ2/l0m89y9SLOpJzvJgl26o/lVwppRozLQaWy3rE0iEmlLdWZtkdRSml6p0WA4u/n3D7hYls2Hec9XuP2R1HKaXqlRYDJxOT44kICdCT0JRSTY4WAydhwQHcNKgDn/2Qy/6jReeeQCmlGgktBmeYMjQREWGunoSmlGpCtBicoV1UM67q05aUtfs5WaInoSmlmgYtBtW4Y1hHCksreO/7/eceWSmlGgEtBtXoGxfFkE4xzPl2N+WVVXbHUUopr9NiUINpwzuRe6KExVty7Y6ilFJep8WgBiO6taZL63Bmf52lF8xSSjV6Wgxq4Ocn3DmsI9tzT7IqM9/uOEop5VVaDM5iXP/2tAwPZvbXeokKpVTjpsXgLEIC/bltaAIrdh4mLfek3XGUUsprtBicw81DEggN8ucN3TpQSjViWgzOISo0iEnnd2DR5gPkHC+2O45SSnmFFoNamDqsIwb0AnZKqUbrnMVAROaIyCER+cGpLUZElorILutntNUuIvKyiGSIyBYRGeg0zRRr/F0iMsWpPUlEtlrTvCwiUtdv0lPto5oxtl873v1+HyeK9BIVSqnGpzZbBv8CRp3RNgNYZozpCiyzXgOMBrpaj2nAa+AoHsBMYDAwCJh5uoBY49zpNN2Zy/IJ04Z3oqisknfW7LU7ilJK1blzFgNjzNfA0TOaxwFzredzgfFO7W8bh9VAlIi0Ba4ElhpjjhpjjgFLgVHWsObGmNXGcWbX207z8ik92jbn4m6t+Oe3uykpr7Q7jlJK1akAN6eLNcacvk7DQSDWet4ecL66W7bVdrb27GraqyUi03BscRAbG0tqaqpb4QsLC92adkhkJSt2lvHHd7/ikg6Bbi3bG7m8TXO5RnO5RnO5xlu53C0GPzLGGBGpl+s1GGNmA7MBkpOTzYgRI9yaT2pqKu5Me7ExLDm4iuUHS/nfycMJ8K/b4+/u5vI2zeUazeUazeUab+Vy99ssz9rFg/XzkNWeA8Q7jRdntZ2tPa6adp8kIvx6RGf2Hy1m8Va9gJ1SqvFwtxgsAk73CJoCfOTUfqvVq2gIcMLanbQEuEJEoq0Dx1cAS6xhJ0VkiNWL6Fanefmky3rE0rV1OK+lZuoF7JRSjUZtupa+C3wHnCci2SIyFXgGuFxEdgGXWa8BPgWygAzgDeDXAMaYo8BTwFrr8aTVhjXOm9Y0mcBndfPWvMPPT7jr4s6kHyxg+Y5D555AKaUagHMeMzDG3FjDoEurGdcA99QwnznAnGra1wG9z5XDl4zt346/LN3Jq8szuaR77LknUEopH6dnILsh0N+PO4d1ZN3eY6zJ0stbK6UaPi0Gbpo0qAMtw4P4+/IMu6MopZTHtBi4KSTQn6kXdeKbXUfYvP+43XGUUsojWgw8cPOQDjQPCeAV3TpQSjVwWgw8EBESyG0XduSL7XnsOFhgdxyllHKbFgMP/XJoIqFB/ryaqlsHSqmGS4uBh6LDgrhlSAIfbz7A7iOn7I6jlFJu0WJQB6YO60igvx+v6rEDpVQDpcWgDrSOCOGmwR1YuDGH/UeL7I6jlFIu02JQR341vDN+Iryamml3FKWUcpkWgzrSJjKEG86PZ8H6/eQcL7Y7jlJKuUSLQR26a0RnAF5foVsHSqmGRYtBHWof1YzrkuJI+X4/B0+U2B1HKaVqTYtBHfv1iC5UGcM/dOtAKdWAaDGoY/ExofxiYBz/+X6fbh0opRoMLQZecO8lXaiqMrymZyUrpRoILQZeEB8TynVJcbyrxw6UUg2EFgMvuWek49iBXrNIKdUQaDHwkviYUCYmO3oWHdDzDpRSPk6LgRfdM7ILBqP3O1BK+TwtBl4UFx3KDefHM3/dfr1mkVLKp2kx8LJ7R3ZFRPjbV7vsjqKUUjXSYuBlbSJDuHlwAh9syNH7HSilfJZHxUBEHhCRbSLyg4i8KyIhItJRRNaISIaIvCciQda4wdbrDGt4otN8HrHad4jIlR6+J59z94jOBPn78dKXO+2OopRqwE4Ul3OgsMor83a7GIhIe2A6kGyM6Q34A5OAZ4EXjTFdgGPAVGuSqcAxq/1FazxEpKc1XS9gFPCqiPi7m8sXtYoI5tahCXy0+QC78vReyUop98z+OpPHvi32yvlLnu4mCgCaiUgAEArkApcAC6zhc4Hx1vNx1mus4ZeKiFjtKcaYUmPMbiADGORhLp9z1/DOhAUF8JelunWglHLd4YJS5qzcw/lt/GkTGVLn8xdjjPsTi9wHzAKKgS+A+4DV1n//iEg88JkxpreI/ACMMsZkW8MygcHA49Y071jtb1nTLKhmedOAaQCxsbFJKSkpbuUuLCwkPDzcrWk98d+MMv6bUc7MC0LoGPnzjR+7cp2L5nKN5nKN5qqdeWmlLNtXwf8MNHRq5X6ukSNHrjfGJJ/ZHuDuDEUkGsd/9R2B48D7OHbzeI0xZjYwGyA5OdmMGDHCrfmkpqbi7rSeSBpSzornlvPVkXD+PW6wz+Q6F83lGs3lGs11btnHilixdAXXJ8fTqcVRr+TyZDfRZcBuY8xhY0w58CFwIRBl7TYCiANyrOc5QDyANTwSyHdur2aaRiUiJJB7Rnbhm11H+C4z3+44SqkG4uVljq7p0y/t6rVleFIM9gFDRCTU2vd/KbAdWA5cZ40zBfjIer7Ieo01/Cvj2Ee1CJhk9TbqCHQFvvcgl0+7eUgCbZqH8PySdDzZRaeUahoyDxeyYH02Nw9JoF1UM68tx+1iYIxZg+NA8AZgqzWv2cDDwIMikgG0AN6yJnkLaGG1PwjMsOazDZiPo5B8DtxjjKl0N5evCwn0Z/qlXdmw7zjL0g7ZHUcp5eNe+GIHIYH+/HpkZ68ux+1jBgDGmJnAzDOas6imN5AxpgSYWMN8ZuE4EN0kTEyO441vsnh+yQ5Gdm+Nv5/YHUkp5YM27z/Op1sPMv3SrrQMD/bqsvQMZBsE+vvx2yu6sSOvgP9ubJSHR5RSHjLG8Ozn6cSEBXHnsI5eX54WA5uM6d2WvnGR/GXpTkrKG+1eMaWUm77ZdYRVmfncO7ILESGBXl+eFgOb+PkJD4/qTs7xYt5ZvdfuOEopH1JV5dgqiItuxuQhHeplmVoMbHRhl5YM69qSV5ZncLKk3O44Sikf8cnWXLYdOMmDl3cjOKB+rs6jxcBmD4/qzrGicl5fkWl3FKWUDyitqOT5Jel0bxPBuP7t6225Wgxs1rt9JGP7teOtlbs5VuKdqxEqpRqOd1bvY//RYh4Z06NeexpqMfABv7vyPKqq4MNduqtIqabsRHE5f/tqFxd1acnwri3rddlaDHxAfEwoU4YmsDKngvSDJ+2Oo5SyyWupmZwoLmfG6O44LuxQf7QY+Ih7RnahWQD86dN0u6MopWyQc7yYOd/u5tr+7endPrLel6/FwEdEhQZxTecgVuw8zMpdR+yOo5SqZy98sQOAB6/oZsvytRj4kMsSAoiLbsasT9OorNKL2CnVVGzJPs6HG3L45YWJxEWH2pJBi4EPCbROREvLPckH67PtjqOUqgfGGJ5enEaLsCDuGdnFthxaDHzM1X3bMrBDFM9/sYNTpRV2x1FKedkX2/P4fvdRHri8G83r4bITNdFi4GNEhP+5uieHC0r1RDSlGrmyiir+9GkaXVuHM+n8+HNP4EVaDHzQwA7RjO3XjtnfZHHgeLHdcZRSXvLv1XvZk1/EH67qQYC/vV/HWgx81O9HnUeVgeeX7LA7ilLKC46eKuOlL3cyrGtLRnRrZXccLQa+Ki46lDuHdWThxhw27DtmdxylVB17celOTpVV8tjVPev9BLPqaDHwYb8e0YXWEcE88fF2qrSrqVKNRvrBk8xbs5ebB3egW2yE3XEALQY+LSw4gBmju7N5/3EW6h3RlGoUjDE89cl2IkICuf8ye04wq44WAx83vn97+sVH8ezn6drVVKlGYOn2PL7NyOfBy7sRHRZkd5wfaTHwcX5+wsxrenKooJRXlmfYHUcp5YGS8kqeXuzoSjp5cP3cway2tBg0AAM7RDNhQHve/GY3e46csjuOUspNb36Txb6jRcy8ppftXUnP5FtpVI1mjO5OUIAfT32y3e4oSik3HDhezCvLMxnVqw0X1fO9CmpDi0ED0bp5CPdd2pVl6Yf4Kj3P7jhKKRfN+jSNKmN49KoedkeplkfFQESiRGSBiKSLSJqIXCAiMSKyVER2WT+jrXFFRF4WkQwR2SIiA53mM8Uaf5eITPH0TTVWU4Ym0rlVGE98vJ2S8kq74yilamlV5hEWb8nl7hGdiY+x56qk5+LplsFLwOfGmO5APyANmAEsM8Z0BZZZrwFGA12txzTgNQARiQFmAoOBQcDM0wVE/VRQgB+Pj+3F3vwi3lq52+44SqlaKK+s4vFF24iLbsZdF3e2O06N3C4GIhIJDAfeAjDGlBljjgPjgLnWaHOB8dbzccDbxmE1ECUibYErgaXGmKPGmGPAUmCUu7kau2FdWzGqVxv+9tUuso8V2R1HKXUO//p2DzvzCvnfq3sSEuhvd5waiTHundkqIv2B2cB2HFsF64H7gBxjTJQ1jgDHjDFRIvIJ8IwxZqU1bBnwMDACCDHGPG21PwYUG2P+XM0yp+HYqiA2NjYpJSXFreyFhYWEh4e7Na031TZXfnEVj6wspncLf6YPDPGZXPVNc7lGc7mmLnIdK6nikW+KOS/Gn/sHBtfJZSc8zTVy5Mj1xpjkM9sDPMgUAAwEfmOMWSMiL/H/u4QAMMYYEamz6ygYY2bjKEAkJyebESNGuDWf1NRU3J3Wm1zJdahZJs9+nk5Vmx5c0j3WZ3LVJ83lGs3lmrrIdc9/NmCklL/dNpwOLermWIG31pcnxwyygWxjzBrr9QIcxSHP2v2D9fOQNTwHcL5gd5zVVlO7OoupF3WkS+twZi7apgeTlfJB3+w6zOItudwzskudFQJvcrsYGGMOAvtF5Dyr6VIcu4wWAad7BE0BPrKeLwJutXoVDQFOGGNygSXAFSISbR04vsJqU2cRFODHU+N6s/9oMa/qmclK+ZTSikpmfrSNxBahTBveye44teLJbiKA3wDzRCQIyAJ+iaPAzBeRqcBe4Hpr3E+BMUAGUGSNizHmqIg8Bay1xnvSGHPUw1xNwgWdWzC+fzv+sSKLcQPa07mV7+13Vaopei01k6wjp5h7+yCfPmjszKNiYIzZBPzsQASOrYQzxzXAPTXMZw4wx5MsTdUfrurBV+mHeHThVt69c4hPXBddqaYs63Ahry7P5Jp+7bjYB25aU1t6BnID1zoihBmje7A66ygfbNBDLUrZyRjD//z3B4ID/Xjsat8807gmWgwagUnnx5OUEM2sxds5eqrM7jhKNVkLN+awKjOfh0d1p3WE97t91yUtBo2An5/wx2v7UFBSwR8/TbM7jlJN0rFTZcxanMaADlHcNMi3Lk9dG1oMGonz2kQwbXgnFqzPZlXGEbvjKNXkPL04jRPF5fzx2j74+TW8Y3daDBqR6Zd2JbFFKI8s3EpxmZ57oFR9WbnrCB9syOZXF3eiR9vmdsdxixaDRiQk0J8/TujD3vwi/rpsp91xlGoSissqeWThFjq1DOM3l3S1O47btBg0MkM7t2TS+fG8+c1ufsg5YXccpRq9F7/cyf6jxfxxQp8Gc05BdbQYNEKPjOlBTFgQD3+whfLKKrvjKNVobck+zpvfZHHjoHiGdGphdxyPaDFohCKbBfLUuF5sO3CS2V9n2R1HqUaprKKK372/hVYRwcwY3bDOKaiOFoNGalTvtlzVpy0vfbmLXXkFdsdRqtH5+/IMduQV8Mdr+xDZLNDuOB7TYtCIPTGuF2HB/vxuwRYqq+rsSuJKNXnbD5zk1eUZXDugPZf28O4l5OuLFoNGrGV4MI+P7cWm/cd5a6XuLlKqLpRXVvG7BZuJCg3if6/uaXecOqPFoJEb268dl/eM5YUvdpJ5uNDuOEo1eK+lZrLtwEmeHt+L6LAgu+PUGS0GjZyIMGt8b5oF+fPb+Zup0N5FSrlt24ETvLxsF2P7tWNU77Z2x6lTWgyagNbNQ3hyXG827T/O69q7SCm3lFZU8tv5m4kOC+LJcb3sjlPntBg0Edf0dfQu+uuXO0nLPWl3HKUanJeX7SL9YAHPTOhDVGjj2T10mhaDJkJEeGp8byKbBfLg/M2UVejuIqVqa+O+Y7yWmsnEpLhG03voTFoMmpCYsCD+NKEvabkn+euXeu0ipWqjqKyCB+dvpm1kMx67pvH0HjqTFoMm5vKesdyQHM8/VmSydo/ealqpc5m1OI09+ad44fp+NA9p+CeX1USLQRP02DU9iYsO5YH3NlFQUm53HKV81lfpecxbs487h3Vq8NceOhctBk1QeHAAL97QnwPHi3ni4+12x1HKJ50sM/x+wVa6t4ngt1d0szuO12kxaKKSEqK5Z2QXFqzPZvGWXLvjKOVTjDHM2VrKyeJy/jqpP8EBDffS1LWlxaAJm35pV/rFR/HIh1vIOV5sdxylfMa/V+9l0+FKHhnTne5tGuady1ylxaAJC/T34+VJ/amsMtyfslEvZqcUsONgAU8vTqNvS39uG5pod5x643ExEBF/EdkoIp9YrzuKyBoRyRCR90QkyGoPtl5nWMMTnebxiNW+Q0Su9DSTqr2EFmE8Nb43a/cc4+9fZdgdRylblZRXMv3djTQPCeSOPsGINLwb27urLrYM7gPSnF4/C7xojOkCHAOmWu1TgWNW+4vWeIhIT2AS0AsYBbwqIo1/B50PmTAwjvH92/HSsp3a3VQ1aU8v3s6OvAL+PLEvzYObTiEAD4uBiMQBVwFvWq8FuARYYI0yFxhvPR9nvcYafqk1/jggxRhTaozZDWQAgzzJpVz31PjexMeEMv3djRw7VWZ3HKXq3eItubyzeh/ThndixHmt7Y5T78QY9/cTi8gC4E9ABPAQcBuw2vrvHxGJBz4zxvQWkR+AUcaYbGtYJjAYeNya5h2r/S1rmgVnLA4RmQZMA4iNjU1KSUlxK3dhYSHh4eFuTetNdufac6KSp1eX0KulP/cP/P9NZLtz1URzuUZz1exQURUzVxXTNsyPPwwOIcBPfCJXdTzNNXLkyPXGmOQz2wPcnaGIXA0cMsasF5ERbidzgTFmNjAbIDk52YwY4d5iU1NTcXdab/KJXC138/jH28nwT+DO4Z18J1c1NJdrNFf1yiqquO4fqwgMKGfur4YRHxPqE7lq4q1cnuwmuhAYKyJ7gBQcu4deAqJE5HSRiQNyrOc5QDyANTwSyHdur2YaVc+mDE3kyl6xPPt5Ohv2HbM7jlJe96fP0tiSfYLnruv3YyFoitwuBsaYR4wxccaYRBwHgL8yxkwGlgPXWaNNAT6yni+yXmMN/8o49lEtAiZZvY06Al2B793NpTwjIjz3i360iQzh3nkbOKrHD1Qj9smWA/zz2z3cNjSRUb3b2B3HVt44z+Bh4EERyQBaAG9Z7W8BLaz2B4EZAMaYbcB8YDvwOXCPMabSC7lULUWGBvLa5CSOFJZxX8pGqjw4rqSUr8o8XMjDC7YwoEMUfxjTw+44tnP7mIEzY0wqkGo9z6Ka3kDGmBJgYg3TzwJm1UUWVTf6xEXy+Nhe/GHhVmKqArlkpN2JlKo7RWUV3P3OeoID/XnlpoEEBej5t7oGVI1uHBTPhIHtWZRZTuqOQ3bHUapOGGP4w4db2XWokL/e0J92Uc3sjuQTtBioGokIs8b3IS7Cj/tSNrEvv8juSEp57J/f7uG/mw7wwGXdGN6tld1xfIYWA3VWzYL8+c2AYACm/XsdRWUVNidSyn2rs/KZ9Wkal/WI5d6RXeyO41O0GKhzah3qx8s3DmBHXgEPf7AVT05UVMouuSeKufc/G0hoEcpfbuiHn1/TutzEuWgxULVycbdW/O7K8/h48wHe+CbL7jhKuaSkvJK7/r2e4rJKZt+S1KhvX+kuLQaq1u6+uDNX9WnLM5+lszxdDyirhsEYw8MfbGFLzglevKE/XVpH2B3JJ2kxULUmIvx5Yj96tG3O9Hc3knGowO5ISp3Taysy+WjTAR664jyu6NW0Tyw7Gy0GyiXNgvx549ZkggP9mTp3HceL9Axl5bu+3J7H80t2cE2/dvx6RGe74/g0LQbKZe2imvH6LUnkHi/h7nc2UFZRZXckpX5m+4GTTE/ZSO92kTz3i75N6kY17tBioNySlBDNM7/ow3dZ+Ty6UHsYKd+Sd7KEqXPX0jwkkDenJNMsSO+XdS51cjkK1TRNGBjHniOnePmrDBJbhnGP9ttWPqCorIKpc9dyoric9++6gNjmIXZHahC0GCiPPHB5N/bkF/H8kh0ktAjl6r7t7I6kmrDKKsN9KZvYfuAkb05Jple7SLsjNRhaDJRHRITnruvLgePFPDh/M63CgxncqYXdsVQTZIzhiY+3sXR7Ho9f05NLusfaHalB0WMGymMhgY4eRnHRzbjz7XXszNMup6r+vf51Fm9/t5dpwztx24Ud7Y7T4GgxUHUiOiyIub8cRHCgP7fN+Z6DJ0rsjqSakI825fDMZ+lc068dM0Z1tztOg6TFQNWZ+JhQ/nnb+ZwoLmfKnO85UVRudyTVBKzYeZiH3t/M4I4x/HliX73mkJu0GKg61bt9JK/fkszuI6e4fe5avcqp8qoN+45x17/X06V1BG9MSSY4QLuQukuLgapzF3VtyUuT+rNx3zE9KU15zc68An75z7XENg/m7dsH6cXnPKTFQHnF6D5tmXVtH1bsPMxv399MZZWelKbqzt78U9zy1hqCA/z499TBtIoItjtSg6ddS5XX3DioAyeKy3nms3RCA/3504Q+uj9XeezA8WJuemMNpRVVvDftAuJjQu2O1ChoMVBeddfFnSkqreDlrzIIDvTjibG99Boxym2HCkqY/OYaThaX8587h3BeG70cdV3RYqC87oHLu1FSUcXsr7MICfTnkdHdtSAol+UXlnLzm2vIO1nCv6cOok+cnl1cl7QYKK8TER4Z3Z2S8kpmf52FCMwYpQVB1V5+YSk3vbGGfUeLmHPb+SQlxNgdqdFx+wCyiMSLyHIR2S4i20TkPqs9RkSWisgu62e01S4i8rKIZIjIFhEZ6DSvKdb4u0RkiudvS/kaEeGJsb24eUgHXl+RxTOfpeuVTlWtnC4Ee4+e4q0p5zO0c0u7IzVKnmwZVAC/NcZsEJEIYL2ILAVuA5YZY54RkRnADOBhYDTQ1XoMBl4DBotIDDATSAaMNZ9FxphjHmRTPkhEeGpcbwTh9a+zMKC7jNRZHS5w7Bo6XQgu7KKFwFvcLgbGmFwg13peICJpQHtgHDDCGm0ukIqjGIwD3jaOfwdXi0iUiLS1xl1qjDkKYBWUUcC77mZTvktEeHJcL0Rg9tdZlJZXMvOaXtrLSP1M7oliJr+xhtwTJcyZcj5DtRB4ldTFprqIJAJfA72BfcaYKKtdgGPGmCgR+QR4xhiz0hq2DEeRGAGEGGOettofA4qNMX+uZjnTgGkAsbGxSSkpKW7lLSwsJDw83K1pvakp5TLG8N6OMj7fU8FF7QO4vXcQfi5uITSl9VUXGlKuQ0VVPLe2hFPlhgeSQugWXf9nFjek9eWKkSNHrjfGJP9sgDHGowcQDqwHJlivj58x/Jj18xPgIqf2ZTh2DT0E/I9T+2PAQ+dablJSknHX8uXL3Z7Wm5parqqqKvPXpTtNwsOfmF+/s96Ullf6RC5PaS7XnJlr58GTZvCsL02/J5aYzfuP2ZLJmIazvlwFrDPVfKd61JtIRAKBD4B5xpgPreY8EWlrjMm1dgMdstpzgHinyeOsthz+f7fS6fZUT3KphkFEuO+yroQF+/P04jROlpTz2s1JhAdrJ7emav3eY0ydu5ZAfz9Spg2he5vmdkdqMjzpTSTAW0CaMeYvToMWAad7BE0BPnJqv9XqVTQEOGEcxx2WAFeISLTV8+gKq001EXcM68Tz1/VlVWY+N72xmvzCUrsjKRss33GIyW+uJqpZIB/ePVQLQT3z5NpEFwK3AJeIyCbrMQZ4BrhcRHYBl1mvAT4FsoAM4A3g1wDGceD4KWCt9XjSalNNyMTkeGbfksTOvAKu+8d37MsvsjuSqkfvr9vPnXPX0blVOO/fNVQvMWEDT3oTrQRqOuJ3aTXjG+CeGuY1B5jjbhbVOFzaI5Z5dwxm6tx1XPvqt7wxJZmBHaLtjqW8yBjDwl1lfJS5hQu7tOAfNycRoVcftYVetVT5lKSEGD68eyjhIQHcOHs1n23NtTuS8pKyiip++/5mPsosZ2JSHP/65SAtBDbSYqB8TqdW4Xx491B6t4/k1//ZwCvLM/Rs5Ubm9HWGPtyQw4SugTx3XV8C/fXryE669pVPahEezLw7BjO2XzueX7KD+1I2UVJeaXcsVQfSD55k7N+/ZXP2cV6+cQBjOwfpWeg+QIuB8lkhgf789Yb+PDyqOx9vOcD1r39H7oliu2MpD3z+Qy6/eHUVFVVVvH/XBYzt187uSMqixUD5NBHh7hGdefPWZLIOn+Lql1eyKuOI3bGUiyoqq/jTZ2nc9c4GusZGsOjei+gbF2V3LOVEi4FqEC7tEctH915ITFgQN7+1hldT9ThCQ3GksJRb53zP6yuyuHlIB9771RBim4fYHUudQU/1VA1G51bh/PeeC5nx4Vae+3wH/Vr5029QGTFhQXZHUzVYlXmE+1M2caK4nD9P7Md1SXF2R1I10C0D1aCEBQfw8qT+PDmuF9uOVDL6pa9ZnZVvdyx1hsoqw1+W7mTym2uICAngv/dcqIXAx2kxUA2OiHDrBYk8dkEIYUEB3PTGav68ZAdlFVV2R1PAvvwibnj9O15etosJA+JYdO9F9Girl5bwdVoMVIOV0Nyfj39zEb8YGMffl2dw7avfsiuvwO5YTZYxhvlr9zP6pa/ZcbCAF2/oxwvX9yNMLzzYIGgxUA1aWHAAz0/sx+u3JJF7ooSr/raS11dkUlGpWwn16eCJEu58ex2//2ALfeOi+PyB4Vw7QHcLNSRaslWjcGWvNgzsEM0fFm7lT5+ls3hrLs/+oq/unvAyYwwpa/fzx8VplFdV8T9X9eD2CzvqnesaIN0yUI1Gq4hgZt+SxN9vGkDOsWKu+dtKnl+STnGZnrnsDRmHCrjpjTU88uFWerVvzuf3DeeOYZ20EDRQumWgGhUR4eq+7biwc0ueWrydV5Zn8t+NB3h8bC8u7xlrd7xGoaisgr99lcGb32TRLNCfP17bh0nnx2sRaOC0GKhGKTosiL9c358bkuN57KMfuPPtdYw4rxWPjulB19gIu+M1SFVVhkWbD/Dc5+kcOFHCdUlxzBjdnZbhwXZHU3VAi4Fq1AZ3asHi6cOYu2oPLy3bxaiXvuHGQfHcf1k3/RJzwbo9R3lqcRqb9x+nd/vmvHTjAM5PjLE7lqpDWgxUoxfo78cdwzoxYWAcL325k3fW7GPhhhymXtSRqcM6EdlMr6Ffk+0HTvLCFztYln6I2ObB/HliPyYMaK+7hBohLQaqyYgJC+KJcb255YJEXly6k5e/ymDud3u5c1hHbh2aSHO9scqP0nJP8vflGSzekkvzkAB+P+o8bhuaSGiQfmU0VvrJqianS+twXpk8kLtzTvCXpTv58xc7eX1FFrdckMDtF3Vs0ruP1u89xqvLM1iWfojw4ADuHdmFO4fr1lNToMVANVm920cy57bz+SHnBK+lZvLaikzeXLmbcf3acduFifRqF2l3xHpRXlnFp1tz+ee3e9i0/zhRoYE8eHk3plyQSGSoFoGmQouBavJ6t4/klckDyTxcyD+/3c0H63N4f302gxJjmDQontG929IsyN/umHVu/9Ei5q/bz/x1+8k7WUrHlmE8MbYX1yXF6SUkmiD9xJWydG4VztPj+/C7K7ozf91+5q3Zy4PzNzPzo22M7d+Ocf3bk5wQ3aAPnhaUlLNkWx4fbcphpXWToIu7teJPExIY0a11g35vyjNaDJQ6Q2RoIHcO78QdwzqyZvdR5q/dzwcbspm3Zh9tI0O4qk9bLu8ZS1JCNAEN4Cbux4vKSN1xmHc2lrDlyy8pq6giLroZ913aleuT42kX1czuiMoHaDFQqgYiwpBOLRjSqQVPje/Nl2l5fLw5l7nf7eHNlbuJCg1kRLdWXNS1FUM7t/CZL9XKKsPWnBN8m3GEb3YdZu2eY1RWGSKDhZsGJTC2fzsGxEfpTejVT/hMMRCRUcBLgD/wpjHmGZsjKfWjsOAAxvVvz7j+7SkoKeebXUf4Mi2PFTsO899NBwBIaBFKUodoQkvKick+TrfYCEICa3msYd48ePRR2LcPOnSAWbNg8uRaTXqksJRtB06ycd8xNu47zoZ9xygoqQCge5sI7rq4E5f3bMPRjI1cMrKXW+9fNX4+UQxExB94BbgcyAbWisgiY8x2e5Mp9XMRIYGM6dOWMX3aUlVl2HmogG8z8lmdlc/Xu45wpLCMd9K+xU8gsUUY3WIjSGgZSnx0KHHRzYhtHkKL8CBiQoMcu5nmzYNp06CoyLGAvXsdrwEmT6a4rJIjhaUcKSwl72QJ+44Wse9oEbuPnGLHwQKOFJYBIALdWkdwdd+2XNC5JUM7t/hJN9nUTN0SUDXziWIADAIyjDFZACKSAowDtBgon+bnJ3Rv05zubZoz9aKOGGP48PPlNIvrQfrBAnblFbAjr4Cv0g9RVs09FkKD/Ak96UezW14moKoSg2AEyv0DKFoXQNH2z6qdLio0kISYUC7p3ppusRH0aNucvnGRROiJc8pNYoyxOwMich0wyhhzh/X6FmCwMebeM8abBkwDiI2NTUpJSXFreYWFhYSHh3sW2gs0l2saUq4qYzhRajhc7PhZUGY4WWYorjCUHjxMqfHD+S/RHwiRKoLbtiY0AJoHC82DhOgQoWUzP8ICXf8vvyGtL1/QWHONHDlyvTEm+WcDjDG2P4DrcBwnOP36FuDvZ5smKSnJuGv58uVuT+tNmss1jSZXQoIx8PNHQoK9ueqJ5nKNp7mAdaaa71Rf6ReXA8Q7vY6z2pRq/GbNgtDQn7aFhjralaonvlIM1gJdRaSjiAQBk4BFNmdSqn5MngyzZ0NCguMocEKC43UtexMpVRd84gCyMaZCRO4FluDYXTrHGLPN5lhK1Z/Jk/XLX9nKJ4oBgDHmU+BTu3MopVRT5Cu7iZRSStlIi4FSSiktBkoppbQYKKWUwkfOQHaHiBwG9ro5eUvgSB3GqSuayzWayzWayzWNNVeCMabVmY0Nthh4QkTWmepOx7aZ5nKN5nKN5nJNU8ulu4mUUkppMVBKKdV0i8FsuwPUQHO5RnO5RnO5pknlapLHDJRSSv1UU90yUEop5USLgVJKqcZdDERkoohsE5EqEUk+Y9gjIpIhIjtE5Eqn9lFWW4aIzKinnO+JyCbrsUdENlntiSJS7DTsH/WRxynX4yKS47T8MU7Dql1/9ZTreRFJF5EtIrJQRKKsdlvXl5Wh3n9/asgRLyLLRWS79Tdwn9Ve42daj9n2iMhWa/nrrLYYEVkqIrusn9H1nOk8p3WySUROisj9dqwvEZkjIodE5AentmrXjzi8bP2+bRGRgW4vuLo73jSWB9ADOA9IBZKd2nsCm4FgoCOQiePS2f7W805AkDVOz3rO/ALwv9bzROAHG9ff48BD1bRXu/7qMdcVQID1/FngWR9ZX7b//jhlaQsMtJ5HADutz63az7Ses+0BWp7R9hwww3o+4/RnauPneBBIsGN9AcOBgc6/yzWtH2AM8BkgwBBgjbvLbdRbBsaYNGPMjmoGjQNSjDGlxpjdQAYwyHpkGGOyjDFlQIo1br0QEQGuB96tr2W6qab1Vy+MMV8YYyqsl6tx3BnPF9j6++PMGJNrjNlgPS8A0oD2dmSppXHAXOv5XGC8fVG4FMg0xrh7hQOPGGO+Bo6e0VzT+hkHvG0cVgNRItLWneU26mJwFu2B/U6vs622mtrryzAgzxizy6mto4hsFJEVIjKsHrOcdq+1+TnHadPd7vXk7HYc/xmdZuf68qX18iMRSQQGAGuspuo+0/pkgC9EZL2ITLPaYo0xudbzg0CsDblOm8RP/yGze31Bzeunzn7nGnwxEJEvReSHah62/EdWk1rmvJGf/hLmAh2MMQOAB4H/iEjzesz1GtAZ6G9leaEul+1BrtPjPApUAPOsJq+vr4ZGRMKBD4D7jTEnsfEzdXKRMWYgMBq4R0SGOw80jv0ftvR5F8dtd8cC71tNvrC+fsJb68dn7nTmLmPMZW5MlgPEO72Os9o4S7tHzpVTRAKACUCS0zSlQKn1fL2IZALdgHV1kak2uZzyvQF8Yr082/qrl1wichtwNXCp9cdRL+vrHLy+XlwhIoE4CsE8Y8yHAMaYPKfhzp9pvTHG5Fg/D4nIQhy71/JEpK0xJtfazXGovnNZRgMbTq8nX1hflprWT539zjX4LQM3LQImiUiwiHQEugLfA2uBriLS0foPYZI1bn24DEg3xmSfbhCRViLibz3vZOXMqqc8nLHv8VrgdO+GmtZffeUaBfweGGuMKXJqt3V9Ye/vz09Yx5/eAtKMMX9xaq/pM62vXGEiEnH6OY7OAD/gWE9TrNGmAB/VZy4nP9k6t3t9Oalp/SwCbrV6FQ0BTjjtTnJNfR+pr+ej8tfi2IdWCuQBS5yGPYqj58cOYLRT+xgcPS8ygUfrMeu/gLvOaPsFsA3YBGwArqnn9fdvYCuwxfqla3uu9VdPuTJw7CfdZD3+4Qvry87fn2pyXIRjV8IWp/U05myfaT3l6oSjl9Vm67N61GpvASwDdgFfAjE2rLMwIB+IdGqr9/WFoxjlAuXW99fUmtYPjl5Er1i/b1tx6jXp6kMvR6GUUqrJ7iZSSinlRIuBUkopLQZKKaW0GCillEKLgVJKKbQYKKWUQouBUkop4P8AZNDl/tal/vsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = torch.linspace(-100, 100, 100)\n",
    "y = func(x)\n",
    "\n",
    "plt.plot(x, y)\n",
    "plt.scatter([10], [14], label='Minimum', c='red')\n",
    "plt.legend()\n",
    "plt.grid()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEICAYAAAC9E5gJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA8dklEQVR4nO3dd3gVZdrH8e+dTgqBAIaSkFClhJpQBFGwAiIgCwo2VITFFevqiruviwV7d9eGiqKyoqIoioqURJogvSUBkkiHAKElkJ7n/eMMeAwJSU7KnCT357rOxTnPtF8mIXdm5plnxBiDUkqp2s3D7gBKKaXsp8VAKaWUFgOllFJaDJRSSqHFQCmlFFoMlFJKocVAKaUUWgxUDSQiF4vIChE5ISJHRWS5iPSwpt0mIssqcdufiogRkZ6F2t8VkQ1FzO8jIrNFZKe1XP9C031F5B0RSbW+lu9EpJnT9BARmSMip0Rkl4jcWElfmqrhtBioGkVE6gLfA/8BQoBmwBNAdhXGOAw845SpGTAB+Esx8y8DbgYOFjHtPuAioDPQFDiG42s7400gBwgFbgLeFpGO5cyvaiEtBqqmaQtgjPnMGJNvjMk0xvxsjNkkIu2Bd4CLRCRDRI7D2b++XxKR3dZf4O+ISB1rWn8R2Ssi/xSRI9Zf8DeVkOEjoKOIDLA+17EyJRee0RiTY4x5zRizDMgvYl0tgPnGmFRjTBbwOdDRyhaAo8A8ZozJsNYxF7il9LtLKQctBqqm2Q7ki8gMERkkIvXPTDDGJAATgV+NMYHGmHrWpOdwFJGuQGscRxP/dlpnY6Ch1T4WmCYiF54nw2ngWeDpwhNEZLyIrC18Guk8PgD6ikhTEfHH8df/j9a0tkCeMWa70/wbsYqFUmWhxUDVKMaYk8DFgAHeAw6LyFwRCS1qfhERHKdwHjDGHDXGpOM4xTO60KyPGWOyjTG/APOA60uI8i4QJiLXFtrW/cB4HKeuSmMHsAfYB5wE2gNPWtMCrTZnJ4CgUq5bqbO0GKgaxxiTYIy5zRgTBkThONf+WjGzNwL8gbUictw6dfST1X7GMWPMKafPu6x1ni9DNjAVeAoQq9kLyMPxy71eKb+cNwFfoAEQAHzNH0cGGUDdQvPXBdJLuW6lztJioGo0Y0wijnP4UWeaCs1yBMgEOhpj6lmvYGNMoNM89a3z82c0B/aXYvPTcfyVfoOVJRfYCqwEPizll9AV+Mg6asnGcfG4p4g0xHFKzEtE2jjN38XahlJlosVA1Sgi0k5E/i4iYdbncGAMjl/AAKk4Tt/4ABhjCnCcTnpVRC6wlmkmIlcXWvUTVjfQfsAQ4Msitl0PxwXfU9a683CcDnrIababgX7GmGlOy/mKiJ/10UdE/KxTSgCrgVtFJFhEvIG/AfuNMUeso5WvgSdFJEBE+gLDgE/KtNOUQouBqnnSgV7AKhE5haMIbAH+bk1fjOMv54MicsRqewRIAlaKyElgIeB8gfggji6d+4GZwETriKOw+wA/YJZT20zgAICIBAIrgK9FZKbTPNtwHJ00A+Zb7yOsaQ8BWTiuHRwGBgPXOS37Nxy9lQ4BnwF3GWP0yECVmejDbZQqnnUT2KfW9Qelaiw9MlBKKaXFQCmllJ4mUkophR4ZKKWUwnETTLXUsGFDExkZ6dKyp06dIiAgoOQZq5jmKhvNVTaaq2xqaq61a9ceMcY0OmeCMaZavqKjo42rYmNjXV62MmmustFcZaO5yqam5gLWmCJ+p+ppIqWUUloMlFJKaTFQSilFNb6ArJSqOLm5uezdu5esrKwq33ZwcDAJCQlVvt2SVPdcfn5+hIWF4e3tXar1ajFQSrF3716CgoKIjIzkjzHyqkZ6ejpBQe73CIbqnMsYQ1paGnv37qVFixalWq+eJlJKkZWVRYMGDaq8EKjKISI0aNCgTEd6WgyUUgBaCGqYsn4/SywGIjJdRA6JyBanthdFJFFENonIHGsc9zPTHhWRJBHZ5jwmvIgMtNqSRGSyU3sLEVlltX9+Zpz5ylBQYPjfqt38djCvsjahlFLVUmmODD4CBhZqWwBEGWM643ja0qMAItIBx7NjO1rLvCUiniLiiePxfYOADsAYa16A54FXjTGtcYwZP65cX9F5iMDnq3czZ0cORsdkUsqtiAg333zz2c95eXk0atSIIUOGADB37lyee+65865j//79jBw5slJz1lQlFgNjzBLgaKG2n43jKU7geHjImbHehwGzjOPB4b/jeGBIT+uVZIxJMcbk4Hj4xzDraU6XAbOt5WcAw8v3JRVPRBjbJ5IDpwzLko6UvIBS6lynTsETT0BkJHh6Ov594glHezkEBASwZcsWMjMzAViwYAHNmjU7O33o0KFMnjy5uMUBaNq0KbNnzz7vPKpoFdGb6A7gc+t9M/54vCDAXqsNHA8Bd27vheMh38edCovz/OcQkQnABIDQ0FDi4uLKHDaowBDkbXh57lryo/1KXqAKZWRkuPQ1VTbNVTbVMVdwcDDp6eklr+TUKfyvuQbPdev+aNu1Cx5/nPy5czk9bx6Ucdyc/Pz8s9u+4oormD17NsOHD+fjjz9mxIgRrFixgvT0dGbOnMm6det4+eWXmThxIkFBQaxfv55Dhw7x5JNPMnz4cHbt2sX111/PqlWrmDlzJt9//z2nT58mOTmZe+65h9zcXGbNmoWPjw+zZ88mJCSEwYMHM3XqVLp3705aWhqXXnopW7Zs4ZNPPuGHH34ocfmq5ry/SpKVlVXqn8VyFQMR+ReQh+PRfpXOOJ4bOw0gJibG9O/f36X1DEj+me9ScmnZqSfNG/hXYMLyiYuLw9WvqTJprrKpjrkSEhJK143ylVfAuRA48Vy3jqB334UpU8qUy7mr5K233sqTTz7JqFGjSEhI4K9//Su//fYbQUFB+Pn54ePjQ1BQEN7e3qSlpfHrr7+SmJjI0KFDueWWWwgMDMTDw+Ps/ImJiaxfv56srCxat27N888/z8aNG3nggQeYM2cO999/P56engQEBBAUFER2djYiQlBQEB4eHqVavqqVpcurn58f3bp1K9W8LvcmEpHbcDwY/Cbzxwn4fUC402xhVltx7WlAPRHxKtReqQaEe+Epwse/7qzsTSlVs3z4Yfmml6Bz587s3LmTzz77jMGDB5933uHDh+Ph4UGHDh1ITU0tcp4BAwYQFBREo0aNCA4O5tprrwWgU6dO7Ny5s8Q85V2+OnGpGIjIQOAfwFBjzGmnSXOB0SLiKyItgDbAb8BqoI3Vc8gHx0XmuVYRiQXOXPEZC3zr2pdSevX9PBjUqQmfr9nDqWztWaRUqe3ZU77ppTB06FAeeughxowZc975fH19z74vrkOI8zweHh5nP3t4eJCX5/i/7+XlRUFBAcA5/fJLs3xNUZqupZ8BvwIXisheERkH/BcIAhaIyAYReQfAGLMV+AKIB34C7jbG5FvXBCYB84EE4AtrXoBHgAdFJAnHNYQPKvQrLMZtfSJIz8pjzvpKPxBRquYIDy/f9FK44447mDJlCp06dSr3ukojMjKStWvXAtTqi8+l6U00xhjTxBjjbYwJM8Z8YIxpbYwJN8Z0tV4TneZ/2hjTyhhzoTHmR6f2H4wxba1pTzu1pxhjelrrHGWMya74L/Nc3ZvXp1OzYD5asVO7mSpVWrffXr7ppRAWFsa9995b7vWU1kMPPcTbb79Nt27dOHKkFvcyLOohB9XhVREPt5m9Zo+JeOR7s2T7IZfXVZFq6sM0KovmKpvz5YqPjy/dSjIyjOnRwxg499Wzp2N6GZ08ebLMy1SFmpCrqO8r+nCbcw3p0oSGgb5MX/a73VGUqh4CAiA2Fh5/HCIiwMPD8e/jj8PixWXuVqrcR60uBr5entzcuzmx2w6TcjjD7jhKVQ8BAY7uozt3Qn6+498pU7QQVHO1uhgA3NQrAh9PDz5asdPuKEopZZtaXwwaBflybZemzF67lxOZuXbHUUopW9T6YgBwe99ITufk88Xq8veRVkqp6kiLARDVLJieLUL4aMVO8vIL7I6jlFJVTouBZdzFLdh3PJP5W4u+rV0pVbkOHjzI6NGjadWqFdHR0QwePJjt27dXaYa4uDhWrFhR5LTU1FSGDBlCly5d6NChQ4nDZezcuZOoqKgSt/nMM8/86XOfPn1KH7gCaTGwXNE+lOYh/nywLMXuKEpVC1OXTKXhCw2ZumRquddljOG6666jf//+JCcns3btWp599tlixxwqSn5+/nk/l8b5isG///1vrrzySjZu3Eh8fHyJz1YorcLFoLjtVzYtBhZPD+GOvpGs232ctbuO2R1HKbeWmpHKY7GPkZaZxmOxj5GaUb4j6tjYWLy9vZk48exgBnTp0oV+/foRFxd39gE3AJMmTeKjjz4CHENJPPLII3Tv3p0vv/zynM8///wzF110Ed27d2fUqFFkZGScXW7KlCl0796dTp06kZiYyM6dO3nnnXd49dVX6dq16zm/lA8cOEBYWNjZz507dwYchezhhx8mKiqKTp068fnnn1PYRx99xKRJk85+HjJkCHFxcUyePJnMzEy6du3KTTfdBEBgYOB517t06VL69+/PyJEjadeuHTfddFOFjKKgxcDJqJhwgvy89CY0pYrxTeI3tH6jNY1fbvyn9sYvN6b1G635NtG1cSa3bNlCdHS0S8s2aNCAdevWMXr06D99vuKKK5g6dSoLFy5k3bp1xMTE8Morr5xdrmHDhqxbt4677rqLl156icjISCZOnMgDDzzAhg0bzjldc/fddzNu3DgGDBjA008/zf79+wH4+uuv2bBhAxs3bmThwoU8/PDDHDhwoFTZn3vuOerUqcOGDRuYOfPPTwI433rXr1/Pa6+9Rnx8PCkpKSxfvtylfedMi4GTAF8vbuzZnB+3HGDP0dMlL6BULfP88udJPpZc5LTkY8k8t7xiTp2UxQ033FDk55UrVxIfH0/fvn3p2rUrM2bMYNeuXWfnGzFiBADR0dGlGo766quvJiUlhfHjx5OYmEi3bt04fPgwy5YtY8yYMXh6ehIaGsqll17K6tWry/11nW+9PXv2JCwsDA8PD7p27Vohw2lrMShkbJ9IRIQZehOaUucYE3X+YaVLml6cjh07nh05tDDnIabh3GGmAwrd+XzmszGGK6+8kg0bNrBhwwbi4+P54IM/BkU+Mxy1p6dnqYejDgkJ4cYbb+STTz6hR48eLFmypFTLlfQ1lJXz0NplyX8+WgwKaVqvDtd0asKs1Xs4maU3oSnl7N5e9zKpx6Qip93T8x7u7eXaaKOXXXYZ2dnZTJs27Wzbpk2bWLp0KREREcTHx5Odnc3x48dZtGhRqdbZu3dvli9fTlJSEgCnTp0qsXdSUFBQsY+UXLx4MadPO84YpKenk5ycTPPmzenXrx+ff/45+fn5HD58mCVLltCzZ88/LRsZGcmGDRsoKChgz549/Pbbb2eneXt7k5t77u+a0qy3ImkxKMKd/VqQkZ3H57/pTWhKFXbo9KGi208V3V4aIsKcOXNYuHAhrVq1omPHjjz66KM0btyY8PBwrr/+eqKiorj++utL/RjHRo0a8dFHHzFmzBg6d+7MRRddRGJi4nmXufbaa5kzZ06RF5DXrl1LTEzM2XXdeeed9OjRg+uuu47OnTvTpUsXLrvsMl544QUaN/7zNZW+ffvSokULOnTowL333kv37t3PTpswYQKdO3c+ewH5jNKst0IVNZRpdXhVxBDW53PDuytM72cWmpy8fJe3U1bVcehjO2musqmQIayNMff/eL/hcQyPY9r9t93Z9/f/eL9LuWrCUNFVqbKGsPYqsVrUUhMuackdH61h3qYDDO/WzO44SrmNZ694lr7N+9KhUQc6NOpA/OF44g/HM6TtkJIXVm5Li0Ex+re9gNYXBDJtSQrDujZFROyOpJRb8PPyY2SHkWc/nykKqnrTawbF8PAQxvdrQfyBk6xITrM7jlKVzujjX2uUsn4/tRicx7CuzWgY6Mu0JTpEharZ/Pz8SEtL04JQQxhjSEtLw8/Pr9TL6Gmi8/Dz9uS2PhG89PN2Eg6cpH2TunZHUqpShIWFsXfvXg4fPlzl287KyirTL62qUt1z+fn5/Wn4jJJoMSjBzb0jeCsumfeWpPDKDV3tjqNUpfD29qZFixa2bDsuLq7U3UWrUm3LpaeJSlDP34fRPZozd+N+9h3PtDuOUkpVCi0GpTCuXwsM6AB2Sqkaq8RiICLTReSQiGxxagsRkQUissP6t77VLiLyhogkicgmEenutMxYa/4dIjLWqT1aRDZby7whbtiHs1m9Ogzt0pTPftvNidM6RIVSquYpzZHBR8DAQm2TgUXGmDbAIuszwCCgjfWaALwNjuIBTAF6AT2BKWcKiDXPeKflCm/LLUy4pCWnc/L5dNWukmdWSqlqpsRiYIxZAhwt1DwMmGG9nwEMd2r/2LrreSVQT0SaAFcDC4wxR40xx4AFwEBrWl1jzErrNumPndblVto3qculbRvx4fLfycot+xOUlFLKnbnamyjUGHPm6Q0HgVDrfTPAeXS3vVbb+dr3FtFeJBGZgOOIg9DQUOLi4lwKn5GR4dKyvYPz+WV7Ds98tpjLmnu7tO3KyFXZNFfZaK6y0VxlU1m5yt211BhjRKRK7lQxxkwDpgHExMSY/v37u7SeuLg4XFn2UmOYf3AFsQez+fdNl+DlWbHX313NVdk0V9lorrLRXGVTWblc/W2Wap3iwfr3zNi1+4Bwp/nCrLbztYcV0e6WRIS/9W/FnqOZzNtcusfaKaVUdeBqMZgLnOkRNBb41qn9VqtXUW/ghHU6aT5wlYjUty4cXwXMt6adFJHeVi+iW53W5ZauaB9KmwsCeTsuWW/dV0rVGKXpWvoZ8CtwoYjsFZFxwHPAlSKyA7jC+gzwA5ACJAHvAX8DMMYcBZ4CVluvJ602rHnet5ZJBn6smC+tcnh4CBMvbUXiwXRit7n+MA+llHInJV4zMMYU91DTy4uY1wB3F7Oe6cD0ItrXAFEl5XAnQ7s25ZUF23krNpnL2oWWvIBSSrk5vQPZBd6eHozv14I1u46xKkWHt1ZKVX9aDFw0umdzGgb68N/YJLujKKVUuWkxcJGftyfjLm7J0h1H2LjnuN1xlFKqXLQYlMPNvZtT18+LN/XoQClVzWkxKIcgP29u69uCn+NT2XYw3e44SinlMi0G5XR7n0j8fTx5K06PDpRS1ZcWg3KqH+DDLb0j+G7jfn4/csruOEop5RItBhVgXL8WeHt68JZeO1BKVVNaDCrABUF+3NirOXPW72PP0dN2x1FKqTLTYlBB/npJKzxEeCsu2e4oSilVZloMKkjjYD9u6BHO7LV72Hc80+44SilVJloMKtDE/q0AePcXPTpQSlUvWgwqULN6dRgZHcas3/Zw8ESW3XGUUqrUtBhUsL/1b02BMbyjRwdKqWpEi0EFCw/x5y/dw/jfb7v16EApVW1oMagEky5rTUGB4W29K1kpVU1oMagE4SH+jIwO4zO9dqCUqia0GFSSuwc4rh3omEVKqepAi0ElCQ/xZ1SMo2fRfr3vQCnl5rQYVKK7B7TGYPR5B0opt6fFoBKF1ffnhh7hfLFmj45ZpJRya1oMKtmkAW0QEf6zeIfdUZRSqlhaDCpZ42A/bu4VwVfr9unzDpRSbqtcxUBEHhCRrSKyRUQ+ExE/EWkhIqtEJElEPhcRH2teX+tzkjU90mk9j1rt20Tk6nJ+TW7nrv6t8PH04PWF2+2OopSqxk5k5rI/o6BS1u1yMRCRZsC9QIwxJgrwBEYDzwOvGmNaA8eAcdYi44BjVvur1nyISAdruY7AQOAtEfF0NZc7ahTky619Ivh24352pOqzkpVSrpm2JJnHlmdWyv1L5T1N5AXUEREvwB84AFwGzLamzwCGW++HWZ+xpl8uImK1zzLGZBtjfgeSgJ7lzOV2Jl7SigAfL15ZoEcHSqmyO5yezfRlO+nR2JPGwX4Vvn4xxri+sMh9wNNAJvAzcB+w0vrrHxEJB340xkSJyBZgoDFmrzUtGegFPG4t86nV/oG1zOwitjcBmAAQGhoaPWvWLJdyZ2RkEBgY6NKy5fFNUg7fJOUy5SI/WgSfe/BjV66SaK6y0Vxlo7lKZ2ZCNot25/F/3Q0tG7mea8CAAWuNMTGF271cXaGI1MfxV30L4DjwJY7TPJXGGDMNmAYQExNj+vfv79J64uLicHXZ8ojuncsvL8Sy+Eggnwzr5Ta5SqK5ykZzlY3mKtneY6f5ZcEvXB8TTssGRyslV3lOE10B/G6MOWyMyQW+BvoC9azTRgBhwD7r/T4gHMCaHgykObcXsUyNEuTnzd0DWrN0xxF+TU6zO45Sqpp4Y5Gja/q9l7eptG2UpxjsBnqLiL917v9yIB6IBUZa84wFvrXez7U+Y01fbBznqOYCo63eRi2ANsBv5cjl1m7uHUHjun68OD+R8pyiU0rVDsmHM5i9di83946gab06lbYdl4uBMWYVjgvB64DN1rqmAY8AD4pIEtAA+MBa5AOggdX+IDDZWs9W4AscheQn4G5jTL6rudydn7cn917ehnW7j7Mo4ZDdcZRSbu7ln7fh5+3J3wa0qtTtuHzNAMAYMwWYUqg5hSJ6AxljsoBRxaznaRwXomuFUTFhvLc0hRfnb2NAuwvw9BC7Iyml3NDGPcf5YfNB7r28DQ0DfSt1W3oHsg28PT34+1Vt2Zaazjfra+TlEaVUORljeP6nREICfBjfr0Wlb0+LgU0GRzWhc1gwryzYTlZujT0rppRy0dIdR1iRnMakAa0J8vOu9O1pMbCJh4fwyMB27Dueyacrd9kdRynlRgoKHEcFYfXrcFPv5lWyTS0GNurbuiH92jTkzdgkTmbl2h1HKeUmvt98gK37T/LglW3x9aqa0Xm0GNjskYHtOHY6l3d/SbY7ilLKDWTn5fPi/ETaNQ5iWNdmVbZdLQY2i2oWzNAuTflg2e8cy6qc0QiVUtXHpyt3s+doJo8Obl+lPQ21GLiBh6++kIIC+HqHnipSqjY7kZnLfxbv4OLWDbmkTcMq3bYWAzcQHuLP2D4RLNuXR+LBk3bHUUrZ5O24ZE5k5jJ5UDscAztUHS0GbuLuAa2p4wXP/pBodxSllA32Hc9k+vLfua5rM6KaBVf59rUYuIl6/j5c28qHX7YfZtmOI3bHUUpVsZd/3gbAg1e1tWX7WgzcyBURXoTVr8PTPySQX6CD2ClVW2zae5yv1+3j9r6RhNX3tyWDFgM34m3diJZw4CRfrd1rdxylVBUwxjB1XgINAny4e0Br23JoMXAzQzo3oXvzerz48zZOZefZHUcpVcl+jk/lt9+P8sCVbalbBcNOFEeLgZsREf5vSAcOp2frjWhK1XA5eQU8+0MCbS4IZHSP8JIXqERaDNxQ9+b1GdqlKdOWprD/eKbdcZRSleSTlbvYmXaaf17THi9Pe38dazFwU/8YeCEFBl6cv83uKEqpSnD0VA6vL9xOvzYN6d+2kd1xtBi4q7D6/ozv14I56/exbvcxu+MopSrYqwu2cyonn8eGdKjyG8yKosXAjf2tf2suCPLlie/iKdCupkrVGIkHTzJz1S5u7tWctqFBdscBtBi4tQBfLyYPasfGPceZo09EU6pGMMbw1PfxBPl5c/8V9txgVhQtBm5ueNdmdAmvx/M/JWpXU6VqgAXxqSxPSuPBK9tSP8DH7jhnaTFwcx4ewpRrO3AoPZs3Y5PsjqOUKoes3HymznN0Jb2pV9U8way0tBhUA92b12dEt2a8v/R3dh45ZXccpZSL3l+awu6jp5lybUfbu5IW5l5pVLEmD2qHj5cHT30fb3cUpZQL9h/P5M3YZAZ2bMzFVfysgtLQYlBNXFDXj/sub8OixEMsTky1O45Sqoye/iGBAmP41zXt7Y5SpHIVAxGpJyKzRSRRRBJE5CIRCRGRBSKyw/q3vjWviMgbIpIkIptEpLvTesZa8+8QkbHl/aJqqrF9ImnVKIAnvosnKzff7jhKqVJakXyEeZsOcFf/VoSH2DMqaUnKe2TwOvCTMaYd0AVIACYDi4wxbYBF1meAQUAb6zUBeBtAREKAKUAvoCcw5UwBUX/m4+XB40M7sivtNB8s+93uOEqpUsjNL+DxuVsJq1+HiZe2sjtOsVwuBiISDFwCfABgjMkxxhwHhgEzrNlmAMOt98OAj43DSqCeiDQBrgYWGGOOGmOOAQuAga7mqun6tWnEwI6N+c/iHew9dtruOEqpEny0fCfbUzP495AO+Hl72h2nWGKMa3e2ikhXYBoQj+OoYC1wH7DPGFPPmkeAY8aYeiLyPfCcMWaZNW0R8AjQH/Azxky12h8DMo0xLxWxzQk4jioIDQ2NnjVrlkvZMzIyCAwMdGnZylTaXGmZBTy6LJOoBp7c293PbXJVNc1VNpqrbCoi17GsAh5dmsmFIZ7c3923QoadKG+uAQMGrDXGxBRu9ypHJi+gO3CPMWaViLzOH6eEADDGGBGpsHEUjDHTcBQgYmJiTP/+/V1aT1xcHK4uW5nKkutQnWSe/ymRgsbtuaxdqNvkqkqaq2w0V9lURK67/7cOI9n857ZLaN6gYq4VVNb+Ks81g73AXmPMKuvzbBzFIdU6/YP17yFr+j7AecDuMKutuHZ1HuMubkHrCwKZMnerXkxWyg0t3XGYeZsOcPeA1hVWCCqTy8XAGHMQ2CMiF1pNl+M4ZTQXONMjaCzwrfV+LnCr1auoN3DCGHMAmA9cJSL1rQvHV1lt6jx8vDx4algUe45m8pbemayUW8nOy2fKt1uJbODPhEta2h2nVMpzmgjgHmCmiPgAKcDtOArMFyIyDtgFXG/N+wMwGEgCTlvzYow5KiJPAaut+Z40xhwtZ65a4aJWDRjetSnv/JLCsG7NaNXI/c67KlUbvR2XTMqRU8y4o6dbXzR2Vq5iYIzZAJxzIQLHUULheQ1wdzHrmQ5ML0+W2uqf17RnceIh/jVnM5+N7+0W46IrVZulHM7grdhkru3SlEvd4KE1paV3IFdzFwT5MXlQe1amHOWrdXqpRSk7GWP4v2+24OvtwWND3PNO4+JoMagBRvcIJzqiPk/Pi+foqRy74yhVa81Zv48VyWk8MrAdFwRVfrfviqTFoAbw8BCeua4T6Vl5PPNDgt1xlKqVjp3K4el5CXRrXo8be7rX8NSlocWghriwcRATLmnJ7LV7WZF0xO44StU6U+clcCIzl2eu64SHR/W7dqfFoAa59/I2RDbw59E5m8nM0XsPlKoqy3Yc4at1e/nrpS1p36Su3XFcosWgBvHz9uSZEZ3YlXaa1xZttzuOUrVCZk4+j87ZRMuGAdxzWRu747hMi0EN06dVQ0b3COf9pb+zZd8Ju+MoVeO9unA7e45m8syITtXmnoKiaDGogR4d3J6QAB8e+WoTufkFdsdRqsbatPc47y9NYUzPcHq3bGB3nHLRYlADBdfx5qlhHdm6/yTTlqTYHUepGiknr4CHv9xEoyBfJg+qXvcUFEWLQQ01MKoJ13RqwusLd7AjNd3uOErVOP+NTWJbajrPXNeJ4DredscpNy0GNdgTwzoS4OvJw7M3kV9QYSOJK1Xrxe8/yVuxSVzXrRmXt6/cIeSrihaDGqxhoC+PD+3Ihj3H+WCZni5SqiLk5hfw8OyN1PP34d9DOtgdp8JoMajhhnZpypUdQnn55+0kH86wO45S1d7bccls3X+SqcM7Uj/Ax+44FUaLQQ0nIjw9PIo6Pp78/YuN5GnvIqVctnX/Cd5YtIOhXZoyMKqJ3XEqlBaDWuCCun48OSyKDXuO8672LlLKJdl5+fz9i43UD/DhyWEd7Y5T4bQY1BLXdnb0Lnpt4XYSDpy0O45S1c4bi3aQeDCd50Z0op5/zTk9dIYWg1pCRHhqeBTBdbx58IuN5OTp6SKlSmv97mO8HZfMqOiwGtN7qDAtBrVISIAPz47oTMKBk7y2UMcuUqo0Tufk8eAXG2kSXIfHrq05vYcK02JQy1zZIZQbYsJ555dkVu/UR00rVZKn5yWwM+0UL1/fhbp+1f/msuJoMaiFHru2A2H1/Xng8w2kZ+XaHUcpt7U4MZWZq3Yzvl/Laj/2UEm0GNRCgb5evHpDV/Yfz+SJ7+LtjqOUWzqZY/jH7M20axzE369qa3ecSqfFoJaKjqjP3QNaM3vtXuZtOmB3HKXcijGG6ZuzOZmZy2uju+LrVX2Hpi4tLQa12L2Xt6FLeD0e/XoT+45n2h1HKbfxycpdbDicz6OD29GucfV8cllZaTGoxbw9PXhjdFfyCwz3z1qvg9kpBWw7mM7UeQl0bujJbX0i7Y5TZcpdDETEU0TWi8j31ucWIrJKRJJE5HMR8bHafa3PSdb0SKd1PGq1bxORq8ubSZVeRIMAnhoexeqdx/jv4iS74yhlq6zcfO79bD11/by5s5MvItXvwfauqogjg/uABKfPzwOvGmNaA8eAcVb7OOCY1f6qNR8i0gEYDXQEBgJviUjNP0HnRkZ0D2N416a8vmi7djdVtdrUefFsS03npVGdqetbewoBlLMYiEgYcA3wvvVZgMuA2dYsM4Dh1vth1mes6Zdb8w8DZhljso0xvwNJQM/y5FJl99TwKMJD/Ln3s/UcO5Vjdxylqty8TQf4dOVuJlzSkv4XXmB3nConxrh+nlhEZgPPAkHAQ8BtwErrr39EJBz40RgTJSJbgIHGmL3WtGSgF/C4tcynVvsH1jKzC20OEZkATAAIDQ2NnjVrlku5MzIyCAwMdGnZymR3rp0n8pm6MouODT25v/sfh8h25yqO5iobzVW8Q6cLmLIikyYBHvyzlx9eHuIWuYpS3lwDBgxYa4yJKdzu5eoKRWQIcMgYs1ZE+rucrAyMMdOAaQAxMTGmf3/XNhsXF4ery1Ymt8jV8Hce/y6eJM8Ixl/S0n1yFUFzlY3mKlpOXgEj31mBt1cuM/7aj/AQf7fIVZzKylWe00R9gaEishOYheP00OtAPRE5U2TCgH3W+31AOIA1PRhIc24vYhlVxcb2ieTqjqE8/1Mi63YfszuOUpXu2R8T2LT3BC+M7HK2ENRGLhcDY8yjxpgwY0wkjgvAi40xNwGxwEhrtrHAt9b7udZnrOmLjeMc1VxgtNXbqAXQBvjN1VyqfESEF/7ShcbBfkyauY6jev1A1WDfb9rPh8t3clufSAZGNbY7jq0q4z6DR4AHRSQJaAB8YLV/ADSw2h8EJgMYY7YCXwDxwE/A3caY/ErIpUop2N+bt2+K5khGDvfNWk9BOa4rKeWukg9n8MjsTXRrXo9/Dm5vdxzbuXzNwJkxJg6Is96nUERvIGNMFjCqmOWfBp6uiCyqYnQKC+bxoR3555zNhBR4c9kAuxMpVXFO5+Rx16dr8fX25M0bu+Pjpfff6h5QxRrTM5wR3ZsxNzmXuG2H7I6jVIUwxvDPrzez41AGr93Qlab16tgdyS1oMVDFEhGeHt6JsCAP7pu1gd1pp+2OpFS5fbh8J99s2M8DV7TlkraN7I7jNrQYqPOq4+PJPd18AZjwyRpO5+TZnEgp161MSePpHxK4on0okwa0tjuOW9FioEp0gb8Hb4zpxrbUdB75ajPluVFRKbscOJHJpP+tI6KBP6/c0AUPj9o13ERJtBioUrm0bSMevvpCvtu4n/eWptgdR6kyycrNZ+Ina8nMyWfaLdE1+vGVrtJioErtrktbcU2nJjz3YyKxiXpBWVUPxhge+WoTm/ad4NUbutL6giC7I7klLQaq1ESEl0Z1oX2Tutz72XqSDqXbHUmpEr39SzLfbtjPQ1ddyFUda/eNZeejxUCVSR0fT967NQZfb0/GzVjD8dN6h7JyXwvjU3lx/jau7dKUv/VvZXcct6bFQJVZ03p1ePeWaA4cz+KuT9eRk1dgdySlzhG//yT3zlpPVNNgXvhL51r1oBpXaDFQLomOqM9zf+nErylp/GuO9jBS7iX1ZBbjZqymrp8374+NoY6PPi+rJBUyHIWqnUZ0D2PnkVO8sTiJyIYB3K39tpUbOJ2Tx7gZqzmRmcuXEy8itK6f3ZGqBS0GqlweuLItO9NO8+L8bUQ08GdI56Z2R1K1WH6B4b5ZG4jff5L3x8bQsWmw3ZGqDS0GqlxEhBdGdmb/8Uwe/GIjjQJ96dWygd2xVC1kjOGJ77ayID6Vx6/twGXtQu2OVK3oNQNVbn7ejh5GYfXrMP7jNWxP1S6nquq9uySFj3/dxYRLWnJb3xZ2x6l2tBioClE/wIcZt/fE19uT26b/xsETWXZHUrXItxv28dyPiVzbpSmTB7azO061pMVAVZjwEH8+vK0HJzJzGTv9N06czrU7kqoFftl+mIe+3EivFiG8NKqzjjnkIi0GqkJFNQvm3Vti+P3IKe6YsVpHOVWVat3uY0z8ZC2tLwjivbEx+HppF1JXaTFQFe7iNg15fXRX1u8+pjelqUqzPTWd2z9cTWhdXz6+o6cOPldOWgxUpRjUqQlPX9eJX7Yf5u9fbiS/QG9KUxVnV9opbvlgFb5eHnwyrheNgnztjlTtaddSVWnG9GzOicxcnvsxEX9vT54d0UnP56py2388kxvfW0V2XgGfT7iI8BB/uyPVCFoMVKWaeGkrTmfn8cbiJHy9PXhiaEcdI0a57FB6Fje9v4qTmbn8b3xvLmysw1FXFC0GqtI9cGVbsvIKmLYkBT9vTx4d1E4LgiqztIxsbn5/Fakns/hkXE86hendxRVJi4GqdCLCo4PakZWbz7QlKYjA5IFaEFTppWVkc+N7q9h99DTTb+tBdESI3ZFqHJcvIItIuIjEiki8iGwVkfus9hARWSAiO6x/61vtIiJviEiSiGwSke5O6xprzb9DRMaW/8tS7kZEeGJoR27u3Zx3f0nhuR8TdaRTVSpnCsGuo6f4YGwP+rRqaHekGqk8RwZ5wN+NMetEJAhYKyILgNuARcaY50RkMjAZeAQYBLSxXr2At4FeIhICTAFiAGOtZ64x5lg5sik3JCI8NSwKQXh3SQoG9JSROq/D6Y5TQ2cKQd/WWggqi8vFwBhzADhgvU8XkQSgGTAM6G/NNgOIw1EMhgEfG8efgytFpJ6INLHmXWCMOQpgFZSBwGeuZlPuS0R4clhHRGDakhSyc/OZcm1H7WWkznHgRCY3vbeKAyeymD62B320EFQqqYhDdRGJBJYAUcBuY0w9q12AY8aYeiLyPfCcMWaZNW0RjiLRH/Azxky12h8DMo0xLxWxnQnABIDQ0NDoWbNmuZQ3IyODwMBAl5atTLUplzGGz7fl8NPOPC5u5sUdUT54lPEIoTbtr4pQnXIdOl3AC6uzOJVreCDaj7b1q/7O4uq0v8piwIABa40xMedMMMaU6wUEAmuBEdbn44WmH7P+/R642Kl9EY5TQw8B/+fU/hjwUEnbjY6ONq6KjY11ednKVNtyFRQUmNcWbDcRj3xv/vbpWpOdm+8WucpLc5VN4VzbD540vZ5eaLo8Md9s3HPMlkzGVJ/9VVbAGlPE79Ry9SYSEW/gK2CmMeZrqzlVRJoYYw5Yp4EOWe37gHCnxcOstn38cVrpTHtceXKp6kFEuO+KNgT4ejJ1XgIns3J5++ZoAn21k1tttXbXMcbNWI23pwezJvSmXeO6dkeqNcrTm0iAD4AEY8wrTpPmAmd6BI0FvnVqv9XqVdQbOGEc1x3mA1eJSH2r59FVVpuqJe7s15IXR3ZmRXIaN763krSMbLsjKRvEbjvETe+vpF4db76+q48WgipWnrGJ+gK3AJeJyAbrNRh4DrhSRHYAV1ifAX4AUoAk4D3gbwDGceH4KWC19XrSalO1yKiYcKbdEs321HRGvvMru9NO2x1JVaEv1+xh/Iw1tGoUyJcT++gQEzYoT2+iZUBxV/wuL2J+A9xdzLqmA9NdzaJqhsvbhzLzzl6Mm7GG695azntjY+jevL7dsVQlMsYwZ0cO3yZvom/rBrxzczRBOvqoLXTUUuVWoiNC+PquPgT6eTFm2kp+3HzA7kiqkuTkFfD3LzfybXIuo6LD+Oj2nloIbKTFQLmdlo0C+fquPkQ1C+Zv/1vHm7FJerdyDXNmnKGv1+1jRBtvXhjZGW9P/XVkJ937yi01CPRl5p29GNqlKS/O38Z9szaQlZtvdyxVARIPnmTof5ezce9x3hjTjaGtfPQudDegxUC5LT9vT167oSuPDGzHd5v2c/27v3LgRKbdsVQ5/LTlAH95awV5BQV8OfEihnZpanckZdFioNyaiHBX/1a8f2sMKYdPMeSNZaxIOmJ3LFVGefkFPPtjAhM/XUeb0CDmTrqYzmH17I6lnGgxUNXC5e1D+XZSX0ICfLj5g1W8FafXEaqLIxnZ3Dr9N979JYWbezfn87/2JrSun92xVCF6q6eqNlo1CuSbu/sy+evNvPDTNro08qRLzxxCAnzsjqaKsSL5CPfP2sCJzFxeGtWFkdFhdkdSxdAjA1WtBPh68cborjw5rCNbj+Qz6PUlrExJszuWKiS/wPDKgu3c9P4qgvy8+ObuvloI3JwWA1XtiAi3XhTJYxf5EeDjxY3vreSl+dvIySuwO5oCdqed5oZ3f+WNRTsY0S2MuZMupn0THVrC3WkxUNVWRF1PvrvnYv7SPYz/xiZx3VvL2ZGabnesWssYwxer9zDo9SVsO5jOqzd04eXruxCgAw9WC1oMVLUW4OvFi6O68O4t0Rw4kcU1/1nGu78kk5evRwlV6eCJLMZ/vIZ/fLWJzmH1+OmBS7ium54Wqk60ZKsa4eqOjenevD7/nLOZZ39MZN7mAzz/l856eqKSGWOYtXoPz8xLILeggP+7pj139G2hT66rhvTIQNUYjYJ8mXZLNP+9sRv7jmVy7X+W8eL8RDJz9M7lypB0KJ0b31vFo19vpmOzuvx03yXc2a+lFoJqSo8MVI0iIgzp3JS+rRry1Lx43oxN5pv1+3l8aEeu7BBqd7wa4XROHv9ZnMT7S1Oo4+3JM9d1YnSPcC0C1ZwWA1Uj1Q/w4ZXru3JDTDiPfbuF8R+vof+FjfjX4Pa0CQ2yO161VFBgmLtxPy/8lMj+E1mMjA5j8qB2NAz0tTuaqgBaDFSN1qtlA+bd248ZK3by+qIdDHx9KWN6hnP/FW31l1gZrNl5lKfmJbBxz3GimtXl9THd6BEZYncsVYG0GKgaz9vTgzv7tWRE9zBeX7idT1ftZs66fYy7uAXj+rUkuI6OoV+c+P0nefnnbSxKPERoXV9eGtWFEd2a6SmhGkiLgao1QgJ8eGJYFLdcFMmrC7bzxuIkZvy6i/H9WnBrn0jq6oNVzko4cJL/xiYxb9MB6vp58Y+BF3Jbn0j8ffRXRk2l31lV67S+IJA3b+rOXftO8MqC7bz083be/SWFWy6K4I6LW9Tq00drdx3jrdgkFiUeItDXi0kDWjP+Ej16qg20GKhaK6pZMNNv68GWfSd4Oy6Zt39J5v1lvzOsS1Nu6xtJx6bBdkesErn5Bfyw+QAfLt/Jhj3HqefvzYNXtmXsRZEE+2sRqC20GKhaL6pZMG/e1J3kwxl8uPx3vlq7jy/X7qVnZAije4YzKKoJdXw87Y5Z4fYcPc0Xa/bwxZo9pJ7MpkXDAJ4Y2pGR0WE6hEQtpN9xpSytGgUydXgnHr6qHV+s2cPMVbt48IuNTPl2K0O7NmVY12bERNSv1hdP07Nymb81lW837GOZ9ZCgS9s24tkREfRve0G1/tpU+WgxUKqQYH9vxl/Skjv7tWDV70f5YvUevlq3l5mrdtMk2I9rOjXhyg6hREfUx6saPMT9+Okc4rYd5tP1WWxauJCcvALC6tfhvsvbcH1MOE3r1bE7onIDWgyUKoaI0LtlA3q3bMBTw6NYmJDKdxsPMOPXnby/7Hfq+XvTv20jLm7TiD6tGrj+S/XUKXjpJfjwQ9izB8LD4fbbmdo3n9fWvcX9ve/n/y75v1KvLr/AsHnfCZYnHWHpjsOs3nmM/AJDsK9wY88IhnZtSrfwevoQevUnblMMRGQg8DrgCbxvjHnO5khKnRXg68Wwrs0Y1rUZ6Vm5LN1xhIUJqfyy7TDfbNgPQEQDf6Kb18c/K5eQvcdpGxqEn3cJ1xpOnYIBA2D16j/adu0i9cXHecz6+FjsY4zvPp7QwKKH0ziSkc3W/SdZv/sY63cfZ93uY6Rn5QHQrnEQEy9tyZUdGnM0aT2XDehY3l2haii3KAYi4gm8CVwJ7AVWi8hcY0y8vcmUOleQnzeDOzVhcKcmFBQYth9KZ3lSGitT0liy4whHMnL4NGE5HgKRDQJoGxpEREN/wuv7E1a/DqF1/WgQ6EOIvw9eL730p0LwTTt46CpIdrq5V4wvzV7sTERQFGM73UfjOh3ZffQ0vx85xbaD6RzJyHHMJ9D2giCGdG7CRa0a0qdVgz91k41L1iMBVTy3KAZATyDJGJMCICKzgGGAFgPl1jw8hHaN69KucV3GXdwCYwxf/xRLnbD2JB5MZ0dqOttS01mceIicIp6x4J/bCf9Jn1AnNxuvgnx2BwtZXtAsywsxfnhQB8HRvTM/G6bHAiRSz9+biBB/Lmt3AW1Dg2jfpC6dw4IJ0hvnlIvEGGN3BkRkJDDQGHOn9fkWoJcxZlKh+SYAEwBCQ0OjZ82a5dL2MjIyCAwMLF/oSqC5yqY65SowhhPZhsOZjn/TcwwncwyZ+1LJNkK28cAAWV5w2hs8BLw9CvD2KMDXMx9/r3zqeOfRql5D2oY0IMC77H/lV6f95Q5qaq4BAwasNcbEnDPBGGP7CxiJ4zrBmc+3AP893zLR0dHGVbGxsS4vW5k0V9nUiFwREcbAn16TBmF4/NzXPT/cU3W5qpDmKpvy5gLWmCJ+p7pLv7h9QLjT5zCrTama7fbbz2k6FFD0rIdOHarkMKo2c5disBpoIyItRMQHGA3MtTmTUpXvoYegR48/NTVN/+N9u5C2Z983CWxSValULeQWF5CNMXkiMgmYj6Nr6XRjzFabYylV+QICIDb2T/cZPLsjjL4X9aLDbf+gQ0QM8YfjiT8cz5C2Q+xOq2owtygGAMaYH4Af7M6hVJULCIApUxwvwA/HRbQzOjTqQIdGHWyJpmoPdzlNpJRSykZaDJRSSmkxUEoppcVAKaUUbnIHsitE5DCwy8XFGwJHKjBORdFcZaO5ykZzlU1NzRVhjGlUuLHaFoPyEJE1pqjbsW2mucpGc5WN5iqb2pZLTxMppZTSYqCUUqr2FoNpdgcohuYqG81VNpqrbGpVrlp5zUAppdSf1dYjA6WUUk60GCillKrZxUBERonIVhEpEJGYQtMeFZEkEdkmIlc7tQ+02pJEZHIV5fxcRDZYr50issFqjxSRTKdp71RFHqdcj4vIPqftD3aaVuT+q6JcL4pIoohsEpE5IlLPard1f1kZqvznp5gc4SISKyLx1v+B+6z2Yr+nVZhtp4hstra/xmoLEZEFIrLD+rd+FWe60GmfbBCRkyJyvx37S0Smi8ghEdni1Fbk/hGHN6yft00i0t3lDRf1xJua8gLaAxcCcUCMU3sHYCPgC7QAknEMne1pvW8J+FjzdKjizC8D/7beRwJbbNx/jwMPFdFe5P6rwlxXAV7W++eB591kf9n+8+OUpQnQ3XofBGy3vm9Ffk+rONtOoGGhtheAydb7yWe+pzZ+Hw8CEXbsL+ASoLvzz3Jx+wcYDPwICNAbWOXqdmv0kYExJsEYs62IScOAWcaYbGPM70AS0NN6JRljUowxOcAsa94qISICXA98VlXbdFFx+69KGGN+NsbkWR9X4ngynjuw9efHmTHmgDFmnfU+HUgAmtmRpZSGATOs9zOA4fZF4XIg2Rjj6ggH5WKMWQIcLdRc3P4ZBnxsHFYC9UTEpacg1ehicB7NgD1On/dabcW1V5V+QKoxZodTWwsRWS8iv4hIvyrMcsYk6/BzutOhu937ydkdOP4yOsPO/eVO++UsEYkEugGrrKaivqdVyQA/i8haEZlgtYUaYw5Y7w8CoTbkOmM0f/6DzO79BcXvnwr7mav2xUBEForIliJetvxFVpxS5hzDn38IDwDNjTHdgAeB/4lI3SrM9TbQCuhqZXm5Irddjlxn5vkXkAfMtJoqfX9VNyISCHwF3G+MOYmN31MnFxtjugODgLtF5BLnicZx/sOWPu/ieOzuUOBLq8kd9tefVNb+cZsnnbnKGHOFC4vtA8KdPodZbZynvVxKyikiXsAIINppmWwg23q/VkSSgbbAmorIVJpcTvneA763Pp5v/1VJLhG5DRgCXG7956iS/VWCSt8vZSEi3jgKwUxjzNcAxphUp+nO39MqY4zZZ/17SETm4Di9lioiTYwxB6zTHIeqOpdlELDuzH5yh/1lKW7/VNjPXLU/MnDRXGC0iPiKSAugDfAbsBpoIyItrL8QRlvzVoUrgERjzN4zDSLSSEQ8rfctrZwpVZSHQucerwPO9G4obv9VVa6BwD+AocaY007ttu4v7P35+RPr+tMHQIIx5hWn9uK+p1WVK0BEgs68x9EZYAuO/TTWmm0s8G1V5nLyp6Nzu/eXk+L2z1zgVqtXUW/ghNPppLKp6iv1VXxV/joc59CygVRgvtO0f+Ho+bENGOTUPhhHz4tk4F9VmPUjYGKhtr8AW4ENwDrg2iref58Am4FN1g9dk5L2XxXlSsJxnnSD9XrHHfaXnT8/ReS4GMephE1O+2nw+b6nVZSrJY5eVhut79W/rPYGwCJgB7AQCLFhnwUAaUCwU1uV7y8cxegAkGv9/hpX3P7B0YvoTevnbTNOvSbL+tLhKJRSStXa00RKKaWcaDFQSimlxUAppZQWA6WUUmgxUEophRYDpZRSaDFQSikF/D/LgCMAVx4nwQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# requires_grad=True is required\n",
    "current_solution = torch.tensor([100.], requires_grad=True)\n",
    "\n",
    "# initialize a simple SGD\n",
    "# SGD will minimize только одну переменную — current_solution\n",
    "optimizer = torch.optim.SGD([current_solution], 1e-2)\n",
    "\n",
    "NUM_STEP = 200\n",
    "\n",
    "\n",
    "for i in range(NUM_STEP):\n",
    "    current_loss = func(current_solution)\n",
    "    \n",
    "    # zero out .grad of `current_solution`\n",
    "    optimizer.zero_grad()\n",
    "    \n",
    "    # compute new gradiat by `current_solution`\n",
    "    current_loss.backward()\n",
    "    \n",
    "    # update current_solution\n",
    "    optimizer.step()\n",
    "    \n",
    "    if i % 20 == 0:\n",
    "        display.clear_output()\n",
    "        \n",
    "        x = torch.linspace(-100, 100, 100)\n",
    "        y = func(x)\n",
    "\n",
    "        plt.plot(x, y)\n",
    "        plt.scatter([10], [14], label='Minimum', c='red', linewidths=3)\n",
    "        plt.scatter([current_solution.item()], [current_loss.item()],\n",
    "                    label='Current Solution', marker='*', c='green', linewidths=3)\n",
    "        \n",
    "        plt.legend()\n",
    "        plt.grid()\n",
    "        plt.title(f'Step №{i}')\n",
    "        plt.show()\n",
    "        \n",
    "        time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}