dev.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 246,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torch import optim\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "\n",
    "from torchvision import models\n",
    "import torchvision.transforms as T\n",
    "from torchvision.datasets import MNIST\n",
    "\n",
    "from torchinfo import summary\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 247,
   "metadata": {},
   "outputs": [],
   "source": [
    "Ya_transforms = T.Compose(\n",
    "    [\n",
    "        T.ToTensor(),\n",
    "        T.RandomChoice(\n",
    "            [\n",
    "                T.ColorJitter(),\n",
    "                T.RandomHorizontalFlip(),\n",
    "                T.GaussianBlur(3),\n",
    "                T.RandomSolarize(.6)\n",
    "            ]\n",
    "        )\n",
    "    ]\n",
    ")\n",
    "\n",
    "Yb_transforms = T.Compose(\n",
    "    [\n",
    "        T.ToTensor(),\n",
    "        T.RandomChoice(\n",
    "            [\n",
    "                T.ColorJitter(),\n",
    "                T.RandomHorizontalFlip(),\n",
    "                T.GaussianBlur(3),\n",
    "                T.RandomSolarize(.6)\n",
    "            ]\n",
    "        )\n",
    "    ]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 248,
   "metadata": {},
   "outputs": [],
   "source": [
    "A_dataset = MNIST(\n",
    "    'data', \n",
    "    'True',\n",
    "    transform=Ya_transforms,\n",
    "    download=True\n",
    ")\n",
    "\n",
    "B_dataset = MNIST(\n",
    "    'data', \n",
    "    'True',\n",
    "    transform=Yb_transforms,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 249,
   "metadata": {},
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "A_loader = DataLoader(A_dataset, batch_size=BATCH_SIZE)\n",
    "B_loader = DataLoader(B_dataset, batch_size=BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "A_sample = next(iter(A_loader))[0]\n",
    "B_sample = next(iter(B_loader))[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 225,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.4980, 0.9686, 1.0000, 0.6510,\n",
       "          0.1020, 0.6863, 0.5333, 0.4941, 0.0706, 0.0706, 0.0706, 0.0118,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.2510, 0.7647, 0.9490, 0.9922,\n",
       "          0.6745, 0.8824, 0.9922, 0.9922, 0.9922, 0.9922, 0.9922, 0.6667,\n",
       "          0.6039, 0.3686, 0.1412, 0.1176, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.1529, 0.2196, 0.3216,\n",
       "          0.3216, 0.3647, 0.9843, 0.9922, 0.9922, 0.9922, 0.9922, 0.9922,\n",
       "          0.9922, 0.9922, 0.9922, 0.9333, 0.1922, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.9451, 0.9686, 0.7137, 0.7765, 0.9922, 0.9922,\n",
       "          0.9922, 0.9922, 0.9922, 0.8588, 0.0706, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.6039, 0.1686, 0.0000, 0.0431, 0.8039, 0.9922,\n",
       "          0.9922, 0.4196, 0.6118, 0.3137, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.3529, 0.9922,\n",
       "          0.6039, 0.0039, 0.0549, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0078, 0.7451, 0.9922,\n",
       "          0.5451, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.2745, 0.9922, 0.7451,\n",
       "          0.0431, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0039, 0.4235, 0.6275, 0.8824, 0.9451, 0.1373,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0980, 0.4667, 0.9922, 0.9922, 0.9412, 0.3176, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.1059, 0.5882, 0.9922, 0.9922, 0.7294, 0.1765, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.7333, 0.9922, 0.9882, 0.3647, 0.0627, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.2510,\n",
       "          0.9765, 0.9922, 0.9765, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0078,\n",
       "          0.8118, 0.9922, 0.9922, 0.7176, 0.5098, 0.1804, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.7137, 0.9804, 0.9922, 0.9922, 0.9922, 0.8980, 0.5804, 0.1529,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.3059, 0.7882, 0.9922, 0.9922, 0.9922, 0.9922, 0.8667,\n",
       "          0.4471, 0.0941, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0078, 0.3176, 0.7765, 0.9922, 0.9922, 0.9922,\n",
       "          0.9922, 0.8353, 0.2588, 0.0902, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0353, 0.3137, 0.7647, 0.9922,\n",
       "          0.9922, 0.9922, 0.9922, 0.8588, 0.6706, 0.0706, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0431, 0.5216,\n",
       "          0.9569, 0.9922, 0.9922, 0.9922, 0.9922, 0.8863, 0.6745, 0.2157,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0627, 0.5176, 0.5294, 0.8314, 0.9922, 0.9922, 0.9922, 0.5333,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000],\n",
       "         [0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000,\n",
       "          0.0000, 0.0000, 0.0000, 0.0000]]])"
      ]
     },
     "execution_count": 225,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A_sample[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_samples(A_batch, B_batch):\n",
    "\n",
    "    fig, ax = plt.subplots(2, 7)\n",
    "    # ax = ax.reshape(14)\n",
    "\n",
    "    for i in range(7):\n",
    "        ax[0, i].imshow(A_batch[i][0])\n",
    "        ax[1, i].imshow(B_batch[i][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Encoder(nn.Module):\n",
    "    def __init__(self) -> None:\n",
    "        super().__init__()\n",
    "\n",
    "        self.conv1 = nn.Sequential(\n",
    "            nn.Conv2d(1, 32, 4, 1),\n",
    "            nn.Dropout(),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm2d(32)\n",
    "        )\n",
    "\n",
    "        self.conv2 = nn.Sequential(\n",
    "            nn.Conv2d(32, 16, 6, 2),\n",
    "            nn.Dropout(),\n",
    "            nn.ReLU(),\n",
    "            nn.BatchNorm2d(16)\n",
    "        )\n",
    "\n",
    "        self.linear = nn.Sequential(\n",
    "            nn.LazyLinear(16)\n",
    "        )\n",
    "    \n",
    "    def forward(self, x):\n",
    "        x = self.conv1(x)\n",
    "        x = self.conv2(x)\n",
    "        x = x.view(x.size(0), -1)\n",
    "        out = self.linear(x)\n",
    "      \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 259,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Projector(nn.Module):\n",
    "    def __init__(self) -> None:\n",
    "        super().__init__()\n",
    "\n",
    "        self.proj = nn.Sequential(\n",
    "            nn.Linear(in_features=16, out_features=32),\n",
    "            nn.BatchNorm1d(32),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(32, 16)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.proj(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 260,
   "metadata": {},
   "outputs": [],
   "source": [
    "class F_theta(nn.Module):\n",
    "    def __init__(self) -> None:\n",
    "        super().__init__()\n",
    "\n",
    "        self.encoder = Encoder()\n",
    "        self.proj = Projector()\n",
    "\n",
    "    def forward(self, x):\n",
    "        encoded = self.encoder(x)\n",
    "        x = self.proj(encoded)\n",
    "        return x, encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [],
   "source": [
    "f_theta = F_theta()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 262,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "=================================================================\n",
       "Layer (type:depth-idx)                   Param #\n",
       "=================================================================\n",
       "F_theta                                  --\n",
       "├─Encoder: 1-1                           --\n",
       "│    └─Sequential: 2-1                   --\n",
       "│    │    └─Conv2d: 3-1                  544\n",
       "│    │    └─Dropout: 3-2                 --\n",
       "│    │    └─ReLU: 3-3                    --\n",
       "│    │    └─BatchNorm2d: 3-4             64\n",
       "│    └─Sequential: 2-2                   --\n",
       "│    │    └─Conv2d: 3-5                  18,448\n",
       "│    │    └─Dropout: 3-6                 --\n",
       "│    │    └─ReLU: 3-7                    --\n",
       "│    │    └─BatchNorm2d: 3-8             32\n",
       "│    └─Sequential: 2-3                   --\n",
       "│    │    └─LazyLinear: 3-9              --\n",
       "├─Projector: 1-2                         --\n",
       "│    └─Sequential: 2-4                   --\n",
       "│    │    └─Linear: 3-10                 544\n",
       "│    │    └─BatchNorm1d: 3-11            64\n",
       "│    │    └─ReLU: 3-12                   --\n",
       "│    │    └─Linear: 3-13                 528\n",
       "=================================================================\n",
       "Total params: 20,224\n",
       "Trainable params: 20,224\n",
       "Non-trainable params: 0\n",
       "================================================================="
      ]
     },
     "execution_count": 262,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "summary(f_theta)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 263,
   "metadata": {},
   "outputs": [],
   "source": [
    "def normalize(x):\n",
    "    return ((x - x.mean(0)) / x.std(0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 264,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimizer = optim.Adam(f_theta.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 265,
   "metadata": {},
   "outputs": [],
   "source": [
    "epoch_loss = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 274,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([ 46.1071,  16.4787,   4.9440,  15.8657,   5.5808, -18.1278,   1.0149,\n",
      "         41.6873, -31.0885, -20.4176, -22.5892,   6.4539, -15.9461, -11.5106,\n",
      "          7.8488,   8.7911], grad_fn=<SelectBackward0>)\n",
      "Epoch 1 finished. Loss: 1.602\n",
      "tensor([ 52.3106,  29.1177,  14.8703,  13.9134,  17.5207,  -7.6311,   1.0507,\n",
      "         36.9610, -35.8788, -24.6493, -12.7419,  10.3941,  -9.7290,  -8.7875,\n",
      "         25.2963,   7.0533], grad_fn=<SelectBackward0>)\n",
      "Epoch 2 finished. Loss: 1.592\n",
      "tensor([ 54.8341,  21.6798,   8.5622,  18.2535,  19.7113,  -8.1439,  10.1015,\n",
      "         34.6644, -44.1203, -23.0919,  -4.8697,  11.9878,   1.6465,  -9.8415,\n",
      "         17.7444,  22.8734], grad_fn=<SelectBackward0>)\n",
      "Epoch 3 finished. Loss: 1.597\n",
      "tensor([ 61.3607,  26.4037,   3.5679,  16.0325,  20.6803, -15.5255,   9.7262,\n",
      "         32.6398, -46.1915, -24.3841, -10.7016,   0.8762, -11.2492,  -2.3860,\n",
      "         20.9871,   0.2347], grad_fn=<SelectBackward0>)\n",
      "Epoch 4 finished. Loss: 1.575\n",
      "tensor([ 62.0398,  23.9060,  15.7224,  11.0735,   9.9599, -23.3155,   2.4798,\n",
      "         54.3497, -48.0217, -25.3169, -11.2040,  15.2046,  -7.9935,  -6.4035,\n",
      "         20.5900,  14.1853], grad_fn=<SelectBackward0>)\n",
      "Epoch 5 finished. Loss: 1.591\n",
      "tensor([ 67.3465,  21.8159,   7.6317,  20.4019,  13.9240, -30.9211,   9.5061,\n",
      "         38.9306, -55.7768, -20.1217, -22.9606,  12.4128, -14.4625,   3.4374,\n",
      "         11.2147,   3.5743], grad_fn=<SelectBackward0>)\n",
      "Epoch 6 finished. Loss: 1.582\n",
      "tensor([ 57.4582,   9.9286,  17.2710,  21.0407,   6.0428, -15.5752,  -9.1197,\n",
      "         57.2369, -41.6173, -18.0261, -33.7178,  17.5762, -10.3037,  -5.0772,\n",
      "         19.1683,  -3.8925], grad_fn=<SelectBackward0>)\n",
      "Epoch 7 finished. Loss: 1.574\n",
      "tensor([ 66.7059,  16.5936,   8.7029,  15.4857,  23.4943, -23.3114,   1.8104,\n",
      "         40.9042, -58.7495, -21.1644, -17.8420,  20.4951, -11.8185,  -5.3560,\n",
      "         22.5148,   8.4680], grad_fn=<SelectBackward0>)\n",
      "Epoch 8 finished. Loss: 1.568\n",
      "tensor([ 62.8414,  13.0860,  -3.6292,  15.5718,  23.2404, -11.5746,   3.1481,\n",
      "         24.2432, -53.4087, -24.5460, -11.7519,  15.7866,  -8.6824,   5.6735,\n",
      "         19.1379,   4.7134], grad_fn=<SelectBackward0>)\n",
      "Epoch 9 finished. Loss: 1.554\n",
      "tensor([ 57.0205,  21.1768,   1.7483,  22.1280,  18.7801,  -7.8583,  28.6297,\n",
      "         38.4006, -45.1846, -23.5016, -32.0024,  17.0780,  -8.8340,   1.0461,\n",
      "         15.5539,   2.0894], grad_fn=<SelectBackward0>)\n",
      "Epoch 10 finished. Loss: 1.556\n",
      "tensor([ 52.9762,  18.8820,   7.2766,  11.1710,   2.0224, -19.7320,  14.8869,\n",
      "         39.9581, -37.4525, -22.4051, -13.8772,  12.3382, -18.5381,  -3.4529,\n",
      "         19.4336,  25.3040], grad_fn=<SelectBackward0>)\n",
      "Epoch 11 finished. Loss: 1.558\n",
      "tensor([ 74.3283,  19.1174,  21.9975,  12.8365,  20.9312,  -8.3406,  10.9683,\n",
      "         61.1073, -39.6049, -24.5788, -19.8557,  24.2655,  -4.1174,  -7.1447,\n",
      "         23.3105,   3.4811], grad_fn=<SelectBackward0>)\n",
      "Epoch 12 finished. Loss: 1.544\n",
      "tensor([ 70.4721,  34.4595,   8.7372,  24.3619,  24.4817, -20.6283,  25.3002,\n",
      "         38.8386, -44.2175, -27.4588, -14.2466,  15.6915,  -3.2858, -17.2093,\n",
      "         28.8843,   2.3202], grad_fn=<SelectBackward0>)\n",
      "Epoch 13 finished. Loss: 1.541\n",
      "tensor([ 76.2925,  16.6631,  23.6010,  16.4576,  11.0985, -15.9924,   2.8452,\n",
      "         38.3204, -36.5643, -18.0433, -26.7469,  30.5666,  -6.9850, -13.8400,\n",
      "         24.0261,   9.4730], grad_fn=<SelectBackward0>)\n",
      "Epoch 14 finished. Loss: 1.547\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m/Users/paveldumin/Documents/barlow_twins/dev.ipynb Ячейка 16\u001b[0m in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/paveldumin/Documents/barlow_twins/dev.ipynb#X22sZmlsZQ%3D%3D?line=14'>15</a>\u001b[0m loss \u001b[39m=\u001b[39m c_diff\u001b[39m.\u001b[39msum()\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/paveldumin/Documents/barlow_twins/dev.ipynb#X22sZmlsZQ%3D%3D?line=15'>16</a>\u001b[0m optimizer\u001b[39m.\u001b[39mzero_grad()\n\u001b[0;32m---> <a href='vscode-notebook-cell:/Users/paveldumin/Documents/barlow_twins/dev.ipynb#X22sZmlsZQ%3D%3D?line=16'>17</a>\u001b[0m loss\u001b[39m.\u001b[39;49mbackward()\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/paveldumin/Documents/barlow_twins/dev.ipynb#X22sZmlsZQ%3D%3D?line=17'>18</a>\u001b[0m optimizer\u001b[39m.\u001b[39mstep()\n\u001b[1;32m     <a href='vscode-notebook-cell:/Users/paveldumin/Documents/barlow_twins/dev.ipynb#X22sZmlsZQ%3D%3D?line=18'>19</a>\u001b[0m batch_loss\u001b[39m.\u001b[39mappend(loss\u001b[39m.\u001b[39mitem())\n",
      "File \u001b[0;32m~/miniforge3/envs/pt/lib/python3.8/site-packages/torch/_tensor.py:396\u001b[0m, in \u001b[0;36mTensor.backward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    387\u001b[0m \u001b[39mif\u001b[39;00m has_torch_function_unary(\u001b[39mself\u001b[39m):\n\u001b[1;32m    388\u001b[0m     \u001b[39mreturn\u001b[39;00m handle_torch_function(\n\u001b[1;32m    389\u001b[0m         Tensor\u001b[39m.\u001b[39mbackward,\n\u001b[1;32m    390\u001b[0m         (\u001b[39mself\u001b[39m,),\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    394\u001b[0m         create_graph\u001b[39m=\u001b[39mcreate_graph,\n\u001b[1;32m    395\u001b[0m         inputs\u001b[39m=\u001b[39minputs)\n\u001b[0;32m--> 396\u001b[0m torch\u001b[39m.\u001b[39;49mautograd\u001b[39m.\u001b[39;49mbackward(\u001b[39mself\u001b[39;49m, gradient, retain_graph, create_graph, inputs\u001b[39m=\u001b[39;49minputs)\n",
      "File \u001b[0;32m~/miniforge3/envs/pt/lib/python3.8/site-packages/torch/autograd/__init__.py:173\u001b[0m, in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    168\u001b[0m     retain_graph \u001b[39m=\u001b[39m create_graph\n\u001b[1;32m    170\u001b[0m \u001b[39m# The reason we repeat same the comment below is that\u001b[39;00m\n\u001b[1;32m    171\u001b[0m \u001b[39m# some Python versions print out the first line of a multi-line function\u001b[39;00m\n\u001b[1;32m    172\u001b[0m \u001b[39m# calls in the traceback and some print out the last line\u001b[39;00m\n\u001b[0;32m--> 173\u001b[0m Variable\u001b[39m.\u001b[39;49m_execution_engine\u001b[39m.\u001b[39;49mrun_backward(  \u001b[39m# Calls into the C++ engine to run the backward pass\u001b[39;49;00m\n\u001b[1;32m    174\u001b[0m     tensors, grad_tensors_, retain_graph, create_graph, inputs,\n\u001b[1;32m    175\u001b[0m     allow_unreachable\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m, accumulate_grad\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "f_theta.train()\n",
    "for epoch in range(50):\n",
    "    batch_loss = []\n",
    "    for x_a, x_b in zip(A_loader, B_loader):\n",
    "        # print(x_a[0].shape, x_b[0].shape)\n",
    "        out_a = normalize(f_theta(x_a[0])[0])\n",
    "        out_b = normalize(f_theta(x_b[0])[0])\n",
    "        emb = f_theta(x_b[0])[0]\n",
    "        # print(f'f_theta out: {out_a.shape}')\n",
    "        cross_corr = torch.matmul(out_a.T, out_b) / BATCH_SIZE\n",
    "        c_diff = (cross_corr - torch.eye(cross_corr.size(0))).pow(2)\n",
    "        # print(f'Cross corr: {cross_corr.shape}')\n",
    "        # print(c_diff.shape)\n",
    "        c_diff.flatten()[:-1].view(c_diff.size(0) - 1, c_diff.size(0)+1)[:, 1:].mul_(.5)\n",
    "        loss = c_diff.sum()\n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        batch_loss.append(loss.item())\n",
    "    epoch_loss.append(np.mean(batch_loss))\n",
    "    print(f_theta(x_b[0])[1][0])\n",
    "    if epoch+1 % 5 == 0:\n",
    "        torch.save(f_theta.state_dict(), f'weights/n_epochs_{epoch}_new.pt')\n",
    "    print(f'Epoch {epoch+1} finished. Loss: {epoch_loss[-1]:.3f}')\n",
    "        \n",
    "        \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 275,
   "metadata": {},
   "outputs": [],
   "source": [
    "torch.save(f_theta.state_dict(), f'weights/n_epochs_{epoch}.pt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/paveldumin/miniforge3/envs/pt/lib/python3.8/site-packages/torch/nn/modules/lazy.py:178: UserWarning: Lazy modules are a new feature under heavy development so changes to the API or functionality can happen at any moment.\n",
      "  warnings.warn('Lazy modules are a new feature under heavy development '\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 216,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f_theta = F_theta()\n",
    "\n",
    "f_theta.load_state_dict(torch.load('weights/n_epochs_23.pt'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 282,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_loader = DataLoader(MNIST('data', train=False, transform=T.ToTensor()), batch_size=32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 285,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_batch_img, test_batch_lbl = (next(iter(valid_loader)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 286,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[-6.9082e+00,  1.4926e+01,  2.0808e+01, -4.7688e+01,  4.3042e+01,\n",
       "          4.5962e+01, -4.9906e+01,  2.5168e+00, -3.1275e+01,  1.5816e+01,\n",
       "         -3.1620e+01,  3.5737e+01, -4.5994e+00,  4.2364e+01,  1.5794e+01,\n",
       "          1.1120e+01],\n",
       "        [ 8.6672e+01, -1.8814e+01,  3.2317e+01, -3.2457e+01, -9.0589e+00,\n",
       "         -8.8214e+00, -1.9338e+01,  5.0588e+01,  3.0347e+01,  7.8374e+00,\n",
       "         -1.2130e+01,  1.3253e+01, -2.5789e+01, -1.8396e+01,  3.6527e+01,\n",
       "          7.4531e+00],\n",
       "        [ 9.0540e+01, -5.9130e+01, -1.4260e+01, -1.2459e+01,  5.2794e+01,\n",
       "         -4.6375e+01, -2.1636e+01, -2.6174e+01,  2.5309e+00,  4.8639e+01,\n",
       "         -2.5860e+01,  1.0535e+01, -4.0442e+01,  1.7193e+01,  2.1324e+01,\n",
       "         -2.7289e+01],\n",
       "        [ 4.1543e+01,  8.1364e+01,  2.1217e+01, -5.3465e+01, -1.4093e+01,\n",
       "         -5.5029e+01,  2.1401e+01,  3.8139e+01, -2.5498e+01,  2.6841e+01,\n",
       "         -5.4498e+01,  1.4192e+01, -3.8766e+01,  2.9353e+01,  5.5838e+01,\n",
       "         -1.0746e+01],\n",
       "        [ 2.3217e+01,  2.7210e+01,  1.0400e+02, -2.2871e+01,  1.1966e+01,\n",
       "         -5.7186e+01,  1.0429e+01,  2.7029e-01, -2.0448e+01,  6.1623e+00,\n",
       "         -4.3571e+01, -1.4634e+01,  3.9368e+00,  3.6733e+00,  3.7684e+01,\n",
       "          6.8133e+00],\n",
       "        [ 7.2807e+01, -5.2573e+01, -1.1967e+00, -9.8960e+00,  4.1997e+01,\n",
       "         -4.1540e+01,  3.7526e+00, -1.7750e+01, -2.7236e+01,  5.9745e+01,\n",
       "         -2.8106e+01,  1.1733e+01, -2.2754e+01,  1.8747e+01,  1.7623e+01,\n",
       "         -5.3018e+01],\n",
       "        [ 1.3483e+01, -2.1592e+00,  1.9066e+01, -2.3327e+01,  1.9564e+01,\n",
       "          1.1003e+01, -1.8513e+01,  1.1540e+00, -2.5884e+01,  3.3724e+00,\n",
       "         -2.2222e+01, -5.7038e+01, -3.2401e+01, -4.9367e+00,  3.4315e+01,\n",
       "         -3.8737e+01],\n",
       "        [ 3.8464e+01,  1.6105e+01,  6.5650e+01,  9.6517e+00,  1.4062e+01,\n",
       "         -5.8420e+01, -1.2202e+01,  2.0002e+00, -1.8810e+01, -1.3314e+01,\n",
       "         -6.6115e+01,  6.5031e+00,  9.9358e+00, -5.4043e+00,  6.4768e+01,\n",
       "         -2.0798e+01],\n",
       "        [ 5.8704e+01,  2.6624e+01,  3.3605e+01,  9.6897e+00,  1.2583e+01,\n",
       "         -1.3891e+01, -5.0482e+00,  2.1261e+01, -3.3936e+01, -3.8699e+00,\n",
       "          2.2634e+00,  1.0288e+01,  3.2210e+01,  1.4741e+01,  6.1009e+01,\n",
       "          1.5985e+01],\n",
       "        [ 5.2876e+00, -4.1087e+00,  2.2719e+00, -2.4596e+01,  1.2288e+01,\n",
       "          4.5235e+00, -6.8424e-01,  2.4233e+01, -5.3403e+01, -1.3263e+01,\n",
       "         -5.0891e+01, -8.3487e+00,  3.8490e+00, -3.0800e+00,  4.5542e+01,\n",
       "         -5.9257e+01],\n",
       "        [ 5.0323e+01,  8.0899e+01,  2.1284e+01, -6.2694e+01, -3.8914e+01,\n",
       "         -3.0416e+01,  3.2471e+01,  1.4032e+01, -6.4655e+00,  6.3751e+01,\n",
       "         -4.6284e+01,  2.0044e+01, -3.4431e+01, -7.5920e+00,  4.6827e+01,\n",
       "         -3.2308e+00],\n",
       "        [ 2.8665e+01,  4.5452e+01,  1.1926e+01, -3.5427e+01, -1.0180e+00,\n",
       "         -5.2164e+01,  1.1552e+01,  7.1101e+01, -1.7407e+01,  3.4921e+01,\n",
       "         -4.6439e+00, -7.7096e+00, -5.3749e+00,  2.5458e+01,  3.6173e+01,\n",
       "         -3.5789e+01],\n",
       "        [ 3.4190e+01, -3.0521e-01,  3.3843e+01, -1.7589e+01,  1.3871e+01,\n",
       "         -1.2059e+01, -1.1831e+01,  1.4372e+01, -4.8941e+01, -8.7362e+00,\n",
       "         -7.2916e+01, -1.0618e+01,  1.9112e+01,  1.1399e+01,  4.7234e+01,\n",
       "         -3.2810e+01],\n",
       "        [ 1.8920e+01,  6.3815e+01,  2.5797e+01, -4.4641e+01, -3.9720e+01,\n",
       "         -7.2751e+01, -2.0103e+00,  9.7547e-01, -4.4311e+01,  4.3629e+01,\n",
       "         -2.9807e+01,  3.2211e+01, -2.5252e+01,  2.5861e+01,  8.5755e+01,\n",
       "         -2.0849e+01],\n",
       "        [ 5.0517e+01, -4.3661e+01, -9.8503e+00, -2.8858e+01,  6.0195e+01,\n",
       "         -4.0207e+01, -6.3046e-02,  1.6837e+01,  9.5648e+00,  3.7547e+01,\n",
       "         -2.8870e+01, -6.5664e+00, -3.3052e+01, -1.7603e+01,  9.6855e+01,\n",
       "          1.0921e+01],\n",
       "        [ 4.0195e+01,  3.2154e+01, -2.0683e+01, -2.1905e+01,  6.0392e-01,\n",
       "          5.7260e+00, -4.4764e+01,  2.9830e+01,  2.5070e+01, -6.9312e+00,\n",
       "         -7.6758e+00, -2.7891e+01, -5.0493e+01, -8.8024e+01, -1.3804e+01,\n",
       "         -7.0698e+00],\n",
       "        [ 1.4442e+01,  9.5534e+00,  8.8365e+01, -2.8591e+01,  1.9613e+01,\n",
       "         -4.2253e+01, -8.7725e+00,  6.1012e+00, -3.1384e+01,  7.2682e+00,\n",
       "         -6.0789e+01,  8.9941e+00,  2.9411e+01, -6.8648e+00,  5.9018e+01,\n",
       "         -2.1730e+01],\n",
       "        [ 6.2168e+00,  4.2237e+01,  5.2119e+00, -6.4870e+01,  3.3855e+01,\n",
       "         -2.2362e+00, -2.7728e+01, -9.3700e+00, -3.6724e+01,  1.3709e+00,\n",
       "         -2.9117e+01,  3.7380e+01, -6.0396e+00,  3.6775e+01,  2.2283e+01,\n",
       "          9.2564e+00],\n",
       "        [ 3.1923e+01,  3.1542e+01, -1.0803e+01,  6.4450e+00, -5.3678e+00,\n",
       "          6.9195e+00,  2.2260e+01,  3.3553e+01, -3.3018e+00, -1.0402e+01,\n",
       "         -1.6648e+01, -2.2304e+00, -3.6345e+01, -4.5108e+01,  6.2914e+01,\n",
       "         -2.0172e+01],\n",
       "        [ 4.7103e+01, -8.9737e-01,  4.9735e+01, -8.1763e+00, -2.3164e+01,\n",
       "         -4.5700e+01, -2.3474e+01, -3.6232e+00, -2.6867e+01, -1.2394e+01,\n",
       "         -4.6373e+01, -4.4191e+01, -7.8901e+00, -1.1927e+01,  1.4563e+01,\n",
       "         -1.4379e+00],\n",
       "        [ 1.1608e+01,  4.8719e+00,  3.4234e+00, -1.7181e+01,  3.4512e+01,\n",
       "          7.2292e+00, -1.7833e+01,  1.0669e+01, -5.0223e+01, -9.5969e-01,\n",
       "         -6.7894e+01, -3.8544e+00, -8.5989e+00,  1.4802e+01,  3.8762e+01,\n",
       "         -3.0815e+01],\n",
       "        [ 4.2500e+01, -7.0039e+00,  4.1836e+01, -1.5595e+01, -3.8250e-01,\n",
       "         -4.4640e+01, -2.1228e+01,  7.7937e+01, -1.3643e+01, -4.0791e+00,\n",
       "         -3.5719e+01,  1.1148e+01, -4.1296e+01,  1.2722e+00,  5.2403e+01,\n",
       "         -9.3579e-01],\n",
       "        [ 2.7556e+01,  2.0823e+01,  3.0007e+01, -3.7561e+01, -2.7059e+00,\n",
       "         -6.9059e+01,  9.2927e+00,  4.8858e+01, -3.3651e+01,  1.2087e+01,\n",
       "         -3.1161e+01,  1.0710e+00, -1.5417e+01,  2.6799e+01,  2.8656e+01,\n",
       "         -2.9983e+01],\n",
       "        [ 6.7818e+01,  4.4887e+00,  2.8332e+01,  1.9146e+01, -2.5931e+00,\n",
       "          1.0167e+01, -2.8075e+01,  2.6450e+01, -2.9035e+01,  9.2721e-01,\n",
       "         -2.7972e+01,  9.4787e+00, -5.2879e+00, -6.3551e+00,  6.8447e+01,\n",
       "          1.4067e+01],\n",
       "        [ 4.4166e+01,  4.6608e+00,  6.3163e+01, -4.8023e-01, -1.3442e+00,\n",
       "         -5.5863e+01, -1.2019e+01, -1.5950e+00, -2.1198e+01, -3.0023e+00,\n",
       "         -6.8724e+01, -2.8731e+01,  1.2504e+01, -1.0514e+01,  3.6324e+01,\n",
       "          6.1424e+00],\n",
       "        [ 4.3868e+01,  5.4571e+01,  1.6907e+01, -1.0515e+01, -5.5607e+01,\n",
       "         -4.8292e+00,  3.7557e+01,  4.0438e+01,  1.7031e+00,  5.7309e+01,\n",
       "         -7.1751e+00, -2.2513e+00,  1.6037e+01,  1.0495e+01,  6.4445e+01,\n",
       "         -3.1635e+01],\n",
       "        [-1.5894e+01,  4.7232e+01,  1.3574e+01, -5.1721e+01,  4.3889e+01,\n",
       "          2.7942e+00, -3.2763e+01, -1.3442e+01, -3.9054e+00, -3.9168e+01,\n",
       "         -2.9333e+01,  2.6035e+01,  1.7217e+00,  4.3431e+01,  2.1446e+01,\n",
       "         -7.9247e+00],\n",
       "        [ 3.2674e+01,  4.2398e-01,  6.9227e+01, -4.1137e+00, -1.9286e+00,\n",
       "         -3.9923e+01, -1.8811e+01, -2.9481e+00, -1.9594e+01,  2.8935e-01,\n",
       "         -4.9432e+01, -3.0096e+01, -2.7221e+00, -1.8076e+01,  5.0387e+01,\n",
       "         -4.1521e+00],\n",
       "        [ 4.9133e+01,  7.3430e+01,  2.9054e+01, -5.1918e+01, -2.6115e+01,\n",
       "         -4.3616e+01, -6.5966e+00,  1.0060e+01, -3.9026e+01,  3.4089e+01,\n",
       "         -3.8827e+01,  2.2200e+01, -7.5509e+01,  1.2646e+01,  5.4322e+01,\n",
       "         -1.1616e+01],\n",
       "        [ 5.8024e+01, -5.6960e+01, -2.2532e+01, -2.8093e+01,  6.7612e+01,\n",
       "         -4.7712e+01,  1.5547e+00, -1.8522e+00,  1.3565e+01,  3.7278e+01,\n",
       "         -4.3887e+01,  2.6843e+00, -2.5789e+01, -1.8685e+01,  8.2705e+01,\n",
       "         -1.2880e+00],\n",
       "        [ 3.7126e+01,  2.0344e+01,  1.4979e+01,  6.6252e+00, -3.2926e-01,\n",
       "         -4.8538e+00, -6.9684e+01,  2.9811e+01, -3.6899e+01,  2.9058e+01,\n",
       "         -4.0723e+01,  8.2762e+00, -1.1097e+01, -1.2341e+01,  5.0384e+01,\n",
       "          4.3316e+00],\n",
       "        [ 4.6289e+01, -3.3058e+01, -6.9035e+00, -3.5140e+01,  6.8168e+01,\n",
       "         -3.7510e+01, -3.1837e+00, -1.2908e-01,  1.7490e+01,  3.6713e+01,\n",
       "         -5.3106e+01,  1.4489e+01, -2.4472e+01, -6.2248e+00,  7.8681e+01,\n",
       "          4.6704e+00]], grad_fn=<AddmmBackward0>)"
      ]
     },
     "execution_count": 286,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "f_theta(test_batch_img)[1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.13 ('pt')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13 | packaged by conda-forge | (default, Mar 25 2022, 06:05:16) \n[Clang 12.0.1 ]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a7df815f77bf8edb0a6a656480e7ee3c360a4b494ba862a12bd6032824eda046"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}