ExploreUnet.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import fastai\n",
    "from fastai.vision import *\n",
    "from fastai.callbacks import *\n",
    "\n",
    "from torchvision.models import vgg16_bn\n",
    "\n",
    "from pathlib import Path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data(bs,size):\n",
    "    data = (src.label_from_func(lambda x: path_clean/x.name)\n",
    "           .transform(get_transforms(max_zoom=2.), size=size, tfm_y=True)\n",
    "           .databunch(bs=bs).normalize(imagenet_stats, do_y=True))\n",
    "\n",
    "    data.c = 3\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = Path('/home/dl/data/vrep/v2/')\n",
    "path_noisy = path/'cam_1_tex' #h\n",
    "path_clean = path/'cam_1_can' #l"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs,size=64,128\n",
    "arch = models.resnet34\n",
    "src = ImageImageList.from_folder(path_noisy).random_split_by_pct(0.1, seed=42)\n",
    "data = get_data(bs,size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class FeatureLoss(nn.Module):\n",
    "    def __init__(self, m_feat, layer_ids, layer_wgts):\n",
    "        super().__init__()\n",
    "        self.m_feat = m_feat\n",
    "        self.loss_features = [self.m_feat[i] for i in layer_ids]\n",
    "        self.hooks = hook_outputs(self.loss_features, detach=False)\n",
    "        self.wgts = layer_wgts\n",
    "        self.metric_names = ['pixel',] + [f'feat_{i}' for i in range(len(layer_ids))\n",
    "              ] + [f'gram_{i}' for i in range(len(layer_ids))]\n",
    "\n",
    "    def make_features(self, x, clone=False):\n",
    "        self.m_feat(x)\n",
    "        return [(o.clone() if clone else o) for o in self.hooks.stored]\n",
    "    \n",
    "    def forward(self, input, target):\n",
    "        out_feat = self.make_features(target, clone=True)\n",
    "        in_feat = self.make_features(input)\n",
    "        self.feat_losses = [base_loss(input,target)]\n",
    "        self.feat_losses += [base_loss(f_in, f_out)*w\n",
    "                             for f_in, f_out, w in zip(in_feat, out_feat, self.wgts)]\n",
    "        self.feat_losses += [base_loss(gram_matrix(f_in), gram_matrix(f_out))*w**2 * 5e3\n",
    "                             for f_in, f_out, w in zip(in_feat, out_feat, self.wgts)]\n",
    "        self.metrics = dict(zip(self.metric_names, self.feat_losses))\n",
    "        return sum(self.feat_losses)\n",
    "    \n",
    "    def __del__(self): self.hooks.remove()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'blocks' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-13-623120cd0f5a>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mvgg_m\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvgg16_bn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfeatures\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcuda\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mrequires_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvgg_m\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mfeat_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFeatureLoss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvgg_m\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mblocks\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m15\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 'blocks' is not defined"
     ]
    }
   ],
   "source": [
    "vgg_m = vgg16_bn(True).features.cuda().eval()\n",
    "requires_grad(vgg_m, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "([5, 12, 22, 32, 42],\n",
       " [ReLU(inplace), ReLU(inplace), ReLU(inplace), ReLU(inplace), ReLU(inplace)])"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "blocks = [i-1 for i,o in enumerate(children(vgg_m)) if isinstance(o,nn.MaxPool2d)]\n",
    "blocks, [vgg_m[i] for i in blocks]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "feat_loss = FeatureLoss(vgg_m, blocks[2:5], [5,15,2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "wd = 1e-3\n",
    "learn = unet_learner(data, arch, wd=wd, loss_func=feat_loss, callback_fns=LossMetrics,\n",
    "                     blur=True, norm_type=NormType.Weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchsummary import summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [],
   "source": [
    "encoder = nn.Sequential(*learn.model[:4])\n",
    "# for layer in learn.model[:5]:\n",
    "#     print(layer)\n",
    "#     print('===')\n",
    "#     break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Sequential(\n",
       "    (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
       "    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "    (2): ReLU(inplace)\n",
       "    (3): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
       "    (4): Sequential(\n",
       "      (0): BasicBlock(\n",
       "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (1): BasicBlock(\n",
       "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): BasicBlock(\n",
       "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (5): Sequential(\n",
       "      (0): BasicBlock(\n",
       "        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (downsample): Sequential(\n",
       "          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (1): BasicBlock(\n",
       "        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): BasicBlock(\n",
       "        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (3): BasicBlock(\n",
       "        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (6): Sequential(\n",
       "      (0): BasicBlock(\n",
       "        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (downsample): Sequential(\n",
       "          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (1): BasicBlock(\n",
       "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): BasicBlock(\n",
       "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (3): BasicBlock(\n",
       "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (4): BasicBlock(\n",
       "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (5): BasicBlock(\n",
       "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "    (7): Sequential(\n",
       "      (0): BasicBlock(\n",
       "        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (downsample): Sequential(\n",
       "          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
       "          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        )\n",
       "      )\n",
       "      (1): BasicBlock(\n",
       "        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "      (2): BasicBlock(\n",
       "        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "        (relu): ReLU(inplace)\n",
       "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
       "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "      )\n",
       "    )\n",
       "  )\n",
       "  (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
       "  (2): ReLU()\n",
       "  (3): Sequential(\n",
       "    (0): Sequential(\n",
       "      (0): Conv2d(512, 1024, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (1): ReLU(inplace)\n",
       "    )\n",
       "    (1): Sequential(\n",
       "      (0): Conv2d(1024, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
       "      (1): ReLU(inplace)\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 60,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1           [-1, 64, 64, 64]           9,408\n",
      "       BatchNorm2d-2           [-1, 64, 64, 64]             128\n",
      "              ReLU-3           [-1, 64, 64, 64]               0\n",
      "         MaxPool2d-4           [-1, 64, 32, 32]               0\n",
      "            Conv2d-5           [-1, 64, 32, 32]          36,864\n",
      "       BatchNorm2d-6           [-1, 64, 32, 32]             128\n",
      "              ReLU-7           [-1, 64, 32, 32]               0\n",
      "            Conv2d-8           [-1, 64, 32, 32]          36,864\n",
      "       BatchNorm2d-9           [-1, 64, 32, 32]             128\n",
      "             ReLU-10           [-1, 64, 32, 32]               0\n",
      "       BasicBlock-11           [-1, 64, 32, 32]               0\n",
      "           Conv2d-12           [-1, 64, 32, 32]          36,864\n",
      "      BatchNorm2d-13           [-1, 64, 32, 32]             128\n",
      "             ReLU-14           [-1, 64, 32, 32]               0\n",
      "           Conv2d-15           [-1, 64, 32, 32]          36,864\n",
      "      BatchNorm2d-16           [-1, 64, 32, 32]             128\n",
      "             ReLU-17           [-1, 64, 32, 32]               0\n",
      "       BasicBlock-18           [-1, 64, 32, 32]               0\n",
      "           Conv2d-19           [-1, 64, 32, 32]          36,864\n",
      "      BatchNorm2d-20           [-1, 64, 32, 32]             128\n",
      "             ReLU-21           [-1, 64, 32, 32]               0\n",
      "           Conv2d-22           [-1, 64, 32, 32]          36,864\n",
      "      BatchNorm2d-23           [-1, 64, 32, 32]             128\n",
      "             ReLU-24           [-1, 64, 32, 32]               0\n",
      "       BasicBlock-25           [-1, 64, 32, 32]               0\n",
      "           Conv2d-26          [-1, 128, 16, 16]          73,728\n",
      "      BatchNorm2d-27          [-1, 128, 16, 16]             256\n",
      "             ReLU-28          [-1, 128, 16, 16]               0\n",
      "           Conv2d-29          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-30          [-1, 128, 16, 16]             256\n",
      "           Conv2d-31          [-1, 128, 16, 16]           8,192\n",
      "      BatchNorm2d-32          [-1, 128, 16, 16]             256\n",
      "             ReLU-33          [-1, 128, 16, 16]               0\n",
      "       BasicBlock-34          [-1, 128, 16, 16]               0\n",
      "           Conv2d-35          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-36          [-1, 128, 16, 16]             256\n",
      "             ReLU-37          [-1, 128, 16, 16]               0\n",
      "           Conv2d-38          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-39          [-1, 128, 16, 16]             256\n",
      "             ReLU-40          [-1, 128, 16, 16]               0\n",
      "       BasicBlock-41          [-1, 128, 16, 16]               0\n",
      "           Conv2d-42          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-43          [-1, 128, 16, 16]             256\n",
      "             ReLU-44          [-1, 128, 16, 16]               0\n",
      "           Conv2d-45          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-46          [-1, 128, 16, 16]             256\n",
      "             ReLU-47          [-1, 128, 16, 16]               0\n",
      "       BasicBlock-48          [-1, 128, 16, 16]               0\n",
      "           Conv2d-49          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-50          [-1, 128, 16, 16]             256\n",
      "             ReLU-51          [-1, 128, 16, 16]               0\n",
      "           Conv2d-52          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-53          [-1, 128, 16, 16]             256\n",
      "             ReLU-54          [-1, 128, 16, 16]               0\n",
      "       BasicBlock-55          [-1, 128, 16, 16]               0\n",
      "           Conv2d-56            [-1, 256, 8, 8]         294,912\n",
      "      BatchNorm2d-57            [-1, 256, 8, 8]             512\n",
      "             ReLU-58            [-1, 256, 8, 8]               0\n",
      "           Conv2d-59            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-60            [-1, 256, 8, 8]             512\n",
      "           Conv2d-61            [-1, 256, 8, 8]          32,768\n",
      "      BatchNorm2d-62            [-1, 256, 8, 8]             512\n",
      "             ReLU-63            [-1, 256, 8, 8]               0\n",
      "       BasicBlock-64            [-1, 256, 8, 8]               0\n",
      "           Conv2d-65            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-66            [-1, 256, 8, 8]             512\n",
      "             ReLU-67            [-1, 256, 8, 8]               0\n",
      "           Conv2d-68            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-69            [-1, 256, 8, 8]             512\n",
      "             ReLU-70            [-1, 256, 8, 8]               0\n",
      "       BasicBlock-71            [-1, 256, 8, 8]               0\n",
      "           Conv2d-72            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-73            [-1, 256, 8, 8]             512\n",
      "             ReLU-74            [-1, 256, 8, 8]               0\n",
      "           Conv2d-75            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-76            [-1, 256, 8, 8]             512\n",
      "             ReLU-77            [-1, 256, 8, 8]               0\n",
      "       BasicBlock-78            [-1, 256, 8, 8]               0\n",
      "           Conv2d-79            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-80            [-1, 256, 8, 8]             512\n",
      "             ReLU-81            [-1, 256, 8, 8]               0\n",
      "           Conv2d-82            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-83            [-1, 256, 8, 8]             512\n",
      "             ReLU-84            [-1, 256, 8, 8]               0\n",
      "       BasicBlock-85            [-1, 256, 8, 8]               0\n",
      "           Conv2d-86            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-87            [-1, 256, 8, 8]             512\n",
      "             ReLU-88            [-1, 256, 8, 8]               0\n",
      "           Conv2d-89            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-90            [-1, 256, 8, 8]             512\n",
      "             ReLU-91            [-1, 256, 8, 8]               0\n",
      "       BasicBlock-92            [-1, 256, 8, 8]               0\n",
      "           Conv2d-93            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-94            [-1, 256, 8, 8]             512\n",
      "             ReLU-95            [-1, 256, 8, 8]               0\n",
      "           Conv2d-96            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-97            [-1, 256, 8, 8]             512\n",
      "             ReLU-98            [-1, 256, 8, 8]               0\n",
      "       BasicBlock-99            [-1, 256, 8, 8]               0\n",
      "          Conv2d-100            [-1, 512, 4, 4]       1,179,648\n",
      "     BatchNorm2d-101            [-1, 512, 4, 4]           1,024\n",
      "            ReLU-102            [-1, 512, 4, 4]               0\n",
      "          Conv2d-103            [-1, 512, 4, 4]       2,359,296\n",
      "     BatchNorm2d-104            [-1, 512, 4, 4]           1,024\n",
      "          Conv2d-105            [-1, 512, 4, 4]         131,072\n",
      "     BatchNorm2d-106            [-1, 512, 4, 4]           1,024\n",
      "            ReLU-107            [-1, 512, 4, 4]               0\n",
      "      BasicBlock-108            [-1, 512, 4, 4]               0\n",
      "          Conv2d-109            [-1, 512, 4, 4]       2,359,296\n",
      "     BatchNorm2d-110            [-1, 512, 4, 4]           1,024\n",
      "            ReLU-111            [-1, 512, 4, 4]               0\n",
      "          Conv2d-112            [-1, 512, 4, 4]       2,359,296\n",
      "     BatchNorm2d-113            [-1, 512, 4, 4]           1,024\n",
      "            ReLU-114            [-1, 512, 4, 4]               0\n",
      "      BasicBlock-115            [-1, 512, 4, 4]               0\n",
      "          Conv2d-116            [-1, 512, 4, 4]       2,359,296\n",
      "     BatchNorm2d-117            [-1, 512, 4, 4]           1,024\n",
      "            ReLU-118            [-1, 512, 4, 4]               0\n",
      "          Conv2d-119            [-1, 512, 4, 4]       2,359,296\n",
      "     BatchNorm2d-120            [-1, 512, 4, 4]           1,024\n",
      "            ReLU-121            [-1, 512, 4, 4]               0\n",
      "      BasicBlock-122            [-1, 512, 4, 4]               0\n",
      "     BatchNorm2d-123            [-1, 512, 4, 4]           1,024\n",
      "            ReLU-124            [-1, 512, 4, 4]               0\n",
      "================================================================\n",
      "Total params: 21,285,696\n",
      "Trainable params: 18,048\n",
      "Non-trainable params: 21,267,648\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.19\n",
      "Forward/backward pass size (MB): 31.56\n",
      "Params size (MB): 81.20\n",
      "Estimated Total Size (MB): 112.95\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "summary(encoder, input_size=(3, 128, 128))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "from torch import nn\n",
    "from torchviz import make_dot, make_dot_from_trace"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.40.1 (20161225.0304)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"694pt\" height=\"4676pt\"\n",
       " viewBox=\"0.00 0.00 694.47 4676.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(.6302 .6302) rotate(0) translate(4 7416)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"#ffffff\" stroke=\"transparent\" points=\"-4,4 -4,-7416 1098,-7416 1098,4 -4,4\"/>\n",
       "<!-- 140254507738448 -->\n",
       "<g id=\"node1\" class=\"node\">\n",
       "<title>140254507738448</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"#000000\" points=\"976.5,-21 857.5,-21 857.5,0 976.5,0 976.5,-21\"/>\n",
       "<text text-anchor=\"middle\" x=\"917\" y=\"-7.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507767176 -->\n",
       "<g id=\"node2\" class=\"node\">\n",
       "<title>140254507767176</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"995.5,-78 838.5,-78 838.5,-57 995.5,-57 995.5,-78\"/>\n",
       "<text text-anchor=\"middle\" x=\"917\" y=\"-64.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507767176&#45;&gt;140254507738448 -->\n",
       "<g id=\"edge1\" class=\"edge\">\n",
       "<title>140254507767176&#45;&gt;140254507738448</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M917,-56.7787C917,-49.6134 917,-39.9517 917,-31.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"920.5001,-31.1732 917,-21.1732 913.5001,-31.1732 920.5001,-31.1732\"/>\n",
       "</g>\n",
       "<!-- 140254507766224 -->\n",
       "<g id=\"node3\" class=\"node\">\n",
       "<title>140254507766224</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"801.5,-141.5 682.5,-141.5 682.5,-120.5 801.5,-120.5 801.5,-141.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"742\" y=\"-127.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507766224&#45;&gt;140254507767176 -->\n",
       "<g id=\"edge2\" class=\"edge\">\n",
       "<title>140254507766224&#45;&gt;140254507767176</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M771.2911,-120.3715C800.6547,-109.7167 846.0246,-93.2539 878.3307,-81.5314\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"879.6987,-84.7584 887.9051,-78.0573 877.3109,-78.1782 879.6987,-84.7584\"/>\n",
       "</g>\n",
       "<!-- 140254507790576 -->\n",
       "<g id=\"node4\" class=\"node\">\n",
       "<title>140254507790576</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"820.5,-211.5 663.5,-211.5 663.5,-190.5 820.5,-190.5 820.5,-211.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"742\" y=\"-197.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507790576&#45;&gt;140254507766224 -->\n",
       "<g id=\"edge3\" class=\"edge\">\n",
       "<title>140254507790576&#45;&gt;140254507766224</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M742,-190.3685C742,-180.1925 742,-164.5606 742,-151.8912\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"745.5001,-151.7315 742,-141.7315 738.5001,-151.7316 745.5001,-151.7315\"/>\n",
       "</g>\n",
       "<!-- 140254507790632 -->\n",
       "<g id=\"node5\" class=\"node\">\n",
       "<title>140254507790632</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"626.5,-281.5 507.5,-281.5 507.5,-260.5 626.5,-260.5 626.5,-281.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-267.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward0</text>\n",
       "</g>\n",
       "<!-- 140254507790632&#45;&gt;140254507790576 -->\n",
       "<g id=\"edge4\" class=\"edge\">\n",
       "<title>140254507790632&#45;&gt;140254507790576</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M593.5787,-260.3685C623.6897,-248.3241 672.909,-228.6364 706.3396,-215.2642\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"707.7596,-218.4658 715.7445,-211.5022 705.1599,-211.9665 707.7596,-218.4658\"/>\n",
       "</g>\n",
       "<!-- 140254507791696 -->\n",
       "<g id=\"node6\" class=\"node\">\n",
       "<title>140254507791696</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"643.5,-351.5 490.5,-351.5 490.5,-330.5 643.5,-330.5 643.5,-351.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-337.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507791696&#45;&gt;140254507790632 -->\n",
       "<g id=\"edge5\" class=\"edge\">\n",
       "<title>140254507791696&#45;&gt;140254507790632</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M567,-330.3685C567,-320.1925 567,-304.5606 567,-291.8912\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"570.5001,-291.7315 567,-281.7315 563.5001,-291.7316 570.5001,-291.7315\"/>\n",
       "</g>\n",
       "<!-- 140254507791976 -->\n",
       "<g id=\"node7\" class=\"node\">\n",
       "<title>140254507791976</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"519.5,-421.5 400.5,-421.5 400.5,-400.5 519.5,-400.5 519.5,-421.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"460\" y=\"-407.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507791976&#45;&gt;140254507791696 -->\n",
       "<g id=\"edge6\" class=\"edge\">\n",
       "<title>140254507791976&#45;&gt;140254507791696</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M476.251,-400.3685C493.8716,-388.841 522.1941,-370.3123 542.5127,-357.0197\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"544.4944,-359.9057 550.9467,-351.5022 540.6622,-354.0479 544.4944,-359.9057\"/>\n",
       "</g>\n",
       "<!-- 140254507791808 -->\n",
       "<g id=\"node8\" class=\"node\">\n",
       "<title>140254507791808</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"506,-485 414,-485 414,-464 506,-464 506,-485\"/>\n",
       "<text text-anchor=\"middle\" x=\"460\" y=\"-471.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 140254507791808&#45;&gt;140254507791976 -->\n",
       "<g id=\"edge7\" class=\"edge\">\n",
       "<title>140254507791808&#45;&gt;140254507791976</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M460,-463.7281C460,-455.0091 460,-442.4699 460,-431.8068\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"463.5001,-431.6128 460,-421.6128 456.5001,-431.6129 463.5001,-431.6128\"/>\n",
       "</g>\n",
       "<!-- 140254507793768 -->\n",
       "<g id=\"node9\" class=\"node\">\n",
       "<title>140254507793768</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"662.5,-542 509.5,-542 509.5,-521 662.5,-521 662.5,-542\"/>\n",
       "<text text-anchor=\"middle\" x=\"586\" y=\"-528.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507793768&#45;&gt;140254507791808 -->\n",
       "<g id=\"edge8\" class=\"edge\">\n",
       "<title>140254507793768&#45;&gt;140254507791808</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M562.5957,-520.9123C542.8814,-511.994 514.468,-499.1403 492.5391,-489.2201\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"493.8327,-485.9638 483.279,-485.031 490.9475,-492.3416 493.8327,-485.9638\"/>\n",
       "</g>\n",
       "<!-- 140254507704952 -->\n",
       "<g id=\"node10\" class=\"node\">\n",
       "<title>140254507704952</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"592.5,-605.5 435.5,-605.5 435.5,-584.5 592.5,-584.5 592.5,-605.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"514\" y=\"-591.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507704952&#45;&gt;140254507793768 -->\n",
       "<g id=\"edge9\" class=\"edge\">\n",
       "<title>140254507704952&#45;&gt;140254507793768</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M526.2138,-584.2281C537.1405,-574.5914 553.3588,-560.2877 566.1477,-549.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"568.7817,-551.3523 573.9666,-542.1128 564.1516,-546.1024 568.7817,-551.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507704728 -->\n",
       "<g id=\"node11\" class=\"node\">\n",
       "<title>140254507704728</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"573.5,-669 454.5,-669 454.5,-648 573.5,-648 573.5,-669\"/>\n",
       "<text text-anchor=\"middle\" x=\"514\" y=\"-655.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507704728&#45;&gt;140254507704952 -->\n",
       "<g id=\"edge10\" class=\"edge\">\n",
       "<title>140254507704728&#45;&gt;140254507704952</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M514,-647.7281C514,-639.0091 514,-626.4699 514,-615.8068\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"517.5001,-615.6128 514,-605.6128 510.5001,-615.6129 517.5001,-615.6128\"/>\n",
       "</g>\n",
       "<!-- 140254507705008 -->\n",
       "<g id=\"node12\" class=\"node\">\n",
       "<title>140254507705008</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"662.5,-726 509.5,-726 509.5,-705 662.5,-705 662.5,-726\"/>\n",
       "<text text-anchor=\"middle\" x=\"586\" y=\"-712.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507705008&#45;&gt;140254507704728 -->\n",
       "<g id=\"edge11\" class=\"edge\">\n",
       "<title>140254507705008&#45;&gt;140254507704728</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M572.4574,-704.7787C562.099,-696.5784 547.6107,-685.1085 535.6496,-675.6393\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"537.4949,-672.6361 527.4819,-669.1732 533.1499,-678.1244 537.4949,-672.6361\"/>\n",
       "</g>\n",
       "<!-- 140254507706352 -->\n",
       "<g id=\"node13\" class=\"node\">\n",
       "<title>140254507706352</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"592.5,-789.5 435.5,-789.5 435.5,-768.5 592.5,-768.5 592.5,-789.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"514\" y=\"-775.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507706352&#45;&gt;140254507705008 -->\n",
       "<g id=\"edge12\" class=\"edge\">\n",
       "<title>140254507706352&#45;&gt;140254507705008</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M526.2138,-768.2281C537.1405,-758.5914 553.3588,-744.2877 566.1477,-733.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"568.7817,-735.3523 573.9666,-726.1128 564.1516,-730.1024 568.7817,-735.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507707248 -->\n",
       "<g id=\"node14\" class=\"node\">\n",
       "<title>140254507707248</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"519.5,-853 400.5,-853 400.5,-832 519.5,-832 519.5,-853\"/>\n",
       "<text text-anchor=\"middle\" x=\"460\" y=\"-839.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507707248&#45;&gt;140254507791808 -->\n",
       "<g id=\"edge214\" class=\"edge\">\n",
       "<title>140254507707248&#45;&gt;140254507791808</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M450.2577,-831.8331C442.5876,-822.885 432.1912,-809.444 426,-796 410.6232,-762.6097 407,-752.2608 407,-715.5 407,-715.5 407,-715.5 407,-595 407,-556.4334 430.0294,-516.352 445.7273,-493.5108\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"448.7671,-495.276 451.7188,-485.1006 443.0659,-491.2144 448.7671,-495.276\"/>\n",
       "</g>\n",
       "<!-- 140254507707248&#45;&gt;140254507706352 -->\n",
       "<g id=\"edge13\" class=\"edge\">\n",
       "<title>140254507707248&#45;&gt;140254507706352</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M469.1604,-831.7281C477.1212,-822.3667 488.8273,-808.6012 498.2825,-797.4826\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"501.1629,-799.4982 504.9749,-789.6128 495.8304,-794.9634 501.1629,-799.4982\"/>\n",
       "</g>\n",
       "<!-- 140254507704896 -->\n",
       "<g id=\"node15\" class=\"node\">\n",
       "<title>140254507704896</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"506,-910 414,-910 414,-889 506,-889 506,-910\"/>\n",
       "<text text-anchor=\"middle\" x=\"460\" y=\"-896.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 140254507704896&#45;&gt;140254507707248 -->\n",
       "<g id=\"edge14\" class=\"edge\">\n",
       "<title>140254507704896&#45;&gt;140254507707248</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M460,-888.7787C460,-881.6134 460,-871.9517 460,-863.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"463.5001,-863.1732 460,-853.1732 456.5001,-863.1732 463.5001,-863.1732\"/>\n",
       "</g>\n",
       "<!-- 140254507705568 -->\n",
       "<g id=\"node16\" class=\"node\">\n",
       "<title>140254507705568</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"662.5,-967 509.5,-967 509.5,-946 662.5,-946 662.5,-967\"/>\n",
       "<text text-anchor=\"middle\" x=\"586\" y=\"-953.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507705568&#45;&gt;140254507704896 -->\n",
       "<g id=\"edge15\" class=\"edge\">\n",
       "<title>140254507705568&#45;&gt;140254507704896</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M562.5957,-945.9123C542.8814,-936.994 514.468,-924.1403 492.5391,-914.2201\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"493.8327,-910.9638 483.279,-910.031 490.9475,-917.3416 493.8327,-910.9638\"/>\n",
       "</g>\n",
       "<!-- 140254507707584 -->\n",
       "<g id=\"node17\" class=\"node\">\n",
       "<title>140254507707584</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"592.5,-1030.5 435.5,-1030.5 435.5,-1009.5 592.5,-1009.5 592.5,-1030.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"514\" y=\"-1016.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507707584&#45;&gt;140254507705568 -->\n",
       "<g id=\"edge16\" class=\"edge\">\n",
       "<title>140254507707584&#45;&gt;140254507705568</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M526.2138,-1009.2281C537.1405,-999.5914 553.3588,-985.2877 566.1477,-974.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"568.7817,-976.3523 573.9666,-967.1128 564.1516,-971.1024 568.7817,-976.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507705904 -->\n",
       "<g id=\"node18\" class=\"node\">\n",
       "<title>140254507705904</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"573.5,-1094 454.5,-1094 454.5,-1073 573.5,-1073 573.5,-1094\"/>\n",
       "<text text-anchor=\"middle\" x=\"514\" y=\"-1080.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507705904&#45;&gt;140254507707584 -->\n",
       "<g id=\"edge17\" class=\"edge\">\n",
       "<title>140254507705904&#45;&gt;140254507707584</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M514,-1072.7281C514,-1064.0091 514,-1051.4699 514,-1040.8068\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"517.5001,-1040.6128 514,-1030.6128 510.5001,-1040.6129 517.5001,-1040.6128\"/>\n",
       "</g>\n",
       "<!-- 140254507800504 -->\n",
       "<g id=\"node19\" class=\"node\">\n",
       "<title>140254507800504</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"662.5,-1151 509.5,-1151 509.5,-1130 662.5,-1130 662.5,-1151\"/>\n",
       "<text text-anchor=\"middle\" x=\"586\" y=\"-1137.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507800504&#45;&gt;140254507705904 -->\n",
       "<g id=\"edge18\" class=\"edge\">\n",
       "<title>140254507800504&#45;&gt;140254507705904</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M572.4574,-1129.7787C562.099,-1121.5784 547.6107,-1110.1085 535.6496,-1100.6393\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"537.4949,-1097.6361 527.4819,-1094.1732 533.1499,-1103.1244 537.4949,-1097.6361\"/>\n",
       "</g>\n",
       "<!-- 140254507802408 -->\n",
       "<g id=\"node20\" class=\"node\">\n",
       "<title>140254507802408</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"592.5,-1214.5 435.5,-1214.5 435.5,-1193.5 592.5,-1193.5 592.5,-1214.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"514\" y=\"-1200.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507802408&#45;&gt;140254507800504 -->\n",
       "<g id=\"edge19\" class=\"edge\">\n",
       "<title>140254507802408&#45;&gt;140254507800504</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M526.2138,-1193.2281C537.1405,-1183.5914 553.3588,-1169.2877 566.1477,-1158.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"568.7817,-1160.3523 573.9666,-1151.1128 564.1516,-1155.1024 568.7817,-1160.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507707416 -->\n",
       "<g id=\"node21\" class=\"node\">\n",
       "<title>140254507707416</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"519.5,-1278 400.5,-1278 400.5,-1257 519.5,-1257 519.5,-1278\"/>\n",
       "<text text-anchor=\"middle\" x=\"460\" y=\"-1264.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507707416&#45;&gt;140254507704896 -->\n",
       "<g id=\"edge209\" class=\"edge\">\n",
       "<title>140254507707416&#45;&gt;140254507704896</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M450.2577,-1256.8331C442.5876,-1247.885 432.1912,-1234.444 426,-1221 410.6232,-1187.6097 407,-1177.2608 407,-1140.5 407,-1140.5 407,-1140.5 407,-1020 407,-981.4334 430.0294,-941.352 445.7273,-918.5108\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"448.7671,-920.276 451.7188,-910.1006 443.0659,-916.2144 448.7671,-920.276\"/>\n",
       "</g>\n",
       "<!-- 140254507707416&#45;&gt;140254507802408 -->\n",
       "<g id=\"edge20\" class=\"edge\">\n",
       "<title>140254507707416&#45;&gt;140254507802408</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M469.1604,-1256.7281C477.1212,-1247.3667 488.8273,-1233.6012 498.2825,-1222.4826\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"501.1629,-1224.4982 504.9749,-1214.6128 495.8304,-1219.9634 501.1629,-1224.4982\"/>\n",
       "</g>\n",
       "<!-- 140254507801624 -->\n",
       "<g id=\"node22\" class=\"node\">\n",
       "<title>140254507801624</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"506,-1335 414,-1335 414,-1314 506,-1314 506,-1335\"/>\n",
       "<text text-anchor=\"middle\" x=\"460\" y=\"-1321.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 140254507801624&#45;&gt;140254507707416 -->\n",
       "<g id=\"edge21\" class=\"edge\">\n",
       "<title>140254507801624&#45;&gt;140254507707416</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M460,-1313.7787C460,-1306.6134 460,-1296.9517 460,-1288.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"463.5001,-1288.1732 460,-1278.1732 456.5001,-1288.1732 463.5001,-1288.1732\"/>\n",
       "</g>\n",
       "<!-- 140254507800392 -->\n",
       "<g id=\"node23\" class=\"node\">\n",
       "<title>140254507800392</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"691.5,-1392 538.5,-1392 538.5,-1371 691.5,-1371 691.5,-1392\"/>\n",
       "<text text-anchor=\"middle\" x=\"615\" y=\"-1378.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507800392&#45;&gt;140254507801624 -->\n",
       "<g id=\"edge22\" class=\"edge\">\n",
       "<title>140254507800392&#45;&gt;140254507801624</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M586.2089,-1370.9123C561.4136,-1361.794 525.4315,-1348.5619 498.2259,-1338.5573\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"499.2304,-1335.1976 488.6369,-1335.031 496.8143,-1341.7674 499.2304,-1335.1976\"/>\n",
       "</g>\n",
       "<!-- 140254507802520 -->\n",
       "<g id=\"node24\" class=\"node\">\n",
       "<title>140254507802520</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"621.5,-1455.5 464.5,-1455.5 464.5,-1434.5 621.5,-1434.5 621.5,-1455.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"543\" y=\"-1441.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507802520&#45;&gt;140254507800392 -->\n",
       "<g id=\"edge23\" class=\"edge\">\n",
       "<title>140254507802520&#45;&gt;140254507800392</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M555.2138,-1434.2281C566.1405,-1424.5914 582.3588,-1410.2877 595.1477,-1399.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"597.7817,-1401.3523 602.9666,-1392.1128 593.1516,-1396.1024 597.7817,-1401.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507801848 -->\n",
       "<g id=\"node25\" class=\"node\">\n",
       "<title>140254507801848</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"616.5,-1519 497.5,-1519 497.5,-1498 616.5,-1498 616.5,-1519\"/>\n",
       "<text text-anchor=\"middle\" x=\"557\" y=\"-1505.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507801848&#45;&gt;140254507802520 -->\n",
       "<g id=\"edge24\" class=\"edge\">\n",
       "<title>140254507801848&#45;&gt;140254507802520</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M554.6251,-1497.7281C552.6826,-1488.9174 549.8799,-1476.2055 547.5132,-1465.4708\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"550.9108,-1464.6247 545.3398,-1455.6128 544.075,-1466.1319 550.9108,-1464.6247\"/>\n",
       "</g>\n",
       "<!-- 140254507801736 -->\n",
       "<g id=\"node26\" class=\"node\">\n",
       "<title>140254507801736</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"720.5,-1576 567.5,-1576 567.5,-1555 720.5,-1555 720.5,-1576\"/>\n",
       "<text text-anchor=\"middle\" x=\"644\" y=\"-1562.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507801736&#45;&gt;140254507801848 -->\n",
       "<g id=\"edge25\" class=\"edge\">\n",
       "<title>140254507801736&#45;&gt;140254507801848</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M627.636,-1554.7787C614.7551,-1546.3396 596.5889,-1534.4376 581.9051,-1524.8171\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"583.5734,-1521.7259 573.2907,-1519.1732 579.7372,-1527.5811 583.5734,-1521.7259\"/>\n",
       "</g>\n",
       "<!-- 140254507800280 -->\n",
       "<g id=\"node27\" class=\"node\">\n",
       "<title>140254507800280</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"650.5,-1639.5 493.5,-1639.5 493.5,-1618.5 650.5,-1618.5 650.5,-1639.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"572\" y=\"-1625.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507800280&#45;&gt;140254507801736 -->\n",
       "<g id=\"edge26\" class=\"edge\">\n",
       "<title>140254507800280&#45;&gt;140254507801736</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M584.2138,-1618.2281C595.1405,-1608.5914 611.3588,-1594.2877 624.1477,-1583.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"626.7817,-1585.3523 631.9666,-1576.1128 622.1516,-1580.1024 626.7817,-1585.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507799216 -->\n",
       "<g id=\"node28\" class=\"node\">\n",
       "<title>140254507799216</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"543.5,-1703 424.5,-1703 424.5,-1682 543.5,-1682 543.5,-1703\"/>\n",
       "<text text-anchor=\"middle\" x=\"484\" y=\"-1689.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507799216&#45;&gt;140254507800280 -->\n",
       "<g id=\"edge27\" class=\"edge\">\n",
       "<title>140254507799216&#45;&gt;140254507800280</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M498.928,-1681.7281C512.6644,-1671.816 533.2432,-1656.9665 549.0654,-1645.5494\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"551.2313,-1648.3026 557.2925,-1639.6128 547.1352,-1642.6262 551.2313,-1648.3026\"/>\n",
       "</g>\n",
       "<!-- 140254507799608 -->\n",
       "<g id=\"node189\" class=\"node\">\n",
       "<title>140254507799608</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"475.5,-1639.5 318.5,-1639.5 318.5,-1618.5 475.5,-1618.5 475.5,-1639.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"397\" y=\"-1625.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507799216&#45;&gt;140254507799608 -->\n",
       "<g id=\"edge202\" class=\"edge\">\n",
       "<title>140254507799216&#45;&gt;140254507799608</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M469.2416,-1681.7281C455.6613,-1671.816 435.3163,-1656.9665 419.674,-1645.5494\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"421.6812,-1642.6813 411.5404,-1639.6128 417.5543,-1648.3354 421.6812,-1642.6813\"/>\n",
       "</g>\n",
       "<!-- 140254507799720 -->\n",
       "<g id=\"node29\" class=\"node\">\n",
       "<title>140254507799720</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"530,-1760 438,-1760 438,-1739 530,-1739 530,-1760\"/>\n",
       "<text text-anchor=\"middle\" x=\"484\" y=\"-1746.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 140254507799720&#45;&gt;140254507799216 -->\n",
       "<g id=\"edge28\" class=\"edge\">\n",
       "<title>140254507799720&#45;&gt;140254507799216</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M484,-1738.7787C484,-1731.6134 484,-1721.9517 484,-1713.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"487.5001,-1713.1732 484,-1703.1732 480.5001,-1713.1732 487.5001,-1713.1732\"/>\n",
       "</g>\n",
       "<!-- 140254507802296 -->\n",
       "<g id=\"node30\" class=\"node\">\n",
       "<title>140254507802296</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"686.5,-1817 533.5,-1817 533.5,-1796 686.5,-1796 686.5,-1817\"/>\n",
       "<text text-anchor=\"middle\" x=\"610\" y=\"-1803.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507802296&#45;&gt;140254507799720 -->\n",
       "<g id=\"edge29\" class=\"edge\">\n",
       "<title>140254507802296&#45;&gt;140254507799720</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M586.5957,-1795.9123C566.8814,-1786.994 538.468,-1774.1403 516.5391,-1764.2201\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"517.8327,-1760.9638 507.279,-1760.031 514.9475,-1767.3416 517.8327,-1760.9638\"/>\n",
       "</g>\n",
       "<!-- 140254507471200 -->\n",
       "<g id=\"node31\" class=\"node\">\n",
       "<title>140254507471200</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"616.5,-1880.5 459.5,-1880.5 459.5,-1859.5 616.5,-1859.5 616.5,-1880.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"538\" y=\"-1866.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507471200&#45;&gt;140254507802296 -->\n",
       "<g id=\"edge30\" class=\"edge\">\n",
       "<title>140254507471200&#45;&gt;140254507802296</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M550.2138,-1859.2281C561.1405,-1849.5914 577.3588,-1835.2877 590.1477,-1824.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"592.7817,-1826.3523 597.9666,-1817.1128 588.1516,-1821.1024 592.7817,-1826.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507471424 -->\n",
       "<g id=\"node32\" class=\"node\">\n",
       "<title>140254507471424</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"597.5,-1944 478.5,-1944 478.5,-1923 597.5,-1923 597.5,-1944\"/>\n",
       "<text text-anchor=\"middle\" x=\"538\" y=\"-1930.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507471424&#45;&gt;140254507471200 -->\n",
       "<g id=\"edge31\" class=\"edge\">\n",
       "<title>140254507471424&#45;&gt;140254507471200</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M538,-1922.7281C538,-1914.0091 538,-1901.4699 538,-1890.8068\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"541.5001,-1890.6128 538,-1880.6128 534.5001,-1890.6129 541.5001,-1890.6128\"/>\n",
       "</g>\n",
       "<!-- 140254507471592 -->\n",
       "<g id=\"node33\" class=\"node\">\n",
       "<title>140254507471592</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"686.5,-2001 533.5,-2001 533.5,-1980 686.5,-1980 686.5,-2001\"/>\n",
       "<text text-anchor=\"middle\" x=\"610\" y=\"-1987.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507471592&#45;&gt;140254507471424 -->\n",
       "<g id=\"edge32\" class=\"edge\">\n",
       "<title>140254507471592&#45;&gt;140254507471424</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M596.4574,-1979.7787C586.099,-1971.5784 571.6107,-1960.1085 559.6496,-1950.6393\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"561.4949,-1947.6361 551.4819,-1944.1732 557.1499,-1953.1244 561.4949,-1947.6361\"/>\n",
       "</g>\n",
       "<!-- 140254507471872 -->\n",
       "<g id=\"node34\" class=\"node\">\n",
       "<title>140254507471872</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"616.5,-2064.5 459.5,-2064.5 459.5,-2043.5 616.5,-2043.5 616.5,-2064.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"538\" y=\"-2050.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507471872&#45;&gt;140254507471592 -->\n",
       "<g id=\"edge33\" class=\"edge\">\n",
       "<title>140254507471872&#45;&gt;140254507471592</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M550.2138,-2043.2281C561.1405,-2033.5914 577.3588,-2019.2877 590.1477,-2008.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"592.7817,-2010.3523 597.9666,-2001.1128 588.1516,-2005.1024 592.7817,-2010.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507470920 -->\n",
       "<g id=\"node35\" class=\"node\">\n",
       "<title>140254507470920</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"543.5,-2128 424.5,-2128 424.5,-2107 543.5,-2107 543.5,-2128\"/>\n",
       "<text text-anchor=\"middle\" x=\"484\" y=\"-2114.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507470920&#45;&gt;140254507799720 -->\n",
       "<g id=\"edge195\" class=\"edge\">\n",
       "<title>140254507470920&#45;&gt;140254507799720</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M474.2577,-2106.8331C466.5876,-2097.885 456.1912,-2084.444 450,-2071 434.6232,-2037.6097 431,-2027.2608 431,-1990.5 431,-1990.5 431,-1990.5 431,-1870 431,-1831.4334 454.0294,-1791.352 469.7273,-1768.5108\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"472.7671,-1770.276 475.7188,-1760.1006 467.0659,-1766.2144 472.7671,-1770.276\"/>\n",
       "</g>\n",
       "<!-- 140254507470920&#45;&gt;140254507471872 -->\n",
       "<g id=\"edge34\" class=\"edge\">\n",
       "<title>140254507470920&#45;&gt;140254507471872</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M493.1604,-2106.7281C501.1212,-2097.3667 512.8273,-2083.6012 522.2825,-2072.4826\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"525.1629,-2074.4982 528.9749,-2064.6128 519.8304,-2069.9634 525.1629,-2074.4982\"/>\n",
       "</g>\n",
       "<!-- 140254507472320 -->\n",
       "<g id=\"node36\" class=\"node\">\n",
       "<title>140254507472320</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"530,-2185 438,-2185 438,-2164 530,-2164 530,-2185\"/>\n",
       "<text text-anchor=\"middle\" x=\"484\" y=\"-2171.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 140254507472320&#45;&gt;140254507470920 -->\n",
       "<g id=\"edge35\" class=\"edge\">\n",
       "<title>140254507472320&#45;&gt;140254507470920</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M484,-2163.7787C484,-2156.6134 484,-2146.9517 484,-2138.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"487.5001,-2138.1732 484,-2128.1732 480.5001,-2138.1732 487.5001,-2138.1732\"/>\n",
       "</g>\n",
       "<!-- 140254507472544 -->\n",
       "<g id=\"node37\" class=\"node\">\n",
       "<title>140254507472544</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"758.5,-2242 605.5,-2242 605.5,-2221 758.5,-2221 758.5,-2242\"/>\n",
       "<text text-anchor=\"middle\" x=\"682\" y=\"-2228.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507472544&#45;&gt;140254507472320 -->\n",
       "<g id=\"edge36\" class=\"edge\">\n",
       "<title>140254507472544&#45;&gt;140254507472320</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M645.2217,-2220.9123C612.853,-2211.5941 565.5606,-2197.9796 530.5606,-2187.9038\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"531.1593,-2184.4341 520.5813,-2185.031 529.2227,-2191.1609 531.1593,-2184.4341\"/>\n",
       "</g>\n",
       "<!-- 140254507472936 -->\n",
       "<g id=\"node38\" class=\"node\">\n",
       "<title>140254507472936</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"688.5,-2305.5 531.5,-2305.5 531.5,-2284.5 688.5,-2284.5 688.5,-2305.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"610\" y=\"-2291.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507472936&#45;&gt;140254507472544 -->\n",
       "<g id=\"edge37\" class=\"edge\">\n",
       "<title>140254507472936&#45;&gt;140254507472544</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M622.2138,-2284.2281C633.1405,-2274.5914 649.3588,-2260.2877 662.1477,-2249.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"664.7817,-2251.3523 669.9666,-2242.1128 660.1516,-2246.1024 664.7817,-2251.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507473160 -->\n",
       "<g id=\"node39\" class=\"node\">\n",
       "<title>140254507473160</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"687.5,-2369 568.5,-2369 568.5,-2348 687.5,-2348 687.5,-2369\"/>\n",
       "<text text-anchor=\"middle\" x=\"628\" y=\"-2355.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507473160&#45;&gt;140254507472936 -->\n",
       "<g id=\"edge38\" class=\"edge\">\n",
       "<title>140254507473160&#45;&gt;140254507472936</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M624.9465,-2347.7281C622.449,-2338.9174 618.8456,-2326.2055 615.8027,-2315.4708\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"619.103,-2314.2792 613.0084,-2305.6128 612.3683,-2316.1883 619.103,-2314.2792\"/>\n",
       "</g>\n",
       "<!-- 140254507473328 -->\n",
       "<g id=\"node40\" class=\"node\">\n",
       "<title>140254507473328</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"740.5,-2426 587.5,-2426 587.5,-2405 740.5,-2405 740.5,-2426\"/>\n",
       "<text text-anchor=\"middle\" x=\"664\" y=\"-2412.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507473328&#45;&gt;140254507473160 -->\n",
       "<g id=\"edge39\" class=\"edge\">\n",
       "<title>140254507473328&#45;&gt;140254507473160</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M657.2287,-2404.7787C652.4518,-2397.2153 645.9182,-2386.8705 640.2397,-2377.8794\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"643.0402,-2375.7591 634.741,-2369.1732 637.1217,-2379.4971 643.0402,-2375.7591\"/>\n",
       "</g>\n",
       "<!-- 140254507473608 -->\n",
       "<g id=\"node41\" class=\"node\">\n",
       "<title>140254507473608</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"616.5,-2489.5 459.5,-2489.5 459.5,-2468.5 616.5,-2468.5 616.5,-2489.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"538\" y=\"-2475.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507473608&#45;&gt;140254507473328 -->\n",
       "<g id=\"edge40\" class=\"edge\">\n",
       "<title>140254507473608&#45;&gt;140254507473328</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M559.0896,-2468.3715C579.5319,-2458.0692 610.7488,-2442.3369 633.8192,-2430.7101\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"635.4867,-2433.7892 642.8416,-2426.1631 632.3364,-2427.5381 635.4867,-2433.7892\"/>\n",
       "</g>\n",
       "<!-- 140254507472712 -->\n",
       "<g id=\"node42\" class=\"node\">\n",
       "<title>140254507472712</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"543.5,-2553 424.5,-2553 424.5,-2532 543.5,-2532 543.5,-2553\"/>\n",
       "<text text-anchor=\"middle\" x=\"484\" y=\"-2539.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507472712&#45;&gt;140254507472320 -->\n",
       "<g id=\"edge190\" class=\"edge\">\n",
       "<title>140254507472712&#45;&gt;140254507472320</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M474.2577,-2531.8331C466.5876,-2522.885 456.1912,-2509.444 450,-2496 434.6232,-2462.6097 431,-2452.2608 431,-2415.5 431,-2415.5 431,-2415.5 431,-2295 431,-2256.4334 454.0294,-2216.352 469.7273,-2193.5108\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"472.7671,-2195.276 475.7188,-2185.1006 467.0659,-2191.2144 472.7671,-2195.276\"/>\n",
       "</g>\n",
       "<!-- 140254507472712&#45;&gt;140254507473608 -->\n",
       "<g id=\"edge41\" class=\"edge\">\n",
       "<title>140254507472712&#45;&gt;140254507473608</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M493.1604,-2531.7281C501.1212,-2522.3667 512.8273,-2508.6012 522.2825,-2497.4826\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"525.1629,-2499.4982 528.9749,-2489.6128 519.8304,-2494.9634 525.1629,-2499.4982\"/>\n",
       "</g>\n",
       "<!-- 140254507473944 -->\n",
       "<g id=\"node43\" class=\"node\">\n",
       "<title>140254507473944</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"530,-2610 438,-2610 438,-2589 530,-2589 530,-2610\"/>\n",
       "<text text-anchor=\"middle\" x=\"484\" y=\"-2596.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 140254507473944&#45;&gt;140254507472712 -->\n",
       "<g id=\"edge42\" class=\"edge\">\n",
       "<title>140254507473944&#45;&gt;140254507472712</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M484,-2588.7787C484,-2581.6134 484,-2571.9517 484,-2563.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"487.5001,-2563.1732 484,-2553.1732 480.5001,-2563.1732 487.5001,-2563.1732\"/>\n",
       "</g>\n",
       "<!-- 140254507474112 -->\n",
       "<g id=\"node44\" class=\"node\">\n",
       "<title>140254507474112</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"758.5,-2667 605.5,-2667 605.5,-2646 758.5,-2646 758.5,-2667\"/>\n",
       "<text text-anchor=\"middle\" x=\"682\" y=\"-2653.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507474112&#45;&gt;140254507473944 -->\n",
       "<g id=\"edge43\" class=\"edge\">\n",
       "<title>140254507474112&#45;&gt;140254507473944</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M645.2217,-2645.9123C612.853,-2636.5941 565.5606,-2622.9796 530.5606,-2612.9038\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"531.1593,-2609.4341 520.5813,-2610.031 529.2227,-2616.1609 531.1593,-2609.4341\"/>\n",
       "</g>\n",
       "<!-- 140254507474504 -->\n",
       "<g id=\"node45\" class=\"node\">\n",
       "<title>140254507474504</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"688.5,-2730.5 531.5,-2730.5 531.5,-2709.5 688.5,-2709.5 688.5,-2730.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"610\" y=\"-2716.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507474504&#45;&gt;140254507474112 -->\n",
       "<g id=\"edge44\" class=\"edge\">\n",
       "<title>140254507474504&#45;&gt;140254507474112</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M622.2138,-2709.2281C633.1405,-2699.5914 649.3588,-2685.2877 662.1477,-2674.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"664.7817,-2676.3523 669.9666,-2667.1128 660.1516,-2671.1024 664.7817,-2676.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507474728 -->\n",
       "<g id=\"node46\" class=\"node\">\n",
       "<title>140254507474728</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"687.5,-2794 568.5,-2794 568.5,-2773 687.5,-2773 687.5,-2794\"/>\n",
       "<text text-anchor=\"middle\" x=\"628\" y=\"-2780.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507474728&#45;&gt;140254507474504 -->\n",
       "<g id=\"edge45\" class=\"edge\">\n",
       "<title>140254507474728&#45;&gt;140254507474504</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M624.9465,-2772.7281C622.449,-2763.9174 618.8456,-2751.2055 615.8027,-2740.4708\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"619.103,-2739.2792 613.0084,-2730.6128 612.3683,-2741.1883 619.103,-2739.2792\"/>\n",
       "</g>\n",
       "<!-- 140254507474896 -->\n",
       "<g id=\"node47\" class=\"node\">\n",
       "<title>140254507474896</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"740.5,-2851 587.5,-2851 587.5,-2830 740.5,-2830 740.5,-2851\"/>\n",
       "<text text-anchor=\"middle\" x=\"664\" y=\"-2837.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507474896&#45;&gt;140254507474728 -->\n",
       "<g id=\"edge46\" class=\"edge\">\n",
       "<title>140254507474896&#45;&gt;140254507474728</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M657.2287,-2829.7787C652.4518,-2822.2153 645.9182,-2811.8705 640.2397,-2802.8794\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"643.0402,-2800.7591 634.741,-2794.1732 637.1217,-2804.4971 643.0402,-2800.7591\"/>\n",
       "</g>\n",
       "<!-- 140254507475128 -->\n",
       "<g id=\"node48\" class=\"node\">\n",
       "<title>140254507475128</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"616.5,-2914.5 459.5,-2914.5 459.5,-2893.5 616.5,-2893.5 616.5,-2914.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"538\" y=\"-2900.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507475128&#45;&gt;140254507474896 -->\n",
       "<g id=\"edge47\" class=\"edge\">\n",
       "<title>140254507475128&#45;&gt;140254507474896</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M559.0896,-2893.3715C579.5319,-2883.0692 610.7488,-2867.3369 633.8192,-2855.7101\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"635.4867,-2858.7892 642.8416,-2851.1631 632.3364,-2852.5381 635.4867,-2858.7892\"/>\n",
       "</g>\n",
       "<!-- 140254507474280 -->\n",
       "<g id=\"node49\" class=\"node\">\n",
       "<title>140254507474280</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"543.5,-2978 424.5,-2978 424.5,-2957 543.5,-2957 543.5,-2978\"/>\n",
       "<text text-anchor=\"middle\" x=\"484\" y=\"-2964.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507474280&#45;&gt;140254507473944 -->\n",
       "<g id=\"edge185\" class=\"edge\">\n",
       "<title>140254507474280&#45;&gt;140254507473944</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M474.2577,-2956.8331C466.5876,-2947.885 456.1912,-2934.444 450,-2921 434.6232,-2887.6097 431,-2877.2608 431,-2840.5 431,-2840.5 431,-2840.5 431,-2720 431,-2681.4334 454.0294,-2641.352 469.7273,-2618.5108\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"472.7671,-2620.276 475.7188,-2610.1006 467.0659,-2616.2144 472.7671,-2620.276\"/>\n",
       "</g>\n",
       "<!-- 140254507474280&#45;&gt;140254507475128 -->\n",
       "<g id=\"edge48\" class=\"edge\">\n",
       "<title>140254507474280&#45;&gt;140254507475128</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M493.1604,-2956.7281C501.1212,-2947.3667 512.8273,-2933.6012 522.2825,-2922.4826\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"525.1629,-2924.4982 528.9749,-2914.6128 519.8304,-2919.9634 525.1629,-2924.4982\"/>\n",
       "</g>\n",
       "<!-- 140254507475296 -->\n",
       "<g id=\"node50\" class=\"node\">\n",
       "<title>140254507475296</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"530,-3035 438,-3035 438,-3014 530,-3014 530,-3035\"/>\n",
       "<text text-anchor=\"middle\" x=\"484\" y=\"-3021.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 140254507475296&#45;&gt;140254507474280 -->\n",
       "<g id=\"edge49\" class=\"edge\">\n",
       "<title>140254507475296&#45;&gt;140254507474280</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M484,-3013.7787C484,-3006.6134 484,-2996.9517 484,-2988.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"487.5001,-2988.1732 484,-2978.1732 480.5001,-2988.1732 487.5001,-2988.1732\"/>\n",
       "</g>\n",
       "<!-- 140254507475408 -->\n",
       "<g id=\"node51\" class=\"node\">\n",
       "<title>140254507475408</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"686.5,-3092 533.5,-3092 533.5,-3071 686.5,-3071 686.5,-3092\"/>\n",
       "<text text-anchor=\"middle\" x=\"610\" y=\"-3078.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507475408&#45;&gt;140254507475296 -->\n",
       "<g id=\"edge50\" class=\"edge\">\n",
       "<title>140254507475408&#45;&gt;140254507475296</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M586.5957,-3070.9123C566.8814,-3061.994 538.468,-3049.1403 516.5391,-3039.2201\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"517.8327,-3035.9638 507.279,-3035.031 514.9475,-3042.3416 517.8327,-3035.9638\"/>\n",
       "</g>\n",
       "<!-- 140254507475576 -->\n",
       "<g id=\"node52\" class=\"node\">\n",
       "<title>140254507475576</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"616.5,-3155.5 459.5,-3155.5 459.5,-3134.5 616.5,-3134.5 616.5,-3155.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"538\" y=\"-3141.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507475576&#45;&gt;140254507475408 -->\n",
       "<g id=\"edge51\" class=\"edge\">\n",
       "<title>140254507475576&#45;&gt;140254507475408</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M550.2138,-3134.2281C561.1405,-3124.5914 577.3588,-3110.2877 590.1477,-3099.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"592.7817,-3101.3523 597.9666,-3092.1128 588.1516,-3096.1024 592.7817,-3101.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507475688 -->\n",
       "<g id=\"node53\" class=\"node\">\n",
       "<title>140254507475688</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"597.5,-3219 478.5,-3219 478.5,-3198 597.5,-3198 597.5,-3219\"/>\n",
       "<text text-anchor=\"middle\" x=\"538\" y=\"-3205.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507475688&#45;&gt;140254507475576 -->\n",
       "<g id=\"edge52\" class=\"edge\">\n",
       "<title>140254507475688&#45;&gt;140254507475576</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M538,-3197.7281C538,-3189.0091 538,-3176.4699 538,-3165.8068\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"541.5001,-3165.6128 538,-3155.6128 534.5001,-3165.6129 541.5001,-3165.6128\"/>\n",
       "</g>\n",
       "<!-- 140254507475800 -->\n",
       "<g id=\"node54\" class=\"node\">\n",
       "<title>140254507475800</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"686.5,-3276 533.5,-3276 533.5,-3255 686.5,-3255 686.5,-3276\"/>\n",
       "<text text-anchor=\"middle\" x=\"610\" y=\"-3262.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507475800&#45;&gt;140254507475688 -->\n",
       "<g id=\"edge53\" class=\"edge\">\n",
       "<title>140254507475800&#45;&gt;140254507475688</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M596.4574,-3254.7787C586.099,-3246.5784 571.6107,-3235.1085 559.6496,-3225.6393\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"561.4949,-3222.6361 551.4819,-3219.1732 557.1499,-3228.1244 561.4949,-3222.6361\"/>\n",
       "</g>\n",
       "<!-- 140254507475912 -->\n",
       "<g id=\"node55\" class=\"node\">\n",
       "<title>140254507475912</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"616.5,-3339.5 459.5,-3339.5 459.5,-3318.5 616.5,-3318.5 616.5,-3339.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"538\" y=\"-3325.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507475912&#45;&gt;140254507475800 -->\n",
       "<g id=\"edge54\" class=\"edge\">\n",
       "<title>140254507475912&#45;&gt;140254507475800</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M550.2138,-3318.2281C561.1405,-3308.5914 577.3588,-3294.2877 590.1477,-3283.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"592.7817,-3285.3523 597.9666,-3276.1128 588.1516,-3280.1024 592.7817,-3285.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507475464 -->\n",
       "<g id=\"node56\" class=\"node\">\n",
       "<title>140254507475464</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"543.5,-3403 424.5,-3403 424.5,-3382 543.5,-3382 543.5,-3403\"/>\n",
       "<text text-anchor=\"middle\" x=\"484\" y=\"-3389.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507475464&#45;&gt;140254507475296 -->\n",
       "<g id=\"edge180\" class=\"edge\">\n",
       "<title>140254507475464&#45;&gt;140254507475296</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M474.2577,-3381.8331C466.5876,-3372.885 456.1912,-3359.444 450,-3346 434.6232,-3312.6097 431,-3302.2608 431,-3265.5 431,-3265.5 431,-3265.5 431,-3145 431,-3106.4334 454.0294,-3066.352 469.7273,-3043.5108\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"472.7671,-3045.276 475.7188,-3035.1006 467.0659,-3041.2144 472.7671,-3045.276\"/>\n",
       "</g>\n",
       "<!-- 140254507475464&#45;&gt;140254507475912 -->\n",
       "<g id=\"edge55\" class=\"edge\">\n",
       "<title>140254507475464&#45;&gt;140254507475912</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M493.1604,-3381.7281C501.1212,-3372.3667 512.8273,-3358.6012 522.2825,-3347.4826\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"525.1629,-3349.4982 528.9749,-3339.6128 519.8304,-3344.9634 525.1629,-3349.4982\"/>\n",
       "</g>\n",
       "<!-- 140254507476080 -->\n",
       "<g id=\"node57\" class=\"node\">\n",
       "<title>140254507476080</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"530,-3460 438,-3460 438,-3439 530,-3439 530,-3460\"/>\n",
       "<text text-anchor=\"middle\" x=\"484\" y=\"-3446.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 140254507476080&#45;&gt;140254507475464 -->\n",
       "<g id=\"edge56\" class=\"edge\">\n",
       "<title>140254507476080&#45;&gt;140254507475464</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M484,-3438.7787C484,-3431.6134 484,-3421.9517 484,-3413.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"487.5001,-3413.1732 484,-3403.1732 480.5001,-3413.1732 487.5001,-3413.1732\"/>\n",
       "</g>\n",
       "<!-- 140254507476192 -->\n",
       "<g id=\"node58\" class=\"node\">\n",
       "<title>140254507476192</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"758.5,-3517 605.5,-3517 605.5,-3496 758.5,-3496 758.5,-3517\"/>\n",
       "<text text-anchor=\"middle\" x=\"682\" y=\"-3503.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507476192&#45;&gt;140254507476080 -->\n",
       "<g id=\"edge57\" class=\"edge\">\n",
       "<title>140254507476192&#45;&gt;140254507476080</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M645.2217,-3495.9123C612.853,-3486.5941 565.5606,-3472.9796 530.5606,-3462.9038\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"531.1593,-3459.4341 520.5813,-3460.031 529.2227,-3466.1609 531.1593,-3459.4341\"/>\n",
       "</g>\n",
       "<!-- 140254507476360 -->\n",
       "<g id=\"node59\" class=\"node\">\n",
       "<title>140254507476360</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"688.5,-3580.5 531.5,-3580.5 531.5,-3559.5 688.5,-3559.5 688.5,-3580.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"610\" y=\"-3566.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507476360&#45;&gt;140254507476192 -->\n",
       "<g id=\"edge58\" class=\"edge\">\n",
       "<title>140254507476360&#45;&gt;140254507476192</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M622.2138,-3559.2281C633.1405,-3549.5914 649.3588,-3535.2877 662.1477,-3524.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"664.7817,-3526.3523 669.9666,-3517.1128 660.1516,-3521.1024 664.7817,-3526.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507476472 -->\n",
       "<g id=\"node60\" class=\"node\">\n",
       "<title>140254507476472</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"687.5,-3644 568.5,-3644 568.5,-3623 687.5,-3623 687.5,-3644\"/>\n",
       "<text text-anchor=\"middle\" x=\"628\" y=\"-3630.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507476472&#45;&gt;140254507476360 -->\n",
       "<g id=\"edge59\" class=\"edge\">\n",
       "<title>140254507476472&#45;&gt;140254507476360</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M624.9465,-3622.7281C622.449,-3613.9174 618.8456,-3601.2055 615.8027,-3590.4708\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"619.103,-3589.2792 613.0084,-3580.6128 612.3683,-3591.1883 619.103,-3589.2792\"/>\n",
       "</g>\n",
       "<!-- 140254507476584 -->\n",
       "<g id=\"node61\" class=\"node\">\n",
       "<title>140254507476584</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"740.5,-3701 587.5,-3701 587.5,-3680 740.5,-3680 740.5,-3701\"/>\n",
       "<text text-anchor=\"middle\" x=\"664\" y=\"-3687.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507476584&#45;&gt;140254507476472 -->\n",
       "<g id=\"edge60\" class=\"edge\">\n",
       "<title>140254507476584&#45;&gt;140254507476472</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M657.2287,-3679.7787C652.4518,-3672.2153 645.9182,-3661.8705 640.2397,-3652.8794\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"643.0402,-3650.7591 634.741,-3644.1732 637.1217,-3654.4971 643.0402,-3650.7591\"/>\n",
       "</g>\n",
       "<!-- 140254507476696 -->\n",
       "<g id=\"node62\" class=\"node\">\n",
       "<title>140254507476696</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"616.5,-3764.5 459.5,-3764.5 459.5,-3743.5 616.5,-3743.5 616.5,-3764.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"538\" y=\"-3750.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507476696&#45;&gt;140254507476584 -->\n",
       "<g id=\"edge61\" class=\"edge\">\n",
       "<title>140254507476696&#45;&gt;140254507476584</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M559.0896,-3743.3715C579.5319,-3733.0692 610.7488,-3717.3369 633.8192,-3705.7101\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"635.4867,-3708.7892 642.8416,-3701.1631 632.3364,-3702.5381 635.4867,-3708.7892\"/>\n",
       "</g>\n",
       "<!-- 140254507476248 -->\n",
       "<g id=\"node63\" class=\"node\">\n",
       "<title>140254507476248</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"543.5,-3828 424.5,-3828 424.5,-3807 543.5,-3807 543.5,-3828\"/>\n",
       "<text text-anchor=\"middle\" x=\"484\" y=\"-3814.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507476248&#45;&gt;140254507476080 -->\n",
       "<g id=\"edge175\" class=\"edge\">\n",
       "<title>140254507476248&#45;&gt;140254507476080</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M474.2577,-3806.8331C466.5876,-3797.885 456.1912,-3784.444 450,-3771 434.6232,-3737.6097 431,-3727.2608 431,-3690.5 431,-3690.5 431,-3690.5 431,-3570 431,-3531.4334 454.0294,-3491.352 469.7273,-3468.5108\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"472.7671,-3470.276 475.7188,-3460.1006 467.0659,-3466.2144 472.7671,-3470.276\"/>\n",
       "</g>\n",
       "<!-- 140254507476248&#45;&gt;140254507476696 -->\n",
       "<g id=\"edge62\" class=\"edge\">\n",
       "<title>140254507476248&#45;&gt;140254507476696</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M493.1604,-3806.7281C501.1212,-3797.3667 512.8273,-3783.6012 522.2825,-3772.4826\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"525.1629,-3774.4982 528.9749,-3764.6128 519.8304,-3769.9634 525.1629,-3774.4982\"/>\n",
       "</g>\n",
       "<!-- 140254507476864 -->\n",
       "<g id=\"node64\" class=\"node\">\n",
       "<title>140254507476864</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"530,-3885 438,-3885 438,-3864 530,-3864 530,-3885\"/>\n",
       "<text text-anchor=\"middle\" x=\"484\" y=\"-3871.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 140254507476864&#45;&gt;140254507476248 -->\n",
       "<g id=\"edge63\" class=\"edge\">\n",
       "<title>140254507476864&#45;&gt;140254507476248</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M484,-3863.7787C484,-3856.6134 484,-3846.9517 484,-3838.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"487.5001,-3838.1732 484,-3828.1732 480.5001,-3838.1732 487.5001,-3838.1732\"/>\n",
       "</g>\n",
       "<!-- 140254507476976 -->\n",
       "<g id=\"node65\" class=\"node\">\n",
       "<title>140254507476976</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"723.5,-3942 570.5,-3942 570.5,-3921 723.5,-3921 723.5,-3942\"/>\n",
       "<text text-anchor=\"middle\" x=\"647\" y=\"-3928.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507476976&#45;&gt;140254507476864 -->\n",
       "<g id=\"edge64\" class=\"edge\">\n",
       "<title>140254507476976&#45;&gt;140254507476864</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M616.723,-3920.9123C590.5334,-3911.754 552.4765,-3898.4458 523.823,-3888.4258\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"524.7097,-3885.0281 514.1149,-3885.031 522.399,-3891.6358 524.7097,-3885.0281\"/>\n",
       "</g>\n",
       "<!-- 140254507477144 -->\n",
       "<g id=\"node66\" class=\"node\">\n",
       "<title>140254507477144</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"653.5,-4005.5 496.5,-4005.5 496.5,-3984.5 653.5,-3984.5 653.5,-4005.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"575\" y=\"-3991.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507477144&#45;&gt;140254507476976 -->\n",
       "<g id=\"edge65\" class=\"edge\">\n",
       "<title>140254507477144&#45;&gt;140254507476976</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M587.2138,-3984.2281C598.1405,-3974.5914 614.3588,-3960.2877 627.1477,-3949.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"629.7817,-3951.3523 634.9666,-3942.1128 625.1516,-3946.1024 629.7817,-3951.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507477256 -->\n",
       "<g id=\"node67\" class=\"node\">\n",
       "<title>140254507477256</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"653.5,-4069 534.5,-4069 534.5,-4048 653.5,-4048 653.5,-4069\"/>\n",
       "<text text-anchor=\"middle\" x=\"594\" y=\"-4055.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507477256&#45;&gt;140254507477144 -->\n",
       "<g id=\"edge66\" class=\"edge\">\n",
       "<title>140254507477256&#45;&gt;140254507477144</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M590.7769,-4047.7281C588.1406,-4038.9174 584.3371,-4026.2055 581.1251,-4015.4708\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"584.3952,-4014.1898 578.1755,-4005.6128 577.689,-4016.1965 584.3952,-4014.1898\"/>\n",
       "</g>\n",
       "<!-- 140254507477368 -->\n",
       "<g id=\"node68\" class=\"node\">\n",
       "<title>140254507477368</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"761.5,-4126 608.5,-4126 608.5,-4105 761.5,-4105 761.5,-4126\"/>\n",
       "<text text-anchor=\"middle\" x=\"685\" y=\"-4112.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507477368&#45;&gt;140254507477256 -->\n",
       "<g id=\"edge67\" class=\"edge\">\n",
       "<title>140254507477368&#45;&gt;140254507477256</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M667.8836,-4104.7787C654.2834,-4096.26 635.0501,-4084.2127 619.6164,-4074.5454\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"621.3724,-4071.5154 611.0397,-4069.1732 617.6565,-4077.4478 621.3724,-4071.5154\"/>\n",
       "</g>\n",
       "<!-- 140254507477480 -->\n",
       "<g id=\"node69\" class=\"node\">\n",
       "<title>140254507477480</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"691.5,-4189.5 534.5,-4189.5 534.5,-4168.5 691.5,-4168.5 691.5,-4189.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"613\" y=\"-4175.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507477480&#45;&gt;140254507477368 -->\n",
       "<g id=\"edge68\" class=\"edge\">\n",
       "<title>140254507477480&#45;&gt;140254507477368</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M625.2138,-4168.2281C636.1405,-4158.5914 652.3588,-4144.2877 665.1477,-4133.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"667.7817,-4135.3523 672.9666,-4126.1128 663.1516,-4130.1024 667.7817,-4135.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507477592 -->\n",
       "<g id=\"node70\" class=\"node\">\n",
       "<title>140254507477592</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"584.5,-4253 465.5,-4253 465.5,-4232 584.5,-4232 584.5,-4253\"/>\n",
       "<text text-anchor=\"middle\" x=\"525\" y=\"-4239.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507477592&#45;&gt;140254507477480 -->\n",
       "<g id=\"edge69\" class=\"edge\">\n",
       "<title>140254507477592&#45;&gt;140254507477480</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M539.928,-4231.7281C553.6644,-4221.816 574.2432,-4206.9665 590.0654,-4195.5494\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"592.2313,-4198.3026 598.2925,-4189.6128 588.1352,-4192.6262 592.2313,-4198.3026\"/>\n",
       "</g>\n",
       "<!-- 140254507477200 -->\n",
       "<g id=\"node161\" class=\"node\">\n",
       "<title>140254507477200</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"516.5,-4189.5 359.5,-4189.5 359.5,-4168.5 516.5,-4168.5 516.5,-4189.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"438\" y=\"-4175.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507477592&#45;&gt;140254507477200 -->\n",
       "<g id=\"edge168\" class=\"edge\">\n",
       "<title>140254507477592&#45;&gt;140254507477200</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M510.2416,-4231.7281C496.6613,-4221.816 476.3163,-4206.9665 460.674,-4195.5494\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"462.6812,-4192.6813 452.5404,-4189.6128 458.5543,-4198.3354 462.6812,-4192.6813\"/>\n",
       "</g>\n",
       "<!-- 140254507477704 -->\n",
       "<g id=\"node71\" class=\"node\">\n",
       "<title>140254507477704</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"571,-4310 479,-4310 479,-4289 571,-4289 571,-4310\"/>\n",
       "<text text-anchor=\"middle\" x=\"525\" y=\"-4296.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 140254507477704&#45;&gt;140254507477592 -->\n",
       "<g id=\"edge70\" class=\"edge\">\n",
       "<title>140254507477704&#45;&gt;140254507477592</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M525,-4288.7787C525,-4281.6134 525,-4271.9517 525,-4263.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"528.5001,-4263.1732 525,-4253.1732 521.5001,-4263.1732 528.5001,-4263.1732\"/>\n",
       "</g>\n",
       "<!-- 140254507477816 -->\n",
       "<g id=\"node72\" class=\"node\">\n",
       "<title>140254507477816</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"727.5,-4367 574.5,-4367 574.5,-4346 727.5,-4346 727.5,-4367\"/>\n",
       "<text text-anchor=\"middle\" x=\"651\" y=\"-4353.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507477816&#45;&gt;140254507477704 -->\n",
       "<g id=\"edge71\" class=\"edge\">\n",
       "<title>140254507477816&#45;&gt;140254507477704</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M627.5957,-4345.9123C607.8814,-4336.994 579.468,-4324.1403 557.5391,-4314.2201\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"558.8327,-4310.9638 548.279,-4310.031 555.9475,-4317.3416 558.8327,-4310.9638\"/>\n",
       "</g>\n",
       "<!-- 140254507477984 -->\n",
       "<g id=\"node73\" class=\"node\">\n",
       "<title>140254507477984</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"657.5,-4430.5 500.5,-4430.5 500.5,-4409.5 657.5,-4409.5 657.5,-4430.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"579\" y=\"-4416.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507477984&#45;&gt;140254507477816 -->\n",
       "<g id=\"edge72\" class=\"edge\">\n",
       "<title>140254507477984&#45;&gt;140254507477816</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M591.2138,-4409.2281C602.1405,-4399.5914 618.3588,-4385.2877 631.1477,-4374.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"633.7817,-4376.3523 638.9666,-4367.1128 629.1516,-4371.1024 633.7817,-4376.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507478096 -->\n",
       "<g id=\"node74\" class=\"node\">\n",
       "<title>140254507478096</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"638.5,-4494 519.5,-4494 519.5,-4473 638.5,-4473 638.5,-4494\"/>\n",
       "<text text-anchor=\"middle\" x=\"579\" y=\"-4480.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507478096&#45;&gt;140254507477984 -->\n",
       "<g id=\"edge73\" class=\"edge\">\n",
       "<title>140254507478096&#45;&gt;140254507477984</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M579,-4472.7281C579,-4464.0091 579,-4451.4699 579,-4440.8068\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"582.5001,-4440.6128 579,-4430.6128 575.5001,-4440.6129 582.5001,-4440.6128\"/>\n",
       "</g>\n",
       "<!-- 140254507478208 -->\n",
       "<g id=\"node75\" class=\"node\">\n",
       "<title>140254507478208</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"727.5,-4551 574.5,-4551 574.5,-4530 727.5,-4530 727.5,-4551\"/>\n",
       "<text text-anchor=\"middle\" x=\"651\" y=\"-4537.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507478208&#45;&gt;140254507478096 -->\n",
       "<g id=\"edge74\" class=\"edge\">\n",
       "<title>140254507478208&#45;&gt;140254507478096</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M637.4574,-4529.7787C627.099,-4521.5784 612.6107,-4510.1085 600.6496,-4500.6393\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"602.4949,-4497.6361 592.4819,-4494.1732 598.1499,-4503.1244 602.4949,-4497.6361\"/>\n",
       "</g>\n",
       "<!-- 140254507478320 -->\n",
       "<g id=\"node76\" class=\"node\">\n",
       "<title>140254507478320</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"657.5,-4614.5 500.5,-4614.5 500.5,-4593.5 657.5,-4593.5 657.5,-4614.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"579\" y=\"-4600.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507478320&#45;&gt;140254507478208 -->\n",
       "<g id=\"edge75\" class=\"edge\">\n",
       "<title>140254507478320&#45;&gt;140254507478208</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M591.2138,-4593.2281C602.1405,-4583.5914 618.3588,-4569.2877 631.1477,-4558.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"633.7817,-4560.3523 638.9666,-4551.1128 629.1516,-4555.1024 633.7817,-4560.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507477872 -->\n",
       "<g id=\"node77\" class=\"node\">\n",
       "<title>140254507477872</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"584.5,-4678 465.5,-4678 465.5,-4657 584.5,-4657 584.5,-4678\"/>\n",
       "<text text-anchor=\"middle\" x=\"525\" y=\"-4664.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507477872&#45;&gt;140254507477704 -->\n",
       "<g id=\"edge161\" class=\"edge\">\n",
       "<title>140254507477872&#45;&gt;140254507477704</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M515.2577,-4656.8331C507.5876,-4647.885 497.1912,-4634.444 491,-4621 475.6232,-4587.6097 472,-4577.2608 472,-4540.5 472,-4540.5 472,-4540.5 472,-4420 472,-4381.4334 495.0294,-4341.352 510.7273,-4318.5108\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"513.7671,-4320.276 516.7188,-4310.1006 508.0659,-4316.2144 513.7671,-4320.276\"/>\n",
       "</g>\n",
       "<!-- 140254507477872&#45;&gt;140254507478320 -->\n",
       "<g id=\"edge76\" class=\"edge\">\n",
       "<title>140254507477872&#45;&gt;140254507478320</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M534.1604,-4656.7281C542.1212,-4647.3667 553.8273,-4633.6012 563.2825,-4622.4826\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"566.1629,-4624.4982 569.9749,-4614.6128 560.8304,-4619.9634 566.1629,-4624.4982\"/>\n",
       "</g>\n",
       "<!-- 140254507478488 -->\n",
       "<g id=\"node78\" class=\"node\">\n",
       "<title>140254507478488</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"571,-4735 479,-4735 479,-4714 571,-4714 571,-4735\"/>\n",
       "<text text-anchor=\"middle\" x=\"525\" y=\"-4721.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 140254507478488&#45;&gt;140254507477872 -->\n",
       "<g id=\"edge77\" class=\"edge\">\n",
       "<title>140254507478488&#45;&gt;140254507477872</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M525,-4713.7787C525,-4706.6134 525,-4696.9517 525,-4688.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"528.5001,-4688.1732 525,-4678.1732 521.5001,-4688.1732 528.5001,-4688.1732\"/>\n",
       "</g>\n",
       "<!-- 140254507478600 -->\n",
       "<g id=\"node79\" class=\"node\">\n",
       "<title>140254507478600</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"799.5,-4792 646.5,-4792 646.5,-4771 799.5,-4771 799.5,-4792\"/>\n",
       "<text text-anchor=\"middle\" x=\"723\" y=\"-4778.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507478600&#45;&gt;140254507478488 -->\n",
       "<g id=\"edge78\" class=\"edge\">\n",
       "<title>140254507478600&#45;&gt;140254507478488</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M686.2217,-4770.9123C653.853,-4761.5941 606.5606,-4747.9796 571.5606,-4737.9038\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"572.1593,-4734.4341 561.5813,-4735.031 570.2227,-4741.1609 572.1593,-4734.4341\"/>\n",
       "</g>\n",
       "<!-- 140254507478768 -->\n",
       "<g id=\"node80\" class=\"node\">\n",
       "<title>140254507478768</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"729.5,-4855.5 572.5,-4855.5 572.5,-4834.5 729.5,-4834.5 729.5,-4855.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"651\" y=\"-4841.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507478768&#45;&gt;140254507478600 -->\n",
       "<g id=\"edge79\" class=\"edge\">\n",
       "<title>140254507478768&#45;&gt;140254507478600</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M663.2138,-4834.2281C674.1405,-4824.5914 690.3588,-4810.2877 703.1477,-4799.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"705.7817,-4801.3523 710.9666,-4792.1128 701.1516,-4796.1024 705.7817,-4801.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507478880 -->\n",
       "<g id=\"node81\" class=\"node\">\n",
       "<title>140254507478880</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"728.5,-4919 609.5,-4919 609.5,-4898 728.5,-4898 728.5,-4919\"/>\n",
       "<text text-anchor=\"middle\" x=\"669\" y=\"-4905.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507478880&#45;&gt;140254507478768 -->\n",
       "<g id=\"edge80\" class=\"edge\">\n",
       "<title>140254507478880&#45;&gt;140254507478768</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M665.9465,-4897.7281C663.449,-4888.9174 659.8456,-4876.2055 656.8027,-4865.4708\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"660.103,-4864.2792 654.0084,-4855.6128 653.3683,-4866.1883 660.103,-4864.2792\"/>\n",
       "</g>\n",
       "<!-- 140254507478992 -->\n",
       "<g id=\"node82\" class=\"node\">\n",
       "<title>140254507478992</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"781.5,-4976 628.5,-4976 628.5,-4955 781.5,-4955 781.5,-4976\"/>\n",
       "<text text-anchor=\"middle\" x=\"705\" y=\"-4962.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507478992&#45;&gt;140254507478880 -->\n",
       "<g id=\"edge81\" class=\"edge\">\n",
       "<title>140254507478992&#45;&gt;140254507478880</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M698.2287,-4954.7787C693.4518,-4947.2153 686.9182,-4936.8705 681.2397,-4927.8794\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"684.0402,-4925.7591 675.741,-4919.1732 678.1217,-4929.4971 684.0402,-4925.7591\"/>\n",
       "</g>\n",
       "<!-- 140254507446400 -->\n",
       "<g id=\"node83\" class=\"node\">\n",
       "<title>140254507446400</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"657.5,-5039.5 500.5,-5039.5 500.5,-5018.5 657.5,-5018.5 657.5,-5039.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"579\" y=\"-5025.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507446400&#45;&gt;140254507478992 -->\n",
       "<g id=\"edge82\" class=\"edge\">\n",
       "<title>140254507446400&#45;&gt;140254507478992</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M600.0896,-5018.3715C620.5319,-5008.0692 651.7488,-4992.3369 674.8192,-4980.7101\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"676.4867,-4983.7892 683.8416,-4976.1631 673.3364,-4977.5381 676.4867,-4983.7892\"/>\n",
       "</g>\n",
       "<!-- 140254507478656 -->\n",
       "<g id=\"node84\" class=\"node\">\n",
       "<title>140254507478656</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"584.5,-5103 465.5,-5103 465.5,-5082 584.5,-5082 584.5,-5103\"/>\n",
       "<text text-anchor=\"middle\" x=\"525\" y=\"-5089.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507478656&#45;&gt;140254507478488 -->\n",
       "<g id=\"edge156\" class=\"edge\">\n",
       "<title>140254507478656&#45;&gt;140254507478488</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M515.2577,-5081.8331C507.5876,-5072.885 497.1912,-5059.444 491,-5046 475.6232,-5012.6097 472,-5002.2608 472,-4965.5 472,-4965.5 472,-4965.5 472,-4845 472,-4806.4334 495.0294,-4766.352 510.7273,-4743.5108\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"513.7671,-4745.276 516.7188,-4735.1006 508.0659,-4741.2144 513.7671,-4745.276\"/>\n",
       "</g>\n",
       "<!-- 140254507478656&#45;&gt;140254507446400 -->\n",
       "<g id=\"edge83\" class=\"edge\">\n",
       "<title>140254507478656&#45;&gt;140254507446400</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M534.1604,-5081.7281C542.1212,-5072.3667 553.8273,-5058.6012 563.2825,-5047.4826\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"566.1629,-5049.4982 569.9749,-5039.6128 560.8304,-5044.9634 566.1629,-5049.4982\"/>\n",
       "</g>\n",
       "<!-- 140254507446568 -->\n",
       "<g id=\"node85\" class=\"node\">\n",
       "<title>140254507446568</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"571,-5160 479,-5160 479,-5139 571,-5139 571,-5160\"/>\n",
       "<text text-anchor=\"middle\" x=\"525\" y=\"-5146.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 140254507446568&#45;&gt;140254507478656 -->\n",
       "<g id=\"edge84\" class=\"edge\">\n",
       "<title>140254507446568&#45;&gt;140254507478656</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M525,-5138.7787C525,-5131.6134 525,-5121.9517 525,-5113.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"528.5001,-5113.1732 525,-5103.1732 521.5001,-5113.1732 528.5001,-5113.1732\"/>\n",
       "</g>\n",
       "<!-- 140254507446680 -->\n",
       "<g id=\"node86\" class=\"node\">\n",
       "<title>140254507446680</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"727.5,-5217 574.5,-5217 574.5,-5196 727.5,-5196 727.5,-5217\"/>\n",
       "<text text-anchor=\"middle\" x=\"651\" y=\"-5203.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507446680&#45;&gt;140254507446568 -->\n",
       "<g id=\"edge85\" class=\"edge\">\n",
       "<title>140254507446680&#45;&gt;140254507446568</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M627.5957,-5195.9123C607.8814,-5186.994 579.468,-5174.1403 557.5391,-5164.2201\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"558.8327,-5160.9638 548.279,-5160.031 555.9475,-5167.3416 558.8327,-5160.9638\"/>\n",
       "</g>\n",
       "<!-- 140254507446848 -->\n",
       "<g id=\"node87\" class=\"node\">\n",
       "<title>140254507446848</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"657.5,-5280.5 500.5,-5280.5 500.5,-5259.5 657.5,-5259.5 657.5,-5280.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"579\" y=\"-5266.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507446848&#45;&gt;140254507446680 -->\n",
       "<g id=\"edge86\" class=\"edge\">\n",
       "<title>140254507446848&#45;&gt;140254507446680</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M591.2138,-5259.2281C602.1405,-5249.5914 618.3588,-5235.2877 631.1477,-5224.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"633.7817,-5226.3523 638.9666,-5217.1128 629.1516,-5221.1024 633.7817,-5226.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507446960 -->\n",
       "<g id=\"node88\" class=\"node\">\n",
       "<title>140254507446960</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"638.5,-5344 519.5,-5344 519.5,-5323 638.5,-5323 638.5,-5344\"/>\n",
       "<text text-anchor=\"middle\" x=\"579\" y=\"-5330.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507446960&#45;&gt;140254507446848 -->\n",
       "<g id=\"edge87\" class=\"edge\">\n",
       "<title>140254507446960&#45;&gt;140254507446848</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M579,-5322.7281C579,-5314.0091 579,-5301.4699 579,-5290.8068\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"582.5001,-5290.6128 579,-5280.6128 575.5001,-5290.6129 582.5001,-5290.6128\"/>\n",
       "</g>\n",
       "<!-- 140254507447072 -->\n",
       "<g id=\"node89\" class=\"node\">\n",
       "<title>140254507447072</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"727.5,-5401 574.5,-5401 574.5,-5380 727.5,-5380 727.5,-5401\"/>\n",
       "<text text-anchor=\"middle\" x=\"651\" y=\"-5387.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507447072&#45;&gt;140254507446960 -->\n",
       "<g id=\"edge88\" class=\"edge\">\n",
       "<title>140254507447072&#45;&gt;140254507446960</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M637.4574,-5379.7787C627.099,-5371.5784 612.6107,-5360.1085 600.6496,-5350.6393\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"602.4949,-5347.6361 592.4819,-5344.1732 598.1499,-5353.1244 602.4949,-5347.6361\"/>\n",
       "</g>\n",
       "<!-- 140254507447184 -->\n",
       "<g id=\"node90\" class=\"node\">\n",
       "<title>140254507447184</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"657.5,-5464.5 500.5,-5464.5 500.5,-5443.5 657.5,-5443.5 657.5,-5464.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"579\" y=\"-5450.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507447184&#45;&gt;140254507447072 -->\n",
       "<g id=\"edge89\" class=\"edge\">\n",
       "<title>140254507447184&#45;&gt;140254507447072</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M591.2138,-5443.2281C602.1405,-5433.5914 618.3588,-5419.2877 631.1477,-5408.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"633.7817,-5410.3523 638.9666,-5401.1128 629.1516,-5405.1024 633.7817,-5410.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507446736 -->\n",
       "<g id=\"node91\" class=\"node\">\n",
       "<title>140254507446736</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"584.5,-5528 465.5,-5528 465.5,-5507 584.5,-5507 584.5,-5528\"/>\n",
       "<text text-anchor=\"middle\" x=\"525\" y=\"-5514.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507446736&#45;&gt;140254507446568 -->\n",
       "<g id=\"edge151\" class=\"edge\">\n",
       "<title>140254507446736&#45;&gt;140254507446568</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M515.2577,-5506.8331C507.5876,-5497.885 497.1912,-5484.444 491,-5471 475.6232,-5437.6097 472,-5427.2608 472,-5390.5 472,-5390.5 472,-5390.5 472,-5270 472,-5231.4334 495.0294,-5191.352 510.7273,-5168.5108\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"513.7671,-5170.276 516.7188,-5160.1006 508.0659,-5166.2144 513.7671,-5170.276\"/>\n",
       "</g>\n",
       "<!-- 140254507446736&#45;&gt;140254507447184 -->\n",
       "<g id=\"edge90\" class=\"edge\">\n",
       "<title>140254507446736&#45;&gt;140254507447184</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M534.1604,-5506.7281C542.1212,-5497.3667 553.8273,-5483.6012 563.2825,-5472.4826\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"566.1629,-5474.4982 569.9749,-5464.6128 560.8304,-5469.9634 566.1629,-5474.4982\"/>\n",
       "</g>\n",
       "<!-- 140254507447352 -->\n",
       "<g id=\"node92\" class=\"node\">\n",
       "<title>140254507447352</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"571,-5585 479,-5585 479,-5564 571,-5564 571,-5585\"/>\n",
       "<text text-anchor=\"middle\" x=\"525\" y=\"-5571.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 140254507447352&#45;&gt;140254507446736 -->\n",
       "<g id=\"edge91\" class=\"edge\">\n",
       "<title>140254507447352&#45;&gt;140254507446736</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M525,-5563.7787C525,-5556.6134 525,-5546.9517 525,-5538.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"528.5001,-5538.1732 525,-5528.1732 521.5001,-5538.1732 528.5001,-5538.1732\"/>\n",
       "</g>\n",
       "<!-- 140254507447464 -->\n",
       "<g id=\"node93\" class=\"node\">\n",
       "<title>140254507447464</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"764.5,-5642 611.5,-5642 611.5,-5621 764.5,-5621 764.5,-5642\"/>\n",
       "<text text-anchor=\"middle\" x=\"688\" y=\"-5628.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507447464&#45;&gt;140254507447352 -->\n",
       "<g id=\"edge92\" class=\"edge\">\n",
       "<title>140254507447464&#45;&gt;140254507447352</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M657.723,-5620.9123C631.5334,-5611.754 593.4765,-5598.4458 564.823,-5588.4258\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"565.7097,-5585.0281 555.1149,-5585.031 563.399,-5591.6358 565.7097,-5585.0281\"/>\n",
       "</g>\n",
       "<!-- 140254507447632 -->\n",
       "<g id=\"node94\" class=\"node\">\n",
       "<title>140254507447632</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"694.5,-5705.5 537.5,-5705.5 537.5,-5684.5 694.5,-5684.5 694.5,-5705.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"616\" y=\"-5691.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507447632&#45;&gt;140254507447464 -->\n",
       "<g id=\"edge93\" class=\"edge\">\n",
       "<title>140254507447632&#45;&gt;140254507447464</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M628.2138,-5684.2281C639.1405,-5674.5914 655.3588,-5660.2877 668.1477,-5649.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"670.7817,-5651.3523 675.9666,-5642.1128 666.1516,-5646.1024 670.7817,-5651.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507447744 -->\n",
       "<g id=\"node95\" class=\"node\">\n",
       "<title>140254507447744</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"694.5,-5769 575.5,-5769 575.5,-5748 694.5,-5748 694.5,-5769\"/>\n",
       "<text text-anchor=\"middle\" x=\"635\" y=\"-5755.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507447744&#45;&gt;140254507447632 -->\n",
       "<g id=\"edge94\" class=\"edge\">\n",
       "<title>140254507447744&#45;&gt;140254507447632</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M631.7769,-5747.7281C629.1406,-5738.9174 625.3371,-5726.2055 622.1251,-5715.4708\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"625.3952,-5714.1898 619.1755,-5705.6128 618.689,-5716.1965 625.3952,-5714.1898\"/>\n",
       "</g>\n",
       "<!-- 140254507447856 -->\n",
       "<g id=\"node96\" class=\"node\">\n",
       "<title>140254507447856</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"802.5,-5826 649.5,-5826 649.5,-5805 802.5,-5805 802.5,-5826\"/>\n",
       "<text text-anchor=\"middle\" x=\"726\" y=\"-5812.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507447856&#45;&gt;140254507447744 -->\n",
       "<g id=\"edge95\" class=\"edge\">\n",
       "<title>140254507447856&#45;&gt;140254507447744</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M708.8836,-5804.7787C695.2834,-5796.26 676.0501,-5784.2127 660.6164,-5774.5454\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"662.3724,-5771.5154 652.0397,-5769.1732 658.6565,-5777.4478 662.3724,-5771.5154\"/>\n",
       "</g>\n",
       "<!-- 140254507447968 -->\n",
       "<g id=\"node97\" class=\"node\">\n",
       "<title>140254507447968</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"732.5,-5889.5 575.5,-5889.5 575.5,-5868.5 732.5,-5868.5 732.5,-5889.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"654\" y=\"-5875.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507447968&#45;&gt;140254507447856 -->\n",
       "<g id=\"edge96\" class=\"edge\">\n",
       "<title>140254507447968&#45;&gt;140254507447856</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M666.2138,-5868.2281C677.1405,-5858.5914 693.3588,-5844.2877 706.1477,-5833.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"708.7817,-5835.3523 713.9666,-5826.1128 704.1516,-5830.1024 708.7817,-5835.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507448080 -->\n",
       "<g id=\"node98\" class=\"node\">\n",
       "<title>140254507448080</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"625.5,-5953 506.5,-5953 506.5,-5932 625.5,-5932 625.5,-5953\"/>\n",
       "<text text-anchor=\"middle\" x=\"566\" y=\"-5939.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507448080&#45;&gt;140254507447968 -->\n",
       "<g id=\"edge97\" class=\"edge\">\n",
       "<title>140254507448080&#45;&gt;140254507447968</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M580.928,-5931.7281C594.6644,-5921.816 615.2432,-5906.9665 631.0654,-5895.5494\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"633.2313,-5898.3026 639.2925,-5889.6128 629.1352,-5892.6262 633.2313,-5898.3026\"/>\n",
       "</g>\n",
       "<!-- 140254507447688 -->\n",
       "<g id=\"node141\" class=\"node\">\n",
       "<title>140254507447688</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"557.5,-5889.5 400.5,-5889.5 400.5,-5868.5 557.5,-5868.5 557.5,-5889.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"479\" y=\"-5875.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507448080&#45;&gt;140254507447688 -->\n",
       "<g id=\"edge144\" class=\"edge\">\n",
       "<title>140254507448080&#45;&gt;140254507447688</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M551.2416,-5931.7281C537.6613,-5921.816 517.3163,-5906.9665 501.674,-5895.5494\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"503.6812,-5892.6813 493.5404,-5889.6128 499.5543,-5898.3354 503.6812,-5892.6813\"/>\n",
       "</g>\n",
       "<!-- 140254507448192 -->\n",
       "<g id=\"node99\" class=\"node\">\n",
       "<title>140254507448192</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"612,-6010 520,-6010 520,-5989 612,-5989 612,-6010\"/>\n",
       "<text text-anchor=\"middle\" x=\"566\" y=\"-5996.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 140254507448192&#45;&gt;140254507448080 -->\n",
       "<g id=\"edge98\" class=\"edge\">\n",
       "<title>140254507448192&#45;&gt;140254507448080</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M566,-5988.7787C566,-5981.6134 566,-5971.9517 566,-5963.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"569.5001,-5963.1732 566,-5953.1732 562.5001,-5963.1732 569.5001,-5963.1732\"/>\n",
       "</g>\n",
       "<!-- 140254507448304 -->\n",
       "<g id=\"node100\" class=\"node\">\n",
       "<title>140254507448304</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"768.5,-6067 615.5,-6067 615.5,-6046 768.5,-6046 768.5,-6067\"/>\n",
       "<text text-anchor=\"middle\" x=\"692\" y=\"-6053.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507448304&#45;&gt;140254507448192 -->\n",
       "<g id=\"edge99\" class=\"edge\">\n",
       "<title>140254507448304&#45;&gt;140254507448192</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M668.5957,-6045.9123C648.8814,-6036.994 620.468,-6024.1403 598.5391,-6014.2201\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"599.8327,-6010.9638 589.279,-6010.031 596.9475,-6017.3416 599.8327,-6010.9638\"/>\n",
       "</g>\n",
       "<!-- 140254507448472 -->\n",
       "<g id=\"node101\" class=\"node\">\n",
       "<title>140254507448472</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"698.5,-6130.5 541.5,-6130.5 541.5,-6109.5 698.5,-6109.5 698.5,-6130.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"620\" y=\"-6116.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507448472&#45;&gt;140254507448304 -->\n",
       "<g id=\"edge100\" class=\"edge\">\n",
       "<title>140254507448472&#45;&gt;140254507448304</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M632.2138,-6109.2281C643.1405,-6099.5914 659.3588,-6085.2877 672.1477,-6074.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"674.7817,-6076.3523 679.9666,-6067.1128 670.1516,-6071.1024 674.7817,-6076.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507448584 -->\n",
       "<g id=\"node102\" class=\"node\">\n",
       "<title>140254507448584</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"679.5,-6194 560.5,-6194 560.5,-6173 679.5,-6173 679.5,-6194\"/>\n",
       "<text text-anchor=\"middle\" x=\"620\" y=\"-6180.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507448584&#45;&gt;140254507448472 -->\n",
       "<g id=\"edge101\" class=\"edge\">\n",
       "<title>140254507448584&#45;&gt;140254507448472</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M620,-6172.7281C620,-6164.0091 620,-6151.4699 620,-6140.8068\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"623.5001,-6140.6128 620,-6130.6128 616.5001,-6140.6129 623.5001,-6140.6128\"/>\n",
       "</g>\n",
       "<!-- 140254507448696 -->\n",
       "<g id=\"node103\" class=\"node\">\n",
       "<title>140254507448696</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"768.5,-6251 615.5,-6251 615.5,-6230 768.5,-6230 768.5,-6251\"/>\n",
       "<text text-anchor=\"middle\" x=\"692\" y=\"-6237.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507448696&#45;&gt;140254507448584 -->\n",
       "<g id=\"edge102\" class=\"edge\">\n",
       "<title>140254507448696&#45;&gt;140254507448584</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M678.4574,-6229.7787C668.099,-6221.5784 653.6107,-6210.1085 641.6496,-6200.6393\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"643.4949,-6197.6361 633.4819,-6194.1732 639.1499,-6203.1244 643.4949,-6197.6361\"/>\n",
       "</g>\n",
       "<!-- 140254507448808 -->\n",
       "<g id=\"node104\" class=\"node\">\n",
       "<title>140254507448808</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"698.5,-6314.5 541.5,-6314.5 541.5,-6293.5 698.5,-6293.5 698.5,-6314.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"620\" y=\"-6300.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507448808&#45;&gt;140254507448696 -->\n",
       "<g id=\"edge103\" class=\"edge\">\n",
       "<title>140254507448808&#45;&gt;140254507448696</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M632.2138,-6293.2281C643.1405,-6283.5914 659.3588,-6269.2877 672.1477,-6258.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"674.7817,-6260.3523 679.9666,-6251.1128 670.1516,-6255.1024 674.7817,-6260.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507448360 -->\n",
       "<g id=\"node105\" class=\"node\">\n",
       "<title>140254507448360</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"625.5,-6378 506.5,-6378 506.5,-6357 625.5,-6357 625.5,-6378\"/>\n",
       "<text text-anchor=\"middle\" x=\"566\" y=\"-6364.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507448360&#45;&gt;140254507448192 -->\n",
       "<g id=\"edge137\" class=\"edge\">\n",
       "<title>140254507448360&#45;&gt;140254507448192</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M556.2577,-6356.8331C548.5876,-6347.885 538.1912,-6334.444 532,-6321 516.6232,-6287.6097 513,-6277.2608 513,-6240.5 513,-6240.5 513,-6240.5 513,-6120 513,-6081.4334 536.0294,-6041.352 551.7273,-6018.5108\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"554.7671,-6020.276 557.7188,-6010.1006 549.0659,-6016.2144 554.7671,-6020.276\"/>\n",
       "</g>\n",
       "<!-- 140254507448360&#45;&gt;140254507448808 -->\n",
       "<g id=\"edge104\" class=\"edge\">\n",
       "<title>140254507448360&#45;&gt;140254507448808</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M575.1604,-6356.7281C583.1212,-6347.3667 594.8273,-6333.6012 604.2825,-6322.4826\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"607.1629,-6324.4982 610.9749,-6314.6128 601.8304,-6319.9634 607.1629,-6324.4982\"/>\n",
       "</g>\n",
       "<!-- 140254507448976 -->\n",
       "<g id=\"node106\" class=\"node\">\n",
       "<title>140254507448976</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"612,-6435 520,-6435 520,-6414 612,-6414 612,-6435\"/>\n",
       "<text text-anchor=\"middle\" x=\"566\" y=\"-6421.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 140254507448976&#45;&gt;140254507448360 -->\n",
       "<g id=\"edge105\" class=\"edge\">\n",
       "<title>140254507448976&#45;&gt;140254507448360</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M566,-6413.7787C566,-6406.6134 566,-6396.9517 566,-6388.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"569.5001,-6388.1732 566,-6378.1732 562.5001,-6388.1732 569.5001,-6388.1732\"/>\n",
       "</g>\n",
       "<!-- 140254507449088 -->\n",
       "<g id=\"node107\" class=\"node\">\n",
       "<title>140254507449088</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"840.5,-6492 687.5,-6492 687.5,-6471 840.5,-6471 840.5,-6492\"/>\n",
       "<text text-anchor=\"middle\" x=\"764\" y=\"-6478.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507449088&#45;&gt;140254507448976 -->\n",
       "<g id=\"edge106\" class=\"edge\">\n",
       "<title>140254507449088&#45;&gt;140254507448976</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M727.2217,-6470.9123C694.853,-6461.5941 647.5606,-6447.9796 612.5606,-6437.9038\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"613.1593,-6434.4341 602.5813,-6435.031 611.2227,-6441.1609 613.1593,-6434.4341\"/>\n",
       "</g>\n",
       "<!-- 140254507449256 -->\n",
       "<g id=\"node108\" class=\"node\">\n",
       "<title>140254507449256</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"770.5,-6555.5 613.5,-6555.5 613.5,-6534.5 770.5,-6534.5 770.5,-6555.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"692\" y=\"-6541.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507449256&#45;&gt;140254507449088 -->\n",
       "<g id=\"edge107\" class=\"edge\">\n",
       "<title>140254507449256&#45;&gt;140254507449088</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M704.2138,-6534.2281C715.1405,-6524.5914 731.3588,-6510.2877 744.1477,-6499.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"746.7817,-6501.3523 751.9666,-6492.1128 742.1516,-6496.1024 746.7817,-6501.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507449368 -->\n",
       "<g id=\"node109\" class=\"node\">\n",
       "<title>140254507449368</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"769.5,-6619 650.5,-6619 650.5,-6598 769.5,-6598 769.5,-6619\"/>\n",
       "<text text-anchor=\"middle\" x=\"710\" y=\"-6605.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507449368&#45;&gt;140254507449256 -->\n",
       "<g id=\"edge108\" class=\"edge\">\n",
       "<title>140254507449368&#45;&gt;140254507449256</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M706.9465,-6597.7281C704.449,-6588.9174 700.8456,-6576.2055 697.8027,-6565.4708\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"701.103,-6564.2792 695.0084,-6555.6128 694.3683,-6566.1883 701.103,-6564.2792\"/>\n",
       "</g>\n",
       "<!-- 140254507449480 -->\n",
       "<g id=\"node110\" class=\"node\">\n",
       "<title>140254507449480</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"822.5,-6676 669.5,-6676 669.5,-6655 822.5,-6655 822.5,-6676\"/>\n",
       "<text text-anchor=\"middle\" x=\"746\" y=\"-6662.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507449480&#45;&gt;140254507449368 -->\n",
       "<g id=\"edge109\" class=\"edge\">\n",
       "<title>140254507449480&#45;&gt;140254507449368</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M739.2287,-6654.7787C734.4518,-6647.2153 727.9182,-6636.8705 722.2397,-6627.8794\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"725.0402,-6625.7591 716.741,-6619.1732 719.1217,-6629.4971 725.0402,-6625.7591\"/>\n",
       "</g>\n",
       "<!-- 140254507449592 -->\n",
       "<g id=\"node111\" class=\"node\">\n",
       "<title>140254507449592</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"698.5,-6739.5 541.5,-6739.5 541.5,-6718.5 698.5,-6718.5 698.5,-6739.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"620\" y=\"-6725.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507449592&#45;&gt;140254507449480 -->\n",
       "<g id=\"edge110\" class=\"edge\">\n",
       "<title>140254507449592&#45;&gt;140254507449480</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M641.0896,-6718.3715C661.5319,-6708.0692 692.7488,-6692.3369 715.8192,-6680.7101\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"717.4867,-6683.7892 724.8416,-6676.1631 714.3364,-6677.5381 717.4867,-6683.7892\"/>\n",
       "</g>\n",
       "<!-- 140254507449144 -->\n",
       "<g id=\"node112\" class=\"node\">\n",
       "<title>140254507449144</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"625.5,-6803 506.5,-6803 506.5,-6782 625.5,-6782 625.5,-6803\"/>\n",
       "<text text-anchor=\"middle\" x=\"566\" y=\"-6789.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507449144&#45;&gt;140254507448976 -->\n",
       "<g id=\"edge132\" class=\"edge\">\n",
       "<title>140254507449144&#45;&gt;140254507448976</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M556.2577,-6781.8331C548.5876,-6772.885 538.1912,-6759.444 532,-6746 516.6232,-6712.6097 513,-6702.2608 513,-6665.5 513,-6665.5 513,-6665.5 513,-6545 513,-6506.4334 536.0294,-6466.352 551.7273,-6443.5108\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"554.7671,-6445.276 557.7188,-6435.1006 549.0659,-6441.2144 554.7671,-6445.276\"/>\n",
       "</g>\n",
       "<!-- 140254507449144&#45;&gt;140254507449592 -->\n",
       "<g id=\"edge111\" class=\"edge\">\n",
       "<title>140254507449144&#45;&gt;140254507449592</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M575.1604,-6781.7281C583.1212,-6772.3667 594.8273,-6758.6012 604.2825,-6747.4826\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"607.1629,-6749.4982 610.9749,-6739.6128 601.8304,-6744.9634 607.1629,-6749.4982\"/>\n",
       "</g>\n",
       "<!-- 140254507449760 -->\n",
       "<g id=\"node113\" class=\"node\">\n",
       "<title>140254507449760</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"612,-6860 520,-6860 520,-6839 612,-6839 612,-6860\"/>\n",
       "<text text-anchor=\"middle\" x=\"566\" y=\"-6846.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">AddBackward0</text>\n",
       "</g>\n",
       "<!-- 140254507449760&#45;&gt;140254507449144 -->\n",
       "<g id=\"edge112\" class=\"edge\">\n",
       "<title>140254507449760&#45;&gt;140254507449144</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M566,-6838.7787C566,-6831.6134 566,-6821.9517 566,-6813.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"569.5001,-6813.1732 566,-6803.1732 562.5001,-6813.1732 569.5001,-6813.1732\"/>\n",
       "</g>\n",
       "<!-- 140254507449872 -->\n",
       "<g id=\"node114\" class=\"node\">\n",
       "<title>140254507449872</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"840.5,-6917 687.5,-6917 687.5,-6896 840.5,-6896 840.5,-6917\"/>\n",
       "<text text-anchor=\"middle\" x=\"764\" y=\"-6903.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507449872&#45;&gt;140254507449760 -->\n",
       "<g id=\"edge113\" class=\"edge\">\n",
       "<title>140254507449872&#45;&gt;140254507449760</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M727.2217,-6895.9123C694.853,-6886.5941 647.5606,-6872.9796 612.5606,-6862.9038\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"613.1593,-6859.4341 602.5813,-6860.031 611.2227,-6866.1609 613.1593,-6859.4341\"/>\n",
       "</g>\n",
       "<!-- 140254507450040 -->\n",
       "<g id=\"node115\" class=\"node\">\n",
       "<title>140254507450040</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"770.5,-6980.5 613.5,-6980.5 613.5,-6959.5 770.5,-6959.5 770.5,-6980.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"692\" y=\"-6966.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507450040&#45;&gt;140254507449872 -->\n",
       "<g id=\"edge114\" class=\"edge\">\n",
       "<title>140254507450040&#45;&gt;140254507449872</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M704.2138,-6959.2281C715.1405,-6949.5914 731.3588,-6935.2877 744.1477,-6924.0086\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"746.7817,-6926.3523 751.9666,-6917.1128 742.1516,-6921.1024 746.7817,-6926.3523\"/>\n",
       "</g>\n",
       "<!-- 140254507450152 -->\n",
       "<g id=\"node116\" class=\"node\">\n",
       "<title>140254507450152</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"769.5,-7044 650.5,-7044 650.5,-7023 769.5,-7023 769.5,-7044\"/>\n",
       "<text text-anchor=\"middle\" x=\"710\" y=\"-7030.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507450152&#45;&gt;140254507450040 -->\n",
       "<g id=\"edge115\" class=\"edge\">\n",
       "<title>140254507450152&#45;&gt;140254507450040</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M706.9465,-7022.7281C704.449,-7013.9174 700.8456,-7001.2055 697.8027,-6990.4708\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"701.103,-6989.2792 695.0084,-6980.6128 694.3683,-6991.1883 701.103,-6989.2792\"/>\n",
       "</g>\n",
       "<!-- 140254507450264 -->\n",
       "<g id=\"node117\" class=\"node\">\n",
       "<title>140254507450264</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"822.5,-7101 669.5,-7101 669.5,-7080 822.5,-7080 822.5,-7101\"/>\n",
       "<text text-anchor=\"middle\" x=\"746\" y=\"-7087.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507450264&#45;&gt;140254507450152 -->\n",
       "<g id=\"edge116\" class=\"edge\">\n",
       "<title>140254507450264&#45;&gt;140254507450152</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M739.2287,-7079.7787C734.4518,-7072.2153 727.9182,-7061.8705 722.2397,-7052.8794\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"725.0402,-7050.7591 716.741,-7044.1732 719.1217,-7054.4971 725.0402,-7050.7591\"/>\n",
       "</g>\n",
       "<!-- 140254507360328 -->\n",
       "<g id=\"node118\" class=\"node\">\n",
       "<title>140254507360328</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"698.5,-7164.5 541.5,-7164.5 541.5,-7143.5 698.5,-7143.5 698.5,-7164.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"620\" y=\"-7150.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 140254507360328&#45;&gt;140254507450264 -->\n",
       "<g id=\"edge117\" class=\"edge\">\n",
       "<title>140254507360328&#45;&gt;140254507450264</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M641.0896,-7143.3715C661.5319,-7133.0692 692.7488,-7117.3369 715.8192,-7105.7101\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"717.4867,-7108.7892 724.8416,-7101.1631 714.3364,-7102.5381 717.4867,-7108.7892\"/>\n",
       "</g>\n",
       "<!-- 140254507449928 -->\n",
       "<g id=\"node119\" class=\"node\">\n",
       "<title>140254507449928</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"656.5,-7228 475.5,-7228 475.5,-7207 656.5,-7207 656.5,-7228\"/>\n",
       "<text text-anchor=\"middle\" x=\"566\" y=\"-7214.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">MaxPool2DWithIndicesBackward</text>\n",
       "</g>\n",
       "<!-- 140254507449928&#45;&gt;140254507449760 -->\n",
       "<g id=\"edge127\" class=\"edge\">\n",
       "<title>140254507449928&#45;&gt;140254507449760</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M556.2577,-7206.8331C548.5876,-7197.885 538.1912,-7184.444 532,-7171 516.6232,-7137.6097 513,-7127.2608 513,-7090.5 513,-7090.5 513,-7090.5 513,-6970 513,-6931.4334 536.0294,-6891.352 551.7273,-6868.5108\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"554.7671,-6870.276 557.7188,-6860.1006 549.0659,-6866.2144 554.7671,-6870.276\"/>\n",
       "</g>\n",
       "<!-- 140254507449928&#45;&gt;140254507360328 -->\n",
       "<g id=\"edge118\" class=\"edge\">\n",
       "<title>140254507449928&#45;&gt;140254507360328</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M575.1604,-7206.7281C583.1212,-7197.3667 594.8273,-7183.6012 604.2825,-7172.4826\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"607.1629,-7174.4982 610.9749,-7164.6128 601.8304,-7169.9634 607.1629,-7174.4982\"/>\n",
       "</g>\n",
       "<!-- 140254507360496 -->\n",
       "<g id=\"node120\" class=\"node\">\n",
       "<title>140254507360496</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"625.5,-7285 506.5,-7285 506.5,-7264 625.5,-7264 625.5,-7285\"/>\n",
       "<text text-anchor=\"middle\" x=\"566\" y=\"-7271.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">ThresholdBackward1</text>\n",
       "</g>\n",
       "<!-- 140254507360496&#45;&gt;140254507449928 -->\n",
       "<g id=\"edge119\" class=\"edge\">\n",
       "<title>140254507360496&#45;&gt;140254507449928</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M566,-7263.7787C566,-7256.6134 566,-7246.9517 566,-7238.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"569.5001,-7238.1732 566,-7228.1732 562.5001,-7238.1732 569.5001,-7238.1732\"/>\n",
       "</g>\n",
       "<!-- 140254507360608 -->\n",
       "<g id=\"node121\" class=\"node\">\n",
       "<title>140254507360608</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"642.5,-7342 489.5,-7342 489.5,-7321 642.5,-7321 642.5,-7342\"/>\n",
       "<text text-anchor=\"middle\" x=\"566\" y=\"-7328.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507360608&#45;&gt;140254507360496 -->\n",
       "<g id=\"edge120\" class=\"edge\">\n",
       "<title>140254507360608&#45;&gt;140254507360496</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M566,-7320.7787C566,-7313.6134 566,-7303.9517 566,-7295.3097\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"569.5001,-7295.1732 566,-7285.1732 562.5001,-7295.1732 569.5001,-7295.1732\"/>\n",
       "</g>\n",
       "<!-- 140254507799496 -->\n",
       "<g id=\"node122\" class=\"node\">\n",
       "<title>140254507799496</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"560.5,-7412 493.5,-7412 493.5,-7378 560.5,-7378 560.5,-7412\"/>\n",
       "<text text-anchor=\"middle\" x=\"527\" y=\"-7398.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"527\" y=\"-7385.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (64)</text>\n",
       "</g>\n",
       "<!-- 140254507799496&#45;&gt;140254507360608 -->\n",
       "<g id=\"edge121\" class=\"edge\">\n",
       "<title>140254507799496&#45;&gt;140254507360608</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M537.4513,-7377.9832C542.5548,-7369.6737 548.7207,-7359.6342 554.0144,-7351.015\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"557.0736,-7352.7216 559.3248,-7342.3687 551.1088,-7349.0581 557.0736,-7352.7216\"/>\n",
       "</g>\n",
       "<!-- 140254507799552 -->\n",
       "<g id=\"node123\" class=\"node\">\n",
       "<title>140254507799552</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"633,-7412 579,-7412 579,-7378 633,-7378 633,-7412\"/>\n",
       "<text text-anchor=\"middle\" x=\"606\" y=\"-7398.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"606\" y=\"-7385.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (64)</text>\n",
       "</g>\n",
       "<!-- 140254507799552&#45;&gt;140254507360608 -->\n",
       "<g id=\"edge122\" class=\"edge\">\n",
       "<title>140254507799552&#45;&gt;140254507360608</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M595.2807,-7377.9832C590.0464,-7369.6737 583.7223,-7359.6342 578.2929,-7351.015\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"581.1378,-7348.9644 572.8464,-7342.3687 575.2149,-7352.6954 581.1378,-7348.9644\"/>\n",
       "</g>\n",
       "<!-- 140254507798936 -->\n",
       "<g id=\"node124\" class=\"node\">\n",
       "<title>140254507798936</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"813.5,-7171 716.5,-7171 716.5,-7137 813.5,-7137 813.5,-7171\"/>\n",
       "<text text-anchor=\"middle\" x=\"765\" y=\"-7157.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.4.0.bn1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"765\" y=\"-7144.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (64)</text>\n",
       "</g>\n",
       "<!-- 140254507798936&#45;&gt;140254507450264 -->\n",
       "<g id=\"edge123\" class=\"edge\">\n",
       "<title>140254507798936&#45;&gt;140254507450264</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M759.9083,-7136.9832C757.5278,-7129.0273 754.6728,-7119.4855 752.1706,-7111.1228\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"755.4718,-7109.9457 749.252,-7101.3687 748.7656,-7111.9523 755.4718,-7109.9457\"/>\n",
       "</g>\n",
       "<!-- 140254507798992 -->\n",
       "<g id=\"node125\" class=\"node\">\n",
       "<title>140254507798992</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"914.5,-7171 831.5,-7171 831.5,-7137 914.5,-7137 914.5,-7171\"/>\n",
       "<text text-anchor=\"middle\" x=\"873\" y=\"-7157.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.4.0.bn1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"873\" y=\"-7144.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (64)</text>\n",
       "</g>\n",
       "<!-- 140254507798992&#45;&gt;140254507450264 -->\n",
       "<g id=\"edge124\" class=\"edge\">\n",
       "<title>140254507798992&#45;&gt;140254507450264</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M838.9663,-7136.9832C819.4812,-7127.2406 795.2399,-7115.12 776.3969,-7105.6985\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"777.8672,-7102.5205 767.3576,-7101.1788 774.7367,-7108.7815 777.8672,-7102.5205\"/>\n",
       "</g>\n",
       "<!-- 140254507794272 -->\n",
       "<g id=\"node126\" class=\"node\">\n",
       "<title>140254507794272</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"885.5,-6987 788.5,-6987 788.5,-6953 885.5,-6953 885.5,-6987\"/>\n",
       "<text text-anchor=\"middle\" x=\"837\" y=\"-6973.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.4.0.bn2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"837\" y=\"-6960.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (64)</text>\n",
       "</g>\n",
       "<!-- 140254507794272&#45;&gt;140254507449872 -->\n",
       "<g id=\"edge125\" class=\"edge\">\n",
       "<title>140254507794272&#45;&gt;140254507449872</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M817.4373,-6952.9832C807.1733,-6944.0549 794.6133,-6933.1294 784.2495,-6924.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"786.3367,-6921.291 776.4947,-6917.3687 781.7426,-6926.5725 786.3367,-6921.291\"/>\n",
       "</g>\n",
       "<!-- 140254507794328 -->\n",
       "<g id=\"node127\" class=\"node\">\n",
       "<title>140254507794328</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"986.5,-6987 903.5,-6987 903.5,-6953 986.5,-6953 986.5,-6987\"/>\n",
       "<text text-anchor=\"middle\" x=\"945\" y=\"-6973.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.4.0.bn2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"945\" y=\"-6960.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (64)</text>\n",
       "</g>\n",
       "<!-- 140254507794328&#45;&gt;140254507449872 -->\n",
       "<g id=\"edge126\" class=\"edge\">\n",
       "<title>140254507794328&#45;&gt;140254507449872</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M903.4749,-6955.4318C873.4686,-6944.9047 833.1831,-6930.7714 803.6238,-6920.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"804.6356,-6917.0471 794.0408,-6917.0392 802.3183,-6923.6524 804.6356,-6917.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507793656 -->\n",
       "<g id=\"node128\" class=\"node\">\n",
       "<title>140254507793656</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"813.5,-6746 716.5,-6746 716.5,-6712 813.5,-6712 813.5,-6746\"/>\n",
       "<text text-anchor=\"middle\" x=\"765\" y=\"-6732.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.4.1.bn1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"765\" y=\"-6719.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (64)</text>\n",
       "</g>\n",
       "<!-- 140254507793656&#45;&gt;140254507449480 -->\n",
       "<g id=\"edge128\" class=\"edge\">\n",
       "<title>140254507793656&#45;&gt;140254507449480</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M759.9083,-6711.9832C757.5278,-6704.0273 754.6728,-6694.4855 752.1706,-6686.1228\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"755.4718,-6684.9457 749.252,-6676.3687 748.7656,-6686.9523 755.4718,-6684.9457\"/>\n",
       "</g>\n",
       "<!-- 140254507793712 -->\n",
       "<g id=\"node129\" class=\"node\">\n",
       "<title>140254507793712</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"914.5,-6746 831.5,-6746 831.5,-6712 914.5,-6712 914.5,-6746\"/>\n",
       "<text text-anchor=\"middle\" x=\"873\" y=\"-6732.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.4.1.bn1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"873\" y=\"-6719.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (64)</text>\n",
       "</g>\n",
       "<!-- 140254507793712&#45;&gt;140254507449480 -->\n",
       "<g id=\"edge129\" class=\"edge\">\n",
       "<title>140254507793712&#45;&gt;140254507449480</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M838.9663,-6711.9832C819.4812,-6702.2406 795.2399,-6690.12 776.3969,-6680.6985\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"777.8672,-6677.5205 767.3576,-6676.1788 774.7367,-6683.7815 777.8672,-6677.5205\"/>\n",
       "</g>\n",
       "<!-- 140254507793152 -->\n",
       "<g id=\"node130\" class=\"node\">\n",
       "<title>140254507793152</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"885.5,-6562 788.5,-6562 788.5,-6528 885.5,-6528 885.5,-6562\"/>\n",
       "<text text-anchor=\"middle\" x=\"837\" y=\"-6548.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.4.1.bn2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"837\" y=\"-6535.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (64)</text>\n",
       "</g>\n",
       "<!-- 140254507793152&#45;&gt;140254507449088 -->\n",
       "<g id=\"edge130\" class=\"edge\">\n",
       "<title>140254507793152&#45;&gt;140254507449088</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M817.4373,-6527.9832C807.1733,-6519.0549 794.6133,-6508.1294 784.2495,-6499.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"786.3367,-6496.291 776.4947,-6492.3687 781.7426,-6501.5725 786.3367,-6496.291\"/>\n",
       "</g>\n",
       "<!-- 140254507793208 -->\n",
       "<g id=\"node131\" class=\"node\">\n",
       "<title>140254507793208</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"986.5,-6562 903.5,-6562 903.5,-6528 986.5,-6528 986.5,-6562\"/>\n",
       "<text text-anchor=\"middle\" x=\"945\" y=\"-6548.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.4.1.bn2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"945\" y=\"-6535.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (64)</text>\n",
       "</g>\n",
       "<!-- 140254507793208&#45;&gt;140254507449088 -->\n",
       "<g id=\"edge131\" class=\"edge\">\n",
       "<title>140254507793208&#45;&gt;140254507449088</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M903.4749,-6530.4318C873.4686,-6519.9047 833.1831,-6505.7714 803.6238,-6495.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"804.6356,-6492.0471 794.0408,-6492.0392 802.3183,-6498.6524 804.6356,-6492.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507792536 -->\n",
       "<g id=\"node132\" class=\"node\">\n",
       "<title>140254507792536</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"813.5,-6321 716.5,-6321 716.5,-6287 813.5,-6287 813.5,-6321\"/>\n",
       "<text text-anchor=\"middle\" x=\"765\" y=\"-6307.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.4.2.bn1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"765\" y=\"-6294.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (64)</text>\n",
       "</g>\n",
       "<!-- 140254507792536&#45;&gt;140254507448696 -->\n",
       "<g id=\"edge133\" class=\"edge\">\n",
       "<title>140254507792536&#45;&gt;140254507448696</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M745.4373,-6286.9832C735.1733,-6278.0549 722.6133,-6267.1294 712.2495,-6258.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"714.3367,-6255.291 704.4947,-6251.3687 709.7426,-6260.5725 714.3367,-6255.291\"/>\n",
       "</g>\n",
       "<!-- 140254507792592 -->\n",
       "<g id=\"node133\" class=\"node\">\n",
       "<title>140254507792592</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"914.5,-6321 831.5,-6321 831.5,-6287 914.5,-6287 914.5,-6321\"/>\n",
       "<text text-anchor=\"middle\" x=\"873\" y=\"-6307.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.4.2.bn1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"873\" y=\"-6294.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (64)</text>\n",
       "</g>\n",
       "<!-- 140254507792592&#45;&gt;140254507448696 -->\n",
       "<g id=\"edge134\" class=\"edge\">\n",
       "<title>140254507792592&#45;&gt;140254507448696</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M831.4749,-6289.4318C801.4686,-6278.9047 761.1831,-6264.7714 731.6238,-6254.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"732.6356,-6251.0471 722.0408,-6251.0392 730.3183,-6257.6524 732.6356,-6251.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507792088 -->\n",
       "<g id=\"node134\" class=\"node\">\n",
       "<title>140254507792088</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"813.5,-6137 716.5,-6137 716.5,-6103 813.5,-6103 813.5,-6137\"/>\n",
       "<text text-anchor=\"middle\" x=\"765\" y=\"-6123.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.4.2.bn2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"765\" y=\"-6110.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (64)</text>\n",
       "</g>\n",
       "<!-- 140254507792088&#45;&gt;140254507448304 -->\n",
       "<g id=\"edge135\" class=\"edge\">\n",
       "<title>140254507792088&#45;&gt;140254507448304</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M745.4373,-6102.9832C735.1733,-6094.0549 722.6133,-6083.1294 712.2495,-6074.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"714.3367,-6071.291 704.4947,-6067.3687 709.7426,-6076.5725 714.3367,-6071.291\"/>\n",
       "</g>\n",
       "<!-- 140254507792144 -->\n",
       "<g id=\"node135\" class=\"node\">\n",
       "<title>140254507792144</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"914.5,-6137 831.5,-6137 831.5,-6103 914.5,-6103 914.5,-6137\"/>\n",
       "<text text-anchor=\"middle\" x=\"873\" y=\"-6123.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.4.2.bn2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"873\" y=\"-6110.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (64)</text>\n",
       "</g>\n",
       "<!-- 140254507792144&#45;&gt;140254507448304 -->\n",
       "<g id=\"edge136\" class=\"edge\">\n",
       "<title>140254507792144&#45;&gt;140254507448304</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M831.4749,-6105.4318C801.4686,-6094.9047 761.1831,-6080.7714 731.6238,-6070.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"732.6356,-6067.0471 722.0408,-6067.0392 730.3183,-6073.6524 732.6356,-6067.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507791472 -->\n",
       "<g id=\"node136\" class=\"node\">\n",
       "<title>140254507791472</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"847.5,-5896 750.5,-5896 750.5,-5862 847.5,-5862 847.5,-5896\"/>\n",
       "<text text-anchor=\"middle\" x=\"799\" y=\"-5882.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.5.0.bn1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"799\" y=\"-5869.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (128)</text>\n",
       "</g>\n",
       "<!-- 140254507791472&#45;&gt;140254507447856 -->\n",
       "<g id=\"edge138\" class=\"edge\">\n",
       "<title>140254507791472&#45;&gt;140254507447856</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M779.4373,-5861.9832C769.1733,-5853.0549 756.6133,-5842.1294 746.2495,-5833.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"748.3367,-5830.291 738.4947,-5826.3687 743.7426,-5835.5725 748.3367,-5830.291\"/>\n",
       "</g>\n",
       "<!-- 140254507791528 -->\n",
       "<g id=\"node137\" class=\"node\">\n",
       "<title>140254507791528</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"948.5,-5896 865.5,-5896 865.5,-5862 948.5,-5862 948.5,-5896\"/>\n",
       "<text text-anchor=\"middle\" x=\"907\" y=\"-5882.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.5.0.bn1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"907\" y=\"-5869.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (128)</text>\n",
       "</g>\n",
       "<!-- 140254507791528&#45;&gt;140254507447856 -->\n",
       "<g id=\"edge139\" class=\"edge\">\n",
       "<title>140254507791528&#45;&gt;140254507447856</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M865.4749,-5864.4318C835.4686,-5853.9047 795.1831,-5839.7714 765.6238,-5829.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"766.6356,-5826.0471 756.0408,-5826.0392 764.3183,-5832.6524 766.6356,-5826.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507791024 -->\n",
       "<g id=\"node138\" class=\"node\">\n",
       "<title>140254507791024</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"809.5,-5712 712.5,-5712 712.5,-5678 809.5,-5678 809.5,-5712\"/>\n",
       "<text text-anchor=\"middle\" x=\"761\" y=\"-5698.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.5.0.bn2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"761\" y=\"-5685.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (128)</text>\n",
       "</g>\n",
       "<!-- 140254507791024&#45;&gt;140254507447464 -->\n",
       "<g id=\"edge140\" class=\"edge\">\n",
       "<title>140254507791024&#45;&gt;140254507447464</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M741.4373,-5677.9832C731.1733,-5669.0549 718.6133,-5658.1294 708.2495,-5649.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"710.3367,-5646.291 700.4947,-5642.3687 705.7426,-5651.5725 710.3367,-5646.291\"/>\n",
       "</g>\n",
       "<!-- 140254507791080 -->\n",
       "<g id=\"node139\" class=\"node\">\n",
       "<title>140254507791080</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"910.5,-5712 827.5,-5712 827.5,-5678 910.5,-5678 910.5,-5712\"/>\n",
       "<text text-anchor=\"middle\" x=\"869\" y=\"-5698.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.5.0.bn2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"869\" y=\"-5685.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (128)</text>\n",
       "</g>\n",
       "<!-- 140254507791080&#45;&gt;140254507447464 -->\n",
       "<g id=\"edge141\" class=\"edge\">\n",
       "<title>140254507791080&#45;&gt;140254507447464</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M827.4749,-5680.4318C797.4686,-5669.9047 757.1831,-5655.7714 727.6238,-5645.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"728.6356,-5642.0471 718.0408,-5642.0392 726.3183,-5648.6524 728.6356,-5642.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507447520 -->\n",
       "<g id=\"node140\" class=\"node\">\n",
       "<title>140254507447520</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"473.5,-5826 320.5,-5826 320.5,-5805 473.5,-5805 473.5,-5826\"/>\n",
       "<text text-anchor=\"middle\" x=\"397\" y=\"-5812.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507447520&#45;&gt;140254507447352 -->\n",
       "<g id=\"edge142\" class=\"edge\">\n",
       "<title>140254507447520&#45;&gt;140254507447352</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M402.6652,-5804.8336C422.4908,-5767.5055 488.9414,-5642.3915 514.6873,-5593.917\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"517.8006,-5595.5166 519.4003,-5585.0433 511.6185,-5592.2331 517.8006,-5595.5166\"/>\n",
       "</g>\n",
       "<!-- 140254507447688&#45;&gt;140254507447520 -->\n",
       "<g id=\"edge143\" class=\"edge\">\n",
       "<title>140254507447688&#45;&gt;140254507447520</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M465.0898,-5868.2281C452.4085,-5858.4078 433.4688,-5843.7411 418.7815,-5832.3674\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"420.7542,-5829.4683 410.7048,-5826.1128 416.4683,-5835.0028 420.7542,-5829.4683\"/>\n",
       "</g>\n",
       "<!-- 140254507791248 -->\n",
       "<g id=\"node142\" class=\"node\">\n",
       "<title>140254507791248</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"230,-5896 82,-5896 82,-5862 230,-5862 230,-5896\"/>\n",
       "<text text-anchor=\"middle\" x=\"156\" y=\"-5882.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.5.0.downsample.1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"156\" y=\"-5869.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (128)</text>\n",
       "</g>\n",
       "<!-- 140254507791248&#45;&gt;140254507447520 -->\n",
       "<g id=\"edge145\" class=\"edge\">\n",
       "<title>140254507791248&#45;&gt;140254507447520</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M220.5836,-5861.9832C260.2192,-5851.5397 310.2258,-5838.3637 346.8685,-5828.7089\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"348.0513,-5832.0168 356.8295,-5826.0843 346.2677,-5825.2478 348.0513,-5832.0168\"/>\n",
       "</g>\n",
       "<!-- 140254507791360 -->\n",
       "<g id=\"node143\" class=\"node\">\n",
       "<title>140254507791360</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"382,-5896 248,-5896 248,-5862 382,-5862 382,-5896\"/>\n",
       "<text text-anchor=\"middle\" x=\"315\" y=\"-5882.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.5.0.downsample.1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"315\" y=\"-5869.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (128)</text>\n",
       "</g>\n",
       "<!-- 140254507791360&#45;&gt;140254507447520 -->\n",
       "<g id=\"edge146\" class=\"edge\">\n",
       "<title>140254507791360&#45;&gt;140254507447520</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M336.9745,-5861.9832C348.7323,-5852.8781 363.1723,-5841.6959 374.9426,-5832.581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"377.2013,-5835.2587 382.9649,-5826.3687 372.9154,-5829.7241 377.2013,-5835.2587\"/>\n",
       "</g>\n",
       "<!-- 140254507790464 -->\n",
       "<g id=\"node144\" class=\"node\">\n",
       "<title>140254507790464</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"772.5,-5471 675.5,-5471 675.5,-5437 772.5,-5437 772.5,-5471\"/>\n",
       "<text text-anchor=\"middle\" x=\"724\" y=\"-5457.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.5.1.bn1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"724\" y=\"-5444.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (128)</text>\n",
       "</g>\n",
       "<!-- 140254507790464&#45;&gt;140254507447072 -->\n",
       "<g id=\"edge147\" class=\"edge\">\n",
       "<title>140254507790464&#45;&gt;140254507447072</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M704.4373,-5436.9832C694.1733,-5428.0549 681.6133,-5417.1294 671.2495,-5408.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"673.3367,-5405.291 663.4947,-5401.3687 668.7426,-5410.5725 673.3367,-5405.291\"/>\n",
       "</g>\n",
       "<!-- 140254507790520 -->\n",
       "<g id=\"node145\" class=\"node\">\n",
       "<title>140254507790520</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"873.5,-5471 790.5,-5471 790.5,-5437 873.5,-5437 873.5,-5471\"/>\n",
       "<text text-anchor=\"middle\" x=\"832\" y=\"-5457.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.5.1.bn1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"832\" y=\"-5444.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (128)</text>\n",
       "</g>\n",
       "<!-- 140254507790520&#45;&gt;140254507447072 -->\n",
       "<g id=\"edge148\" class=\"edge\">\n",
       "<title>140254507790520&#45;&gt;140254507447072</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M790.4749,-5439.4318C760.4686,-5428.9047 720.1831,-5414.7714 690.6238,-5404.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"691.6356,-5401.0471 681.0408,-5401.0392 689.3183,-5407.6524 691.6356,-5401.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507781760 -->\n",
       "<g id=\"node146\" class=\"node\">\n",
       "<title>140254507781760</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"772.5,-5287 675.5,-5287 675.5,-5253 772.5,-5253 772.5,-5287\"/>\n",
       "<text text-anchor=\"middle\" x=\"724\" y=\"-5273.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.5.1.bn2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"724\" y=\"-5260.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (128)</text>\n",
       "</g>\n",
       "<!-- 140254507781760&#45;&gt;140254507446680 -->\n",
       "<g id=\"edge149\" class=\"edge\">\n",
       "<title>140254507781760&#45;&gt;140254507446680</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M704.4373,-5252.9832C694.1733,-5244.0549 681.6133,-5233.1294 671.2495,-5224.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"673.3367,-5221.291 663.4947,-5217.3687 668.7426,-5226.5725 673.3367,-5221.291\"/>\n",
       "</g>\n",
       "<!-- 140254507781816 -->\n",
       "<g id=\"node147\" class=\"node\">\n",
       "<title>140254507781816</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"873.5,-5287 790.5,-5287 790.5,-5253 873.5,-5253 873.5,-5287\"/>\n",
       "<text text-anchor=\"middle\" x=\"832\" y=\"-5273.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.5.1.bn2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"832\" y=\"-5260.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (128)</text>\n",
       "</g>\n",
       "<!-- 140254507781816&#45;&gt;140254507446680 -->\n",
       "<g id=\"edge150\" class=\"edge\">\n",
       "<title>140254507781816&#45;&gt;140254507446680</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M790.4749,-5255.4318C760.4686,-5244.9047 720.1831,-5230.7714 690.6238,-5220.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"691.6356,-5217.0471 681.0408,-5217.0392 689.3183,-5223.6524 691.6356,-5217.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507781200 -->\n",
       "<g id=\"node148\" class=\"node\">\n",
       "<title>140254507781200</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"772.5,-5046 675.5,-5046 675.5,-5012 772.5,-5012 772.5,-5046\"/>\n",
       "<text text-anchor=\"middle\" x=\"724\" y=\"-5032.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.5.2.bn1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"724\" y=\"-5019.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (128)</text>\n",
       "</g>\n",
       "<!-- 140254507781200&#45;&gt;140254507478992 -->\n",
       "<g id=\"edge152\" class=\"edge\">\n",
       "<title>140254507781200&#45;&gt;140254507478992</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M718.9083,-5011.9832C716.5278,-5004.0273 713.6728,-4994.4855 711.1706,-4986.1228\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"714.4718,-4984.9457 708.252,-4976.3687 707.7656,-4986.9523 714.4718,-4984.9457\"/>\n",
       "</g>\n",
       "<!-- 140254507781256 -->\n",
       "<g id=\"node149\" class=\"node\">\n",
       "<title>140254507781256</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"873.5,-5046 790.5,-5046 790.5,-5012 873.5,-5012 873.5,-5046\"/>\n",
       "<text text-anchor=\"middle\" x=\"832\" y=\"-5032.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.5.2.bn1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"832\" y=\"-5019.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (128)</text>\n",
       "</g>\n",
       "<!-- 140254507781256&#45;&gt;140254507478992 -->\n",
       "<g id=\"edge153\" class=\"edge\">\n",
       "<title>140254507781256&#45;&gt;140254507478992</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M797.9663,-5011.9832C778.4812,-5002.2406 754.2399,-4990.12 735.3969,-4980.6985\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"736.8672,-4977.5205 726.3576,-4976.1788 733.7367,-4983.7815 736.8672,-4977.5205\"/>\n",
       "</g>\n",
       "<!-- 140254507780752 -->\n",
       "<g id=\"node150\" class=\"node\">\n",
       "<title>140254507780752</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"844.5,-4862 747.5,-4862 747.5,-4828 844.5,-4828 844.5,-4862\"/>\n",
       "<text text-anchor=\"middle\" x=\"796\" y=\"-4848.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.5.2.bn2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"796\" y=\"-4835.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (128)</text>\n",
       "</g>\n",
       "<!-- 140254507780752&#45;&gt;140254507478600 -->\n",
       "<g id=\"edge154\" class=\"edge\">\n",
       "<title>140254507780752&#45;&gt;140254507478600</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M776.4373,-4827.9832C766.1733,-4819.0549 753.6133,-4808.1294 743.2495,-4799.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"745.3367,-4796.291 735.4947,-4792.3687 740.7426,-4801.5725 745.3367,-4796.291\"/>\n",
       "</g>\n",
       "<!-- 140254507780808 -->\n",
       "<g id=\"node151\" class=\"node\">\n",
       "<title>140254507780808</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"945.5,-4862 862.5,-4862 862.5,-4828 945.5,-4828 945.5,-4862\"/>\n",
       "<text text-anchor=\"middle\" x=\"904\" y=\"-4848.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.5.2.bn2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"904\" y=\"-4835.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (128)</text>\n",
       "</g>\n",
       "<!-- 140254507780808&#45;&gt;140254507478600 -->\n",
       "<g id=\"edge155\" class=\"edge\">\n",
       "<title>140254507780808&#45;&gt;140254507478600</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M862.4749,-4830.4318C832.4686,-4819.9047 792.1831,-4805.7714 762.6238,-4795.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"763.6356,-4792.0471 753.0408,-4792.0392 761.3183,-4798.6524 763.6356,-4792.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507780192 -->\n",
       "<g id=\"node152\" class=\"node\">\n",
       "<title>140254507780192</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"772.5,-4621 675.5,-4621 675.5,-4587 772.5,-4587 772.5,-4621\"/>\n",
       "<text text-anchor=\"middle\" x=\"724\" y=\"-4607.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.5.3.bn1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"724\" y=\"-4594.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (128)</text>\n",
       "</g>\n",
       "<!-- 140254507780192&#45;&gt;140254507478208 -->\n",
       "<g id=\"edge157\" class=\"edge\">\n",
       "<title>140254507780192&#45;&gt;140254507478208</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M704.4373,-4586.9832C694.1733,-4578.0549 681.6133,-4567.1294 671.2495,-4558.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"673.3367,-4555.291 663.4947,-4551.3687 668.7426,-4560.5725 673.3367,-4555.291\"/>\n",
       "</g>\n",
       "<!-- 140254507780248 -->\n",
       "<g id=\"node153\" class=\"node\">\n",
       "<title>140254507780248</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"873.5,-4621 790.5,-4621 790.5,-4587 873.5,-4587 873.5,-4621\"/>\n",
       "<text text-anchor=\"middle\" x=\"832\" y=\"-4607.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.5.3.bn1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"832\" y=\"-4594.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (128)</text>\n",
       "</g>\n",
       "<!-- 140254507780248&#45;&gt;140254507478208 -->\n",
       "<g id=\"edge158\" class=\"edge\">\n",
       "<title>140254507780248&#45;&gt;140254507478208</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M790.4749,-4589.4318C760.4686,-4578.9047 720.1831,-4564.7714 690.6238,-4554.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"691.6356,-4551.0471 681.0408,-4551.0392 689.3183,-4557.6524 691.6356,-4551.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507779744 -->\n",
       "<g id=\"node154\" class=\"node\">\n",
       "<title>140254507779744</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"772.5,-4437 675.5,-4437 675.5,-4403 772.5,-4403 772.5,-4437\"/>\n",
       "<text text-anchor=\"middle\" x=\"724\" y=\"-4423.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.5.3.bn2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"724\" y=\"-4410.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (128)</text>\n",
       "</g>\n",
       "<!-- 140254507779744&#45;&gt;140254507477816 -->\n",
       "<g id=\"edge159\" class=\"edge\">\n",
       "<title>140254507779744&#45;&gt;140254507477816</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M704.4373,-4402.9832C694.1733,-4394.0549 681.6133,-4383.1294 671.2495,-4374.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"673.3367,-4371.291 663.4947,-4367.3687 668.7426,-4376.5725 673.3367,-4371.291\"/>\n",
       "</g>\n",
       "<!-- 140254507779800 -->\n",
       "<g id=\"node155\" class=\"node\">\n",
       "<title>140254507779800</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"873.5,-4437 790.5,-4437 790.5,-4403 873.5,-4403 873.5,-4437\"/>\n",
       "<text text-anchor=\"middle\" x=\"832\" y=\"-4423.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.5.3.bn2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"832\" y=\"-4410.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (128)</text>\n",
       "</g>\n",
       "<!-- 140254507779800&#45;&gt;140254507477816 -->\n",
       "<g id=\"edge160\" class=\"edge\">\n",
       "<title>140254507779800&#45;&gt;140254507477816</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M790.4749,-4405.4318C760.4686,-4394.9047 720.1831,-4380.7714 690.6238,-4370.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"691.6356,-4367.0471 681.0408,-4367.0392 689.3183,-4373.6524 691.6356,-4367.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507779128 -->\n",
       "<g id=\"node156\" class=\"node\">\n",
       "<title>140254507779128</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"806.5,-4196 709.5,-4196 709.5,-4162 806.5,-4162 806.5,-4196\"/>\n",
       "<text text-anchor=\"middle\" x=\"758\" y=\"-4182.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.0.bn1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"758\" y=\"-4169.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507779128&#45;&gt;140254507477368 -->\n",
       "<g id=\"edge162\" class=\"edge\">\n",
       "<title>140254507779128&#45;&gt;140254507477368</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M738.4373,-4161.9832C728.1733,-4153.0549 715.6133,-4142.1294 705.2495,-4133.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"707.3367,-4130.291 697.4947,-4126.3687 702.7426,-4135.5725 707.3367,-4130.291\"/>\n",
       "</g>\n",
       "<!-- 140254507779184 -->\n",
       "<g id=\"node157\" class=\"node\">\n",
       "<title>140254507779184</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"907.5,-4196 824.5,-4196 824.5,-4162 907.5,-4162 907.5,-4196\"/>\n",
       "<text text-anchor=\"middle\" x=\"866\" y=\"-4182.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.0.bn1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"866\" y=\"-4169.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507779184&#45;&gt;140254507477368 -->\n",
       "<g id=\"edge163\" class=\"edge\">\n",
       "<title>140254507779184&#45;&gt;140254507477368</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M824.4749,-4164.4318C794.4686,-4153.9047 754.1831,-4139.7714 724.6238,-4129.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"725.6356,-4126.0471 715.0408,-4126.0392 723.3183,-4132.6524 725.6356,-4126.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507778680 -->\n",
       "<g id=\"node158\" class=\"node\">\n",
       "<title>140254507778680</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"768.5,-4012 671.5,-4012 671.5,-3978 768.5,-3978 768.5,-4012\"/>\n",
       "<text text-anchor=\"middle\" x=\"720\" y=\"-3998.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.0.bn2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"720\" y=\"-3985.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507778680&#45;&gt;140254507476976 -->\n",
       "<g id=\"edge164\" class=\"edge\">\n",
       "<title>140254507778680&#45;&gt;140254507476976</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M700.4373,-3977.9832C690.1733,-3969.0549 677.6133,-3958.1294 667.2495,-3949.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"669.3367,-3946.291 659.4947,-3942.3687 664.7426,-3951.5725 669.3367,-3946.291\"/>\n",
       "</g>\n",
       "<!-- 140254507778736 -->\n",
       "<g id=\"node159\" class=\"node\">\n",
       "<title>140254507778736</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"869.5,-4012 786.5,-4012 786.5,-3978 869.5,-3978 869.5,-4012\"/>\n",
       "<text text-anchor=\"middle\" x=\"828\" y=\"-3998.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.0.bn2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"828\" y=\"-3985.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507778736&#45;&gt;140254507476976 -->\n",
       "<g id=\"edge165\" class=\"edge\">\n",
       "<title>140254507778736&#45;&gt;140254507476976</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M786.4749,-3980.4318C756.4686,-3969.9047 716.1831,-3955.7714 686.6238,-3945.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"687.6356,-3942.0471 677.0408,-3942.0392 685.3183,-3948.6524 687.6356,-3942.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507477032 -->\n",
       "<g id=\"node160\" class=\"node\">\n",
       "<title>140254507477032</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"432.5,-4126 279.5,-4126 279.5,-4105 432.5,-4105 432.5,-4126\"/>\n",
       "<text text-anchor=\"middle\" x=\"356\" y=\"-4112.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507477032&#45;&gt;140254507476864 -->\n",
       "<g id=\"edge166\" class=\"edge\">\n",
       "<title>140254507477032&#45;&gt;140254507476864</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M361.6652,-4104.8336C381.4908,-4067.5055 447.9414,-3942.3915 473.6873,-3893.917\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"476.8006,-3895.5166 478.4003,-3885.0433 470.6185,-3892.2331 476.8006,-3895.5166\"/>\n",
       "</g>\n",
       "<!-- 140254507477200&#45;&gt;140254507477032 -->\n",
       "<g id=\"edge167\" class=\"edge\">\n",
       "<title>140254507477200&#45;&gt;140254507477032</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M424.0898,-4168.2281C411.4085,-4158.4078 392.4688,-4143.7411 377.7815,-4132.3674\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"379.7542,-4129.4683 369.7048,-4126.1128 375.4683,-4135.0028 379.7542,-4129.4683\"/>\n",
       "</g>\n",
       "<!-- 140254507778904 -->\n",
       "<g id=\"node162\" class=\"node\">\n",
       "<title>140254507778904</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"189,-4196 41,-4196 41,-4162 189,-4162 189,-4196\"/>\n",
       "<text text-anchor=\"middle\" x=\"115\" y=\"-4182.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.0.downsample.1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"115\" y=\"-4169.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507778904&#45;&gt;140254507477032 -->\n",
       "<g id=\"edge169\" class=\"edge\">\n",
       "<title>140254507778904&#45;&gt;140254507477032</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M179.5836,-4161.9832C219.2192,-4151.5397 269.2258,-4138.3637 305.8685,-4128.7089\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"307.0513,-4132.0168 315.8295,-4126.0843 305.2677,-4125.2478 307.0513,-4132.0168\"/>\n",
       "</g>\n",
       "<!-- 140254507779016 -->\n",
       "<g id=\"node163\" class=\"node\">\n",
       "<title>140254507779016</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"341,-4196 207,-4196 207,-4162 341,-4162 341,-4196\"/>\n",
       "<text text-anchor=\"middle\" x=\"274\" y=\"-4182.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.0.downsample.1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"274\" y=\"-4169.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507779016&#45;&gt;140254507477032 -->\n",
       "<g id=\"edge170\" class=\"edge\">\n",
       "<title>140254507779016&#45;&gt;140254507477032</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M295.9745,-4161.9832C307.7323,-4152.8781 322.1723,-4141.6959 333.9426,-4132.581\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"336.2013,-4135.2587 341.9649,-4126.3687 331.9154,-4129.7241 336.2013,-4135.2587\"/>\n",
       "</g>\n",
       "<!-- 140254507778120 -->\n",
       "<g id=\"node164\" class=\"node\">\n",
       "<title>140254507778120</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"731.5,-3771 634.5,-3771 634.5,-3737 731.5,-3737 731.5,-3771\"/>\n",
       "<text text-anchor=\"middle\" x=\"683\" y=\"-3757.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.1.bn1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"683\" y=\"-3744.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507778120&#45;&gt;140254507476584 -->\n",
       "<g id=\"edge171\" class=\"edge\">\n",
       "<title>140254507778120&#45;&gt;140254507476584</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M677.9083,-3736.9832C675.5278,-3729.0273 672.6728,-3719.4855 670.1706,-3711.1228\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"673.4718,-3709.9457 667.252,-3701.3687 666.7656,-3711.9523 673.4718,-3709.9457\"/>\n",
       "</g>\n",
       "<!-- 140254507778176 -->\n",
       "<g id=\"node165\" class=\"node\">\n",
       "<title>140254507778176</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"832.5,-3771 749.5,-3771 749.5,-3737 832.5,-3737 832.5,-3771\"/>\n",
       "<text text-anchor=\"middle\" x=\"791\" y=\"-3757.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.1.bn1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"791\" y=\"-3744.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507778176&#45;&gt;140254507476584 -->\n",
       "<g id=\"edge172\" class=\"edge\">\n",
       "<title>140254507778176&#45;&gt;140254507476584</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M756.9663,-3736.9832C737.4812,-3727.2406 713.2399,-3715.12 694.3969,-3705.6985\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"695.8672,-3702.5205 685.3576,-3701.1788 692.7367,-3708.7815 695.8672,-3702.5205\"/>\n",
       "</g>\n",
       "<!-- 140254507773512 -->\n",
       "<g id=\"node166\" class=\"node\">\n",
       "<title>140254507773512</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"803.5,-3587 706.5,-3587 706.5,-3553 803.5,-3553 803.5,-3587\"/>\n",
       "<text text-anchor=\"middle\" x=\"755\" y=\"-3573.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.1.bn2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"755\" y=\"-3560.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507773512&#45;&gt;140254507476192 -->\n",
       "<g id=\"edge173\" class=\"edge\">\n",
       "<title>140254507773512&#45;&gt;140254507476192</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M735.4373,-3552.9832C725.1733,-3544.0549 712.6133,-3533.1294 702.2495,-3524.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"704.3367,-3521.291 694.4947,-3517.3687 699.7426,-3526.5725 704.3367,-3521.291\"/>\n",
       "</g>\n",
       "<!-- 140254507773568 -->\n",
       "<g id=\"node167\" class=\"node\">\n",
       "<title>140254507773568</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"904.5,-3587 821.5,-3587 821.5,-3553 904.5,-3553 904.5,-3587\"/>\n",
       "<text text-anchor=\"middle\" x=\"863\" y=\"-3573.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.1.bn2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"863\" y=\"-3560.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507773568&#45;&gt;140254507476192 -->\n",
       "<g id=\"edge174\" class=\"edge\">\n",
       "<title>140254507773568&#45;&gt;140254507476192</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M821.4749,-3555.4318C791.4686,-3544.9047 751.1831,-3530.7714 721.6238,-3520.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"722.6356,-3517.0471 712.0408,-3517.0392 720.3183,-3523.6524 722.6356,-3517.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507772952 -->\n",
       "<g id=\"node168\" class=\"node\">\n",
       "<title>140254507772952</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"731.5,-3346 634.5,-3346 634.5,-3312 731.5,-3312 731.5,-3346\"/>\n",
       "<text text-anchor=\"middle\" x=\"683\" y=\"-3332.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.2.bn1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"683\" y=\"-3319.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507772952&#45;&gt;140254507475800 -->\n",
       "<g id=\"edge176\" class=\"edge\">\n",
       "<title>140254507772952&#45;&gt;140254507475800</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M663.4373,-3311.9832C653.1733,-3303.0549 640.6133,-3292.1294 630.2495,-3283.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"632.3367,-3280.291 622.4947,-3276.3687 627.7426,-3285.5725 632.3367,-3280.291\"/>\n",
       "</g>\n",
       "<!-- 140254507773008 -->\n",
       "<g id=\"node169\" class=\"node\">\n",
       "<title>140254507773008</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"832.5,-3346 749.5,-3346 749.5,-3312 832.5,-3312 832.5,-3346\"/>\n",
       "<text text-anchor=\"middle\" x=\"791\" y=\"-3332.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.2.bn1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"791\" y=\"-3319.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507773008&#45;&gt;140254507475800 -->\n",
       "<g id=\"edge177\" class=\"edge\">\n",
       "<title>140254507773008&#45;&gt;140254507475800</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M749.4749,-3314.4318C719.4686,-3303.9047 679.1831,-3289.7714 649.6238,-3279.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"650.6356,-3276.0471 640.0408,-3276.0392 648.3183,-3282.6524 650.6356,-3276.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507772504 -->\n",
       "<g id=\"node170\" class=\"node\">\n",
       "<title>140254507772504</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"731.5,-3162 634.5,-3162 634.5,-3128 731.5,-3128 731.5,-3162\"/>\n",
       "<text text-anchor=\"middle\" x=\"683\" y=\"-3148.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.2.bn2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"683\" y=\"-3135.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507772504&#45;&gt;140254507475408 -->\n",
       "<g id=\"edge178\" class=\"edge\">\n",
       "<title>140254507772504&#45;&gt;140254507475408</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M663.4373,-3127.9832C653.1733,-3119.0549 640.6133,-3108.1294 630.2495,-3099.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"632.3367,-3096.291 622.4947,-3092.3687 627.7426,-3101.5725 632.3367,-3096.291\"/>\n",
       "</g>\n",
       "<!-- 140254507772560 -->\n",
       "<g id=\"node171\" class=\"node\">\n",
       "<title>140254507772560</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"832.5,-3162 749.5,-3162 749.5,-3128 832.5,-3128 832.5,-3162\"/>\n",
       "<text text-anchor=\"middle\" x=\"791\" y=\"-3148.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.2.bn2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"791\" y=\"-3135.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507772560&#45;&gt;140254507475408 -->\n",
       "<g id=\"edge179\" class=\"edge\">\n",
       "<title>140254507772560&#45;&gt;140254507475408</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M749.4749,-3130.4318C719.4686,-3119.9047 679.1831,-3105.7714 649.6238,-3095.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"650.6356,-3092.0471 640.0408,-3092.0392 648.3183,-3098.6524 650.6356,-3092.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507771944 -->\n",
       "<g id=\"node172\" class=\"node\">\n",
       "<title>140254507771944</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"731.5,-2921 634.5,-2921 634.5,-2887 731.5,-2887 731.5,-2921\"/>\n",
       "<text text-anchor=\"middle\" x=\"683\" y=\"-2907.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.3.bn1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"683\" y=\"-2894.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507771944&#45;&gt;140254507474896 -->\n",
       "<g id=\"edge181\" class=\"edge\">\n",
       "<title>140254507771944&#45;&gt;140254507474896</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M677.9083,-2886.9832C675.5278,-2879.0273 672.6728,-2869.4855 670.1706,-2861.1228\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"673.4718,-2859.9457 667.252,-2851.3687 666.7656,-2861.9523 673.4718,-2859.9457\"/>\n",
       "</g>\n",
       "<!-- 140254507772000 -->\n",
       "<g id=\"node173\" class=\"node\">\n",
       "<title>140254507772000</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"832.5,-2921 749.5,-2921 749.5,-2887 832.5,-2887 832.5,-2921\"/>\n",
       "<text text-anchor=\"middle\" x=\"791\" y=\"-2907.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.3.bn1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"791\" y=\"-2894.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507772000&#45;&gt;140254507474896 -->\n",
       "<g id=\"edge182\" class=\"edge\">\n",
       "<title>140254507772000&#45;&gt;140254507474896</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M756.9663,-2886.9832C737.4812,-2877.2406 713.2399,-2865.12 694.3969,-2855.6985\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"695.8672,-2852.5205 685.3576,-2851.1788 692.7367,-2858.7815 695.8672,-2852.5205\"/>\n",
       "</g>\n",
       "<!-- 140254507771496 -->\n",
       "<g id=\"node174\" class=\"node\">\n",
       "<title>140254507771496</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"803.5,-2737 706.5,-2737 706.5,-2703 803.5,-2703 803.5,-2737\"/>\n",
       "<text text-anchor=\"middle\" x=\"755\" y=\"-2723.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.3.bn2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"755\" y=\"-2710.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507771496&#45;&gt;140254507474112 -->\n",
       "<g id=\"edge183\" class=\"edge\">\n",
       "<title>140254507771496&#45;&gt;140254507474112</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M735.4373,-2702.9832C725.1733,-2694.0549 712.6133,-2683.1294 702.2495,-2674.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"704.3367,-2671.291 694.4947,-2667.3687 699.7426,-2676.5725 704.3367,-2671.291\"/>\n",
       "</g>\n",
       "<!-- 140254507771552 -->\n",
       "<g id=\"node175\" class=\"node\">\n",
       "<title>140254507771552</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"904.5,-2737 821.5,-2737 821.5,-2703 904.5,-2703 904.5,-2737\"/>\n",
       "<text text-anchor=\"middle\" x=\"863\" y=\"-2723.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.3.bn2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"863\" y=\"-2710.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507771552&#45;&gt;140254507474112 -->\n",
       "<g id=\"edge184\" class=\"edge\">\n",
       "<title>140254507771552&#45;&gt;140254507474112</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M821.4749,-2705.4318C791.4686,-2694.9047 751.1831,-2680.7714 721.6238,-2670.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"722.6356,-2667.0471 712.0408,-2667.0392 720.3183,-2673.6524 722.6356,-2667.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507770936 -->\n",
       "<g id=\"node176\" class=\"node\">\n",
       "<title>140254507770936</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"731.5,-2496 634.5,-2496 634.5,-2462 731.5,-2462 731.5,-2496\"/>\n",
       "<text text-anchor=\"middle\" x=\"683\" y=\"-2482.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.4.bn1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"683\" y=\"-2469.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507770936&#45;&gt;140254507473328 -->\n",
       "<g id=\"edge186\" class=\"edge\">\n",
       "<title>140254507770936&#45;&gt;140254507473328</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M677.9083,-2461.9832C675.5278,-2454.0273 672.6728,-2444.4855 670.1706,-2436.1228\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"673.4718,-2434.9457 667.252,-2426.3687 666.7656,-2436.9523 673.4718,-2434.9457\"/>\n",
       "</g>\n",
       "<!-- 140254507770992 -->\n",
       "<g id=\"node177\" class=\"node\">\n",
       "<title>140254507770992</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"832.5,-2496 749.5,-2496 749.5,-2462 832.5,-2462 832.5,-2496\"/>\n",
       "<text text-anchor=\"middle\" x=\"791\" y=\"-2482.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.4.bn1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"791\" y=\"-2469.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507770992&#45;&gt;140254507473328 -->\n",
       "<g id=\"edge187\" class=\"edge\">\n",
       "<title>140254507770992&#45;&gt;140254507473328</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M756.9663,-2461.9832C737.4812,-2452.2406 713.2399,-2440.12 694.3969,-2430.6985\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"695.8672,-2427.5205 685.3576,-2426.1788 692.7367,-2433.7815 695.8672,-2427.5205\"/>\n",
       "</g>\n",
       "<!-- 140254507770488 -->\n",
       "<g id=\"node178\" class=\"node\">\n",
       "<title>140254507770488</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"803.5,-2312 706.5,-2312 706.5,-2278 803.5,-2278 803.5,-2312\"/>\n",
       "<text text-anchor=\"middle\" x=\"755\" y=\"-2298.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.4.bn2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"755\" y=\"-2285.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507770488&#45;&gt;140254507472544 -->\n",
       "<g id=\"edge188\" class=\"edge\">\n",
       "<title>140254507770488&#45;&gt;140254507472544</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M735.4373,-2277.9832C725.1733,-2269.0549 712.6133,-2258.1294 702.2495,-2249.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"704.3367,-2246.291 694.4947,-2242.3687 699.7426,-2251.5725 704.3367,-2246.291\"/>\n",
       "</g>\n",
       "<!-- 140254507770544 -->\n",
       "<g id=\"node179\" class=\"node\">\n",
       "<title>140254507770544</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"904.5,-2312 821.5,-2312 821.5,-2278 904.5,-2278 904.5,-2312\"/>\n",
       "<text text-anchor=\"middle\" x=\"863\" y=\"-2298.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.4.bn2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"863\" y=\"-2285.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507770544&#45;&gt;140254507472544 -->\n",
       "<g id=\"edge189\" class=\"edge\">\n",
       "<title>140254507770544&#45;&gt;140254507472544</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M821.4749,-2280.4318C791.4686,-2269.9047 751.1831,-2255.7714 721.6238,-2245.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"722.6356,-2242.0471 712.0408,-2242.0392 720.3183,-2248.6524 722.6356,-2242.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507769928 -->\n",
       "<g id=\"node180\" class=\"node\">\n",
       "<title>140254507769928</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"731.5,-2071 634.5,-2071 634.5,-2037 731.5,-2037 731.5,-2071\"/>\n",
       "<text text-anchor=\"middle\" x=\"683\" y=\"-2057.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.5.bn1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"683\" y=\"-2044.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507769928&#45;&gt;140254507471592 -->\n",
       "<g id=\"edge191\" class=\"edge\">\n",
       "<title>140254507769928&#45;&gt;140254507471592</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M663.4373,-2036.9832C653.1733,-2028.0549 640.6133,-2017.1294 630.2495,-2008.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"632.3367,-2005.291 622.4947,-2001.3687 627.7426,-2010.5725 632.3367,-2005.291\"/>\n",
       "</g>\n",
       "<!-- 140254507769984 -->\n",
       "<g id=\"node181\" class=\"node\">\n",
       "<title>140254507769984</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"832.5,-2071 749.5,-2071 749.5,-2037 832.5,-2037 832.5,-2071\"/>\n",
       "<text text-anchor=\"middle\" x=\"791\" y=\"-2057.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.5.bn1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"791\" y=\"-2044.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507769984&#45;&gt;140254507471592 -->\n",
       "<g id=\"edge192\" class=\"edge\">\n",
       "<title>140254507769984&#45;&gt;140254507471592</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M749.4749,-2039.4318C719.4686,-2028.9047 679.1831,-2014.7714 649.6238,-2004.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"650.6356,-2001.0471 640.0408,-2001.0392 648.3183,-2007.6524 650.6356,-2001.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507769416 -->\n",
       "<g id=\"node182\" class=\"node\">\n",
       "<title>140254507769416</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"731.5,-1887 634.5,-1887 634.5,-1853 731.5,-1853 731.5,-1887\"/>\n",
       "<text text-anchor=\"middle\" x=\"683\" y=\"-1873.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.5.bn2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"683\" y=\"-1860.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507769416&#45;&gt;140254507802296 -->\n",
       "<g id=\"edge193\" class=\"edge\">\n",
       "<title>140254507769416&#45;&gt;140254507802296</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M663.4373,-1852.9832C653.1733,-1844.0549 640.6133,-1833.1294 630.2495,-1824.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"632.3367,-1821.291 622.4947,-1817.3687 627.7426,-1826.5725 632.3367,-1821.291\"/>\n",
       "</g>\n",
       "<!-- 140254507769472 -->\n",
       "<g id=\"node183\" class=\"node\">\n",
       "<title>140254507769472</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"832.5,-1887 749.5,-1887 749.5,-1853 832.5,-1853 832.5,-1887\"/>\n",
       "<text text-anchor=\"middle\" x=\"791\" y=\"-1873.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.6.5.bn2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"791\" y=\"-1860.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (256)</text>\n",
       "</g>\n",
       "<!-- 140254507769472&#45;&gt;140254507802296 -->\n",
       "<g id=\"edge194\" class=\"edge\">\n",
       "<title>140254507769472&#45;&gt;140254507802296</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M749.4749,-1855.4318C719.4686,-1844.9047 679.1831,-1830.7714 649.6238,-1820.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"650.6356,-1817.0471 640.0408,-1817.0392 648.3183,-1823.6524 650.6356,-1817.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507768800 -->\n",
       "<g id=\"node184\" class=\"node\">\n",
       "<title>140254507768800</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"765.5,-1646 668.5,-1646 668.5,-1612 765.5,-1612 765.5,-1646\"/>\n",
       "<text text-anchor=\"middle\" x=\"717\" y=\"-1632.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.7.0.bn1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"717\" y=\"-1619.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140254507768800&#45;&gt;140254507801736 -->\n",
       "<g id=\"edge196\" class=\"edge\">\n",
       "<title>140254507768800&#45;&gt;140254507801736</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M697.4373,-1611.9832C687.1733,-1603.0549 674.6133,-1592.1294 664.2495,-1583.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"666.3367,-1580.291 656.4947,-1576.3687 661.7426,-1585.5725 666.3367,-1580.291\"/>\n",
       "</g>\n",
       "<!-- 140254507768856 -->\n",
       "<g id=\"node185\" class=\"node\">\n",
       "<title>140254507768856</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"866.5,-1646 783.5,-1646 783.5,-1612 866.5,-1612 866.5,-1646\"/>\n",
       "<text text-anchor=\"middle\" x=\"825\" y=\"-1632.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.7.0.bn1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"825\" y=\"-1619.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140254507768856&#45;&gt;140254507801736 -->\n",
       "<g id=\"edge197\" class=\"edge\">\n",
       "<title>140254507768856&#45;&gt;140254507801736</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M783.4749,-1614.4318C753.4686,-1603.9047 713.1831,-1589.7714 683.6238,-1579.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"684.6356,-1576.0471 674.0408,-1576.0392 682.3183,-1582.6524 684.6356,-1576.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507768352 -->\n",
       "<g id=\"node186\" class=\"node\">\n",
       "<title>140254507768352</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"736.5,-1462 639.5,-1462 639.5,-1428 736.5,-1428 736.5,-1462\"/>\n",
       "<text text-anchor=\"middle\" x=\"688\" y=\"-1448.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.7.0.bn2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"688\" y=\"-1435.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140254507768352&#45;&gt;140254507800392 -->\n",
       "<g id=\"edge198\" class=\"edge\">\n",
       "<title>140254507768352&#45;&gt;140254507800392</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M668.4373,-1427.9832C658.1733,-1419.0549 645.6133,-1408.1294 635.2495,-1399.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"637.3367,-1396.291 627.4947,-1392.3687 632.7426,-1401.5725 637.3367,-1396.291\"/>\n",
       "</g>\n",
       "<!-- 140254507768408 -->\n",
       "<g id=\"node187\" class=\"node\">\n",
       "<title>140254507768408</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"837.5,-1462 754.5,-1462 754.5,-1428 837.5,-1428 837.5,-1462\"/>\n",
       "<text text-anchor=\"middle\" x=\"796\" y=\"-1448.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.7.0.bn2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"796\" y=\"-1435.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140254507768408&#45;&gt;140254507800392 -->\n",
       "<g id=\"edge199\" class=\"edge\">\n",
       "<title>140254507768408&#45;&gt;140254507800392</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M754.4749,-1430.4318C724.4686,-1419.9047 684.1831,-1405.7714 654.6238,-1395.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"655.6356,-1392.0471 645.0408,-1392.0392 653.3183,-1398.6524 655.6356,-1392.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507799048 -->\n",
       "<g id=\"node188\" class=\"node\">\n",
       "<title>140254507799048</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"367.5,-1576 214.5,-1576 214.5,-1555 367.5,-1555 367.5,-1576\"/>\n",
       "<text text-anchor=\"middle\" x=\"291\" y=\"-1562.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">CudnnBatchNormBackward</text>\n",
       "</g>\n",
       "<!-- 140254507799048&#45;&gt;140254507801624 -->\n",
       "<g id=\"edge200\" class=\"edge\">\n",
       "<title>140254507799048&#45;&gt;140254507801624</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M298.4798,-1554.8336C324.7652,-1517.3497 413.1251,-1391.3452 446.8069,-1343.3138\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"449.7307,-1345.2403 452.6066,-1335.0433 443.9995,-1341.2213 449.7307,-1345.2403\"/>\n",
       "</g>\n",
       "<!-- 140254507799608&#45;&gt;140254507799048 -->\n",
       "<g id=\"edge201\" class=\"edge\">\n",
       "<title>140254507799608&#45;&gt;140254507799048</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M379.0185,-1618.2281C362.0894,-1608.0866 336.5323,-1592.7764 317.3116,-1581.2622\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"319.0931,-1578.2494 308.7159,-1576.1128 315.4957,-1584.2544 319.0931,-1578.2494\"/>\n",
       "</g>\n",
       "<!-- 140254507768576 -->\n",
       "<g id=\"node190\" class=\"node\">\n",
       "<title>140254507768576</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"148,-1646 0,-1646 0,-1612 148,-1612 148,-1646\"/>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-1632.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.7.0.downsample.1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"74\" y=\"-1619.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140254507768576&#45;&gt;140254507799048 -->\n",
       "<g id=\"edge203\" class=\"edge\">\n",
       "<title>140254507768576&#45;&gt;140254507799048</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M132.152,-1611.9832C167.5355,-1601.629 212.0982,-1588.5888 245.0122,-1578.9573\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"246.2153,-1582.252 254.8299,-1576.0843 244.2493,-1575.5338 246.2153,-1582.252\"/>\n",
       "</g>\n",
       "<!-- 140254507768688 -->\n",
       "<g id=\"node191\" class=\"node\">\n",
       "<title>140254507768688</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"300,-1646 166,-1646 166,-1612 300,-1612 300,-1646\"/>\n",
       "<text text-anchor=\"middle\" x=\"233\" y=\"-1632.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.7.0.downsample.1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"233\" y=\"-1619.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140254507768688&#45;&gt;140254507799048 -->\n",
       "<g id=\"edge204\" class=\"edge\">\n",
       "<title>140254507768688&#45;&gt;140254507799048</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M248.5429,-1611.9832C256.4557,-1603.3201 266.0859,-1592.7766 274.1731,-1583.9226\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"276.9129,-1586.1127 281.0727,-1576.3687 271.7443,-1581.3919 276.9129,-1586.1127\"/>\n",
       "</g>\n",
       "<!-- 140254554077112 -->\n",
       "<g id=\"node192\" class=\"node\">\n",
       "<title>140254554077112</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"707.5,-1221 610.5,-1221 610.5,-1187 707.5,-1187 707.5,-1221\"/>\n",
       "<text text-anchor=\"middle\" x=\"659\" y=\"-1207.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.7.1.bn1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"659\" y=\"-1194.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140254554077112&#45;&gt;140254507800504 -->\n",
       "<g id=\"edge205\" class=\"edge\">\n",
       "<title>140254554077112&#45;&gt;140254507800504</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M639.4373,-1186.9832C629.1733,-1178.0549 616.6133,-1167.1294 606.2495,-1158.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"608.3367,-1155.291 598.4947,-1151.3687 603.7426,-1160.5725 608.3367,-1155.291\"/>\n",
       "</g>\n",
       "<!-- 140254554768328 -->\n",
       "<g id=\"node193\" class=\"node\">\n",
       "<title>140254554768328</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"808.5,-1221 725.5,-1221 725.5,-1187 808.5,-1187 808.5,-1221\"/>\n",
       "<text text-anchor=\"middle\" x=\"767\" y=\"-1207.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.7.1.bn1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"767\" y=\"-1194.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140254554768328&#45;&gt;140254507800504 -->\n",
       "<g id=\"edge206\" class=\"edge\">\n",
       "<title>140254554768328&#45;&gt;140254507800504</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M725.4749,-1189.4318C695.4686,-1178.9047 655.1831,-1164.7714 625.6238,-1154.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"626.6356,-1151.0471 616.0408,-1151.0392 624.3183,-1157.6524 626.6356,-1151.0471\"/>\n",
       "</g>\n",
       "<!-- 140254558748344 -->\n",
       "<g id=\"node194\" class=\"node\">\n",
       "<title>140254558748344</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"707.5,-1037 610.5,-1037 610.5,-1003 707.5,-1003 707.5,-1037\"/>\n",
       "<text text-anchor=\"middle\" x=\"659\" y=\"-1023.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.7.1.bn2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"659\" y=\"-1010.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140254558748344&#45;&gt;140254507705568 -->\n",
       "<g id=\"edge207\" class=\"edge\">\n",
       "<title>140254558748344&#45;&gt;140254507705568</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M639.4373,-1002.9832C629.1733,-994.0549 616.6133,-983.1294 606.2495,-974.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"608.3367,-971.291 598.4947,-967.3687 603.7426,-976.5725 608.3367,-971.291\"/>\n",
       "</g>\n",
       "<!-- 140254558748512 -->\n",
       "<g id=\"node195\" class=\"node\">\n",
       "<title>140254558748512</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"808.5,-1037 725.5,-1037 725.5,-1003 808.5,-1003 808.5,-1037\"/>\n",
       "<text text-anchor=\"middle\" x=\"767\" y=\"-1023.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.7.1.bn2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"767\" y=\"-1010.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140254558748512&#45;&gt;140254507705568 -->\n",
       "<g id=\"edge208\" class=\"edge\">\n",
       "<title>140254558748512&#45;&gt;140254507705568</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M725.4749,-1005.4318C695.4686,-994.9047 655.1831,-980.7714 625.6238,-970.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"626.6356,-967.0471 616.0408,-967.0392 624.3183,-973.6524 626.6356,-967.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507767456 -->\n",
       "<g id=\"node196\" class=\"node\">\n",
       "<title>140254507767456</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"707.5,-796 610.5,-796 610.5,-762 707.5,-762 707.5,-796\"/>\n",
       "<text text-anchor=\"middle\" x=\"659\" y=\"-782.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.7.2.bn1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"659\" y=\"-769.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140254507767456&#45;&gt;140254507705008 -->\n",
       "<g id=\"edge210\" class=\"edge\">\n",
       "<title>140254507767456&#45;&gt;140254507705008</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M639.4373,-761.9832C629.1733,-753.0549 616.6133,-742.1294 606.2495,-733.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"608.3367,-730.291 598.4947,-726.3687 603.7426,-735.5725 608.3367,-730.291\"/>\n",
       "</g>\n",
       "<!-- 140254507767512 -->\n",
       "<g id=\"node197\" class=\"node\">\n",
       "<title>140254507767512</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"808.5,-796 725.5,-796 725.5,-762 808.5,-762 808.5,-796\"/>\n",
       "<text text-anchor=\"middle\" x=\"767\" y=\"-782.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.7.2.bn1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"767\" y=\"-769.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140254507767512&#45;&gt;140254507705008 -->\n",
       "<g id=\"edge211\" class=\"edge\">\n",
       "<title>140254507767512&#45;&gt;140254507705008</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M725.4749,-764.4318C695.4686,-753.9047 655.1831,-739.7714 625.6238,-729.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"626.6356,-726.0471 616.0408,-726.0392 624.3183,-732.6524 626.6356,-726.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507766952 -->\n",
       "<g id=\"node198\" class=\"node\">\n",
       "<title>140254507766952</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"707.5,-612 610.5,-612 610.5,-578 707.5,-578 707.5,-612\"/>\n",
       "<text text-anchor=\"middle\" x=\"659\" y=\"-598.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.7.2.bn2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"659\" y=\"-585.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140254507766952&#45;&gt;140254507793768 -->\n",
       "<g id=\"edge212\" class=\"edge\">\n",
       "<title>140254507766952&#45;&gt;140254507793768</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M639.4373,-577.9832C629.1733,-569.0549 616.6133,-558.1294 606.2495,-549.1143\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"608.3367,-546.291 598.4947,-542.3687 603.7426,-551.5725 608.3367,-546.291\"/>\n",
       "</g>\n",
       "<!-- 140254507767008 -->\n",
       "<g id=\"node199\" class=\"node\">\n",
       "<title>140254507767008</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"808.5,-612 725.5,-612 725.5,-578 808.5,-578 808.5,-612\"/>\n",
       "<text text-anchor=\"middle\" x=\"767\" y=\"-598.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">0.7.2.bn2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"767\" y=\"-585.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140254507767008&#45;&gt;140254507793768 -->\n",
       "<g id=\"edge213\" class=\"edge\">\n",
       "<title>140254507767008&#45;&gt;140254507793768</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M725.4749,-580.4318C695.4686,-569.9047 655.1831,-555.7714 625.6238,-545.4012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"626.6356,-542.0471 616.0408,-542.0392 624.3183,-548.6524 626.6356,-542.0471\"/>\n",
       "</g>\n",
       "<!-- 140254507766392 -->\n",
       "<g id=\"node200\" class=\"node\">\n",
       "<title>140254507766392</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"596,-428 538,-428 538,-394 596,-394 596,-428\"/>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-414.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"567\" y=\"-401.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140254507766392&#45;&gt;140254507791696 -->\n",
       "<g id=\"edge215\" class=\"edge\">\n",
       "<title>140254507766392&#45;&gt;140254507791696</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M567,-393.6966C567,-384.0634 567,-372.003 567,-361.8518\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"570.5001,-361.7912 567,-351.7913 563.5001,-361.7913 570.5001,-361.7912\"/>\n",
       "</g>\n",
       "<!-- 140254507766448 -->\n",
       "<g id=\"node201\" class=\"node\">\n",
       "<title>140254507766448</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"668,-428 614,-428 614,-394 668,-394 668,-428\"/>\n",
       "<text text-anchor=\"middle\" x=\"641\" y=\"-414.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"641\" y=\"-401.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140254507766448&#45;&gt;140254507791696 -->\n",
       "<g id=\"edge216\" class=\"edge\">\n",
       "<title>140254507766448&#45;&gt;140254507791696</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M622.7079,-393.6966C611.4408,-383.0386 597.0329,-369.4095 585.6926,-358.6821\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"588.0778,-356.1206 578.4079,-351.7913 583.2674,-361.2059 588.0778,-356.1206\"/>\n",
       "</g>\n",
       "<!-- 140254507790912 -->\n",
       "<g id=\"node202\" class=\"node\">\n",
       "<title>140254507790912</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"839,-281.5 645,-281.5 645,-260.5 839,-260.5 839,-281.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"742\" y=\"-267.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">WeightNormCudaInterfaceBackward</text>\n",
       "</g>\n",
       "<!-- 140254507790912&#45;&gt;140254507790576 -->\n",
       "<g id=\"edge217\" class=\"edge\">\n",
       "<title>140254507790912&#45;&gt;140254507790576</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M742,-260.3685C742,-250.1925 742,-234.5606 742,-221.8912\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"745.5001,-221.7315 742,-211.7315 738.5001,-221.7316 745.5001,-221.7315\"/>\n",
       "</g>\n",
       "<!-- 140254558397440 -->\n",
       "<g id=\"node203\" class=\"node\">\n",
       "<title>140254558397440</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"775.5,-358 676.5,-358 676.5,-324 775.5,-324 775.5,-358\"/>\n",
       "<text text-anchor=\"middle\" x=\"726\" y=\"-344.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">3.0.0.weight_v</text>\n",
       "<text text-anchor=\"middle\" x=\"726\" y=\"-331.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1024, 512, 3, 3)</text>\n",
       "</g>\n",
       "<!-- 140254558397440&#45;&gt;140254507790912 -->\n",
       "<g id=\"edge218\" class=\"edge\">\n",
       "<title>140254558397440&#45;&gt;140254507790912</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M729.9551,-323.6966C732.1569,-314.0634 734.9136,-302.003 737.2339,-291.8518\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"740.7171,-292.3198 739.5334,-281.7913 733.8931,-290.7599 740.7171,-292.3198\"/>\n",
       "</g>\n",
       "<!-- 140254554442440 -->\n",
       "<g id=\"node204\" class=\"node\">\n",
       "<title>140254554442440</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"882,-358 794,-358 794,-324 882,-324 882,-358\"/>\n",
       "<text text-anchor=\"middle\" x=\"838\" y=\"-344.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">3.0.0.weight_g</text>\n",
       "<text text-anchor=\"middle\" x=\"838\" y=\"-331.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1024, 1, 1, 1)</text>\n",
       "</g>\n",
       "<!-- 140254554442440&#45;&gt;140254507790912 -->\n",
       "<g id=\"edge219\" class=\"edge\">\n",
       "<title>140254554442440&#45;&gt;140254507790912</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M814.2697,-323.6966C799.1385,-312.6635 779.6408,-298.4464 764.7115,-287.5605\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"766.659,-284.6489 756.5168,-281.5852 762.5348,-290.305 766.659,-284.6489\"/>\n",
       "</g>\n",
       "<!-- 140254507766056 -->\n",
       "<g id=\"node205\" class=\"node\">\n",
       "<title>140254507766056</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"919,-288 857,-288 857,-254 919,-254 919,-288\"/>\n",
       "<text text-anchor=\"middle\" x=\"888\" y=\"-274.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">3.0.0.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"888\" y=\"-261.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (1024)</text>\n",
       "</g>\n",
       "<!-- 140254507766056&#45;&gt;140254507790576 -->\n",
       "<g id=\"edge220\" class=\"edge\">\n",
       "<title>140254507766056&#45;&gt;140254507790576</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M856.6793,-255.9832C832.1215,-244.2089 798.0997,-227.8971 773.4968,-216.1012\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"774.7716,-212.831 764.2413,-211.6636 771.7452,-219.143 774.7716,-212.831\"/>\n",
       "</g>\n",
       "<!-- 140254507766840 -->\n",
       "<g id=\"node206\" class=\"node\">\n",
       "<title>140254507766840</title>\n",
       "<polygon fill=\"#d3d3d3\" stroke=\"#000000\" points=\"1014,-141.5 820,-141.5 820,-120.5 1014,-120.5 1014,-141.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"917\" y=\"-127.9\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">WeightNormCudaInterfaceBackward</text>\n",
       "</g>\n",
       "<!-- 140254507766840&#45;&gt;140254507767176 -->\n",
       "<g id=\"edge221\" class=\"edge\">\n",
       "<title>140254507766840&#45;&gt;140254507767176</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M917,-120.2281C917,-111.5091 917,-98.9699 917,-88.3068\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"920.5001,-88.1128 917,-78.1128 913.5001,-88.1129 920.5001,-88.1128\"/>\n",
       "</g>\n",
       "<!-- 140254554279216 -->\n",
       "<g id=\"node207\" class=\"node\">\n",
       "<title>140254554279216</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"951.5,-218 852.5,-218 852.5,-184 951.5,-184 951.5,-218\"/>\n",
       "<text text-anchor=\"middle\" x=\"902\" y=\"-204.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">3.1.0.weight_v</text>\n",
       "<text text-anchor=\"middle\" x=\"902\" y=\"-191.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512, 1024, 3, 3)</text>\n",
       "</g>\n",
       "<!-- 140254554279216&#45;&gt;140254507766840 -->\n",
       "<g id=\"edge222\" class=\"edge\">\n",
       "<title>140254554279216&#45;&gt;140254507766840</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M905.7079,-183.6966C907.7721,-174.0634 910.3565,-162.003 912.5317,-151.8518\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"916.0145,-152.3027 914.6876,-141.7913 909.1699,-150.8359 916.0145,-152.3027\"/>\n",
       "</g>\n",
       "<!-- 140254554194048 -->\n",
       "<g id=\"node208\" class=\"node\">\n",
       "<title>140254554194048</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"1058,-218 970,-218 970,-184 1058,-184 1058,-218\"/>\n",
       "<text text-anchor=\"middle\" x=\"1014\" y=\"-204.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">3.1.0.weight_g</text>\n",
       "<text text-anchor=\"middle\" x=\"1014\" y=\"-191.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512, 1, 1, 1)</text>\n",
       "</g>\n",
       "<!-- 140254554194048&#45;&gt;140254507766840 -->\n",
       "<g id=\"edge223\" class=\"edge\">\n",
       "<title>140254554194048&#45;&gt;140254507766840</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M990.0225,-183.6966C974.7337,-172.6635 955.0329,-158.4464 939.9481,-147.5605\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"941.8252,-144.5989 931.668,-141.5852 937.7289,-150.2752 941.8252,-144.5989\"/>\n",
       "</g>\n",
       "<!-- 140254507753312 -->\n",
       "<g id=\"node209\" class=\"node\">\n",
       "<title>140254507753312</title>\n",
       "<polygon fill=\"#add8e6\" stroke=\"#000000\" points=\"1094,-148 1032,-148 1032,-114 1094,-114 1094,-148\"/>\n",
       "<text text-anchor=\"middle\" x=\"1063\" y=\"-134.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\">3.1.0.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"1063\" y=\"-121.4\" font-family=\"Times,serif\" font-size=\"12.00\" fill=\"#000000\"> (512)</text>\n",
       "</g>\n",
       "<!-- 140254507753312&#45;&gt;140254507767176 -->\n",
       "<g id=\"edge224\" class=\"edge\">\n",
       "<title>140254507753312&#45;&gt;140254507767176</title>\n",
       "<path fill=\"none\" stroke=\"#000000\" d=\"M1031.6793,-117.3777C1007.8612,-107.0184 975.1407,-92.7872 950.7451,-82.1768\"/>\n",
       "<polygon fill=\"#000000\" stroke=\"#000000\" points=\"952.0766,-78.9392 941.5104,-78.1603 949.2847,-85.3584 952.0766,-78.9392\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f8f8c1cac50>"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "make_dot(encoder(torch.randn(1, 3, 128, 128).to(device)), params=dict(encoder.named_parameters()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = encoder(torch.randn(1, 3, 128, 128).to(device))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 512, 4, 4])"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "res.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dl/anaconda3/envs/fastai/lib/python3.7/site-packages/torchvision-0.2.1-py3.7.egg/torchvision/models/squeezenet.py:94: UserWarning: nn.init.kaiming_uniform is now deprecated in favor of nn.init.kaiming_uniform_.\n",
      "  init.kaiming_uniform(m.weight.data)\n",
      "/home/dl/anaconda3/envs/fastai/lib/python3.7/site-packages/torchvision-0.2.1-py3.7.egg/torchvision/models/squeezenet.py:92: UserWarning: nn.init.normal is now deprecated in favor of nn.init.normal_.\n",
      "  init.normal(m.weight.data, mean=0.0, std=0.01)\n"
     ]
    }
   ],
   "source": [
    "import torchvision.models as models\n",
    "net = models.squeezenet1_0(pretrained=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [],
   "source": [
    "_=net.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = torch.randn(1, 3, 224, 224).to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[3.8900, 4.3447, 0.8647, 0.9475, 3.9939, 4.7409, 4.5169, 6.0487, 4.1048,\n",
       "         3.3984, 1.2202, 2.4193, 1.7004, 1.9143, 4.0369, 3.7702, 1.1948, 4.7033,\n",
       "         6.5576, 1.8975, 1.1037, 3.1495, 1.4911, 4.2212, 2.7740, 1.3039, 2.2415,\n",
       "         3.3967, 1.5833, 1.3222, 4.1339, 3.4977, 2.1649, 1.4839, 2.5340, 2.9412,\n",
       "         3.4808, 2.9188, 2.5402, 7.4968, 5.0729, 3.9720, 4.4870, 2.6553, 2.4817,\n",
       "         3.3902, 6.3804, 4.7172, 5.5394, 2.5900, 4.9458, 1.2878, 2.1832, 1.8202,\n",
       "         3.4688, 4.2959, 3.6123, 3.1116, 3.2600, 2.6428, 3.0195, 3.8237, 2.9711,\n",
       "         2.2707, 3.4332, 3.8719, 2.3842, 4.3661, 2.3761, 2.4951, 2.4894, 2.6180,\n",
       "         0.6260, 0.5487, 0.9018, 2.0091, 2.9226, 4.0619, 3.3530, 2.6094, 2.2348,\n",
       "         2.4864, 4.4442, 0.7489, 7.8479, 5.4125, 4.4483, 2.8519, 1.9949, 3.9809,\n",
       "         2.8501, 2.9643, 2.5181, 2.5199, 3.3338, 0.5659, 1.9904, 3.1426, 1.5445,\n",
       "         3.4651, 2.5519, 2.8298, 2.3348, 2.8920, 4.4913, 2.0949, 3.0544, 4.7355,\n",
       "         3.7873, 4.2821, 3.5156, 3.0582, 2.2186, 3.2508, 2.4541, 2.2138, 2.2261,\n",
       "         1.6752, 3.9539, 3.2094, 2.9434, 1.6475, 2.7521, 2.3412, 4.9038, 3.3472,\n",
       "         2.2833, 3.4878, 4.1756, 2.0592, 2.6428, 3.3908, 4.2968, 4.5030, 5.7090,\n",
       "         1.9165, 3.6009, 1.9797, 4.4568, 1.8613, 1.1842, 1.4706, 0.9591, 3.8061,\n",
       "         1.2705, 0.9964, 1.2522, 0.4223, 1.0970, 1.8120, 0.6419, 6.4924, 2.8762,\n",
       "         6.8525, 5.4506, 6.9107, 4.9039, 6.3762, 5.7913, 5.1874, 2.4104, 5.1615,\n",
       "         6.8882, 4.2319, 2.4826, 3.3667, 3.1789, 2.6317, 5.1434, 3.7223, 3.8224,\n",
       "         5.7341, 3.4283, 4.0241, 5.0017, 1.9942, 3.9320, 2.9775, 5.2301, 5.0826,\n",
       "         6.2524, 3.7930, 5.0075, 4.5629, 4.5630, 5.1415, 4.5804, 7.3141, 2.2698,\n",
       "         4.4753, 2.7551, 3.7209, 5.9100, 3.8797, 2.2967, 7.7412, 5.7316, 4.4093,\n",
       "         5.1265, 5.5459, 4.1572, 7.2839, 4.6457, 6.6862, 8.7475, 4.3653, 3.1578,\n",
       "         5.3422, 4.5863, 3.9859, 1.9613, 4.4191, 5.9971, 4.7267, 3.7512, 4.3999,\n",
       "         5.6693, 5.0521, 5.9181, 6.5633, 1.6842, 3.1322, 2.5315, 3.6671, 3.7465,\n",
       "         4.2350, 3.3001, 3.4828, 1.3142, 5.1855, 6.9942, 5.3205, 6.1403, 3.1823,\n",
       "         5.0272, 4.7049, 4.3565, 4.7226, 4.9928, 4.3607, 5.1814, 4.3947, 5.7717,\n",
       "         2.0111, 1.6727, 3.6872, 3.6556, 4.7984, 2.8906, 4.7637, 3.7600, 4.2146,\n",
       "         3.1146, 5.4053, 5.2266, 4.3383, 3.4946, 5.5457, 4.1606, 6.4646, 3.2621,\n",
       "         4.7223, 2.5508, 6.3051, 4.6436, 6.2672, 5.6134, 3.5650, 2.2876, 3.3219,\n",
       "         4.3829, 3.6161, 4.8327, 3.1653, 3.0591, 2.3436, 2.6962, 4.2634, 2.5021,\n",
       "         1.8363, 4.8164, 3.4953, 4.3992, 5.1482, 5.8513, 3.3535, 3.6115, 3.1103,\n",
       "         2.3570, 1.0338, 2.1111, 1.8099, 2.4625, 4.1362, 2.5042, 3.9613, 1.8159,\n",
       "         0.7415, 4.2736, 2.2772, 3.5201, 4.0988, 1.5339, 1.1354, 2.2953, 2.1485,\n",
       "         1.9581, 2.6202, 4.5070, 3.9023, 4.7931, 3.2992, 3.5612, 4.1215, 3.3903,\n",
       "         3.7991, 3.2334, 5.7971, 2.6213, 1.9111, 1.5169, 1.3116, 0.7572, 1.4861,\n",
       "         2.5654, 1.6999, 0.8045, 4.0023, 4.7099, 3.2136, 6.5073, 4.8472, 5.4170,\n",
       "         1.5029, 5.2505, 3.8827, 4.3084, 2.2332, 3.9943, 3.7520, 2.4888, 2.1128,\n",
       "         2.4487, 2.8708, 3.3298, 2.8258, 2.0601, 3.4181, 2.5354, 3.7703, 2.7090,\n",
       "         2.2339, 3.0335, 4.9288, 5.2243, 3.6383, 4.0071, 2.2845, 4.5720, 5.0439,\n",
       "         2.8282, 6.6839, 3.6645, 3.6044, 0.4732, 3.6848, 2.7666, 1.4841, 1.4118,\n",
       "         2.7868, 2.4234, 2.4668, 2.6884, 2.3881, 1.4889, 1.9801, 1.3697, 0.7717,\n",
       "         1.8609, 1.1263, 1.8089, 2.6052, 2.0332, 3.9165, 1.2675, 3.3120, 2.3856,\n",
       "         1.3160, 0.8914, 2.0715, 5.1458, 2.8519, 2.0558, 3.1003, 1.2669, 2.7395,\n",
       "         4.4046, 3.6786, 1.5361, 2.2347, 5.3041, 3.5718, 1.4060, 1.1020, 0.3060,\n",
       "         0.9735, 2.3849, 1.2825, 1.6872, 2.9329, 5.4339, 5.8692, 4.7771, 5.7707,\n",
       "         5.7018, 1.0573, 3.5443, 4.4964, 3.6909, 5.1525, 1.4828, 3.7267, 0.2296,\n",
       "         0.8013, 0.8486, 5.3339, 0.6454, 3.7209, 4.6091, 6.1880, 4.0753, 2.1365,\n",
       "         5.0665, 2.1733, 3.9464, 2.8641, 2.2530, 1.9816, 0.7195, 1.2160, 3.4541,\n",
       "         3.1702, 1.5142, 6.2604, 2.1744, 3.0608, 2.9507, 4.7974, 4.9355, 2.4504,\n",
       "         0.2032, 2.0373, 6.1704, 2.7170, 1.4932, 5.1465, 2.8975, 3.3406, 5.6725,\n",
       "         3.6332, 3.0564, 3.0755, 2.8383, 5.1488, 3.2539, 2.6305, 1.2397, 0.1408,\n",
       "         4.0800, 4.8254, 1.2542, 5.5587, 1.5659, 1.0734, 7.0372, 2.5010, 0.9676,\n",
       "         0.7276, 1.1974, 2.0005, 2.9644, 2.3467, 4.2001, 4.8287, 2.8546, 1.4899,\n",
       "         2.9409, 1.6001, 5.4992, 7.9819, 6.4584, 3.4360, 3.9706, 2.1591, 1.7075,\n",
       "         0.7561, 5.4417, 4.7093, 1.9132, 1.9307, 1.1053, 3.5298, 5.2184, 2.0243,\n",
       "         3.1277, 3.3123, 6.1919, 1.4141, 1.4960, 3.9835, 1.6303, 3.1748, 1.1525,\n",
       "         4.2780, 2.9332, 4.9476, 3.3256, 1.6385, 2.8224, 2.5141, 2.0650, 2.5111,\n",
       "         6.8073, 5.9068, 4.0249, 1.6607, 0.9291, 1.3260, 1.7366, 5.7384, 3.3033,\n",
       "         0.8102, 1.5696, 8.2183, 1.8279, 0.0281, 1.2208, 1.8031, 5.5215, 9.0362,\n",
       "         0.8252, 2.4041, 2.7945, 2.1301, 2.3716, 2.4950, 1.3322, 0.5260, 0.5846,\n",
       "         4.8235, 1.5636, 1.9028, 5.1016, 2.6992, 0.7895, 3.2137, 2.7080, 3.4957,\n",
       "         4.4549, 5.4659, 7.2520, 0.7611, 4.4005, 3.9724, 2.1456, 2.1002, 3.9598,\n",
       "         2.8517, 2.5710, 2.4862, 2.9729, 1.4943, 2.3992, 1.2466, 6.5313, 3.9188,\n",
       "         0.7299, 2.4597, 4.7788, 2.4091, 4.5209, 3.7370, 1.5521, 1.9954, 2.7172,\n",
       "         3.2188, 3.1494, 4.0060, 3.1515, 3.3641, 0.8798, 2.2762, 1.7873, 3.9321,\n",
       "         2.0859, 3.5130, 2.9898, 2.6573, 1.9137, 5.4496, 4.1773, 4.2570, 3.7509,\n",
       "         2.9589, 3.0377, 2.0262, 3.4193, 2.1939, 6.6591, 2.8456, 1.3753, 4.1439,\n",
       "         1.1970, 2.3775, 3.1214, 3.8981, 5.8744, 2.2379, 3.4788, 3.2054, 1.3590,\n",
       "         5.7059, 6.4435, 4.1874, 2.3867, 0.4708, 2.7996, 3.2577, 2.8502, 2.4908,\n",
       "         5.5165, 2.6320, 3.4993, 4.2444, 0.8580, 4.6864, 5.7117, 3.2727, 3.7284,\n",
       "         3.7266, 4.6324, 3.2709, 1.8973, 4.4722, 3.9583, 3.6930, 4.9500, 1.5268,\n",
       "         1.2654, 3.9537, 2.2444, 4.7907, 4.5682, 4.6914, 2.1611, 5.3208, 1.9730,\n",
       "         1.6334, 6.5016, 3.5444, 2.4938, 2.8558, 2.2827, 2.5274, 2.0473, 2.3295,\n",
       "         2.1198, 3.4509, 1.6323, 4.0694, 3.0579, 1.4803, 1.5030, 1.5669, 1.9551,\n",
       "         0.4992, 1.2005, 3.8638, 1.8327, 4.2390, 1.8130, 2.3338, 3.8506, 3.0903,\n",
       "         1.9014, 0.9429, 0.7354, 0.9977, 0.5837, 3.9543, 3.4702, 0.7297, 3.4550,\n",
       "         1.7637, 0.7148, 4.0286, 1.9621, 2.7077, 4.2554, 1.7480, 3.0977, 2.5926,\n",
       "         4.1911, 6.5809, 4.7840, 1.4016, 3.8925, 3.4812, 2.6017, 1.3699, 2.2165,\n",
       "         3.0160, 2.7205, 0.9858, 1.8999, 0.5429, 4.8883, 2.1908, 3.7575, 5.2881,\n",
       "         1.6427, 4.6772, 2.7648, 5.4317, 1.9267, 2.8569, 1.2885, 4.3916, 4.6831,\n",
       "         0.6728, 3.5094, 4.9448, 3.2312, 6.2380, 1.5615, 8.6431, 3.6239, 2.9401,\n",
       "         2.8209, 0.0618, 2.9216, 7.7297, 2.0129, 1.8414, 2.4561, 1.0892, 2.0302,\n",
       "         2.2705, 4.3188, 5.7187, 4.4108, 3.7261, 5.0767, 1.6579, 2.6642, 1.8037,\n",
       "         2.6828, 1.1232, 3.8041, 1.7467, 1.2657, 2.3846, 2.2442, 5.2483, 6.2280,\n",
       "         3.3493, 2.4394, 1.3304, 6.1875, 1.6074, 5.1190, 3.1090, 4.4081, 3.7882,\n",
       "         5.0418, 3.7801, 4.6991, 3.7073, 1.4654, 2.5973, 0.9660, 4.8254, 2.3720,\n",
       "         2.9196, 3.5856, 2.7755, 3.7058, 4.2384, 3.6207, 1.5439, 3.3795, 6.2854,\n",
       "         4.6712, 2.5557, 4.2122, 2.7053, 3.0794, 5.4811, 0.5022, 2.7648, 1.3816,\n",
       "         1.8648, 0.5037, 0.7993, 4.5834, 8.8922, 6.0901, 1.0030, 3.3316, 1.9282,\n",
       "         0.9633, 3.9146, 1.3893, 2.6828, 1.7909, 2.5203, 4.0123, 3.1203, 3.9219,\n",
       "         2.6811, 1.2114, 2.1431, 2.0163, 5.2827, 6.5565, 4.9499, 3.1271, 1.2218,\n",
       "         4.0840, 1.7541, 3.0575, 3.9060, 1.4754, 1.6650, 4.3085, 5.3373, 2.7003,\n",
       "         4.7336, 3.3892, 2.6075, 2.6682, 4.5207, 3.2174, 4.3049, 5.3583, 3.8642,\n",
       "         1.4268, 2.3428, 4.4268, 2.8847, 3.2654, 4.6010, 5.7249, 5.7061, 3.2530,\n",
       "         1.8033, 1.3960, 1.6185, 5.3890, 4.0182, 0.5505, 4.0160, 1.8561, 1.6849,\n",
       "         3.1356, 2.0724, 4.8725, 2.3336, 3.1963, 6.0369, 6.7485, 1.8348, 5.8989,\n",
       "         1.8003, 0.4103, 3.6872, 2.4657, 2.9143, 0.7614, 8.1690, 2.9726, 2.5169,\n",
       "         6.1307, 1.8707, 2.7709, 5.5592, 1.1553, 1.1311, 2.5390, 2.1213, 1.4250,\n",
       "         1.7142, 5.3910, 3.2158, 3.1445, 0.9658, 3.1483, 4.9180, 6.5479, 2.3059,\n",
       "         2.1000, 1.2580, 2.2296, 1.1129, 5.7426, 1.3189, 2.4756, 1.9946, 2.8521,\n",
       "         2.5169, 3.8075, 6.4496, 5.4603, 1.8274, 0.2648, 3.2283, 3.3895, 3.8933,\n",
       "         2.7669, 4.1063, 3.3272, 2.2843, 2.5345, 0.4412, 0.3689, 1.5732, 0.6429,\n",
       "         2.0570, 2.0065, 3.5929, 1.8204, 2.5340, 1.6636, 0.5728, 0.8633, 1.2087,\n",
       "         1.0380, 4.3547, 2.2247, 4.5399, 1.0136, 0.6480, 0.4766, 1.6159, 0.5082,\n",
       "         1.4678, 1.3647, 2.7876, 3.5036, 1.9265, 2.8576, 3.9993, 1.2151, 2.6666,\n",
       "         3.4051, 5.9645, 1.1884, 3.1672, 4.5137, 0.2897, 2.3217, 0.3488, 1.0285,\n",
       "         1.1795, 0.4881, 0.3435, 3.1041, 2.6723, 2.2709, 2.3435, 1.7819, 6.8944,\n",
       "         2.6755, 5.2853, 0.9181, 5.0877, 3.1980, 4.1490, 3.0897, 2.3618, 5.1112,\n",
       "         8.6795, 3.7019, 2.1699, 7.9268, 4.4244, 2.2384, 4.1602, 1.7098, 1.0568,\n",
       "         3.6378, 1.7548, 1.8115, 0.5311, 0.9517, 0.7218, 1.3810, 2.2413, 3.2067,\n",
       "         3.6728]], device='cuda:0', grad_fn=<ViewBackward>)"
      ]
     },
     "execution_count": 132,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [],
   "source": [
    "def weight_init(m):\n",
    "    if isinstance(m, nn.Conv2d):\n",
    "        torch.nn.init.xavier_uniform_(m.weight.data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[[[-0.0399]],\n",
       "\n",
       "         [[ 0.0013]],\n",
       "\n",
       "         [[-0.0372]],\n",
       "\n",
       "         ...,\n",
       "\n",
       "         [[-0.0017]],\n",
       "\n",
       "         [[-0.0199]],\n",
       "\n",
       "         [[-0.0169]]],\n",
       "\n",
       "\n",
       "        [[[-0.0714]],\n",
       "\n",
       "         [[ 0.0289]],\n",
       "\n",
       "         [[ 0.0246]],\n",
       "\n",
       "         ...,\n",
       "\n",
       "         [[-0.0359]],\n",
       "\n",
       "         [[-0.0143]],\n",
       "\n",
       "         [[ 0.0153]]],\n",
       "\n",
       "\n",
       "        [[[ 0.1270]],\n",
       "\n",
       "         [[ 0.0315]],\n",
       "\n",
       "         [[ 0.0192]],\n",
       "\n",
       "         ...,\n",
       "\n",
       "         [[-0.0372]],\n",
       "\n",
       "         [[-0.0792]],\n",
       "\n",
       "         [[ 0.0002]]],\n",
       "\n",
       "\n",
       "        ...,\n",
       "\n",
       "\n",
       "        [[[-0.0551]],\n",
       "\n",
       "         [[ 0.1428]],\n",
       "\n",
       "         [[-0.0128]],\n",
       "\n",
       "         ...,\n",
       "\n",
       "         [[ 0.0227]],\n",
       "\n",
       "         [[ 0.0029]],\n",
       "\n",
       "         [[ 0.0995]]],\n",
       "\n",
       "\n",
       "        [[[ 0.0078]],\n",
       "\n",
       "         [[ 0.0949]],\n",
       "\n",
       "         [[ 0.0491]],\n",
       "\n",
       "         ...,\n",
       "\n",
       "         [[-0.0079]],\n",
       "\n",
       "         [[ 0.0145]],\n",
       "\n",
       "         [[ 0.0145]]],\n",
       "\n",
       "\n",
       "        [[[-0.0131]],\n",
       "\n",
       "         [[-0.0341]],\n",
       "\n",
       "         [[-0.0332]],\n",
       "\n",
       "         ...,\n",
       "\n",
       "         [[-0.0196]],\n",
       "\n",
       "         [[ 0.0489]],\n",
       "\n",
       "         [[-0.0035]]]], device='cuda:0', requires_grad=True)"
      ]
     },
     "execution_count": 134,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(net.classifier.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Dropout(p=0.5)\n",
       "  (1): Conv2d(512, 1000, kernel_size=(1, 1), stride=(1, 1))\n",
       "  (2): ReLU(inplace)\n",
       "  (3): AvgPool2d(kernel_size=13, stride=1, padding=0)\n",
       ")"
      ]
     },
     "execution_count": 135,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net.classifier.apply(weight_init)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Parameter containing:\n",
       "tensor([[[[ 0.0155]],\n",
       "\n",
       "         [[-0.0615]],\n",
       "\n",
       "         [[-0.0021]],\n",
       "\n",
       "         ...,\n",
       "\n",
       "         [[ 0.0471]],\n",
       "\n",
       "         [[-0.0605]],\n",
       "\n",
       "         [[-0.0052]]],\n",
       "\n",
       "\n",
       "        [[[-0.0399]],\n",
       "\n",
       "         [[-0.0120]],\n",
       "\n",
       "         [[-0.0365]],\n",
       "\n",
       "         ...,\n",
       "\n",
       "         [[ 0.0163]],\n",
       "\n",
       "         [[-0.0598]],\n",
       "\n",
       "         [[-0.0370]]],\n",
       "\n",
       "\n",
       "        [[[-0.0118]],\n",
       "\n",
       "         [[ 0.0339]],\n",
       "\n",
       "         [[ 0.0394]],\n",
       "\n",
       "         ...,\n",
       "\n",
       "         [[ 0.0153]],\n",
       "\n",
       "         [[ 0.0106]],\n",
       "\n",
       "         [[-0.0183]]],\n",
       "\n",
       "\n",
       "        ...,\n",
       "\n",
       "\n",
       "        [[[ 0.0125]],\n",
       "\n",
       "         [[-0.0429]],\n",
       "\n",
       "         [[-0.0529]],\n",
       "\n",
       "         ...,\n",
       "\n",
       "         [[-0.0179]],\n",
       "\n",
       "         [[-0.0527]],\n",
       "\n",
       "         [[ 0.0213]]],\n",
       "\n",
       "\n",
       "        [[[-0.0501]],\n",
       "\n",
       "         [[ 0.0141]],\n",
       "\n",
       "         [[-0.0143]],\n",
       "\n",
       "         ...,\n",
       "\n",
       "         [[-0.0569]],\n",
       "\n",
       "         [[-0.0493]],\n",
       "\n",
       "         [[-0.0316]]],\n",
       "\n",
       "\n",
       "        [[[ 0.0222]],\n",
       "\n",
       "         [[ 0.0306]],\n",
       "\n",
       "         [[ 0.0368]],\n",
       "\n",
       "         ...,\n",
       "\n",
       "         [[ 0.0194]],\n",
       "\n",
       "         [[-0.0182]],\n",
       "\n",
       "         [[-0.0116]]]], device='cuda:0', requires_grad=True)"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "next(net.classifier.parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[2.1977, 0.8858, 3.0146, 0.3871, 0.7168, 2.9611, 0.7913, 1.9176, 1.2350,\n",
       "         2.1126, 0.9560, 1.3755, 1.8517, 0.2523, 1.4842, 1.0934, 1.0219, 1.6590,\n",
       "         1.8762, 1.1277, 1.5842, 0.8387, 6.7865, 0.1309, 0.4788, 1.3618, 0.4664,\n",
       "         2.2427, 2.1271, 1.4623, 0.9160, 1.5845, 1.2738, 0.5905, 1.2119, 0.1987,\n",
       "         1.4864, 4.0972, 0.9740, 1.0180, 0.2837, 1.0019, 0.7745, 1.0031, 3.0682,\n",
       "         2.7953, 1.0389, 0.1035, 0.9242, 3.6534, 3.6378, 1.1779, 1.7273, 0.8377,\n",
       "         0.4165, 1.9858, 1.3087, 2.8538, 1.8663, 1.0403, 2.1950, 1.1237, 1.6313,\n",
       "         0.2742, 2.0056, 0.5137, 2.7286, 0.5897, 0.9020, 1.5613, 2.0260, 0.1537,\n",
       "         3.2864, 2.7103, 0.7705, 2.1996, 0.6047, 0.8989, 2.1782, 1.4858, 0.8023,\n",
       "         0.7473, 1.0368, 2.3746, 3.1938, 0.5129, 0.7252, 1.3327, 1.8767, 0.5735,\n",
       "         0.9266, 2.4687, 1.4652, 0.9850, 0.8935, 0.7294, 0.9194, 3.2521, 1.5078,\n",
       "         2.3257, 2.1263, 1.3433, 2.2820, 0.4801, 1.7057, 0.9013, 1.7407, 0.7395,\n",
       "         0.1895, 3.1192, 1.2259, 1.4199, 1.9616, 2.6070, 0.9555, 0.9697, 0.4930,\n",
       "         2.1586, 1.5243, 0.6072, 0.9618, 0.5112, 0.7349, 1.8285, 4.1546, 1.8526,\n",
       "         0.9718, 0.7895, 0.1435, 3.5439, 1.5140, 0.6256, 1.6559, 2.8618, 3.8247,\n",
       "         0.9587, 1.4406, 5.0086, 0.4086, 4.2248, 0.4944, 1.0358, 2.0499, 0.8918,\n",
       "         1.9989, 2.2161, 0.7201, 0.4776, 1.4840, 0.9284, 3.0534, 0.4750, 0.4155,\n",
       "         1.0627, 0.4329, 0.5680, 2.0031, 0.7946, 0.5044, 0.7848, 0.7656, 1.1450,\n",
       "         0.9914, 1.3376, 0.4617, 0.7773, 1.8108, 2.1089, 2.8242, 1.4643, 1.1109,\n",
       "         1.9974, 0.3339, 1.8066, 0.7050, 1.0160, 1.5432, 1.1170, 1.6795, 0.5161,\n",
       "         3.0717, 1.2023, 0.3808, 0.8138, 3.0430, 1.3084, 2.7016, 1.5525, 0.3343,\n",
       "         0.3088, 0.6519, 1.9085, 0.3540, 1.6613, 0.6941, 1.9824, 0.9372, 0.8633,\n",
       "         0.6557, 1.3025, 1.4487, 2.7387, 1.0907, 1.9648, 1.5921, 1.4562, 0.7986,\n",
       "         0.6375, 1.3895, 1.1898, 4.9072, 2.4598, 0.1772, 1.7511, 1.0371, 1.5656,\n",
       "         2.6544, 2.7163, 0.7672, 0.6909, 1.0884, 2.4390, 1.9621, 2.1643, 3.1626,\n",
       "         1.0927, 0.6773, 0.4171, 0.5872, 0.6073, 2.4105, 4.8399, 1.8719, 2.1266,\n",
       "         2.2780, 0.5142, 0.6858, 0.2024, 0.3012, 1.6640, 0.5468, 1.6615, 0.5285,\n",
       "         1.7613, 0.8408, 0.4122, 1.0553, 0.4492, 3.4411, 0.4667, 0.5768, 1.2358,\n",
       "         2.0469, 0.5343, 0.5232, 0.1200, 1.1746, 0.3172, 0.7844, 1.7360, 1.6338,\n",
       "         0.2129, 0.5296, 0.1097, 1.4583, 0.8014, 0.7696, 0.2437, 0.7832, 2.7233,\n",
       "         3.8300, 0.5368, 1.6288, 0.9069, 1.2913, 2.4065, 2.8122, 0.3034, 4.1600,\n",
       "         1.7726, 3.0575, 0.6991, 0.9688, 2.6098, 0.0854, 1.5306, 0.9482, 1.5063,\n",
       "         0.5952, 0.5380, 2.3000, 3.0982, 0.3514, 0.5501, 2.4143, 3.7544, 2.6628,\n",
       "         2.4546, 1.7726, 1.8824, 0.6747, 1.2583, 1.0358, 0.4263, 1.2746, 1.3253,\n",
       "         1.0645, 0.4161, 1.8379, 0.4180, 2.5851, 1.4639, 2.0798, 0.9863, 5.3540,\n",
       "         1.2571, 1.8110, 1.9111, 1.3530, 1.8693, 0.3203, 2.5729, 0.2532, 1.4697,\n",
       "         0.9551, 3.1952, 2.5256, 3.2549, 4.2168, 0.4076, 1.6103, 1.4833, 1.8359,\n",
       "         0.8671, 0.5018, 2.1690, 1.6005, 0.9118, 0.9536, 0.9584, 0.9117, 0.2129,\n",
       "         2.5844, 1.5558, 0.3262, 1.5241, 5.0416, 3.3655, 1.1864, 1.0882, 2.0274,\n",
       "         0.9022, 0.9687, 0.9862, 1.3359, 1.1821, 1.7824, 0.4252, 2.6902, 1.4923,\n",
       "         2.0498, 0.5074, 0.1141, 1.3705, 0.4635, 3.4251, 0.6830, 1.4098, 1.1699,\n",
       "         1.7301, 1.1063, 0.8021, 3.4827, 3.2349, 2.2710, 1.8227, 0.4782, 1.9050,\n",
       "         2.8922, 0.7858, 2.5579, 3.4946, 0.8553, 1.9538, 0.3033, 1.0445, 1.9406,\n",
       "         2.2025, 1.1115, 2.4220, 0.9194, 0.2832, 1.0617, 2.2033, 0.9639, 1.2821,\n",
       "         0.9314, 0.9758, 2.3424, 0.3813, 0.0444, 0.3511, 1.3797, 0.6403, 1.3291,\n",
       "         1.1639, 2.9816, 1.9543, 0.8970, 4.5797, 1.2002, 1.9068, 0.5010, 2.4173,\n",
       "         0.1732, 2.4987, 0.9559, 0.6212, 1.0576, 3.4789, 2.2673, 0.3808, 4.3078,\n",
       "         2.8538, 1.8807, 0.9441, 2.9561, 1.9107, 1.1437, 2.4389, 0.5775, 3.1244,\n",
       "         3.3736, 5.8055, 3.2870, 0.2246, 0.4455, 3.1960, 1.3310, 0.3110, 0.2792,\n",
       "         2.6299, 0.6191, 2.0063, 1.8008, 0.4461, 1.1388, 1.7436, 0.2881, 1.5800,\n",
       "         2.7457, 2.4412, 2.1305, 2.7012, 2.8119, 0.4472, 0.9969, 0.7110, 0.7894,\n",
       "         2.2022, 1.0265, 1.5516, 2.7529, 3.1875, 1.0091, 2.3029, 1.7852, 0.4569,\n",
       "         0.7382, 2.1027, 1.9683, 1.5567, 2.2156, 0.7356, 3.9149, 2.8004, 0.2827,\n",
       "         0.9906, 0.3817, 2.1380, 1.4120, 3.4841, 0.2932, 1.7837, 2.8210, 0.2636,\n",
       "         0.8278, 0.9787, 0.5076, 1.3348, 2.1696, 0.1090, 1.7281, 1.8235, 1.2731,\n",
       "         2.4481, 1.1976, 0.4650, 1.9420, 0.4301, 0.0370, 1.4786, 0.5945, 0.9109,\n",
       "         0.4837, 0.5541, 0.2441, 0.8896, 2.3770, 1.5119, 0.9314, 1.5500, 1.6751,\n",
       "         1.1092, 0.5938, 2.2521, 0.3441, 1.5767, 0.3991, 2.3511, 2.3668, 1.0387,\n",
       "         0.6909, 2.9162, 0.7621, 3.5219, 2.5932, 0.3220, 0.5031, 2.0585, 3.3740,\n",
       "         1.8718, 1.1626, 0.7880, 1.6379, 1.6475, 1.5003, 1.6875, 0.4197, 1.9567,\n",
       "         2.7362, 3.7398, 1.2598, 0.8475, 1.0981, 3.5967, 3.2787, 1.5792, 0.9514,\n",
       "         3.3298, 1.2410, 0.8531, 0.4745, 1.1068, 0.3750, 3.7420, 0.7647, 2.6529,\n",
       "         0.6379, 0.9129, 0.4543, 3.7907, 0.2778, 1.8290, 1.8380, 0.2686, 0.9226,\n",
       "         1.6487, 1.4098, 2.2834, 1.7582, 0.3095, 0.5816, 0.7109, 0.2331, 0.3086,\n",
       "         1.1071, 2.7609, 3.0812, 4.3609, 1.9503, 2.6425, 1.4609, 0.4956, 1.4832,\n",
       "         1.0843, 0.4817, 0.8134, 2.1748, 0.3017, 2.3726, 2.5351, 2.0620, 1.0322,\n",
       "         0.5809, 2.5443, 1.2526, 0.9003, 2.6024, 0.2606, 0.3657, 1.2309, 2.3602,\n",
       "         2.4146, 1.3993, 0.3460, 0.5792, 1.1781, 0.5031, 1.0712, 4.3456, 0.9730,\n",
       "         0.9724, 2.1494, 0.4498, 3.1059, 0.2199, 0.6976, 3.0180, 1.7123, 1.2156,\n",
       "         1.1949, 1.1146, 0.9869, 0.9659, 0.3451, 0.8916, 2.8323, 1.6711, 1.4269,\n",
       "         0.4854, 1.7440, 1.6608, 2.3720, 1.0027, 1.0853, 0.4013, 1.6123, 3.4755,\n",
       "         1.6789, 2.2612, 3.4963, 1.0516, 2.9734, 0.4743, 1.9006, 2.9557, 0.7053,\n",
       "         1.8646, 1.5067, 2.5365, 2.5589, 1.3215, 1.3780, 0.3532, 0.3600, 0.6359,\n",
       "         0.9201, 0.4968, 0.4562, 1.0837, 2.4215, 2.5558, 0.5313, 0.2935, 0.5643,\n",
       "         2.2799, 3.0169, 1.7906, 3.2780, 0.3421, 2.5221, 0.9525, 1.2900, 1.2596,\n",
       "         1.6288, 2.7591, 1.6844, 0.9841, 1.0306, 1.8345, 2.2654, 2.1154, 0.0624,\n",
       "         1.0538, 0.6502, 0.3418, 1.0645, 3.3550, 0.8980, 2.2907, 1.0019, 1.2693,\n",
       "         0.5517, 0.2179, 2.1765, 1.3342, 2.1004, 1.0223, 1.6568, 0.3238, 4.0765,\n",
       "         1.0056, 0.3005, 1.5365, 0.8850, 2.6404, 1.0582, 4.1094, 1.6386, 0.8773,\n",
       "         0.9474, 2.7494, 0.1433, 0.5394, 4.3059, 2.8426, 1.4279, 1.6461, 1.9249,\n",
       "         1.0851, 0.8558, 1.8882, 1.7951, 1.2684, 1.7195, 2.7329, 0.1355, 1.4627,\n",
       "         2.0738, 1.6577, 0.8874, 0.0755, 1.1014, 1.2620, 2.7677, 0.7245, 0.2328,\n",
       "         1.1457, 1.1552, 0.9388, 4.0554, 1.6763, 0.3465, 1.5417, 0.1164, 1.0732,\n",
       "         2.1842, 0.8547, 2.2374, 2.1023, 3.3727, 0.5932, 0.8989, 1.2460, 1.4461,\n",
       "         2.5236, 0.3054, 1.7156, 0.6499, 1.6177, 1.3198, 0.1630, 1.2616, 0.4028,\n",
       "         1.3087, 0.8508, 0.6413, 0.5098, 1.7454, 3.2903, 2.3837, 1.4637, 0.6534,\n",
       "         1.5606, 1.2692, 0.8774, 1.9781, 0.9045, 1.1994, 1.7844, 1.9417, 2.1040,\n",
       "         1.2233, 0.6515, 2.3792, 2.3538, 1.9748, 1.3555, 1.6687, 2.1293, 2.3576,\n",
       "         1.4273, 0.9761, 1.9054, 0.9366, 2.0724, 2.1472, 1.8003, 1.4928, 2.2187,\n",
       "         0.4538, 1.1026, 0.8888, 2.1133, 0.5049, 1.9234, 0.2261, 3.6249, 0.7416,\n",
       "         1.2946, 1.1715, 0.4521, 2.4671, 1.9674, 0.4260, 2.2975, 2.0573, 1.4573,\n",
       "         1.1506, 1.5994, 0.5275, 1.5361, 1.2748, 1.8474, 0.9271, 2.0230, 1.4051,\n",
       "         0.5912, 2.1087, 3.0419, 0.7990, 1.3450, 1.7511, 1.2397, 1.1292, 2.7722,\n",
       "         1.8676, 1.9022, 1.6417, 0.4030, 2.0275, 0.9784, 2.1531, 1.4294, 2.2454,\n",
       "         2.4558, 0.9721, 2.3994, 2.7692, 0.2823, 0.5639, 1.5719, 1.6616, 2.0710,\n",
       "         0.8056, 3.6922, 0.5196, 0.9153, 1.1871, 2.6455, 1.8290, 1.5344, 1.6422,\n",
       "         1.4215, 0.1194, 0.5766, 1.5349, 4.0687, 0.3622, 1.0788, 0.2310, 1.2391,\n",
       "         1.6384, 0.6391, 1.2188, 1.7591, 1.2111, 2.6341, 1.1052, 0.3010, 3.1588,\n",
       "         3.8969, 1.2000, 2.4875, 1.7611, 2.7180, 0.5585, 1.4425, 0.9598, 0.9703,\n",
       "         2.2541, 0.8111, 1.8159, 2.2486, 2.2883, 0.9172, 1.5966, 0.5708, 0.2650,\n",
       "         1.5284, 0.9664, 1.1060, 2.9599, 1.7995, 1.1831, 2.6539, 0.6790, 1.0898,\n",
       "         0.4200, 1.5496, 2.4208, 0.5641, 1.8406, 1.1392, 1.6581, 2.2580, 0.7696,\n",
       "         0.2622, 2.7104, 0.8895, 1.3289, 0.8917, 1.0157, 1.1244, 4.3802, 3.6995,\n",
       "         1.7756, 0.0498, 0.6279, 0.6796, 2.4898, 2.4872, 1.2099, 2.8241, 2.3181,\n",
       "         0.9830, 1.0806, 2.4750, 1.5212, 0.6289, 2.9095, 1.2460, 2.1310, 0.8320,\n",
       "         0.3383, 3.1895, 2.1408, 2.4794, 1.0101, 1.9241, 2.2680, 0.8668, 1.6788,\n",
       "         0.1304, 1.8995, 1.4720, 1.2140, 1.4802, 0.6939, 0.3701, 1.2361, 1.0219,\n",
       "         0.5342, 1.4919, 2.9326, 1.5833, 3.4592, 0.2417, 0.8694, 1.9832, 2.2280,\n",
       "         0.3919, 4.1181, 1.1117, 2.0381, 2.8146, 0.6335, 0.5698, 3.3040, 1.5065,\n",
       "         3.5949, 2.0201, 2.1096, 0.2450, 0.9486, 0.5629, 1.0820, 1.5767, 0.4529,\n",
       "         2.5046, 1.1345, 1.1024, 0.9730, 0.8482, 0.1311, 1.9178, 0.3752, 0.5021,\n",
       "         0.8511]], device='cuda:0', grad_fn=<ViewBackward>)"
      ]
     },
     "execution_count": 137,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "net(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:fastai]",
   "language": "python",
   "name": "conda-env-fastai-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}