diff --git a/CHANGELOG.md b/CHANGELOG.md index 50b3e7eb..26c67deb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,17 +1,13 @@ ## Added -1. Added Patching Support for `torch.nn.Sequential` containers. -2. Added support for modeling source and line resistances for passive crossbars/tiles. -3. Added C++ and CUDA bindings for modeling source and line resistances for passive crossbars/tiles\*. -4. Added a new MemTorch logo to `README.md` -5. Added the `set_cuda_malloc_heap_size` routine to patched `torch.mn` modules. -6. Added unit tests for source and line resistance modeling. -7. Relaxed requirements for programming passive crossbars/tiles. +1. Partial support for the `groups` argument for convolutional layers. -**\*Note** it is strongly suggested to set `cuda_malloc_heap_size` using `m.set_cuda_malloc_heap_size` manually when simulating source and line resistances using CUDA bindings. +## Fixed + +1. Patching procedure in `memtorch.mn.module.patch_model` and `memtorch.bh.nonideality.apply_nonidealities` to fix semantic error in `Tutorial.ipynb`. +2. Import statement in `Exemplar_Simulations.ipynb`. ## Enhanced -1. Modularized patching logic in `memtorch.bh.nonideality.NonIdeality` and `memtorch.mn.Module`. -2. Updated `ReadTheDocs` documentation. -3. Transitioned from `Gitter` to `GitHub Discussions` for general discussion. +1. Further modularized patching logic in `memtorch.bh.nonideality.NonIdeality` and `memtorch.mn.Module`. +2. Modified default number of worker in `memtorch.utils` from 2 to 1. diff --git a/docs/conf.py b/docs/conf.py index f02cbfa8..e25b6467 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -21,7 +21,7 @@ author = "Corey Lammie" # The full version, including alpha/beta/rc tags -release = "1.1.4" +release = "1.1.5" autodoc_inherit_docstrings = False # -- General configuration --------------------------------------------------- diff --git a/memtorch/bh/nonideality/NonIdeality.py b/memtorch/bh/nonideality/NonIdeality.py index 08ef6bc6..909a4ee3 100644 --- a/memtorch/bh/nonideality/NonIdeality.py +++ b/memtorch/bh/nonideality/NonIdeality.py @@ -44,20 +44,7 @@ def apply_nonidealities(model, non_idealities, **kwargs): """ def apply_patched_module(model, patched_module, name, m): - if name.__contains__("."): - sequence_container, module = name.split(".") - if module.isdigit(): - module = int(module) - model._modules[sequence_container][module] = patched_module - else: - setattr( - model._modules[sequence_container], - "%s" % module, - patched_module, - ) - else: - model._modules[name] = patched_module - + model._modules[name] = patched_module return model for _, (name, m) in enumerate(list(model.named_modules())): diff --git a/memtorch/examples/Exemplar_Simulations.ipynb b/memtorch/examples/Exemplar_Simulations.ipynb index be8f92ae..9c7802b7 100644 --- a/memtorch/examples/Exemplar_Simulations.ipynb +++ b/memtorch/examples/Exemplar_Simulations.ipynb @@ -2,596 +2,591 @@ "cells": [ { "cell_type": "markdown", - "metadata": {}, "source": [ "# Exemplar Simulations\n", "Exemplar simulations investigating the performance degradation of Memristive Deep Neural Networks (MDNNs) when non-ideal device characteristics are accounted for using the CIFAR-10 dataset are provided below. Results can be plotted using `plot_all_exemplar.m`." - ] + ], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "## 1. Define the MobileNetV2 Network Architecture" - ] + ], + "metadata": {} }, { "cell_type": "code", "execution_count": null, - "metadata": {}, - "outputs": [], "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import torch.nn.functional as F\n", - "\n", - "\n", - "class Block(nn.Module):\n", - " def __init__(self, in_planes, out_planes, expansion, stride):\n", - " super(Block, self).__init__()\n", - " self.stride = stride\n", - " planes = expansion * in_planes\n", - " self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, stride=1, padding=0)\n", - " self.bn1 = nn.BatchNorm2d(planes, affine=False)\n", - " self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, groups=planes)\n", - " self.bn2 = nn.BatchNorm2d(planes, affine=False)\n", - " self.conv3 = nn.Conv2d(planes, out_planes, kernel_size=1, stride=1, padding=0)\n", - " self.bn3 = nn.BatchNorm2d(out_planes, affine=False)\n", - "\n", - " self.shortcut = nn.Sequential()\n", - " if stride == 1 and in_planes != out_planes:\n", - " self.shortcut = nn.Sequential(\n", - " nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0),\n", - " nn.BatchNorm2d(out_planes, affine=False),\n", - " )\n", - "\n", - " def forward(self, x):\n", - " out = F.relu(self.bn1(self.conv1(x)))\n", - " out = F.relu(self.bn2(self.conv2(out)))\n", - " out = self.bn3(self.conv3(out))\n", - " out = out + self.shortcut(x) if self.stride==1 else out\n", - " return out\n", - "\n", - "\n", - "class MobileNetV2(nn.Module):\n", - " cfg = [(1, 16, 1, 1),\n", - " (6, 24, 2, 1),\n", - " (6, 32, 3, 2),\n", - " (6, 64, 4, 2),\n", - " (6, 96, 3, 1),\n", - " (6, 160, 3, 2),\n", - " (6, 320, 1, 1)]\n", - "\n", - " def __init__(self, num_classes=10):\n", - " super(MobileNetV2, self).__init__()\n", - " self.conv1 = nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1)\n", - " self.bn1 = nn.BatchNorm2d(32, affine=False)\n", - " self.layers = self._make_layers(in_planes=32)\n", - " self.conv2 = nn.Conv2d(320, 1280, kernel_size=1, stride=1, padding=0)\n", - " self.bn2 = nn.BatchNorm2d(1280, affine=False)\n", - " self.linear = nn.Linear(1280, num_classes)\n", - "\n", - " def _make_layers(self, in_planes):\n", - " layers = []\n", - " for expansion, out_planes, num_blocks, stride in self.cfg:\n", - " strides = [stride] + [1]*(num_blocks-1)\n", - " for stride in strides:\n", - " layers.append(Block(in_planes, out_planes, expansion, stride))\n", - " in_planes = out_planes\n", - " return nn.Sequential(*layers)\n", - "\n", - " def forward(self, x):\n", - " out = F.relu(self.bn1(self.conv1(x)))\n", - " out = self.layers(out)\n", - " out = F.relu(self.bn2(self.conv2(out)))\n", - " out = F.avg_pool2d(out, 4)\n", - " out = out.view(out.size(0), -1)\n", - " out = self.linear(out)\n", + "import torch\r\n", + "import torch.nn as nn\r\n", + "import torch.nn.functional as F\r\n", + "\r\n", + "\r\n", + "class Block(nn.Module):\r\n", + " def __init__(self, in_planes, out_planes, expansion, stride):\r\n", + " super(Block, self).__init__()\r\n", + " self.stride = stride\r\n", + " planes = expansion * in_planes\r\n", + " self.conv1 = nn.Conv2d(in_planes, planes, kernel_size=1, stride=1, padding=0)\r\n", + " self.bn1 = nn.BatchNorm2d(planes, affine=False)\r\n", + " self.conv2 = nn.Conv2d(planes, planes, kernel_size=3, stride=stride, padding=1, groups=planes)\r\n", + " self.bn2 = nn.BatchNorm2d(planes, affine=False)\r\n", + " self.conv3 = nn.Conv2d(planes, out_planes, kernel_size=1, stride=1, padding=0)\r\n", + " self.bn3 = nn.BatchNorm2d(out_planes, affine=False)\r\n", + "\r\n", + " self.shortcut = nn.Sequential()\r\n", + " if stride == 1 and in_planes != out_planes:\r\n", + " self.shortcut = nn.Sequential(\r\n", + " nn.Conv2d(in_planes, out_planes, kernel_size=1, stride=1, padding=0),\r\n", + " nn.BatchNorm2d(out_planes, affine=False),\r\n", + " )\r\n", + "\r\n", + " def forward(self, x):\r\n", + " out = F.relu(self.bn1(self.conv1(x)))\r\n", + " out = F.relu(self.bn2(self.conv2(out)))\r\n", + " out = self.bn3(self.conv3(out))\r\n", + " out = out + self.shortcut(x) if self.stride==1 else out\r\n", + " return out\r\n", + "\r\n", + "\r\n", + "class MobileNetV2(nn.Module):\r\n", + " cfg = [(1, 16, 1, 1),\r\n", + " (6, 24, 2, 1),\r\n", + " (6, 32, 3, 2),\r\n", + " (6, 64, 4, 2),\r\n", + " (6, 96, 3, 1),\r\n", + " (6, 160, 3, 2),\r\n", + " (6, 320, 1, 1)]\r\n", + "\r\n", + " def __init__(self, num_classes=10):\r\n", + " super(MobileNetV2, self).__init__()\r\n", + " self.conv1 = nn.Conv2d(3, 32, kernel_size=3, stride=1, padding=1)\r\n", + " self.bn1 = nn.BatchNorm2d(32, affine=False)\r\n", + " self.layers = self._make_layers(in_planes=32)\r\n", + " self.conv2 = nn.Conv2d(320, 1280, kernel_size=1, stride=1, padding=0)\r\n", + " self.bn2 = nn.BatchNorm2d(1280, affine=False)\r\n", + " self.linear = nn.Linear(1280, num_classes)\r\n", + "\r\n", + " def _make_layers(self, in_planes):\r\n", + " layers = []\r\n", + " for expansion, out_planes, num_blocks, stride in self.cfg:\r\n", + " strides = [stride] + [1]*(num_blocks-1)\r\n", + " for stride in strides:\r\n", + " layers.append(Block(in_planes, out_planes, expansion, stride))\r\n", + " in_planes = out_planes\r\n", + " return nn.Sequential(*layers)\r\n", + "\r\n", + " def forward(self, x):\r\n", + " out = F.relu(self.bn1(self.conv1(x)))\r\n", + " out = self.layers(out)\r\n", + " out = F.relu(self.bn2(self.conv2(out)))\r\n", + " out = F.avg_pool2d(out, 4)\r\n", + " out = out.view(out.size(0), -1)\r\n", + " out = self.linear(out)\r\n", " return out" - ] + ], + "outputs": [], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "## 2. Train MobileNetV2 Using CIFAR-10" - ] + ], + "metadata": {} }, { "cell_type": "code", "execution_count": null, - "metadata": {}, - "outputs": [], "source": [ - "import torch\n", - "from torch.autograd import Variable\n", - "import torch.nn as nn\n", - "import torch.nn.functional as F\n", - "import torch.optim as optim\n", - "import numpy as np\n", - "import random\n", - "import torchvision\n", - "import torchvision.transforms as transforms\n", - "from mobilenetv2 import MobileNetV2\n", - "\n", - "\n", - "def set_all_seeds(seed):\n", - " random.seed(seed)\n", - " np.random.seed(seed)\n", - " torch.manual_seed(seed)\n", - " torch.cuda.manual_seed(seed)\n", - " torch.backends.cudnn.deterministic = True\n", - "\n", - "def test(model, test_loader):\n", - " correct = 0\n", - " for batch_idx, (data, target) in enumerate(test_loader):\n", - " output = model(data.to(device))\n", - " pred = output.data.max(1)[1]\n", - " correct += pred.eq(target.to(device).data.view_as(pred)).cpu().sum()\n", - "\n", - " return 100. * float(correct) / float(len(test_loader.dataset))\n", - "\n", - "set_all_seeds(0)\n", - "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\n", - "epochs = 100\n", - "transform_train = transforms.Compose([\n", - " transforms.RandomCrop(32, padding=4),\n", - " transforms.RandomHorizontalFlip(),\n", - " transforms.ToTensor(),\n", - " transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),\n", - "])\n", - "transform_test = transforms.Compose([\n", - " transforms.ToTensor(),\n", - " transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),\n", - "])\n", - "train_set = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train)\n", - "train_loader = torch.utils.data.DataLoader(train_set, batch_size=256, shuffle=True, num_workers=1)\n", - "test_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)\n", - "test_loader = torch.utils.data.DataLoader(test_set, batch_size=128, shuffle=False, num_workers=1)\n", - "model = MobileNetV2().to(device)\n", - "criterion = nn.CrossEntropyLoss()\n", - "learning_rate = 0.1\n", - "optimizer = optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9)\n", - "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=40, gamma=0.1)\n", - "best_accuracy = 0\n", - "for epoch in range(0, epochs):\n", - " print('Epoch: [%d]\\t\\t' % (epoch + 1), end='')\n", - " model.train()\n", - " for batch_idx, (data, target) in enumerate(train_loader):\n", - " optimizer.zero_grad()\n", - " output = model(data.to(device))\n", - " loss = criterion(output, target.to(device))\n", - " loss.backward()\n", - " optimizer.step()\n", - "\n", - " scheduler.step()\n", - " model.eval()\n", - " accuracy = test(model, test_loader)\n", - " print('%2.2f%%' % accuracy)\n", - " if accuracy > best_accuracy:\n", - " print('Saving model...')\n", - " torch.save(model.state_dict(), 'trained_model.pt')\n", + "import torch\r\n", + "from torch.autograd import Variable\r\n", + "import torch.nn as nn\r\n", + "import torch.nn.functional as F\r\n", + "import torch.optim as optim\r\n", + "import numpy as np\r\n", + "import random\r\n", + "import torchvision\r\n", + "import torchvision.transforms as transforms\r\n", + "\r\n", + "\r\n", + "def set_all_seeds(seed):\r\n", + " random.seed(seed)\r\n", + " np.random.seed(seed)\r\n", + " torch.manual_seed(seed)\r\n", + " torch.cuda.manual_seed(seed)\r\n", + " torch.backends.cudnn.deterministic = True\r\n", + "\r\n", + "def test(model, test_loader):\r\n", + " correct = 0\r\n", + " for batch_idx, (data, target) in enumerate(test_loader):\r\n", + " output = model(data.to(device))\r\n", + " pred = output.data.max(1)[1]\r\n", + " correct += pred.eq(target.to(device).data.view_as(pred)).cpu().sum()\r\n", + "\r\n", + " return 100. * float(correct) / float(len(test_loader.dataset))\r\n", + "\r\n", + "set_all_seeds(0)\r\n", + "device = torch.device('cuda') if torch.cuda.is_available() else torch.device('cpu')\r\n", + "epochs = 100\r\n", + "transform_train = transforms.Compose([\r\n", + " transforms.RandomCrop(32, padding=4),\r\n", + " transforms.RandomHorizontalFlip(),\r\n", + " transforms.ToTensor(),\r\n", + " transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),\r\n", + "])\r\n", + "transform_test = transforms.Compose([\r\n", + " transforms.ToTensor(),\r\n", + " transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),\r\n", + "])\r\n", + "train_set = torchvision.datasets.CIFAR10(root='./data', train=True, download=True, transform=transform_train)\r\n", + "train_loader = torch.utils.data.DataLoader(train_set, batch_size=256, shuffle=True, num_workers=1)\r\n", + "test_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)\r\n", + "test_loader = torch.utils.data.DataLoader(test_set, batch_size=128, shuffle=False, num_workers=1)\r\n", + "model = MobileNetV2().to(device)\r\n", + "criterion = nn.CrossEntropyLoss()\r\n", + "learning_rate = 0.1\r\n", + "optimizer = optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9)\r\n", + "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=40, gamma=0.1)\r\n", + "best_accuracy = 0\r\n", + "for epoch in range(0, epochs):\r\n", + " print('Epoch: [%d]\\t\\t' % (epoch + 1), end='')\r\n", + " model.train()\r\n", + " for batch_idx, (data, target) in enumerate(train_loader):\r\n", + " optimizer.zero_grad()\r\n", + " output = model(data.to(device))\r\n", + " loss = criterion(output, target.to(device))\r\n", + " loss.backward()\r\n", + " optimizer.step()\r\n", + "\r\n", + " scheduler.step()\r\n", + " model.eval()\r\n", + " accuracy = test(model, test_loader)\r\n", + " print('%2.2f%%' % accuracy)\r\n", + " if accuracy > best_accuracy:\r\n", + " print('Saving model...')\r\n", + " torch.save(model.state_dict(), 'trained_model.pt')\r\n", " best_accuracy = accuracy" - ] + ], + "outputs": [], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "## 3. Figure 1 [A,E]" - ] + ], + "metadata": {} }, { "cell_type": "code", "execution_count": null, - "metadata": {}, - "outputs": [], "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import torch.nn.functional as F\n", - "import torch.optim as optim\n", - "from torchvision import datasets, transforms\n", - "import memtorch\n", - "from memtorch.utils import LoadCIFAR10\n", - "import numpy as np\n", - "import pandas as pd\n", - "from mobilenetv2 import MobileNetV2\n", - "import torchvision\n", - "import copy\n", - "from memtorch.mn.Module import patch_model\n", - "from memtorch.map.Parameter import naive_map\n", - "from memtorch.bh.crossbar.Program import naive_program\n", - "from memtorch.bh.nonideality.NonIdeality import apply_nonidealities\n", - "\n", - "\n", - "def test(model, test_loader):\n", - " correct = 0\n", - " for batch_idx, (data, target) in enumerate(test_loader):\n", - " output = model(data.to(device))\n", - " pred = output.data.max(1)[1]\n", - " correct += pred.eq(target.to(device).data.view_as(pred)).cpu().sum()\n", - "\n", - " return 100. * float(correct) / float(len(test_loader.dataset))\n", - "\n", - "device = torch.device('cuda')\n", - "transform_test = transforms.Compose([\n", - " transforms.ToTensor(),\n", - " transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),\n", - "])\n", - "test_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)\n", - "test_loader = torch.utils.data.DataLoader(test_set, batch_size=256, shuffle=False, num_workers=1)\n", - "model = MobileNetV2().to(device)\n", - "try:\n", - " model.load_state_dict(torch.load('trained_model.pt'), strict=False)\n", - " model.eval()\n", - "except:\n", - " raise Exception('trained_model.pt has not been found.')\n", - "\n", - "print('Test Set Accuracy: \\t%2.2f%%' % test(model, test_loader))\n", - "\n", - "model = MobileNetV2().to(device)\n", - "model.load_state_dict(torch.load('trained_model.pt'), strict=True)\n", - "model.eval()\n", - "print(test(model, test_loader))\n", - "r_on = 1.4e4\n", - "r_off = 5e7\n", - "\n", - "def trial(r_on, r_off, tile_shape, ADC_resolution, sigma):\n", - " model_ = copy.deepcopy(model)\n", - " reference_memristor = memtorch.bh.memristor.VTEAM\n", - " if sigma == 0.:\n", - " reference_memristor_params = {'time_series_resolution': 1e-10, 'r_off': r_off, 'r_on': r_on}\n", - " else:\n", - " reference_memristor_params = {'time_series_resolution': 1e-10,\n", - " 'r_off': memtorch.bh.StochasticParameter(loc=r_off, scale=sigma*2, min=1),\n", - " 'r_on': memtorch.bh.StochasticParameter(loc=r_on, scale=sigma, min=1)}\n", - "\n", - " patched_model = patch_model(copy.deepcopy(model_),\n", - " memristor_model=reference_memristor,\n", - " memristor_model_params=reference_memristor_params,\n", - " module_parameters_to_patch=[torch.nn.Linear, torch.nn.Conv2d],\n", - " mapping_routine=naive_map,\n", - " transistor=True,\n", - " programming_routine=None,\n", - " scheme=memtorch.bh.Scheme.DoubleColumn,\n", - " tile_shape=tile_shape,\n", - " max_input_voltage=0.3,\n", - " ADC_resolution=int(ADC_resolution),\n", - " ADC_overflow_rate=0.,\n", - " quant_method='linear')\n", - " \n", - " patched_model.tune_()\n", - " return test(patched_model, test_loader)\n", - "\n", - "df = pd.DataFrame(columns=['tile_shape', 'ADC_resolution', 'sigma', 'test_set_accuracy'])\n", - "tile_shape = [(256, 64)]\n", - "ADC_resolution = np.linspace(2, 10, num=5, endpoint=True, dtype=int)\n", - "sigma = np.logspace(6, 7, endpoint=True, num=5)\n", - "for tile_shape_ in tile_shape:\n", - " for ADC_resolution_ in ADC_resolution:\n", - " for sigma_ in sigma:\n", - " print('tile_shape: %s; ADC_resolution: %d; sigma: %d' % (tile_shape_, ADC_resolution_, sigma_))\n", - " df = df.append({'tile_shape': tile_shape_, \n", - " 'ADC_resolution': ADC_resolution_, \n", - " 'sigma': sigma_, \n", - " 'test_set_accuracy': trial(r_on, r_off, tile_shape_, ADC_resolution_, sigma_)}, ignore_index=True)\n", + "import torch\r\n", + "import torch.nn as nn\r\n", + "import torch.nn.functional as F\r\n", + "import torch.optim as optim\r\n", + "from torchvision import datasets, transforms\r\n", + "import memtorch\r\n", + "from memtorch.utils import LoadCIFAR10\r\n", + "import numpy as np\r\n", + "import pandas as pd\r\n", + "import torchvision\r\n", + "import copy\r\n", + "from memtorch.mn.Module import patch_model\r\n", + "from memtorch.map.Parameter import naive_map\r\n", + "from memtorch.bh.crossbar.Program import naive_program\r\n", + "from memtorch.bh.nonideality.NonIdeality import apply_nonidealities\r\n", + "\r\n", + "\r\n", + "def test(model, test_loader):\r\n", + " correct = 0\r\n", + " for batch_idx, (data, target) in enumerate(test_loader):\r\n", + " output = model(data.to(device))\r\n", + " pred = output.data.max(1)[1]\r\n", + " correct += pred.eq(target.to(device).data.view_as(pred)).cpu().sum()\r\n", + "\r\n", + " return 100. * float(correct) / float(len(test_loader.dataset))\r\n", + "\r\n", + "device = torch.device('cuda')\r\n", + "transform_test = transforms.Compose([\r\n", + " transforms.ToTensor(),\r\n", + " transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),\r\n", + "])\r\n", + "test_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)\r\n", + "test_loader = torch.utils.data.DataLoader(test_set, batch_size=256, shuffle=False, num_workers=1)\r\n", + "model = MobileNetV2().to(device)\r\n", + "try:\r\n", + " model.load_state_dict(torch.load('trained_model.pt'), strict=False)\r\n", + " model.eval()\r\n", + "except:\r\n", + " raise Exception('trained_model.pt has not been found.')\r\n", + "\r\n", + "print('Test Set Accuracy: \\t%2.2f%%' % test(model, test_loader))\r\n", + "\r\n", + "model = MobileNetV2().to(device)\r\n", + "model.load_state_dict(torch.load('trained_model.pt'), strict=True)\r\n", + "model.eval()\r\n", + "print(test(model, test_loader))\r\n", + "r_on = 1.4e4\r\n", + "r_off = 5e7\r\n", + "\r\n", + "def trial(r_on, r_off, tile_shape, ADC_resolution, sigma):\r\n", + " model_ = copy.deepcopy(model)\r\n", + " reference_memristor = memtorch.bh.memristor.VTEAM\r\n", + " if sigma == 0.:\r\n", + " reference_memristor_params = {'time_series_resolution': 1e-10, 'r_off': r_off, 'r_on': r_on}\r\n", + " else:\r\n", + " reference_memristor_params = {'time_series_resolution': 1e-10,\r\n", + " 'r_off': memtorch.bh.StochasticParameter(loc=r_off, scale=sigma*2, min=1),\r\n", + " 'r_on': memtorch.bh.StochasticParameter(loc=r_on, scale=sigma, min=1)}\r\n", + "\r\n", + " patched_model = patch_model(copy.deepcopy(model_),\r\n", + " memristor_model=reference_memristor,\r\n", + " memristor_model_params=reference_memristor_params,\r\n", + " module_parameters_to_patch=[torch.nn.Linear, torch.nn.Conv2d],\r\n", + " mapping_routine=naive_map,\r\n", + " transistor=True,\r\n", + " programming_routine=None,\r\n", + " scheme=memtorch.bh.Scheme.DoubleColumn,\r\n", + " tile_shape=tile_shape,\r\n", + " max_input_voltage=0.3,\r\n", + " ADC_resolution=int(ADC_resolution),\r\n", + " ADC_overflow_rate=0.,\r\n", + " quant_method='linear')\r\n", + " \r\n", + " patched_model.tune_()\r\n", + " return test(patched_model, test_loader)\r\n", + "\r\n", + "df = pd.DataFrame(columns=['tile_shape', 'ADC_resolution', 'sigma', 'test_set_accuracy'])\r\n", + "tile_shape = [(256, 64)]\r\n", + "ADC_resolution = np.linspace(2, 10, num=5, endpoint=True, dtype=int)\r\n", + "sigma = np.logspace(6, 7, endpoint=True, num=5)\r\n", + "for tile_shape_ in tile_shape:\r\n", + " for ADC_resolution_ in ADC_resolution:\r\n", + " for sigma_ in sigma:\r\n", + " print('tile_shape: %s; ADC_resolution: %d; sigma: %d' % (tile_shape_, ADC_resolution_, sigma_))\r\n", + " df = df.append({'tile_shape': tile_shape_, \r\n", + " 'ADC_resolution': ADC_resolution_, \r\n", + " 'sigma': sigma_, \r\n", + " 'test_set_accuracy': trial(r_on, r_off, tile_shape_, ADC_resolution_, sigma_)}, ignore_index=True)\r\n", " df.to_csv('1_AE.csv', index=False)" - ] + ], + "outputs": [], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "## 4. Figure 1 [B,F]" - ] + ], + "metadata": {} }, { "cell_type": "code", "execution_count": null, - "metadata": {}, - "outputs": [], "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import torch.nn.functional as F\n", - "import torch.optim as optim\n", - "from torchvision import datasets, transforms\n", - "import memtorch\n", - "from memtorch.utils import LoadCIFAR10\n", - "import numpy as np\n", - "import pandas as pd\n", - "from mobilenetv2 import MobileNetV2\n", - "import torchvision\n", - "import copy\n", - "from memtorch.mn.Module import patch_model\n", - "from memtorch.map.Parameter import naive_map\n", - "from memtorch.bh.crossbar.Program import naive_program\n", - "from memtorch.bh.nonideality.NonIdeality import apply_nonidealities\n", - "\n", - "\n", - "def test(model, test_loader):\n", - " correct = 0\n", - " for batch_idx, (data, target) in enumerate(test_loader):\n", - " output = model(data.to(device))\n", - " pred = output.data.max(1)[1]\n", - " correct += pred.eq(target.to(device).data.view_as(pred)).cpu().sum()\n", - "\n", - " return 100. * float(correct) / float(len(test_loader.dataset))\n", - "\n", - "device = torch.device('cuda')\n", - "transform_test = transforms.Compose([\n", - " transforms.ToTensor(),\n", - " transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),\n", - "])\n", - "test_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)\n", - "test_loader = torch.utils.data.DataLoader(test_set, batch_size=256, shuffle=False, num_workers=1)\n", - "model = MobileNetV2().to(device)\n", - "try:\n", - " model.load_state_dict(torch.load('trained_model.pt'), strict=False)\n", - " model.eval()\n", - "except:\n", - " raise Exception('trained_model.pt has not been found.')\n", - "\n", - "print('Test Set Accuracy: \\t%2.2f%%' % test(model, test_loader))\n", - "\n", - "model = MobileNetV2().to(device)\n", - "model.load_state_dict(torch.load('trained_model.pt'), strict=True)\n", - "model.eval()\n", - "print(test(model, test_loader))\n", - "r_on = 1.4e4\n", - "r_off = 5e7\n", - "\n", - "def trial(r_on, r_off, tile_shape, ADC_resolution, conductance_states):\n", - " model_ = copy.deepcopy(model)\n", - " reference_memristor = memtorch.bh.memristor.VTEAM\n", - " reference_memristor_params = {'time_series_resolution': 1e-10, 'r_off': r_off, 'r_on': r_on}\n", - " patched_model = patch_model(copy.deepcopy(model_),\n", - " memristor_model=reference_memristor,\n", - " memristor_model_params=reference_memristor_params,\n", - " module_parameters_to_patch=[torch.nn.Linear, torch.nn.Conv2d],\n", - " mapping_routine=naive_map,\n", - " transistor=True,\n", - " programming_routine=None,\n", - " scheme=memtorch.bh.Scheme.DoubleColumn,\n", - " tile_shape=tile_shape,\n", - " max_input_voltage=0.3,\n", - " ADC_resolution=int(ADC_resolution),\n", - " ADC_overflow_rate=0.,\n", - " quant_method='linear')\n", - "\n", - " patched_model = apply_nonidealities(patched_model,\n", - " non_idealities=[memtorch.bh.nonideality.NonIdeality.FiniteConductanceStates],\n", - " conductance_states = int(conductance_states))\n", - " \n", - " patched_model.tune_()\n", - " return test(patched_model, test_loader)\n", - "\n", - "df = pd.DataFrame(columns=['tile_shape', 'ADC_resolution', 'conductance_states', 'test_set_accuracy'])\n", - "torch.backends.cudnn.benchmark = False\n", - "tile_shape = [(128, 128), (256, 64)]\n", - "ADC_resolution = np.linspace(2, 10, num=5, endpoint=True, dtype=int)\n", - "conductance_states = np.linspace(2, 10, num=5, endpoint=True, dtype=int)\n", - "for tile_shape_ in tile_shape:\n", - " for ADC_resolution_ in ADC_resolution:\n", - " for conductance_states_ in conductance_states:\n", - " print('tile_shape: %s; ADC_resolution: %d; conductance_states: %d' % (tile_shape_, ADC_resolution_, conductance_states_))\n", - " df = df.append({'tile_shape': tile_shape_, \n", - " 'ADC_resolution': ADC_resolution_, \n", - " 'conductance_states': conductance_states_, \n", - " 'test_set_accuracy': trial(r_on, r_off, tile_shape_, ADC_resolution_, conductance_states_)}, ignore_index=True)\n", + "import torch\r\n", + "import torch.nn as nn\r\n", + "import torch.nn.functional as F\r\n", + "import torch.optim as optim\r\n", + "from torchvision import datasets, transforms\r\n", + "import memtorch\r\n", + "from memtorch.utils import LoadCIFAR10\r\n", + "import numpy as np\r\n", + "import pandas as pd\r\n", + "import torchvision\r\n", + "import copy\r\n", + "from memtorch.mn.Module import patch_model\r\n", + "from memtorch.map.Parameter import naive_map\r\n", + "from memtorch.bh.crossbar.Program import naive_program\r\n", + "from memtorch.bh.nonideality.NonIdeality import apply_nonidealities\r\n", + "\r\n", + "\r\n", + "def test(model, test_loader):\r\n", + " correct = 0\r\n", + " for batch_idx, (data, target) in enumerate(test_loader):\r\n", + " output = model(data.to(device))\r\n", + " pred = output.data.max(1)[1]\r\n", + " correct += pred.eq(target.to(device).data.view_as(pred)).cpu().sum()\r\n", + "\r\n", + " return 100. * float(correct) / float(len(test_loader.dataset))\r\n", + "\r\n", + "device = torch.device('cuda')\r\n", + "transform_test = transforms.Compose([\r\n", + " transforms.ToTensor(),\r\n", + " transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),\r\n", + "])\r\n", + "test_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)\r\n", + "test_loader = torch.utils.data.DataLoader(test_set, batch_size=256, shuffle=False, num_workers=1)\r\n", + "model = MobileNetV2().to(device)\r\n", + "try:\r\n", + " model.load_state_dict(torch.load('trained_model.pt'), strict=False)\r\n", + " model.eval()\r\n", + "except:\r\n", + " raise Exception('trained_model.pt has not been found.')\r\n", + "\r\n", + "print('Test Set Accuracy: \\t%2.2f%%' % test(model, test_loader))\r\n", + "\r\n", + "model = MobileNetV2().to(device)\r\n", + "model.load_state_dict(torch.load('trained_model.pt'), strict=True)\r\n", + "model.eval()\r\n", + "print(test(model, test_loader))\r\n", + "r_on = 1.4e4\r\n", + "r_off = 5e7\r\n", + "\r\n", + "def trial(r_on, r_off, tile_shape, ADC_resolution, conductance_states):\r\n", + " model_ = copy.deepcopy(model)\r\n", + " reference_memristor = memtorch.bh.memristor.VTEAM\r\n", + " reference_memristor_params = {'time_series_resolution': 1e-10, 'r_off': r_off, 'r_on': r_on}\r\n", + " patched_model = patch_model(copy.deepcopy(model_),\r\n", + " memristor_model=reference_memristor,\r\n", + " memristor_model_params=reference_memristor_params,\r\n", + " module_parameters_to_patch=[torch.nn.Linear, torch.nn.Conv2d],\r\n", + " mapping_routine=naive_map,\r\n", + " transistor=True,\r\n", + " programming_routine=None,\r\n", + " scheme=memtorch.bh.Scheme.DoubleColumn,\r\n", + " tile_shape=tile_shape,\r\n", + " max_input_voltage=0.3,\r\n", + " ADC_resolution=int(ADC_resolution),\r\n", + " ADC_overflow_rate=0.,\r\n", + " quant_method='linear')\r\n", + "\r\n", + " patched_model = apply_nonidealities(patched_model,\r\n", + " non_idealities=[memtorch.bh.nonideality.NonIdeality.FiniteConductanceStates],\r\n", + " conductance_states = int(conductance_states))\r\n", + " \r\n", + " patched_model.tune_()\r\n", + " return test(patched_model, test_loader)\r\n", + "\r\n", + "df = pd.DataFrame(columns=['tile_shape', 'ADC_resolution', 'conductance_states', 'test_set_accuracy'])\r\n", + "torch.backends.cudnn.benchmark = False\r\n", + "tile_shape = [(128, 128), (256, 64)]\r\n", + "ADC_resolution = np.linspace(2, 10, num=5, endpoint=True, dtype=int)\r\n", + "conductance_states = np.linspace(2, 10, num=5, endpoint=True, dtype=int)\r\n", + "for tile_shape_ in tile_shape:\r\n", + " for ADC_resolution_ in ADC_resolution:\r\n", + " for conductance_states_ in conductance_states:\r\n", + " print('tile_shape: %s; ADC_resolution: %d; conductance_states: %d' % (tile_shape_, ADC_resolution_, conductance_states_))\r\n", + " df = df.append({'tile_shape': tile_shape_, \r\n", + " 'ADC_resolution': ADC_resolution_, \r\n", + " 'conductance_states': conductance_states_, \r\n", + " 'test_set_accuracy': trial(r_on, r_off, tile_shape_, ADC_resolution_, conductance_states_)}, ignore_index=True)\r\n", " df.to_csv('1_BF.csv', index=False)" - ] + ], + "outputs": [], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "## 5. Figure 1 [C,G]" - ] + ], + "metadata": {} }, { "cell_type": "code", "execution_count": null, - "metadata": {}, - "outputs": [], "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import torch.nn.functional as F\n", - "import torch.optim as optim\n", - "from torchvision import datasets, transforms\n", - "import memtorch\n", - "from memtorch.utils import LoadCIFAR10\n", - "import numpy as np\n", - "import pandas as pd\n", - "from mobilenetv2 import MobileNetV2\n", - "import torchvision\n", - "import copy\n", - "from memtorch.mn.Module import patch_model\n", - "from memtorch.map.Parameter import naive_map\n", - "from memtorch.bh.crossbar.Program import naive_program\n", - "from memtorch.bh.nonideality.NonIdeality import apply_nonidealities\n", - "\n", - "\n", - "def test(model, test_loader):\n", - " correct = 0\n", - " for batch_idx, (data, target) in enumerate(test_loader):\n", - " output = model(data.to(device))\n", - " pred = output.data.max(1)[1]\n", - " correct += pred.eq(target.to(device).data.view_as(pred)).cpu().sum()\n", - "\n", - " return 100. * float(correct) / float(len(test_loader.dataset))\n", - "\n", - "device = torch.device('cuda')\n", - "transform_test = transforms.Compose([\n", - " transforms.ToTensor(),\n", - " transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),\n", - "])\n", - "test_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)\n", - "test_loader = torch.utils.data.DataLoader(test_set, batch_size=256, shuffle=False, num_workers=1)\n", - "model = MobileNetV2().to(device)\n", - "try:\n", - " model.load_state_dict(torch.load('trained_model.pt'), strict=False)\n", - " model.eval()\n", - "except:\n", - " raise Exception('trained_model.pt has not been found.')\n", - "\n", - "print('Test Set Accuracy: \\t%2.2f%%' % test(model, test_loader))\n", - "\n", - "model = MobileNetV2().to(device)\n", - "model.load_state_dict(torch.load('trained_model.pt'), strict=True)\n", - "model.eval()\n", - "print(test(model, test_loader))\n", - "r_on = 1.4e4\n", - "r_off = 5e7\n", - "\n", - "def trial(r_on, r_off, tile_shape, ADC_resolution, failure_percentage):\n", - " model_ = copy.deepcopy(model)\n", - " reference_memristor = memtorch.bh.memristor.VTEAM\n", - " reference_memristor_params = {'time_series_resolution': 1e-10, 'r_off': r_off, 'r_on': r_on}\n", - " patched_model = patch_model(copy.deepcopy(model_),\n", - " memristor_model=reference_memristor,\n", - " memristor_model_params=reference_memristor_params,\n", - " module_parameters_to_patch=[torch.nn.Linear, torch.nn.Conv2d],\n", - " mapping_routine=naive_map,\n", - " transistor=True,\n", - " programming_routine=None,\n", - " scheme=memtorch.bh.Scheme.DoubleColumn,\n", - " tile_shape=tile_shape,\n", - " max_input_voltage=0.3,\n", - " ADC_resolution=int(ADC_resolution),\n", - " ADC_overflow_rate=0.,\n", - " quant_method='linear')\n", - "\n", - " patched_model = apply_nonidealities(patched_model,\n", - " non_idealities=[memtorch.bh.nonideality.NonIdeality.DeviceFaults],\n", - " lrs_proportion=failure_percentage,\n", - " hrs_proportion=0.,\n", - " electroform_proportion=0.)\n", - " \n", - " patched_model.tune_()\n", - " return test(patched_model, test_loader)\n", - "\n", - "df = pd.DataFrame(columns=['tile_shape', 'ADC_resolution', 'failure_percentage', 'test_set_accuracy'])\n", - "tile_shape = [(128, 128), (256, 64)]\n", - "ADC_resolution = np.linspace(2, 10, num=5, endpoint=True, dtype=int)\n", - "failure_percentage = np.linspace(0, 0.25, 5)\n", - "for tile_shape_ in tile_shape:\n", - " for ADC_resolution_ in ADC_resolution:\n", - " for failure_percentage_ in failure_percentage:\n", - " print('tile_shape: %s; ADC_resolution: %d; failure_percentage: %d' % (tile_shape_, ADC_resolution_, failure_percentage_))\n", - " df = df.append({'tile_shape': tile_shape_, \n", - " 'ADC_resolution': ADC_resolution_, \n", - " 'failure_percentage': failure_percentage_, \n", - " 'test_set_accuracy': trial(r_on, r_off, tile_shape_, ADC_resolution_, failure_percentage_)}, ignore_index=True)\n", + "import torch\r\n", + "import torch.nn as nn\r\n", + "import torch.nn.functional as F\r\n", + "import torch.optim as optim\r\n", + "from torchvision import datasets, transforms\r\n", + "import memtorch\r\n", + "from memtorch.utils import LoadCIFAR10\r\n", + "import numpy as np\r\n", + "import pandas as pd\r\n", + "import torchvision\r\n", + "import copy\r\n", + "from memtorch.mn.Module import patch_model\r\n", + "from memtorch.map.Parameter import naive_map\r\n", + "from memtorch.bh.crossbar.Program import naive_program\r\n", + "from memtorch.bh.nonideality.NonIdeality import apply_nonidealities\r\n", + "\r\n", + "\r\n", + "def test(model, test_loader):\r\n", + " correct = 0\r\n", + " for batch_idx, (data, target) in enumerate(test_loader):\r\n", + " output = model(data.to(device))\r\n", + " pred = output.data.max(1)[1]\r\n", + " correct += pred.eq(target.to(device).data.view_as(pred)).cpu().sum()\r\n", + "\r\n", + " return 100. * float(correct) / float(len(test_loader.dataset))\r\n", + "\r\n", + "device = torch.device('cuda')\r\n", + "transform_test = transforms.Compose([\r\n", + " transforms.ToTensor(),\r\n", + " transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),\r\n", + "])\r\n", + "test_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)\r\n", + "test_loader = torch.utils.data.DataLoader(test_set, batch_size=256, shuffle=False, num_workers=1)\r\n", + "model = MobileNetV2().to(device)\r\n", + "try:\r\n", + " model.load_state_dict(torch.load('trained_model.pt'), strict=False)\r\n", + " model.eval()\r\n", + "except:\r\n", + " raise Exception('trained_model.pt has not been found.')\r\n", + "\r\n", + "print('Test Set Accuracy: \\t%2.2f%%' % test(model, test_loader))\r\n", + "\r\n", + "model = MobileNetV2().to(device)\r\n", + "model.load_state_dict(torch.load('trained_model.pt'), strict=True)\r\n", + "model.eval()\r\n", + "print(test(model, test_loader))\r\n", + "r_on = 1.4e4\r\n", + "r_off = 5e7\r\n", + "\r\n", + "def trial(r_on, r_off, tile_shape, ADC_resolution, failure_percentage):\r\n", + " model_ = copy.deepcopy(model)\r\n", + " reference_memristor = memtorch.bh.memristor.VTEAM\r\n", + " reference_memristor_params = {'time_series_resolution': 1e-10, 'r_off': r_off, 'r_on': r_on}\r\n", + " patched_model = patch_model(copy.deepcopy(model_),\r\n", + " memristor_model=reference_memristor,\r\n", + " memristor_model_params=reference_memristor_params,\r\n", + " module_parameters_to_patch=[torch.nn.Linear, torch.nn.Conv2d],\r\n", + " mapping_routine=naive_map,\r\n", + " transistor=True,\r\n", + " programming_routine=None,\r\n", + " scheme=memtorch.bh.Scheme.DoubleColumn,\r\n", + " tile_shape=tile_shape,\r\n", + " max_input_voltage=0.3,\r\n", + " ADC_resolution=int(ADC_resolution),\r\n", + " ADC_overflow_rate=0.,\r\n", + " quant_method='linear')\r\n", + "\r\n", + " patched_model = apply_nonidealities(patched_model,\r\n", + " non_idealities=[memtorch.bh.nonideality.NonIdeality.DeviceFaults],\r\n", + " lrs_proportion=failure_percentage,\r\n", + " hrs_proportion=0.,\r\n", + " electroform_proportion=0.)\r\n", + " \r\n", + " patched_model.tune_()\r\n", + " return test(patched_model, test_loader)\r\n", + "\r\n", + "df = pd.DataFrame(columns=['tile_shape', 'ADC_resolution', 'failure_percentage', 'test_set_accuracy'])\r\n", + "tile_shape = [(128, 128), (256, 64)]\r\n", + "ADC_resolution = np.linspace(2, 10, num=5, endpoint=True, dtype=int)\r\n", + "failure_percentage = np.linspace(0, 0.25, 5)\r\n", + "for tile_shape_ in tile_shape:\r\n", + " for ADC_resolution_ in ADC_resolution:\r\n", + " for failure_percentage_ in failure_percentage:\r\n", + " print('tile_shape: %s; ADC_resolution: %d; failure_percentage: %d' % (tile_shape_, ADC_resolution_, failure_percentage_))\r\n", + " df = df.append({'tile_shape': tile_shape_, \r\n", + " 'ADC_resolution': ADC_resolution_, \r\n", + " 'failure_percentage': failure_percentage_, \r\n", + " 'test_set_accuracy': trial(r_on, r_off, tile_shape_, ADC_resolution_, failure_percentage_)}, ignore_index=True)\r\n", " df.to_csv('1_CG.csv', index=False)" - ] + ], + "outputs": [], + "metadata": {} }, { "cell_type": "markdown", - "metadata": {}, "source": [ "## 6. Figure 1 [D,H]" - ] + ], + "metadata": {} }, { "cell_type": "code", "execution_count": null, - "metadata": {}, - "outputs": [], "source": [ - "import torch\n", - "import torch.nn as nn\n", - "import torch.nn.functional as F\n", - "import torch.optim as optim\n", - "from torchvision import datasets, transforms\n", - "import memtorch\n", - "from memtorch.utils import LoadCIFAR10\n", - "import numpy as np\n", - "import pandas as pd\n", - "from mobilenetv2 import MobileNetV2\n", - "import torchvision\n", - "import copy\n", - "from memtorch.mn.Module import patch_model\n", - "from memtorch.map.Parameter import naive_map\n", - "from memtorch.bh.crossbar.Program import naive_program\n", - "from memtorch.bh.nonideality.NonIdeality import apply_nonidealities\n", - "\n", - "\n", - "def test(model, test_loader):\n", - " correct = 0\n", - " for batch_idx, (data, target) in enumerate(test_loader):\n", - " output = model(data.to(device))\n", - " pred = output.data.max(1)[1]\n", - " correct += pred.eq(target.to(device).data.view_as(pred)).cpu().sum()\n", - "\n", - " return 100. * float(correct) / float(len(test_loader.dataset))\n", - "\n", - "device = torch.device('cuda')\n", - "transform_test = transforms.Compose([\n", - " transforms.ToTensor(),\n", - " transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),\n", - "])\n", - "test_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)\n", - "test_loader = torch.utils.data.DataLoader(test_set, batch_size=256, shuffle=False, num_workers=1)\n", - "model = MobileNetV2().to(device)\n", - "try:\n", - " model.load_state_dict(torch.load('trained_model.pt'), strict=False)\n", - " model.eval()\n", - "except:\n", - " raise Exception('trained_model.pt has not been found.')\n", - "\n", - "print('Test Set Accuracy: \\t%2.2f%%' % test(model, test_loader))\n", - "\n", - "model = MobileNetV2().to(device)\n", - "model.load_state_dict(torch.load('trained_model.pt'), strict=True)\n", - "model.eval()\n", - "print(test(model, test_loader))\n", - "r_on = 1.4e4\n", - "r_off = 5e7\n", - "\n", - "def trial(r_on, r_off, tile_shape, ADC_resolution, failure_percentage):\n", - " model_ = copy.deepcopy(model)\n", - " reference_memristor = memtorch.bh.memristor.VTEAM\n", - " reference_memristor_params = {'time_series_resolution': 1e-10, 'r_off': r_off, 'r_on': r_on}\n", - " patched_model = patch_model(copy.deepcopy(model_),\n", - " memristor_model=reference_memristor,\n", - " memristor_model_params=reference_memristor_params,\n", - " module_parameters_to_patch=[torch.nn.Linear, torch.nn.Conv2d],\n", - " mapping_routine=naive_map,\n", - " transistor=True,\n", - " programming_routine=None,\n", - " scheme=memtorch.bh.Scheme.DoubleColumn,\n", - " tile_shape=tile_shape,\n", - " max_input_voltage=0.3,\n", - " ADC_resolution=int(ADC_resolution),\n", - " ADC_overflow_rate=0.,\n", - " quant_method='linear')\n", - "\n", - " patched_model = apply_nonidealities(patched_model,\n", - " non_idealities=[memtorch.bh.nonideality.NonIdeality.DeviceFaults],\n", - " lrs_proportion=0.,\n", - " hrs_proportion=failure_percentage,\n", - " electroform_proportion=0.)\n", - " \n", - " patched_model.tune_()\n", - " return test(patched_model, test_loader)\n", - "\n", - "df = pd.DataFrame(columns=['tile_shape', 'ADC_resolution', 'failure_percentage', 'test_set_accuracy'])\n", - "tile_shape = [(128, 128), (256, 64)]\n", - "ADC_resolution = np.linspace(2, 10, num=5, endpoint=True, dtype=int)\n", - "failure_percentage = np.linspace(0, 0.25, 5)\n", - "for tile_shape_ in tile_shape:\n", - " for ADC_resolution_ in ADC_resolution:\n", - " for failure_percentage_ in failure_percentage:\n", - " print('tile_shape: %s; ADC_resolution: %d; failure_percentage: %d' % (tile_shape_, ADC_resolution_, failure_percentage_))\n", - " df = df.append({'tile_shape': tile_shape_, \n", - " 'ADC_resolution': ADC_resolution_, \n", - " 'failure_percentage': failure_percentage_, \n", - " 'test_set_accuracy': trial(r_on, r_off, tile_shape_, ADC_resolution_, failure_percentage_)}, ignore_index=True)\n", + "import torch\r\n", + "import torch.nn as nn\r\n", + "import torch.nn.functional as F\r\n", + "import torch.optim as optim\r\n", + "from torchvision import datasets, transforms\r\n", + "import memtorch\r\n", + "from memtorch.utils import LoadCIFAR10\r\n", + "import numpy as np\r\n", + "import pandas as pd\r\n", + "import torchvision\r\n", + "import copy\r\n", + "from memtorch.mn.Module import patch_model\r\n", + "from memtorch.map.Parameter import naive_map\r\n", + "from memtorch.bh.crossbar.Program import naive_program\r\n", + "from memtorch.bh.nonideality.NonIdeality import apply_nonidealities\r\n", + "\r\n", + "\r\n", + "def test(model, test_loader):\r\n", + " correct = 0\r\n", + " for batch_idx, (data, target) in enumerate(test_loader):\r\n", + " output = model(data.to(device))\r\n", + " pred = output.data.max(1)[1]\r\n", + " correct += pred.eq(target.to(device).data.view_as(pred)).cpu().sum()\r\n", + "\r\n", + " return 100. * float(correct) / float(len(test_loader.dataset))\r\n", + "\r\n", + "device = torch.device('cuda')\r\n", + "transform_test = transforms.Compose([\r\n", + " transforms.ToTensor(),\r\n", + " transforms.Normalize(mean=(0.5, 0.5, 0.5), std=(0.5, 0.5, 0.5)),\r\n", + "])\r\n", + "test_set = torchvision.datasets.CIFAR10(root='./data', train=False, download=True, transform=transform_test)\r\n", + "test_loader = torch.utils.data.DataLoader(test_set, batch_size=256, shuffle=False, num_workers=1)\r\n", + "model = MobileNetV2().to(device)\r\n", + "try:\r\n", + " model.load_state_dict(torch.load('trained_model.pt'), strict=False)\r\n", + " model.eval()\r\n", + "except:\r\n", + " raise Exception('trained_model.pt has not been found.')\r\n", + "\r\n", + "print('Test Set Accuracy: \\t%2.2f%%' % test(model, test_loader))\r\n", + "\r\n", + "model = MobileNetV2().to(device)\r\n", + "model.load_state_dict(torch.load('trained_model.pt'), strict=True)\r\n", + "model.eval()\r\n", + "print(test(model, test_loader))\r\n", + "r_on = 1.4e4\r\n", + "r_off = 5e7\r\n", + "\r\n", + "def trial(r_on, r_off, tile_shape, ADC_resolution, failure_percentage):\r\n", + " model_ = copy.deepcopy(model)\r\n", + " reference_memristor = memtorch.bh.memristor.VTEAM\r\n", + " reference_memristor_params = {'time_series_resolution': 1e-10, 'r_off': r_off, 'r_on': r_on}\r\n", + " patched_model = patch_model(copy.deepcopy(model_),\r\n", + " memristor_model=reference_memristor,\r\n", + " memristor_model_params=reference_memristor_params,\r\n", + " module_parameters_to_patch=[torch.nn.Linear, torch.nn.Conv2d],\r\n", + " mapping_routine=naive_map,\r\n", + " transistor=True,\r\n", + " programming_routine=None,\r\n", + " scheme=memtorch.bh.Scheme.DoubleColumn,\r\n", + " tile_shape=tile_shape,\r\n", + " max_input_voltage=0.3,\r\n", + " ADC_resolution=int(ADC_resolution),\r\n", + " ADC_overflow_rate=0.,\r\n", + " quant_method='linear')\r\n", + "\r\n", + " patched_model = apply_nonidealities(patched_model,\r\n", + " non_idealities=[memtorch.bh.nonideality.NonIdeality.DeviceFaults],\r\n", + " lrs_proportion=0.,\r\n", + " hrs_proportion=failure_percentage,\r\n", + " electroform_proportion=0.)\r\n", + " \r\n", + " patched_model.tune_()\r\n", + " return test(patched_model, test_loader)\r\n", + "\r\n", + "df = pd.DataFrame(columns=['tile_shape', 'ADC_resolution', 'failure_percentage', 'test_set_accuracy'])\r\n", + "tile_shape = [(128, 128), (256, 64)]\r\n", + "ADC_resolution = np.linspace(2, 10, num=5, endpoint=True, dtype=int)\r\n", + "failure_percentage = np.linspace(0, 0.25, 5)\r\n", + "for tile_shape_ in tile_shape:\r\n", + " for ADC_resolution_ in ADC_resolution:\r\n", + " for failure_percentage_ in failure_percentage:\r\n", + " print('tile_shape: %s; ADC_resolution: %d; failure_percentage: %d' % (tile_shape_, ADC_resolution_, failure_percentage_))\r\n", + " df = df.append({'tile_shape': tile_shape_, \r\n", + " 'ADC_resolution': ADC_resolution_, \r\n", + " 'failure_percentage': failure_percentage_, \r\n", + " 'test_set_accuracy': trial(r_on, r_off, tile_shape_, ADC_resolution_, failure_percentage_)}, ignore_index=True)\r\n", " df.to_csv('1_DH.csv', index=False)" - ] + ], + "outputs": [], + "metadata": {} } ], "metadata": { @@ -615,4 +610,4 @@ }, "nbformat": 4, "nbformat_minor": 4 -} +} \ No newline at end of file diff --git a/memtorch/mn/Conv1d.py b/memtorch/mn/Conv1d.py index a633c77e..1725793c 100644 --- a/memtorch/mn/Conv1d.py +++ b/memtorch/mn/Conv1d.py @@ -88,6 +88,9 @@ def __init__( assert isinstance( convolutional_layer, nn.Conv1d ), "convolutional_layer is not an instance of nn.Conv1d." + assert ( + convolutional_layer.groups != 2 + ), "groups=2 is not currently supported for convolutional layers." self.device = torch.device("cpu" if "cpu" in memtorch.__version__ else "cuda") self.transistor = transistor self.scheme = scheme @@ -134,10 +137,12 @@ def __init__( convolutional_layer.in_channels, convolutional_layer.out_channels, convolutional_layer.kernel_size, + stride=convolutional_layer.stride, + padding=convolutional_layer.padding, + dilation=convolutional_layer.dilation, + groups=convolutional_layer.groups, **kwargs ) - self.padding = convolutional_layer.padding - self.stride = convolutional_layer.stride self.weight.data = convolutional_layer.weight.data if convolutional_layer.bias is not None: self.bias.data = convolutional_layer.bias.data @@ -205,7 +210,9 @@ def forward(self, input): input[batch] .unfold(-1, size=self.kernel_size[0], step=self.stride[0]) .permute(1, 0, 2) - .reshape(-1, self.in_channels * self.kernel_size[0]) + .reshape( + -1, (self.in_channels // self.groups) * self.kernel_size[0] + ) ) if hasattr(self, "non_linear"): warnings.warn( @@ -294,7 +301,9 @@ def forward(self, input): def tune(self, input_batch_size=8, input_shape=32): """Tuning method.""" self.transform_output = naive_tune( - self, (input_batch_size, self.in_channels, input_shape), self.verbose + self, + (input_batch_size, (self.in_channels // self.groups), input_shape), + self.verbose, ) def __str__(self): diff --git a/memtorch/mn/Conv2d.py b/memtorch/mn/Conv2d.py index fe497eb8..18517088 100644 --- a/memtorch/mn/Conv2d.py +++ b/memtorch/mn/Conv2d.py @@ -4,6 +4,7 @@ import numpy as np import torch import torch.nn as nn +from torch.nn.modules import conv import memtorch from memtorch.bh.crossbar.Crossbar import init_crossbar, simulate_matmul @@ -88,6 +89,9 @@ def __init__( assert isinstance( convolutional_layer, nn.Conv2d ), "convolutional_layer is not an instance of nn.Conv2d." + assert ( + convolutional_layer.groups != 2 + ), "groups=2 is not currently supported for convolutional layers." self.device = torch.device("cpu" if "cpu" in memtorch.__version__ else "cuda") self.transistor = transistor self.scheme = scheme @@ -134,10 +138,12 @@ def __init__( convolutional_layer.in_channels, convolutional_layer.out_channels, convolutional_layer.kernel_size, + stride=convolutional_layer.stride, + padding=convolutional_layer.padding, + dilation=convolutional_layer.dilation, + groups=convolutional_layer.groups, **kwargs ) - self.padding = convolutional_layer.padding - self.stride = convolutional_layer.stride self.weight.data = convolutional_layer.weight.data if convolutional_layer.bias is not None: self.bias.data = convolutional_layer.bias.data @@ -226,7 +232,10 @@ def forward(self, input): .unfold(2, size=self.kernel_size[0], step=self.stride[0]) .permute(1, 2, 0, 3, 4) .reshape( - -1, self.in_channels * self.kernel_size[0] * self.kernel_size[1] + -1, + (self.in_channels // self.groups) + * self.kernel_size[0] + * self.kernel_size[1], ) ) if hasattr(self, "non_linear"): @@ -324,7 +333,12 @@ def tune(self, input_batch_size=8, input_shape=32): """Tuning method.""" self.transform_output = naive_tune( self, - (input_batch_size, self.in_channels, input_shape, input_shape), + ( + input_batch_size, + (self.in_channels // self.groups), + input_shape, + input_shape, + ), self.verbose, ) diff --git a/memtorch/mn/Conv3d.py b/memtorch/mn/Conv3d.py index 1a627070..a58bcb95 100644 --- a/memtorch/mn/Conv3d.py +++ b/memtorch/mn/Conv3d.py @@ -88,6 +88,9 @@ def __init__( assert isinstance( convolutional_layer, nn.Conv3d ), "convolutional_layer is not an instance of nn.Conv3d." + assert ( + convolutional_layer.groups != 2 + ), "groups=2 is not currently supported for convolutional layers." self.device = torch.device("cpu" if "cpu" in memtorch.__version__ else "cuda") self.transistor = transistor self.scheme = scheme @@ -134,10 +137,12 @@ def __init__( convolutional_layer.in_channels, convolutional_layer.out_channels, convolutional_layer.kernel_size, + stride=convolutional_layer.stride, + padding=convolutional_layer.padding, + dilation=convolutional_layer.dilation, + groups=convolutional_layer.groups, **kwargs ) - self.padding = convolutional_layer.padding - self.stride = convolutional_layer.stride self.weight.data = convolutional_layer.weight.data if convolutional_layer.bias is not None: self.bias.data = convolutional_layer.bias.data @@ -243,7 +248,7 @@ def forward(self, input): .permute(1, 2, 3, 0, 4, 5, 6) .reshape( -1, - self.in_channels + (self.in_channels // self.groups) * self.kernel_size[0] * self.kernel_size[1] * self.kernel_size[2], @@ -337,7 +342,13 @@ def tune(self, input_batch_size=4, input_shape=32): """Tuning method.""" self.transform_output = naive_tune( self, - (input_batch_size, self.in_channels, input_shape, input_shape, input_shape), + ( + input_batch_size, + (self.in_channels // self.groups), + input_shape, + input_shape, + input_shape, + ), self.verbose, ) diff --git a/memtorch/mn/Module.py b/memtorch/mn/Module.py index 88efe73a..877bbaab 100644 --- a/memtorch/mn/Module.py +++ b/memtorch/mn/Module.py @@ -3,6 +3,8 @@ import torch import torch.functional as F +from torch.nn import modules +from torch.nn.modules import module import memtorch from memtorch.map.Input import naive_scale @@ -97,58 +99,75 @@ def patch_model( torch.nn.Module Patched torch.nn.Module. """ - model.map = mapping_routine - for _, (name, m) in enumerate(list(model.named_modules())): - for parameter in module_parameters_to_patch: - if isinstance(m, parameter): - parameter_type = str(type(m)) - patch = supported_module_parameters.get(parameter_type) - patched_module = patch( - m, - memristor_model=memristor_model, - memristor_model_params=memristor_model_params, - mapping_routine=mapping_routine, - transistor=transistor, - programming_routine=programming_routine, - programming_routine_params=programming_routine_params, - p_l=p_l, - scheme=scheme, - tile_shape=tile_shape, - max_input_voltage=max_input_voltage, - scaling_routine=scaling_routine, - scaling_routine_params=scaling_routine_params, - source_resistance=source_resistance, - line_resistance=line_resistance, - ADC_resolution=ADC_resolution, - ADC_overflow_rate=ADC_overflow_rate, - quant_method=quant_method, - use_bindings=use_bindings, - verbose=verbose, - **kwargs - ) - if name.__contains__("."): - sequence_container, module = name.split(".") - if module.isdigit(): - module = int(module) - model._modules[sequence_container][module] = patched_module + + def patch_module(target_attr): + parameter_type = str(type(target_attr)) + patch = supported_module_parameters.get(parameter_type) + return patch( + target_attr, + memristor_model=memristor_model, + memristor_model_params=memristor_model_params, + mapping_routine=mapping_routine, + transistor=transistor, + programming_routine=programming_routine, + programming_routine_params=programming_routine_params, + p_l=p_l, + scheme=scheme, + tile_shape=tile_shape, + max_input_voltage=max_input_voltage, + scaling_routine=scaling_routine, + scaling_routine_params=scaling_routine_params, + source_resistance=source_resistance, + line_resistance=line_resistance, + ADC_resolution=ADC_resolution, + ADC_overflow_rate=ADC_overflow_rate, + quant_method=quant_method, + use_bindings=use_bindings, + verbose=verbose, + **kwargs + ) + + def patch_modules(module, name=""): + for attr_str in dir(module): + target_attr = getattr(module, attr_str) + if any( + isinstance(target_attr, module_parameter) + and not hasattr(target_attr, "transistor") + for module_parameter in module_parameters_to_patch + ): + new_bn = patch_module(target_attr) + setattr(module, attr_str, new_bn) + + if isinstance(module, torch.nn.Module): + if type(module) == torch.nn.modules.container.Sequential: + for idx, (name, child) in enumerate(module.named_children()): + if any( + isinstance(child, module_parameter) + and not hasattr(child, "transistor") + for module_parameter in module_parameters_to_patch + ): + target_attr = module[idx] + new_bn = patch_module(target_attr) + module[idx] = new_bn else: - setattr( - model._modules[sequence_container], - "%s" % module, - patched_module, - ) - else: - model._modules[name] = patched_module + patch_modules(child, name) + else: + for name, child in module.named_children(): + patch_modules(child, name) + else: + for child in module: + patch_modules(child, name) + + patch_modules(model) def tune_(self, tune_kwargs=None): """Method to tune a memristive layer. - Parameters ---------- tune_kwargs : dict Dictionary of **kwargs for different layer types for .tune(). """ - for i, (name, m) in enumerate(list(self.named_modules())): + for _, (name, m) in enumerate(list(self.named_modules())): if hasattr(m, "tune"): if tune_kwargs is not None: module_type = str(type(m)) diff --git a/memtorch/utils.py b/memtorch/utils.py index d0aa7a94..54ecbb6f 100644 --- a/memtorch/utils.py +++ b/memtorch/utils.py @@ -105,7 +105,7 @@ def pad_tensor(tensor, tile_shape): return tensor_padded -def LoadMNIST(batch_size=32, validation=True): +def LoadMNIST(batch_size=32, validation=True, num_workers=1): """Method to load the MNIST dataset. Parameters @@ -114,6 +114,8 @@ def LoadMNIST(batch_size=32, validation=True): Batch size. validation : bool Load the validation set (True). + num_workers : int + Number of workers to use. Returns ------- @@ -135,24 +137,24 @@ def LoadMNIST(batch_size=32, validation=True): full_train_set, [train_size, validation_size] ) train_loader = torch.utils.data.DataLoader( - train_set, batch_size=batch_size, shuffle=True, num_workers=2 + train_set, batch_size=batch_size, shuffle=True, num_workers=num_workers ) validation_loader = torch.utils.data.DataLoader( - validation_set, batch_size=batch_size, shuffle=True, num_workers=2 + validation_set, batch_size=batch_size, shuffle=True, num_workers=num_workers ) else: train_loader = torch.utils.data.DataLoader( - full_train_set, batch_size=batch_size, shuffle=True, num_workers=2 + full_train_set, batch_size=batch_size, shuffle=True, num_workers=num_workers ) validation_loader = None test_loader = torch.utils.data.DataLoader( - test_set, batch_size=int(batch_size / 2), shuffle=False, num_workers=2 + test_set, batch_size=int(batch_size / 2), shuffle=False, num_workers=num_workers ) return train_loader, validation_loader, test_loader -def LoadCIFAR10(batch_size=32, validation=True): +def LoadCIFAR10(batch_size=32, validation=True, num_workers=1): """Method to load the CIFAR-10 dataset. Parameters @@ -161,6 +163,8 @@ def LoadCIFAR10(batch_size=32, validation=True): Batch size. validation : bool Load the validation set (True). + num_workers : int + Number of workers to use. Returns ------- @@ -182,18 +186,18 @@ def LoadCIFAR10(batch_size=32, validation=True): full_train_set, [train_size, validation_size] ) train_loader = torch.utils.data.DataLoader( - train_set, batch_size=batch_size, shuffle=True, num_workers=2 + train_set, batch_size=batch_size, shuffle=True, num_workers=num_workers ) validation_loader = torch.utils.data.DataLoader( - validation_set, batch_size=batch_size, shuffle=True, num_workers=2 + validation_set, batch_size=batch_size, shuffle=True, num_workers=num_workers ) else: train_loader = torch.utils.data.DataLoader( - full_train_set, batch_size=batch_size, shuffle=True, num_workers=2 + full_train_set, batch_size=batch_size, shuffle=True, num_workers=num_workers ) validation_loader = None test_loader = torch.utils.data.DataLoader( - test_set, batch_size=int(batch_size / 2), shuffle=False, num_workers=2 + test_set, batch_size=int(batch_size / 2), shuffle=False, num_workers=num_workers ) return train_loader, validation_loader, test_loader diff --git a/memtorch/version.py b/memtorch/version.py index 88fbfbbf..d14425a9 100644 --- a/memtorch/version.py +++ b/memtorch/version.py @@ -1 +1 @@ -__version__ = "1.1.4-cpu" +__version__ = "1.1.5-cpu" diff --git a/setup.py b/setup.py index c9427b88..bdfda3dd 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ from setuptools import find_packages, setup from torch.utils.cpp_extension import include_paths, library_paths -version = "1.1.4" +version = "1.1.5" CUDA = False