From 77a4e7030887d7743f87ea70f0f783fdfd981d11 Mon Sep 17 00:00:00 2001 From: Aditya Agrawal Date: Fri, 12 Jun 2020 04:30:28 +0000 Subject: [PATCH] Fused Adam and Layer Norm are now monkey patched inside pyprof.init(). The examples in this directory are no longer useful. Signed-off-by: Aditya Agrawal --- pyprof/examples/apex/README.md | 18 ---------- pyprof/examples/apex/fused_adam.py | 37 ------------------- pyprof/examples/apex/fused_layer_norm.py | 45 ------------------------ pyprof/examples/apex/test.sh | 43 ---------------------- 4 files changed, 143 deletions(-) delete mode 100644 pyprof/examples/apex/README.md delete mode 100644 pyprof/examples/apex/fused_adam.py delete mode 100644 pyprof/examples/apex/fused_layer_norm.py delete mode 100755 pyprof/examples/apex/test.sh diff --git a/pyprof/examples/apex/README.md b/pyprof/examples/apex/README.md deleted file mode 100644 index e1e4dbc..0000000 --- a/pyprof/examples/apex/README.md +++ /dev/null @@ -1,18 +0,0 @@ - - -This directory has examples of how to use `pyprof` with APEX extensions e.g. -`fused_adam_cuda` and `fused_layer_norm_cuda`. diff --git a/pyprof/examples/apex/fused_adam.py b/pyprof/examples/apex/fused_adam.py deleted file mode 100644 index c86b848..0000000 --- a/pyprof/examples/apex/fused_adam.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -import fused_adam_cuda -from apex.optimizers import FusedAdam, FP16_Optimizer -import pyprof - -pyprof.init() -pyprof.wrap(fused_adam_cuda, 'adam') - -model = torch.nn.Linear(10, 20).cuda().half() -criterion = torch.nn.CrossEntropyLoss().cuda() -optimizer = FusedAdam(model.parameters()) -optimizer = FP16_Optimizer(optimizer) - -x = torch.ones(32, 10).cuda().half() -target = torch.empty(32, dtype=torch.long).random_(20).cuda() -y = model(x) -loss = criterion(y, target) -optimizer.zero_grad() -loss.backward() -optimizer.step() diff --git a/pyprof/examples/apex/fused_layer_norm.py b/pyprof/examples/apex/fused_layer_norm.py deleted file mode 100644 index 6452a95..0000000 --- a/pyprof/examples/apex/fused_layer_norm.py +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- - -# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import torch -import fused_layer_norm_cuda -from apex.normalization import FusedLayerNorm -import pyprof - -pyprof.init() -pyprof.wrap(fused_layer_norm_cuda, 'forward') -pyprof.wrap(fused_layer_norm_cuda, 'backward') -pyprof.wrap(fused_layer_norm_cuda, 'forward_affine') -pyprof.wrap(fused_layer_norm_cuda, 'backward_affine') - -input = torch.randn(20, 5, 10, 10).cuda() - -# With Learnable Parameters -m = FusedLayerNorm(input.size()[1:]).cuda() -output = m(input) - -# Without Learnable Parameters -m = FusedLayerNorm(input.size()[1:], elementwise_affine=False).cuda() -output = m(input) - -# Normalize over last two dimensions -m = FusedLayerNorm([10, 10]).cuda() -output = m(input) - -# Normalize over last dimension of size 10 -m = FusedLayerNorm(10).cuda() -output = m(input) diff --git a/pyprof/examples/apex/test.sh b/pyprof/examples/apex/test.sh deleted file mode 100755 index 2ca2f00..0000000 --- a/pyprof/examples/apex/test.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -SCRIPT=`realpath $0` -SCRIPTPATH=`dirname $SCRIPT` -PYPROF="$SCRIPTPATH/../.." - -parse="python $PYPROF/parse/parse.py" -prof="python $PYPROF/prof/prof.py" - -for f in *.py -do - base=`basename $f .py` - sql=$base.sql - dict=$base.dict - - #NVprof - echo "nvprof -fo $sql python $f" - nvprof -fo $sql python $f - - #Parse - echo $parse $sql - $parse $sql > $dict - - #Prof - echo $prof $dict - $prof -w 130 $dict - \rm $sql $dict -done