diff --git a/BUILD.md b/BUILD.md index 0269b67b72f95..ee5adb5910b88 100644 --- a/BUILD.md +++ b/BUILD.md @@ -1103,12 +1103,13 @@ Dockerfile instructions are available [here](./dockerfiles#migraphx) The default NVIDIA GPU build requires CUDA runtime libraries installed on the system: -* CUDA 10.2 -* cuDNN 7.6.5 -* NCCL v2.7.8 -* OpenMPI 4.0.4 +* [CUDA](https://developer.nvidia.com/cuda-toolkit) 10.2 +* [cuDNN](https://developer.nvidia.com/cudnn) 8.0 +* [NCCL](https://developer.nvidia.com/nccl) 2.7 +* [OpenMPI](https://www.open-mpi.org/) 4.0.4 + * See [install_openmpi.sh](./tools/ci_build/github/linux/docker/scripts/install_openmpi.sh) -The official dependency versions are specified in [Dockerfile.training](./dockerfiles/Dockerfile.training). +These dependency versions should reflect what is in [Dockerfile.training](./dockerfiles/Dockerfile.training). ## Build instructions diff --git a/cmake/external/dml.cmake b/cmake/external/dml.cmake index 80a57b0c104c7..af1e5c940f6de 100644 --- a/cmake/external/dml.cmake +++ b/cmake/external/dml.cmake @@ -20,7 +20,7 @@ if (NOT onnxruntime_USE_CUSTOM_DIRECTML) set(NUGET_CONFIG ${PROJECT_SOURCE_DIR}/../NuGet.config) set(PACKAGES_CONFIG ${PROJECT_SOURCE_DIR}/../packages.config) get_filename_component(PACKAGES_DIR ${CMAKE_CURRENT_BINARY_DIR}/../packages ABSOLUTE) - set(DML_PACKAGE_DIR ${PACKAGES_DIR}/DirectML.3.0.0) + set(DML_PACKAGE_DIR ${PACKAGES_DIR}/DirectML.1.3.0) set(DML_SHARED_LIB DirectML.dll) # Restore nuget packages, which will pull down the DirectML redist package diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index 64a4c16343312..20349383a2db8 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -397,9 +397,7 @@ set(ONNXRUNTIME_TEST_LIBS ${ONNXRUNTIME_INTEROP_TEST_LIBS} ${onnxruntime_libs} ${PROVIDERS_CUDA} -# These providers are shared libraries now, so aren't linked this way anymore: - ${PROVIDERS_DNNL} - ${PROVIDERS_TENSORRT} + # TENSORRT and DNNL are explicitly linked at runtime ${PROVIDERS_MIGRAPHX} ${PROVIDERS_NGRAPH} ${PROVIDERS_OPENVINO} @@ -433,7 +431,6 @@ if(onnxruntime_USE_TENSORRT) list(APPEND onnxruntime_test_framework_src_patterns ${TEST_SRC_DIR}/providers/tensorrt/*) list(APPEND onnxruntime_test_framework_libs onnxruntime_providers_tensorrt) list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_tensorrt onnxruntime_providers_shared) - list(APPEND onnxruntime_test_providers_libs onnxruntime_providers_tensorrt) endif() if(onnxruntime_USE_NNAPI_BUILTIN) diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh index a30327835a897..19ade4bb82219 100755 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh @@ -27,6 +27,6 @@ docker run --gpus all --rm \ -e "PackageName=$PackageName" \ -e "RunTestCsharp=$RunTestCsharp" \ -e "RunTestNative=$RunTestNative" \ - onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimegpubuild:ch35 \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentosgpubuild:ch5h \ /bin/bash /onnxruntime_src/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \ /home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev $CurrentOnnxRuntimeVersion diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh index 61c5014667800..1b361a2a2e751 100755 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh @@ -35,6 +35,6 @@ docker run --rm \ -e "DisableMlOps=$DISABLEMLOPS" \ -e "RunTestCsharp=$RunTestCsharp" \ -e "RunTestNative=$RunTestNative" \ - onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch36 \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch5g \ /bin/bash /onnxruntime_src/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \ /home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev $CurrentOnnxRuntimeVersion diff --git a/dockerfiles/Dockerfile.training b/dockerfiles/Dockerfile.training index 5c8f0b497c3aa..8ddb0d82cb206 100644 --- a/dockerfiles/Dockerfile.training +++ b/dockerfiles/Dockerfile.training @@ -14,7 +14,7 @@ ARG OPENMPI_PATH=/opt/openmpi-${OPENMPI_VERSION} ARG COMMIT=master # cuda development image for building sources -FROM nvidia/cuda:10.2-cudnn7-devel-ubuntu18.04 as builder +FROM nvidia/cuda:10.2-cudnn8-devel-ubuntu18.04 as builder # set location for builds WORKDIR /stage @@ -155,7 +155,7 @@ RUN pip install azureml-defaults transformers==2.11.0 msgpack==1.0.0 tensorboard # switch to cuda runtime environment # note: launch with --gpus all or nvidia-docker -FROM nvidia/cuda:10.2-cudnn7-runtime-ubuntu18.04 +FROM nvidia/cuda:10.2-cudnn8-runtime-ubuntu18.04 WORKDIR /stage # install ucx diff --git a/docs/execution_providers/DirectML-ExecutionProvider.md b/docs/execution_providers/DirectML-ExecutionProvider.md index ce6268b16f622..21cd046578dc1 100644 --- a/docs/execution_providers/DirectML-ExecutionProvider.md +++ b/docs/execution_providers/DirectML-ExecutionProvider.md @@ -6,7 +6,7 @@ When used standalone, the DirectML API is a low-level DirectX 12 library and is The *DirectML Execution Provider* is an optional component of ONNX Runtime that uses DirectML to accelerate inference of ONNX models. The DirectML execution provider is capable of greatly improving evaluation time of models using commodity GPU hardware, without sacrificing broad hardware support or requiring vendor-specific extensions to be installed. -The DirectML Execution Provider currently uses DirectML version 2.1.0. +The DirectML Execution Provider currently uses DirectML version 1.3.0. ## Table of contents diff --git a/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc index b835ae467f7df..e6ec0eb8dccae 100644 --- a/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc +++ b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc @@ -162,7 +162,57 @@ "^test_operator_mm", "^test_operator_non_float_params", "^test_operator_params", - "^test_operator_pow" + "^test_operator_pow", + "^test_nllloss_NC", + "^test_nllloss_NCd1", + "^test_nllloss_NCd1d2", + "^test_nllloss_NCd1d2d3d4d5_none_no_weight", + "^test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded", + "^test_nllloss_NCd1d2d3_none_no_weight_negative_ii", + "^test_nllloss_NCd1d2d3_none_no_weight_negative_ii_expanded", + "^test_nllloss_NCd1d2d3_sum_weight_high_ii", + "^test_nllloss_NCd1d2d3_sum_weight_high_ii_expanded", + "^test_nllloss_NCd1d2_expanded", + "^test_nllloss_NCd1d2_reduction_mean", + "^test_nllloss_NCd1d2_reduction_mean_expanded", + "^test_nllloss_NCd1d2_reduction_sum", + "^test_nllloss_NCd1d2_reduction_sum_expanded", + "^test_nllloss_NCd1d2_with_weight_reduction_sum_ii", + "^test_nllloss_NCd1d2_with_weight_reduction_sum_ii_expanded", + "^test_nllloss_NCd1_expanded", + "^test_nllloss_NC_expanded", + "^test_sce_mean_3d", + "^test_sce_mean_3d_expanded", + "^test_sce_mean_3d_log_prob", + "^test_sce_mean_3d_log_prob_expanded", + "^test_sce_mean", + "^test_sce_mean_expanded", + "^test_sce_mean_log_prob", + "^test_sce_mean_log_prob_expanded", + "^test_sce_NCd1d2d3d4d5_mean_weight", + "^test_sce_NCd1d2d3d4d5_mean_weight_expanded", + "^test_sce_NCd1d2d3d4d5_mean_weight_log_prob", + "^test_sce_NCd1d2d3d4d5_mean_weight_log_prob_expanded", + "^test_sce_NCd1d2d3d4d5_none_no_weight", + "^test_sce_NCd1d2d3d4d5_none_no_weight_expanded", + "^test_sce_NCd1d2d3d4d5_none_no_weight_log_prob", + "^test_sce_NCd1d2d3d4d5_none_no_weight_log_prob_expanded", + "^test_sce_NCd1d2d3_none_no_weight_negative_ii", + "^test_sce_NCd1d2d3_none_no_weight_negative_ii_expanded", + "^test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob", + "^test_sce_NCd1d2d3_none_no_weight_negative_ii_log_prob_expanded", + "^test_sce_NCd1d2d3_sum_weight_high_ii", + "^test_sce_NCd1d2d3_sum_weight_high_ii_expanded", + "^test_sce_NCd1d2d3_sum_weight_high_ii_log_prob", + "^test_sce_NCd1d2d3_sum_weight_high_ii_log_prob_expanded", + "^test_sce_none", + "^test_sce_none_expanded", + "^test_sce_none_log_prob", + "^test_sce_none_log_prob_expanded", + "^test_sce_sum", + "^test_sce_sum_expanded", + "^test_sce_sum_log_prob", + "^test_sce_sum_log_prob_expanded" ], "unsupported_usages": [ "^test_convtranspose_1d", // ConvTransponse supports 4-D only @@ -178,4 +228,4 @@ "^test_bitshift_right_uint16", "^test_bitshift_left_uint16" ] -} \ No newline at end of file +} diff --git a/orttraining/orttraining/test/gradient/gradient_ops_test.cc b/orttraining/orttraining/test/gradient/gradient_ops_test.cc index 8e12312071f2f..15a3ed19f5f55 100644 --- a/orttraining/orttraining/test/gradient/gradient_ops_test.cc +++ b/orttraining/orttraining/test/gradient/gradient_ops_test.cc @@ -289,7 +289,8 @@ TEST(GradientCheckerTest, SubGrad) { TestBroadcastableBinaryOpGrad("Sub"); } -TEST(GradientCheckerTest, MulGrad) { +//flaky +TEST(GradientCheckerTest, DISABLED_MulGrad) { TestBroadcastableBinaryOpGrad("Mul"); } diff --git a/packages.config b/packages.config index 91960d191c725..6fce6d078a397 100644 --- a/packages.config +++ b/packages.config @@ -1,5 +1,5 @@  - + diff --git a/samples/python/pytorch_transformer/README.md b/samples/python/pytorch_transformer/README.md index 569b94aa70b56..7514d6cf70eeb 100644 --- a/samples/python/pytorch_transformer/README.md +++ b/samples/python/pytorch_transformer/README.md @@ -10,12 +10,24 @@ This example was adapted from Pytorch's [Sequence-to-Sequence Modeling with nn.T ## Running PyTorch version -```python +```bash python pt_model.py ``` ## Running ONNX Runtime version -```python +```bash python ort_model.py ``` + +## Optional arguments + +| Argument | Description | Default | +| :---------------- | :-----------------------------------------------------: | --------: | +| --batch-size | input batch size for training | 20 | +| --test-batch-size | input batch size for testing | 20 | +| --epochs | number of epochs to train | 2 | +| --lr | learning rate | 0.001 | +| --no-cuda | disables CUDA training | False | +| --seed | random seed | 1 | +| --log-interval | how many batches to wait before logging training status | 200 | diff --git a/samples/python/pytorch_transformer/ort_train.py b/samples/python/pytorch_transformer/ort_train.py new file mode 100644 index 0000000000000..98301067992ec --- /dev/null +++ b/samples/python/pytorch_transformer/ort_train.py @@ -0,0 +1,85 @@ +import argparse +import math +import torch +import onnxruntime + +from utils import prepare_data, get_batch +from ort_utils import my_loss, transformer_model_description_dynamic_axes +from pt_model import TransformerModel + + +def train(trainer, data_source, device, epoch, args, bptt=35): + total_loss = 0. + for batch, i in enumerate(range(0, data_source.size(0) - 1, bptt)): + data, targets = get_batch(data_source, i) + + loss, pred = trainer.train_step(data, targets) + total_loss += loss.item() + if batch % args.log_interval == 0 and batch > 0: + cur_loss = total_loss / args.log_interval + print('epoch {:3d} | {:5d}/{:5d} batches | loss {:5.2f}'.format(epoch, + batch, + len(data_source) // bptt, + cur_loss)) + total_loss = 0 + + +def evaluate(trainer, data_source, bptt=35): + total_loss = 0. + with torch.no_grad(): + for i in range(0, data_source.size(0) - 1, bptt): + data, targets = get_batch(data_source, i) + loss, pred = trainer.eval_step(data, targets) + total_loss += len(data) * loss.item() + return total_loss / (len(data_source) - 1) + + +if __name__ == "__main__": + # Training settings + parser = argparse.ArgumentParser(description='PyTorch TransformerModel example') + parser.add_argument('--batch-size', type=int, default=20, metavar='N', + help='input batch size for training (default: 20)') + parser.add_argument('--test-batch-size', type=int, default=20, metavar='N', + help='input batch size for testing (default: 20)') + parser.add_argument('--epochs', type=int, default=2, metavar='N', + help='number of epochs to train (default: 2)') + parser.add_argument('--lr', type=float, default=0.001, metavar='LR', + help='learning rate (default: 0.001)') + parser.add_argument('--no-cuda', action='store_true', default=False, + help='disables CUDA training') + parser.add_argument('--seed', type=int, default=1, metavar='S', + help='random seed (default: 1)') + parser.add_argument('--log-interval', type=int, default=200, metavar='N', + help='how many batches to wait before logging training status (default: 200)') + + # Basic setup + args = parser.parse_args() + if not args.no_cuda and torch.cuda.is_available(): + device = "cuda" + else: + device = "cpu" + torch.manual_seed(args.seed) + onnxruntime.set_seed(args.seed) + + # Model + optim_config = onnxruntime.training.optim.SGDConfig(lr=args.lr) + model_desc = transformer_model_description_dynamic_axes() + model = TransformerModel(28785, 200, 2, 200, 2, 0.2).to(device) + + # Preparing data + train_data, val_data, test_data = prepare_data(device, args.batch_size, args.test_batch_size) + trainer = onnxruntime.training.ORTTrainer(model, model_desc, optim_config, loss_fn=my_loss) + + # Train + for epoch in range(1, args.epochs + 1): + train(trainer, train_data, device, epoch, args) + val_loss = evaluate(trainer, val_data) + print('-' * 89) + print('| end of epoch {:3d} | valid loss {:5.2f} | '.format(epoch, val_loss)) + print('-' * 89) + + # Evaluate + test_loss = evaluate(trainer, test_data) + print('=' * 89) + print('| End of training | test loss {:5.2f}'.format(test_loss)) + print('=' * 89) diff --git a/samples/python/pytorch_transformer/pt_train.py b/samples/python/pytorch_transformer/pt_train.py new file mode 100644 index 0000000000000..7d3e8851c9e84 --- /dev/null +++ b/samples/python/pytorch_transformer/pt_train.py @@ -0,0 +1,92 @@ +import argparse +import math +import torch +import torch.nn as nn +import torch.nn.functional as F + +from utils import prepare_data, get_batch +from pt_model import TransformerModel + + +def train(model, data_source, device, epoch, args, bptt=35): + total_loss = 0. + model.train() + for batch, i in enumerate(range(0, data_source.size(0) - 1, bptt)): + data, targets = get_batch(data_source, i) + + optimizer.zero_grad() + output = model(data) + loss = criterion(output.view(-1, 28785), targets) + loss.backward() + optimizer.step() + + total_loss += loss.item() + if batch % args.log_interval == 0 and batch > 0: + cur_loss = total_loss / args.log_interval + print('epoch {:3d} | {:5d}/{:5d} batches | loss {:5.2f}'.format(epoch, + batch, + len(data_source) // bptt, + cur_loss)) + total_loss = 0 + + +def evaluate(model, data_source, criterion, bptt=35): + total_loss = 0. + model.eval() + with torch.no_grad(): + for i in range(0, data_source.size(0) - 1, bptt): + data, targets = get_batch(data_source, i) + output = model(data) + output_flat = output.view(-1, 28785) + total_loss += len(data) * criterion(output_flat, targets).item() + return total_loss / (len(data_source) - 1) + + +if __name__ == "__main__": + # Training settings + parser = argparse.ArgumentParser(description='PyTorch TransformerModel example') + parser.add_argument('--batch-size', type=int, default=20, metavar='N', + help='input batch size for training (default: 20)') + parser.add_argument('--test-batch-size', type=int, default=20, metavar='N', + help='input batch size for testing (default: 20)') + parser.add_argument('--epochs', type=int, default=2, metavar='N', + help='number of epochs to train (default: 2)') + parser.add_argument('--lr', type=float, default=0.001, metavar='LR', + help='learning rate (default: 0.001)') + parser.add_argument('--no-cuda', action='store_true', default=False, + help='disables CUDA training') + parser.add_argument('--seed', type=int, default=1, metavar='S', + help='random seed (default: 1)') + parser.add_argument('--log-interval', type=int, default=200, metavar='N', + help='how many batches to wait before logging training status (default: 200)') + + # Basic setup + args = parser.parse_args() + if not args.no_cuda and torch.cuda.is_available(): + device = "cuda" + else: + device = "cpu" + torch.manual_seed(args.seed) + + # Model + criterion = nn.CrossEntropyLoss() + lr = 0.001 + model = TransformerModel(28785, 200, 2, 200, 2, 0.2).to(device) + optimizer = torch.optim.SGD(model.parameters(), lr=lr) + + # Preparing data + train_data, val_data, test_data = prepare_data(device, args.batch_size, args.test_batch_size) + + # Train + for epoch in range(1, args.epochs + 1): + train(model, train_data, device, epoch, args) + val_loss = evaluate(model, val_data, criterion) + print('-' * 89) + print('| end of epoch {:3d} | valid loss {:5.2f} | '.format(epoch, val_loss)) + print('-' * 89) + + # Evaluate + test_loss = evaluate(model, test_data, criterion) + print('=' * 89) + print('| End of training | test loss {:5.2f}'.format(test_loss)) + print('=' * 89) diff --git a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml index b1e856254f283..a0b1ea5ba5344 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml @@ -18,8 +18,8 @@ jobs: script: | mkdir -p $HOME/.onnx docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch36 /opt/python/cp37-cp37m/bin/python3 \ - /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --enable_lto \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch5g python3 \ + /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \ --skip_submodule_sync --parallel --build_shared_lib --use_openmp workingDirectory: $(Build.SourcesDirectory) - task: Docker@2 @@ -58,8 +58,8 @@ jobs: script: | mkdir -p $HOME/.onnx docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ - --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimegpubuild:ch35 \ - /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \ + --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentosgpubuild:ch5h \ + python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \ --skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2 workingDirectory: $(Build.SourcesDirectory) - task: Docker@2 @@ -90,13 +90,24 @@ jobs: - template: templates/set-version-number-variables-step.yml - script: | + set -e + pushd . + cd $(Build.SourcesDirectory)/cmake/external/protobuf + cmake ./cmake -DCMAKE_INSTALL_PREFIX=$(Build.BinariesDirectory)/protobuf -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Relwithdebinfo + make -j$(getconf _NPROCESSORS_ONLN) + make install + popd + export PATH=$(Build.BinariesDirectory)/protobuf/bin:$PATH + export ONNX_ML=1 + export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer - python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --use_openmp --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config RelWithDebInfo + brew install libomp + python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --use_openmp --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config Release displayName: 'Build and Test MacOS' - template: templates/c-api-artifacts-package-and-publish-steps-posix.yml parameters: - buildConfig: 'RelWithDebInfo' + buildConfig: 'Release' artifactName: 'onnxruntime-osx-x64-$(OnnxRuntimeVersion)' libraryName: 'libonnxruntime.$(OnnxRuntimeVersion).dylib' commitId: $(OnnxRuntimeGitCommitHash) diff --git a/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines-gpu.yml b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines-gpu.yml index bfda9c38d7540..b895c72a9170f 100644 --- a/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines-gpu.yml +++ b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines-gpu.yml @@ -24,7 +24,7 @@ jobs: inputs: script: | mkdir -p $HOME/.onnx - docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimegpubuild:ch35 /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_java --build_shared_lib --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2 + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentosgpubuild:ch5h python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_java --build_shared_lib --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2 workingDirectory: $(Build.SourcesDirectory) - task: Docker@2 displayName: logout @@ -58,6 +58,8 @@ jobs: msbuildPlatform: x64 buildparameter: --use_cuda --cuda_version=10.2 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2" --cudnn_home="C:\local\cudnn-10.2-windows10-x64-v8.0.3.33\cuda" steps: + - template: templates/telemetry-steps.yml + - task: UsePythonVersion@0 inputs: versionSpec: '3.7' @@ -92,7 +94,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_java --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_lto --disable_rtti --build_shared_lib --enable_onnx_tests $(buildparameter)' + arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_java --use_telemetry --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_lto --disable_rtti --build_shared_lib --enable_onnx_tests $(buildparameter)' workingDirectory: '$(Build.BinariesDirectory)' @@ -256,7 +258,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimegpubuild:ch35 /onnxruntime_src/tools/ci_build/github/linux/java_linux_final_test.sh -v $(OnnxRuntimeVersion) -r /build + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentosgpubuild:ch5h /onnxruntime_src/tools/ci_build/github/linux/java_linux_final_test.sh -v $(OnnxRuntimeVersion) -r /build workingDirectory: $(Build.BinariesDirectory)/final-jar - task: Docker@2 diff --git a/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml index 4091e50aee8bd..c2695cd8c7159 100644 --- a/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml @@ -26,7 +26,7 @@ jobs: inputs: script: | mkdir -p $HOME/.onnx - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch36 /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --build_java --use_openmp --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch5g /bin/bash -c "python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --build_java --use_openmp --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) displayName: 'Run build and test' - task: Docker@2 @@ -103,7 +103,7 @@ jobs: steps: - template: templates/telemetry-steps.yml - + - task: UsePythonVersion@0 inputs: versionSpec: '3.7' @@ -137,7 +137,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --enable_wcos --build_java --update --cmake_generator "Visual Studio 16 2019" --enable_lto --disable_rtti --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)' + arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --enable_wcos --build_java --use_telemetry --update --cmake_generator "Visual Studio 16 2019" --enable_lto --disable_rtti --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)' workingDirectory: '$(Build.BinariesDirectory)' diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index e708f05eb2728..c284906865bd8 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -28,8 +28,8 @@ jobs: -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch36 \ - /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch5g \ + python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator Ninja \ --config Debug Release \ --skip_submodule_sync \ @@ -38,8 +38,7 @@ jobs: --build_wheel \ --use_openmp \ --enable_onnx_tests \ - --use_mklml --enable_pybind --build_java --build_nodejs \ - --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so + --use_mklml --enable_pybind --build_java --build_nodejs workingDirectory: $(Build.SourcesDirectory) - task: Docker@2 displayName: logout diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml index a57021520d00d..a7d0753429079 100644 --- a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml @@ -1,5 +1,14 @@ +# This CI has 3 major steps +# 1. Build full ORT, install the full ORT python wheel and use it to generate ort format test models +# and include ops config file for step 3 +# 2. Build minimal ORT including all the kernels and disable exceptions. +# This step is build only to safe-guard the --disable_exceptions option. +# 3. Build minimal ORT include only the kernels using the include ops config file from step 1, +# and the models from /onnxruntime/test/testdata/, run UT, and use onnx_test_runner to +# test the ort format models generated in step 1. +# Exceptions are enabled in this step to help debugging in case of CI failure. jobs: -- job: Linux_CPU_Minimal_Build +- job: Linux_CPU_Minimal_Build_E2E timeoutInMinutes: 120 workspace: clean: all @@ -16,35 +25,57 @@ jobs: command: login addPipelineData: false - task: CmdLine@2 + displayName: Build full onnxruntime and generate ORT format test files inputs: script: | - # We need copy the related test files to a separated folder since the --include_ops_by_model will search the testdata folder recursively - # and include many unnecessary ops, minimal build UT currently uses .ort format models converted from the models we copied below, - # which will be used as the input of --include_ops_by_model to have ops to be included for the minimal build UT. - mkdir -p $HOME/.models - cp $(Build.SourcesDirectory)/onnxruntime/test/testdata/ort_github_issue_4031.onnx $HOME/.models/ - mkdir -p $HOME/.onnx + # Create a folder for all test data + mkdir -p $HOME/.test_data docker run --rm \ - --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ - --volume /data/models:/build/models:ro \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ - --volume $HOME/.models:/home/onnxruntimedev/.models \ + --volume $HOME/.test_data:/home/onnxruntimedev/.test_data \ -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch36 \ - /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch5g \ + /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh + workingDirectory: $(Build.SourcesDirectory) + - task: CmdLine@2 + displayName: Build minimal onnxruntime and disable exception + inputs: + script: | + # We will try to build the ORT minimal with exception disabled + # Only the building process is verified here, no test will be performed + docker run --rm \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch5g \ + python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator Ninja \ --config Debug\ --skip_submodule_sync \ --build_shared_lib \ --parallel \ + --skip_tests \ --minimal_build \ - --disable_ml_ops \ - --disable_exceptions \ - --include_ops_by_model /home/onnxruntimedev/.models/ + --disable_exceptions + workingDirectory: $(Build.SourcesDirectory) + - task: CmdLine@2 + displayName: Build minimal onnxruntime and run tests, exception enabled + inputs: + script: | + docker run --rm \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume $HOME/.test_data:/home/onnxruntimedev/.test_data \ + -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch5g \ + /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh workingDirectory: $(Build.SourcesDirectory) - task: Docker@2 displayName: logout diff --git a/tools/ci_build/github/azure-pipelines/linux-dnnl-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-dnnl-ci-pipeline.yml index 941a00eff8f6b..65d300bca9f8c 100644 --- a/tools/ci_build/github/azure-pipelines/linux-dnnl-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-dnnl-ci-pipeline.yml @@ -27,7 +27,7 @@ jobs: --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch36 \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch5e \ /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build \ --config Debug Release \ diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml index 3e20d8771ec60..72cd550ed101d 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml @@ -28,8 +28,8 @@ jobs: -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimegpubuild:ch35 \ - /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentosgpubuild:ch5h \ + python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator Ninja \ --config Debug Release \ --skip_submodule_sync \ @@ -38,7 +38,7 @@ jobs: --build_wheel \ --enable_onnx_tests --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2 \ --enable_pybind --build_java --build_nodejs \ - --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so CMAKE_CUDA_ARCHITECTURES=52 + --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=52 workingDirectory: $(Build.SourcesDirectory) - task: Docker@2 displayName: logout diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-cuda-11-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-cuda-11-pipeline.yml index 6adfe01c81f5b..043f1bac766c0 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-cuda-11-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-cuda-11-pipeline.yml @@ -28,7 +28,7 @@ jobs: -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecuda11build:ch2e \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecuda11build:ch5j \ /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator Ninja \ --config Debug Release \ diff --git a/tools/ci_build/github/azure-pipelines/linux-multi-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-multi-gpu-ci-pipeline.yml index 07ca1c3b7e537..2752379f551a5 100644 --- a/tools/ci_build/github/azure-pipelines/linux-multi-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-multi-gpu-ci-pipeline.yml @@ -27,8 +27,8 @@ jobs: --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimegpubuild:ch35 \ - /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentosgpubuild:ch5h \ + python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator Ninja \ --config Debug Release \ --skip_submodule_sync \ @@ -36,8 +36,7 @@ jobs: --parallel \ --build_wheel \ --enable_onnx_tests --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2 \ - --enable_pybind --build_java --build_nodejs --enable_multi_device_test \ - --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so + --enable_pybind --build_java --build_nodejs --enable_multi_device_test workingDirectory: $(Build.SourcesDirectory) - task: Docker@2 displayName: logout diff --git a/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml index 7b14b25d74ff1..1a0efaa14dcea 100644 --- a/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml @@ -28,8 +28,8 @@ jobs: -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch36 \ - /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch5g \ + python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator Ninja \ --config Debug Release \ --skip_submodule_sync \ @@ -37,8 +37,7 @@ jobs: --parallel \ --build_wheel \ --enable_onnx_tests \ - --disable_contrib_ops --disable_ml_ops \ - --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so + --disable_contrib_ops --disable_ml_ops workingDirectory: $(Build.SourcesDirectory) - task: Docker@2 displayName: logout diff --git a/tools/ci_build/github/azure-pipelines/linux-nuphar-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-nuphar-ci-pipeline.yml index 5fb4244f3c68e..cafae402d5345 100644 --- a/tools/ci_build/github/azure-pipelines/linux-nuphar-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-nuphar-ci-pipeline.yml @@ -3,7 +3,7 @@ jobs: parameters: AgentPool : 'Linux-CPU' JobName: 'Linux_CI_Dev' - BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--enable_pybind --use_nuphar"' + BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu18.04 -d cpu -r $(Build.BinariesDirectory) -x "--enable_pybind --use_nuphar"' DoNugetPack: 'false' ArtifactName: 'drop-linux' - TimeoutInMinutes: 180 \ No newline at end of file + TimeoutInMinutes: 180 diff --git a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml index a118c2ea79f37..7f431954054ef 100644 --- a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml @@ -3,7 +3,7 @@ jobs: parameters: AgentPool : 'Linux-CPU' JobName: 'Linux_CI_Dev' - BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d openvino -v 2020.4 -r $(Build.BinariesDirectory) -x "--use_openvino CPU_FP32 --build_wheel"' + BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu18.04 -d openvino -v 2020.4 -r $(Build.BinariesDirectory) -x "--use_openvino CPU_FP32 --build_wheel"' DoNugetPack: 'false' ArtifactName: 'drop-linux' TimeoutInMinutes: 120 diff --git a/tools/ci_build/github/azure-pipelines/nodejs/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nodejs/templates/cpu.yml index f5dba4178ad43..9aa591ff27bf5 100644 --- a/tools/ci_build/github/azure-pipelines/nodejs/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nodejs/templates/cpu.yml @@ -48,8 +48,8 @@ jobs: inputs: script: | mkdir -p $HOME/.onnx && docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch36 /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 \ - /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --enable_lto --build_nodejs \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch5g /bin/bash -c "python3 \ + /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --build_nodejs \ --skip_submodule_sync --parallel --build_shared_lib --use_openmp && cd /onnxruntime_src/nodejs && npm pack" workingDirectory: $(Build.SourcesDirectory) - task: Docker@2 diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml index 6af1c7b349439..df2c9c5ecf7a0 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml @@ -40,7 +40,7 @@ jobs: inputs: script: | mkdir -p $HOME/.onnx - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch36 /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --enable_lto --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests --use_mklml && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch5g /bin/bash -c "python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests --use_mklml && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) - task: Docker@2 displayName: logout diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml index b0c59a4287e0e..9108afce13ddf 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml @@ -82,7 +82,7 @@ jobs: inputs: script: | mkdir -p $HOME/.onnx - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch36 /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --enable_lto --skip_submodule_sync --parallel --build_shared_lib --enable_onnx_tests --disable_contrib_ops --disable_ml_ops && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch5g /bin/bash -c "python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --enable_onnx_tests --disable_contrib_ops --disable_ml_ops && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) - task: Docker@2 displayName: logout diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 41cbc938c9f70..cd49b1198c149 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -105,7 +105,7 @@ jobs: inputs: script: | mkdir -p $HOME/.onnx - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch36 /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --enable_lto --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch5g /bin/bash -c "python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) - task: Docker@2 displayName: logout diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml index 755865adcc8fe..63d6538b17dbb 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml @@ -89,8 +89,8 @@ jobs: inputs: script: | mkdir -p $HOME/.onnx - docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimegpubuild:ch35 \ - /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2 --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentosgpubuild:ch5h \ + /bin/bash -c "python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2 --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" - task: Docker@2 displayName: logout inputs: diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml index 1fee4e2b9bb1e..bf4234149884f 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml @@ -28,7 +28,7 @@ jobs: -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch36 \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch5e \ /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator Ninja \ --config Debug Release \ diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-ci.yml b/tools/ci_build/github/azure-pipelines/templates/mac-ci.yml index c135fc2508188..5d722da93aa70 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-ci.yml @@ -27,8 +27,16 @@ jobs: inputs: versionSpec: '12.x' - script: | - export JAVA_HOME=$(java_home -v 11) - + set -e + pushd . + cd $(Build.SourcesDirectory)/cmake/external/protobuf + cmake ./cmake -DCMAKE_INSTALL_PREFIX=$(Build.BinariesDirectory)/protobuf -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Relwithdebinfo + make -j$(getconf _NPROCESSORS_ONLN) + make install + popd + export PATH=$(Build.BinariesDirectory)/protobuf/bin:$PATH + export ONNX_ML=1 + export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer brew install libomp diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 1571d6944115e..14ba068b344e8 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -104,7 +104,7 @@ stages: --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch36 \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch5e \ $(python.manylinux.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator Ninja \ --config Release \ @@ -182,7 +182,7 @@ stages: --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimegpubuild:ch35 \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimegpubuild:ch5f \ $(python.manylinux.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator Ninja \ --config Release \ @@ -394,8 +394,8 @@ stages: - task: CopyFiles@2 displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' inputs: - SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' - Contents: '**\dist\*.whl' + SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist' + Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - script: | @@ -516,8 +516,8 @@ stages: - task: CopyFiles@2 displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' inputs: - SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' - Contents: '**\dist\*.whl' + SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist' + Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - script: | @@ -573,15 +573,26 @@ stages: versionSpec: $(python.version) - script: | - sudo python -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' + set -e + pushd . + cd $(Build.SourcesDirectory)/cmake/external/protobuf + cmake ./cmake -DCMAKE_INSTALL_PREFIX=$(Build.BinariesDirectory)/protobuf -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Relwithdebinfo + make -j$(getconf _NPROCESSORS_ONLN) + make install + popd + export PATH=$(Build.BinariesDirectory)/protobuf/bin:$PATH + export ONNX_ML=1 + export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" + sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer - ./build.sh --config Release --skip_submodule_sync --parallel --build_wheel --use_openmp ${{ parameters.build_py_parameters }} + brew install libomp + python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --use_openmp --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --config Release --build_wheel ${{ parameters.build_py_parameters }} displayName: 'Command Line Script' - task: CopyFiles@2 displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' inputs: - SourceFolder: '$(Build.SourcesDirectory)/build/Linux/Release/dist' + SourceFolder: '$(Build.BinariesDirectory)/Release/dist' Contents: '*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' diff --git a/tools/ci_build/github/linux/docker/Dockerfile.centos b/tools/ci_build/github/linux/docker/Dockerfile.centos index 196cf6c0bd3dd..2169682d3b50e 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.centos +++ b/tools/ci_build/github/linux/docker/Dockerfile.centos @@ -1,11 +1,11 @@ -FROM mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-50f0d02-20190918214028 +FROM centos:7 -ARG PYTHON_VERSION ADD scripts /tmp/scripts -RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh -p $PYTHON_VERSION && rm -rf /tmp/scripts +RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh && /tmp/scripts/manylinux/install_deps.sh && rm -rf /tmp/scripts -ARG BUILD_UID=1000 +ARG BUILD_UID=1001 ARG BUILD_USER=onnxruntimedev -RUN adduser --uid $BUILD_UID $BUILD_USER +RUN adduser --uid $BUILD_UID $BUILD_USER WORKDIR /home/$BUILD_USER USER $BUILD_USER +ENV PATH /usr/local/gradle/bin:$PATH diff --git a/tools/ci_build/github/linux/docker/Dockerfile.centos_gpu b/tools/ci_build/github/linux/docker/Dockerfile.centos_gpu new file mode 100644 index 0000000000000..4efd489bd6e1b --- /dev/null +++ b/tools/ci_build/github/linux/docker/Dockerfile.centos_gpu @@ -0,0 +1,11 @@ +FROM nvcr.io/nvidia/cuda:10.2-cudnn8-devel-centos7 + +ADD scripts /tmp/scripts +RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh && /tmp/scripts/manylinux/install_deps.sh && rm -rf /tmp/scripts + +ARG BUILD_UID=1001 +ARG BUILD_USER=onnxruntimedev +RUN adduser --uid $BUILD_UID $BUILD_USER +WORKDIR /home/$BUILD_USER +USER $BUILD_USER +ENV PATH /usr/local/gradle/bin:$PATH diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu index a567b4f746a5f..9a6e67e749976 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu @@ -1,11 +1,11 @@ FROM quay.io/pypa/manylinux2014_x86_64:latest -ADD manylinux /tmp/scripts -RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts +ADD scripts /tmp/scripts +RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh && /tmp/scripts/manylinux/install_deps.sh && rm -rf /tmp/scripts -ARG BUILD_UID=1000 +ARG BUILD_UID=1001 ARG BUILD_USER=onnxruntimedev RUN adduser --uid $BUILD_UID $BUILD_USER WORKDIR /home/$BUILD_USER USER $BUILD_USER -ENV PATH /usr/local/gradle/bin:/usr/local/dotnet:$PATH +ENV PATH /usr/local/gradle/bin:$PATH diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda10_2 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda10_2 index 448baf0664a94..7d4b85e12574f 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda10_2 +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda10_2 @@ -24,8 +24,8 @@ ENV SSL_CERT_FILE=/opt/_internal/certs.pem #Build manylinux2014 docker image end #Add our own dependencies -ADD manylinux /tmp/scripts -RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts +ADD scripts /tmp/scripts +RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh && /tmp/scripts/manylinux/install_deps.sh && rm -rf /tmp/scripts ARG BUILD_UID=1001 ARG BUILD_USER=onnxruntimedev diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11 index 9fa573cf918b3..f26e199e70797 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11 +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11 @@ -24,8 +24,8 @@ ENV SSL_CERT_FILE=/opt/_internal/certs.pem #Build manylinux2014 docker image end #Add our own dependencies -ADD manylinux /tmp/scripts -RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts +ADD scripts /tmp/scripts +RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh && /tmp/scripts/manylinux/install_deps.sh && rm -rf /tmp/scripts ARG BUILD_UID=1001 ARG BUILD_USER=onnxruntimedev diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_gpu b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_gpu index e42e438186b9b..da5e53eaa639b 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_gpu +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_gpu @@ -1,99 +1,43 @@ -FROM quay.io/pypa/manylinux2014_x86_64:2020-04-04-c0949b6 +# TODO unify this with Dockerfile.manylinux2014_cuda10_2 + +FROM nvcr.io/nvidia/cuda:10.2-cudnn8-devel-centos7 + +#We need both CUDA and manylinux. But the CUDA Toolkit End User License Agreement says NVIDIA CUDA Driver Libraries(libcuda.so, libnvidia-ptxjitcompiler.so) are only distributable in applications that meet this criteria: +#1. The application was developed starting from a NVIDIA CUDA container obtained from Docker Hub or the NVIDIA GPU Cloud, and +#2. The resulting application is packaged as a Docker container and distributed to users on Docker Hub or the NVIDIA GPU Cloud only. +#So we use CUDA as the base image then add manylinux on top of it. + +#Build manylinux2014 docker image begin +ENV AUDITWHEEL_ARCH x86_64 +ENV AUDITWHEEL_PLAT manylinux2014_$AUDITWHEEL_ARCH +ENV LC_ALL en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US.UTF-8 +ENV DEVTOOLSET_ROOTPATH /opt/rh/devtoolset-8/root +ENV PATH $DEVTOOLSET_ROOTPATH/usr/bin:$PATH +ENV LD_LIBRARY_PATH $DEVTOOLSET_ROOTPATH/usr/lib64:$DEVTOOLSET_ROOTPATH/usr/lib:$DEVTOOLSET_ROOTPATH/usr/lib64/dyninst:$DEVTOOLSET_ROOTPATH/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib +ENV PKG_CONFIG_PATH /usr/local/lib/pkgconfig + +COPY manylinux2014_build_scripts /manylinux2014_build_scripts +RUN bash /manylinux2014_build_scripts/build.sh 8 && rm -r manylinux2014_build_scripts + +ENV SSL_CERT_FILE=/opt/_internal/certs.pem + +#Build manylinux2014 docker image end ARG PYTHON_VERSION=3.5 ARG BUILD_EXTR_PAR +#Add our own dependencies ADD scripts /tmp/scripts RUN cd /tmp/scripts && \ /tmp/scripts/install_centos.sh && \ /tmp/scripts/install_deps.sh -d gpu -p $PYTHON_VERSION -x "$BUILD_EXTR_PAR" && \ rm -rf /tmp/scripts -RUN yum remove -y devtoolset-\* && \ - yum install -y \ - devtoolset-7-binutils \ - devtoolset-7-gcc \ - devtoolset-7-gcc-c++ \ - devtoolset-7-gcc-gfortran \ - devtoolset-7-libquadmath-devel \ - devtoolset-7-libstdc++-devel \ - devtoolset-7-runtime - - -# Copied and adapted from https://gitlab.com/nvidia/container-images/cuda/-/blob/d442ff6975fb8310da90e6c3f35a988b6920b017/dist/centos7/10.1/base/Dockerfile - -RUN NVIDIA_GPGKEY_SUM=d1be581509378368edeec8c1eb2958702feedf3bc3d17011adbf24efacce4ab5 && \ -curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/7fa2af80.pub | sed '/^Version/d' > /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA && \ - echo "$NVIDIA_GPGKEY_SUM /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA" | sha256sum -c --strict - - -COPY cuda_manylinux2014.repo /etc/yum.repos.d/cuda.repo - -ENV CUDA_VERSION 10.1.243 -ENV CUDA_PKG_VERSION 10-1-$CUDA_VERSION-1 -RUN yum install -y \ -cuda-cudart-$CUDA_PKG_VERSION \ -cuda-compat-10-1 \ -cuda-libraries-$CUDA_PKG_VERSION \ -cuda-nvtx-$CUDA_PKG_VERSION \ -libcublas10-10.2.1.243-1 \ -cuda-nvml-dev-$CUDA_PKG_VERSION \ -cuda-command-line-tools-$CUDA_PKG_VERSION \ -cuda-libraries-dev-$CUDA_PKG_VERSION \ -cuda-minimal-build-$CUDA_PKG_VERSION \ -libcublas-devel-10.2.1.243-1 \ -&& \ - ln -s cuda-10.1 /usr/local/cuda && \ - rm -rf /var/cache/yum/* - -# nvidia-docker 1.0 -RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \ - echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf - -ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:/opt/rh/devtoolset-7/root/usr/bin:${PATH} -ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH} - -# nvidia-container-runtime -ENV NVIDIA_VISIBLE_DEVICES all -ENV NVIDIA_DRIVER_CAPABILITIES compute,utility -ENV NVIDIA_REQUIRE_CUDA "cuda>=10.1 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411" - - -ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs - -ENV CUDNN_VERSION 7.6.5.32 -LABEL com.nvidia.cudnn.version="${CUDNN_VERSION}" - -# cuDNN license: https://developer.nvidia.com/cudnn/license_agreement -RUN CUDNN_DOWNLOAD_SUM=7eaec8039a2c30ab0bc758d303588767693def6bf49b22485a2c00bf2e136cb3 && \ - curl -fsSL http://developer.download.nvidia.com/compute/redist/cudnn/v7.6.5/cudnn-10.1-linux-x64-v7.6.5.32.tgz -O && \ - echo "$CUDNN_DOWNLOAD_SUM cudnn-10.1-linux-x64-v7.6.5.32.tgz" | sha256sum -c - && \ - tar --no-same-owner -xzf cudnn-10.1-linux-x64-v7.6.5.32.tgz -C /usr/local && \ - rm cudnn-10.1-linux-x64-v7.6.5.32.tgz && \ - ldconfig - - -# install MPI 4.0.0 -RUN MPI_SUM=36f10daa3f1b1d37530f686bf7f70966b2a13c0bc6e2e05aebc7e85e3d21b10d && \ - curl -fsSL https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.0.tar.gz -O && \ - echo "$MPI_SUM openmpi-4.0.0.tar.gz" | sha256sum -c - && \ - tar zxf openmpi-4.0.0.tar.gz && \ - cd openmpi-4.0.0 && \ - ./configure --enable-orterun-prefix-by-default && \ - make -j $(nproc) all && \ - make install && \ - ldconfig - - -# install NCCL 2.4.8 -RUN RPM_SUM=ce8da101c05f0a52e01a569acc7e6845b3883f7e5760eefee85dccb279454d5d && \ - curl -fsSL https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm -O && \ - echo "$RPM_SUM nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm" | sha256sum -c - && \ - rpm -i nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm && \ - yum install -y libnccl-2.4.8-1+cuda10.1 libnccl-devel-2.4.8-1+cuda10.1 libnccl-static-2.4.8-1+cuda10.1 - - -ARG BUILD_UID=1000 +ARG BUILD_UID=1001 ARG BUILD_USER=onnxruntimedev -RUN adduser --comment 'onnxruntime Build User' --uid $BUILD_UID $BUILD_USER +RUN adduser --uid $BUILD_UID $BUILD_USER WORKDIR /home/$BUILD_USER USER $BUILD_USER +ENV PATH /usr/local/gradle/bin:/usr/local/dotnet:$PATH diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino index 14810d236ce04..d8660c47455db 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_openvino @@ -1,11 +1,10 @@ -ARG OS_VERSION=16.04 +ARG OS_VERSION=18.04 FROM ubuntu:${OS_VERSION} ARG PYTHON_VERSION=3.5 ARG OPENVINO_VERSION=2020.4 ADD scripts /tmp/scripts -ENV PATH="/opt/cmake/bin:${PATH}" RUN /tmp/scripts/install_ubuntu.sh -p $PYTHON_VERSION -d EdgeDevice && \ /tmp/scripts/install_deps.sh -p $PYTHON_VERSION -d EdgeDevice diff --git a/tools/ci_build/github/linux/docker/manylinux/install_centos.sh b/tools/ci_build/github/linux/docker/manylinux/install_centos.sh deleted file mode 100755 index f4319a0c532ae..0000000000000 --- a/tools/ci_build/github/linux/docker/manylinux/install_centos.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -set -e - -os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1) - - -if ! rpm -q --quiet epel-release ; then - yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$os_major_version.noarch.rpm -fi - -echo "installing for os major version : $os_major_version" -yum install -y redhat-lsb-core expat-devel libcurl-devel tar unzip curl zlib-devel make libunwind icu aria2 rsync bzip2 git bzip2-devel - -if [ "$os_major_version" == "6" ]; then - # See https://github.com/dotnet/core/blob/master/Documentation/build-and-install-rhel6-prerequisites.md - yum install -y epel-release libunwind openssl libnghttp2 libidn krb5-libs libuuid lttng-ust zlib - curl -sSL -o /tmp/1.tgz https://github.com/unicode-org/icu/releases/download/release-57-1/icu4c-57_1-RHEL6-x64.tgz - mkdir /tmp/icu - tar -zxf /tmp/1.tgz --strip=2 -C /tmp/icu - mv /tmp/icu/lib /tmp/icu/lib64 - /bin/cp -r /tmp/icu/* /usr/ - ldconfig /usr/lib64 - rm -rf /tmp/icu - rm /tmp/1.tgz - - curl -o /tmp/d.sh -sSL https://dot.net/v1/dotnet-install.sh - chmod +x /tmp/d.sh - /tmp/d.sh --install-dir /usr/local/dotnet -c 2.1 - rm /tmp/d.sh -fi -if [ "$os_major_version" == "7" ]; then - # install dotnet core dependencies - yum install -y lttng-ust openssl-libs krb5-libs libicu libuuid - # install dotnet runtimes - yum install -y https://packages.microsoft.com/config/centos/7/packages-microsoft-prod.rpm - yum install -y dotnet-sdk-2.1 -fi - -yum install -y java-1.8.0-openjdk-devel - diff --git a/tools/ci_build/github/linux/docker/manylinux/install_protobuf.sh b/tools/ci_build/github/linux/docker/manylinux/install_protobuf.sh deleted file mode 100755 index 95c457ce8704f..0000000000000 --- a/tools/ci_build/github/linux/docker/manylinux/install_protobuf.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -set -e -#Download a file from internet -function GetFile { - local uri=$1 - local path=$2 - local force=${3:-false} - local download_retries=${4:-5} - local retry_wait_time_seconds=${5:-30} - - if [[ -f $path ]]; then - if [[ $force = false ]]; then - echo "File '$path' already exists. Skipping download" - return 0 - else - rm -rf $path - fi - fi - - if [[ -f $uri ]]; then - echo "'$uri' is a file path, copying file to '$path'" - cp $uri $path - return $? - fi - - echo "Downloading $uri" - # Use aria2c if available, otherwise use curl - if command -v aria2c > /dev/null; then - aria2c -q -d $(dirname $path) -o $(basename $path) "$uri" - else - curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail - fi - - return $? -} - -GetFile https://github.com/protocolbuffers/protobuf/archive/v3.13.0.tar.gz /tmp/src/v3.13.0.tar.gz -tar -xf /tmp/src/v3.13.0.tar.gz -C /tmp/src -cd /tmp/src/protobuf-3.13.0 -cmake ./cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Relwithdebinfo -make -j$(getconf _NPROCESSORS_ONLN) -make install diff --git a/tools/ci_build/github/linux/docker/manylinux/requirements.txt b/tools/ci_build/github/linux/docker/manylinux/requirements.txt deleted file mode 100644 index 24113e3daf8cc..0000000000000 --- a/tools/ci_build/github/linux/docker/manylinux/requirements.txt +++ /dev/null @@ -1,6 +0,0 @@ -numpy==1.16.6 -mypy -pytest -onnx==1.7.0 -protobuf -sympy==1.1.1 diff --git a/tools/ci_build/github/linux/docker/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/scripts/install_deps.sh index 087d5193b7794..835dfdad58b4e 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_deps.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_deps.sh @@ -102,28 +102,19 @@ cd /tmp/src unzip gradle-6.3-bin.zip mv /tmp/src/gradle-6.3 /usr/local/gradle +if ! [ -x "$(command -v protoc)" ]; then + source ${0/%install_deps\.sh/install_protobuf\.sh} +fi -#Don't update 'wheel' to the latest version. see: https://github.com/pypa/auditwheel/issues/102 +export ONNX_ML=1 +export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" ${PYTHON_EXE} -m pip install -r ${0/%install_deps\.sh/requirements\.txt} -if [ $DEVICE_TYPE = "Normal" ]; then - ${PYTHON_EXE} -m pip install sympy==1.1.1 -elif [ $DEVICE_TYPE = "gpu" ]; then - ${PYTHON_EXE} -m pip install sympy==1.1.1 +if [ $DEVICE_TYPE = "gpu" ]; then if [[ $BUILD_EXTR_PAR = *--enable_training* ]]; then - ${PYTHON_EXE} -m pip install --upgrade --pre torch==1.6.0.dev20200610 torchvision==0.7.0.dev20200610 torchtext==0.6.0.dev20200610 -f https://download.pytorch.org/whl/nightly/cu101/torch_nightly.html - ${PYTHON_EXE} -m pip install transformers==v2.10.0 - # transformers requires sklearn - ${PYTHON_EXE} -m pip install sklearn - - if [[ $BUILD_EXTR_PAR = *--enable_training_python_frontend_e2e_tests* ]]; then - echo "install openmpi" - curl -fsSL https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.0.tar.gz -O - tar zxf openmpi-4.0.0.tar.gz - cd openmpi-4.0.0 - ./configure --enable-orterun-prefix-by-default - make all - make install - ldconfig + ${PYTHON_EXE} -m pip install -r ${0/%install_deps.sh/training\/requirements.txt} + + if [[ $BUILD_EXTR_PAR = *--enable_training_python_frontend_e2e_tests* || $BUILD_EXTR_PAR = *enable_training_pipeline_e2e_tests* ]]; then + source ${0/%install_deps.sh/install_openmpi.sh} fi fi fi diff --git a/tools/ci_build/github/linux/docker/scripts/install_openmpi.sh b/tools/ci_build/github/linux/docker/scripts/install_openmpi.sh new file mode 100755 index 0000000000000..35ae8c4aedb2a --- /dev/null +++ b/tools/ci_build/github/linux/docker/scripts/install_openmpi.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +# install Open MPI +echo "Installing Open MPI" +MPI_SHA256_SUM=dca264f420411f540a496bdd131bffd83e325fc9006286b39dd19b62d7368233 +curl -fsSL https://download.open-mpi.org/release/open-mpi/v4.0/openmpi-4.0.4.tar.gz -O +echo "$MPI_SHA256_SUM openmpi-4.0.4.tar.gz" | sha256sum -c - +tar zxf openmpi-4.0.4.tar.gz +cd openmpi-4.0.4 +./configure --enable-orterun-prefix-by-default +make -j $(nproc) all +make install +ldconfig diff --git a/tools/ci_build/github/linux/docker/scripts/install_openvino.sh b/tools/ci_build/github/linux/docker/scripts/install_openvino.sh index 2a8146dcd2f9f..838ac696b6524 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_openvino.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_openvino.sh @@ -18,10 +18,43 @@ cd $INTEL_OPENVINO_SRC_DIR git checkout tags/$OPENVINO_VERSION -b $OPENVINO_VERSION git submodule init git submodule update --recursive -chmod +x install_dependencies.sh -./install_dependencies.sh +host_cpu=$(uname -m) +sudo -E apt update +sudo -E apt-get install -y \ + build-essential \ + curl \ + wget \ + libssl-dev \ + ca-certificates \ + git \ + libboost-regex-dev \ + gcc-multilib g++-multilib \ + libgtk2.0-dev \ + pkg-config \ + unzip \ + automake \ + libtool \ + autoconf \ + libcairo2-dev \ + libpango1.0-dev \ + libglib2.0-dev \ + libgtk2.0-dev \ + libswscale-dev \ + libavcodec-dev \ + libavformat-dev \ + libgstreamer1.0-0 \ + gstreamer1.0-plugins-base \ + libusb-1.0-0-dev \ + libopenblas-dev + +if apt-cache search --names-only '^libpng12-dev'| grep -q libpng12; then + sudo -E apt-get install -y libpng12-dev +else + sudo -E apt-get install -y libpng-dev +fi + mkdir -p build cd build diff --git a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh index 6e70f424a9f2f..85f07637d9f8c 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh @@ -104,7 +104,7 @@ if [ "$SYS_LONG_BIT" = "64" ]; then aria2c -q -d /tmp/dotnet https://packages.microsoft.com/config/ubuntu/${OS_VERSION}/packages-microsoft-prod.deb dpkg -i /tmp/dotnet/packages-microsoft-prod.deb apt-get update - apt-get install -y dotnet-sdk-2.2 + apt-get install -y dotnet-sdk-2.1 rm -rf /tmp/dotnet fi diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_centos.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_centos.sh new file mode 100755 index 0000000000000..5bed6e5778e9b --- /dev/null +++ b/tools/ci_build/github/linux/docker/scripts/manylinux/install_centos.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -e + +os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1) + + +if ! rpm -q --quiet epel-release ; then + yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$os_major_version.noarch.rpm +fi + +echo "installing for os major version : $os_major_version" +yum install -y which gdb redhat-lsb-core expat-devel libcurl-devel tar unzip curl zlib-devel make libunwind icu aria2 rsync bzip2 git bzip2-devel + +if [ "$os_major_version" == "7" ]; then + # install dotnet core dependencies + yum install -y lttng-ust openssl-libs krb5-libs libicu libuuid + # install dotnet runtimes + yum install -y https://packages.microsoft.com/config/centos/7/packages-microsoft-prod.rpm + yum install -y dotnet-sdk-2.1 +fi + +yum install -y java-1.8.0-openjdk-devel + +#If the /opt/python folder exists, we assume this is the manylinux docker image +if [ ! -d "/opt/python/cp35-cp35m" ]; then + yum install -y ccache gcc gcc-c++ python3 python3-devel python3-pip +fi diff --git a/tools/ci_build/github/linux/docker/manylinux/install_deps.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh similarity index 69% rename from tools/ci_build/github/linux/docker/manylinux/install_deps.sh rename to tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh index bf604f668b6af..d7d1679ec10c1 100755 --- a/tools/ci_build/github/linux/docker/manylinux/install_deps.sh +++ b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh @@ -35,7 +35,12 @@ function GetFile { return $? } -PYTHON_EXES=("/opt/python/cp35-cp35m/bin/python3.5" "/opt/python/cp36-cp36m/bin/python3.6" "/opt/python/cp37-cp37m/bin/python3.7" "/opt/python/cp38-cp38/bin/python3.8") +if [ ! -d "/opt/python/cp35-cp35m" ]; then + PYTHON_EXES=("/usr/bin/python3") +else + PYTHON_EXES=("/opt/python/cp35-cp35m/bin/python3.5" "/opt/python/cp36-cp36m/bin/python3.6" "/opt/python/cp37-cp37m/bin/python3.7" "/opt/python/cp38-cp38/bin/python3.8") +fi + os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1) SYS_LONG_BIT=$(getconf LONG_BIT) @@ -86,35 +91,26 @@ unzip /tmp/src/gradle-6.3-bin.zip mv /tmp/src/gradle-6.3 /usr/local/gradle if ! [ -x "$(command -v protoc)" ]; then - source ${0/%install_deps\.sh/install_protobuf\.sh} + source ${0/%install_deps.sh/..\/install_protobuf.sh} fi export ONNX_ML=1 +export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" for PYTHON_EXE in "${PYTHON_EXES[@]}" do ${PYTHON_EXE} -m pip install -r ${0/%install_deps\.sh/requirements\.txt} - onnx_version="c443abd2acad2411103593600319ff81a676afbc" - onnx_tag="onnxtip" - GetFile https://github.com/onnx/onnx/archive/$onnx_version.tar.gz /tmp/src/$onnx_version.tar.gz - tar -xf /tmp/src/$onnx_version.tar.gz -C /tmp/src - cd /tmp/src/onnx-$onnx_version - if [ ! -d "third_party/pybind11/pybind11" ]; then - git clone https://github.com/pybind/pybind11.git third_party/pybind11 - fi - # We need to make the adjustment only for CentOS6 OR we substitue this only for - # ${PYTHON_EXE} where we'd need to escape slashes - # Make sure we do not hit pyhon2 as on CentOS 6 it does not work - ESCAPED_PY=$(echo "${PYTHON_EXE}" | sed 's/\//\\\//g') - sed "1,1 s/\/usr\/bin\/env python/$ESCAPED_PY/" /tmp/src/onnx-$onnx_version/tools/protoc-gen-mypy.py > /tmp/src/onnx-$onnx_version/tools/repl_protoc-gen-mypy.py - chmod a+w /tmp/src/onnx-$onnx_version/tools/protoc-gen-mypy.py - mv /tmp/src/onnx-$onnx_version/tools/repl_protoc-gen-mypy.py /tmp/src/onnx-$onnx_version/tools/protoc-gen-mypy.py - mkdir -p /data/onnx/${onnx_tag} - ${PYTHON_EXE} -m pip install . cd /tmp - ${PYTHON_EXE} -m onnx.backend.test.cmd_tools generate-data -o /data/onnx/$onnx_tag + ${PYTHON_EXE} -m onnx.backend.test.cmd_tools generate-data -o /data/onnx/onnxtip done +cd /tmp/src +GetFile 'https://sourceware.org/pub/valgrind/valgrind-3.16.1.tar.bz2' /tmp/src/valgrind-3.16.1.tar.bz2 +tar -jxvf valgrind-3.16.1.tar.bz2 +cd valgrind-3.16.1 +./configure --prefix=/usr --libdir=/usr/lib64 --enable-only64bit --enable-tls +make -j$(getconf _NPROCESSORS_ONLN) +make install cd / rm -rf /tmp/src diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt b/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt new file mode 100644 index 0000000000000..ef1eac2293f72 --- /dev/null +++ b/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt @@ -0,0 +1,9 @@ +numpy==1.16.6 +mypy +pytest +setuptools>=41.4.0 +wheel +git+http://github.com/onnx/onnx.git@b71de776d2847b6e0fed54a888a8213a7812ad12#egg=onnx +protobuf +sympy==1.1.1 +flake8 diff --git a/tools/ci_build/github/linux/docker/scripts/requirements.txt b/tools/ci_build/github/linux/docker/scripts/requirements.txt index 00350822b04cb..33fe9170ca5e2 100644 --- a/tools/ci_build/github/linux/docker/scripts/requirements.txt +++ b/tools/ci_build/github/linux/docker/scripts/requirements.txt @@ -4,5 +4,7 @@ mypy pytest setuptools>=41.4.0 wheel -onnx==1.7.0 +git+http://github.com/onnx/onnx.git@b71de776d2847b6e0fed54a888a8213a7812ad12#egg=onnx argparse +sympy==1.1.1 +flake8 diff --git a/tools/ci_build/github/linux/docker/scripts/training/requirements.txt b/tools/ci_build/github/linux/docker/scripts/training/requirements.txt new file mode 100644 index 0000000000000..e3a80e4c7d74d --- /dev/null +++ b/tools/ci_build/github/linux/docker/scripts/training/requirements.txt @@ -0,0 +1,8 @@ +--pre +-f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html +# transformers requires sklearn +sklearn +transformers==v2.10.0 +torch==1.6.0.dev20200610 +torchvision==0.7.0.dev20200610 +torchtext==0.6.0.dev20200610 diff --git a/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh b/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh new file mode 100644 index 0000000000000..118f6d4eaa568 --- /dev/null +++ b/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +# This script will run a full ORT build and use the python package built to generate ort format test files, +# and the exclude ops config file, which will be used in the build_minimal_ort_and_run_tests.sh + +set -e + +# Run a full build of ORT +# Since we need the ORT python package to generate the ORT format files and the include ops config files +# Will not run tests since those are covered by other CIs +python3 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir /build --cmake_generator Ninja \ + --config Debug \ + --skip_submodule_sync \ + --parallel \ + --build_wheel \ + --skip_tests \ + --enable_pybind + +# Install the ORT python wheel +python3 -m pip install --user /build/Debug/dist/* + +# Copy the test data to a separated folder +cp -Rf /onnxruntime_src/onnxruntime/test/testdata/ort_minimal_e2e_test_data /home/onnxruntimedev/.test_data + +# Convert all the onnx models in the $HOME/.test_data/ort_minimal_e2e_test_data to ort model +# and generate the included ops config file as $HOME/.test_data/ort_minimal_e2e_test_data/required_operators.config +python3 /onnxruntime_src/tools/python/convert_onnx_models_to_ort.py \ + /home/onnxruntimedev/.test_data/ort_minimal_e2e_test_data + +# Delete all the .onnx files, because the minimal build tests will not work on onnx files +find /home/onnxruntimedev/.test_data/ort_minimal_e2e_test_data -type f -name "*.onnx" -delete + +# Uninstall the ORT python wheel +python3 -m pip uninstall -y onnxruntime + +# Clear the build +rm -rf /build/Debug diff --git a/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh new file mode 100644 index 0000000000000..464ab04f9261e --- /dev/null +++ b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# This script will build a ORT minimal using +# 1. The included ops config file generated by build_full_ort_and_create_ort_files.sh, +# 2. The included models copied from /onnxruntime/test/testdata/ +# The build will run the unit tests for the minimal build +# Extra E2E test cases (converted by build_full_ort_and_create_ort_files.sh) will be run by onnx_test_runner + +set -e + +# Clear the previous build +rm -rf /build/Debug + +# We need copy the related test files to a separated folder since the --include_ops_by_model will search the testdata folder recursively +# and include many unnecessary ops, minimal build UT currently uses .ort format models converted from the models we copied below, +# which will be used as the input of --include_ops_by_model to have ops to be included for the minimal build UT. +mkdir -p /home/onnxruntimedev/.test_data/models_to_exclude +cp /onnxruntime_src/onnxruntime/test/testdata/ort_github_issue_4031.onnx /home/onnxruntimedev/.test_data/models_to_exclude + +# Build a minimal build with included ops and models +# then run ORT minimal UTs +python3 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir /build --cmake_generator Ninja \ + --config Debug \ + --skip_submodule_sync \ + --build_shared_lib \ + --parallel \ + --minimal_build \ + --disable_ml_ops \ + --include_ops_by_model /home/onnxruntimedev/.test_data/models_to_exclude/ \ + --include_ops_by_config /home/onnxruntimedev/.test_data/ort_minimal_e2e_test_data/required_operators.config + +# Run the e2e test cases +/build/Debug/onnx_test_runner /home/onnxruntimedev/.test_data/ort_minimal_e2e_test_data diff --git a/tools/nuget/generate_nuspec_for_native_nuget.py b/tools/nuget/generate_nuspec_for_native_nuget.py index 6fd00f3bc8e72..8f05c7ff36f60 100644 --- a/tools/nuget/generate_nuspec_for_native_nuget.py +++ b/tools/nuget/generate_nuspec_for_native_nuget.py @@ -294,7 +294,7 @@ def generate_files(list, args): runtimes + ' />') files_list.append('') - files_list.append('') if includes_winml: diff --git a/tools/python/example_operator_perf_test.py b/tools/python/example_operator_perf_test.py index 4192a616d00cc..6bcca0f8210a7 100644 --- a/tools/python/example_operator_perf_test.py +++ b/tools/python/example_operator_perf_test.py @@ -103,7 +103,7 @@ def run_test(): total_iters += iters # Adjust the output you want as needed - print(f'n={n},items={num_items},k={k},avg:{total / total_iters:.4f}') + print('n={},items={},k={},avg:{:.4f}'.format(n, num_items, k, total / total_iters)) # combine the various input parameters and create input for each test for n in batches: diff --git a/tools/python/ort_test_dir_utils.py b/tools/python/ort_test_dir_utils.py index 29ff497e97baf..890fa1cb467a3 100644 --- a/tools/python/ort_test_dir_utils.py +++ b/tools/python/ort_test_dir_utils.py @@ -16,9 +16,9 @@ def _get_numpy_type(model_info, name): if type_name == 'tensor_type': return onnx.mapping.TENSOR_TYPE_TO_NP_TYPE[i.type.tensor_type.elem_type] else: - raise ValueError(f"Type is not handled: {type_name}") + raise ValueError("Type is not handled: {}".format(type_name)) - raise ValueError(f"{name} was not found in the model info.") + raise ValueError("{} was not found in the model info.".format(name)) def _create_missing_input_data(model_inputs, name_input_map, symbolic_dim_values_map): @@ -35,7 +35,7 @@ def _create_missing_input_data(model_inputs, name_input_map, symbolic_dim_values input_type = input.type.WhichOneof('value') if input_type != 'tensor_type': - raise ValueError(f'Unsupported model. Need to handle input type of {input_type}') + raise ValueError('Unsupported model. Need to handle input type of {}'.format(input_type)) shape = input.type.tensor_type.shape dims = [] @@ -45,7 +45,7 @@ def _create_missing_input_data(model_inputs, name_input_map, symbolic_dim_values dims.append(dim.dim_value) elif dim_type == 'dim_param': if dim.dim_param not in symbolic_dim_values_map: - raise ValueError(f"Value for symbolic dim {dim.dim_param} was not provided.") + raise ValueError("Value for symbolic dim {} was not provided.".format(dim.dim_param)) dims.append(symbolic_dim_values_map[dim.dim_param]) else: @@ -107,7 +107,7 @@ def save_data(prefix, name_data_map, model_info): else: np_type = _get_numpy_type(model_info, name) tensor = numpy_helper.from_array(data.astype(np_type), name) - filename = os.path.join(test_data_dir, f"{prefix}_{idx}.pb") + filename = os.path.join(test_data_dir, "{}_{}.pb".format(prefix, idx)) with open(filename, 'wb') as f: f.write(tensor.SerializeToString()) @@ -178,21 +178,21 @@ def run_test_dir(model_or_dir): ort_models = glob.glob(os.path.join(model_dir, '*.ort')) models = onnx_models + ort_models if len(models) > 1: - raise ValueError(f"'Multiple .onnx and/or .ort files found in {model_dir}. '" - "'Please provide specific .onnx or .ort file as input.") + raise ValueError("'Multiple .onnx and/or .ort files found in {}. '" + "'Please provide specific .onnx or .ort file as input.".format(model_dir)) elif len(models) == 0: - raise ValueError(f"'No .onnx or .ort files found in {model_dir}.") + raise ValueError("'No .onnx or .ort files found in {}.".format(model_dir)) model_path = models[0] else: model_path = os.path.abspath(model_or_dir) model_dir = os.path.dirname(model_path) - print(f'Running tests in {model_dir} for {model_path}') + print('Running tests in {} for {}'.format(model_dir, model_path)) test_dirs = [d for d in glob.glob(os.path.join(model_dir, 'test*')) if os.path.isdir(d)] if not test_dirs: - raise ValueError(f"No directories with name starting with 'test' were found in {model_dir}.") + raise ValueError("No directories with name starting with 'test' were found in {}.".format(model_dir)) sess = ort.InferenceSession(model_path) @@ -222,11 +222,11 @@ def run_test_dir(model_or_dir): if expected.dtype.char in np.typecodes['AllFloat']: if not np.isclose(expected, actual, rtol=1.e-3, atol=1.e-3).all(): - print(f'Mismatch for {output_names[idx]}:\nExpected:{expected}\nGot:{actual}') + print('Mismatch for {}:\nExpected:{}\nGot:{}'.format(output_names[idx], expected, actual)) failed = True else: if not np.equal(expected, actual).all(): - print(f'Mismatch for {output_names[idx]}:\nExpected:{expected}\nGot:{actual}') + print('Mismatch for {}:\nExpected:{}\nGot:{}'.format(output_names[idx], expected, actual)) failed = True print('FAILED' if failed else 'PASS')