From 87b1fddd97e68892bfcf4ecdff1a2b031e01941e Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 27 Oct 2021 19:00:43 -0700 Subject: [PATCH] Add Linux/MacOS ARM64 support to nuget packaging pipeline (#9570) --- .../c-api-noopenmp-packaging-pipelines.yml | 2 +- .../linux-gpu-tensorrt-packaging-pipeline.yml | 2 +- .../nuget/templates/test_linux.yml | 4 +- .../nuget/templates/test_macos.yml | 3 +- ...ifacts-package-and-publish-steps-posix.yml | 16 +-- .../azure-pipelines/templates/c-api-cpu.yml | 116 +++++++++--------- .../templates/mac-packaging.yml | 52 ++++++++ .../inference/x86_64/default/cpu/Dockerfile | 20 +++ .../default/cpu/scripts/install_centos.sh | 22 ++++ .../default/cpu/scripts/install_deps.sh | 91 ++++++++++++++ .../default/cpu/scripts/requirements.txt | 4 + .../github/linux/java_copy_strip_binary.sh | 5 +- .../ci_build/github/windows/jar_packaging.ps1 | 4 +- .../nuget/generate_nuspec_for_native_nuget.py | 4 +- 14 files changed, 266 insertions(+), 79 deletions(-) create mode 100644 tools/ci_build/github/azure-pipelines/templates/mac-packaging.yml create mode 100644 tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile create mode 100755 tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh create mode 100755 tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_deps.sh create mode 100644 tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/requirements.txt diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index 5f27468132da3..f93b076004f3c 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -71,7 +71,6 @@ jobs: artifactName: 'onnxruntime-linux-x64-cuda-$(OnnxRuntimeVersion)' artifactNameNoVersionString: 'onnxruntime-linux-x64-cuda' libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)' - commitId: $(OnnxRuntimeGitCommitHash) - template: templates/component-governance-component-detection-steps.yml parameters : @@ -560,3 +559,4 @@ jobs: AgentPool : Onnxruntime-Linux-GPU ArtifactSuffix: 'GPU' NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu' + diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-packaging-pipeline.yml index e3c58d3f356f5..c97fb7acf0c8e 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-packaging-pipeline.yml @@ -58,7 +58,7 @@ jobs: artifactName: ${{ parameters.artifactName }} artifactNameNoVersionString: ${{ parameters.artifactNameNoVersionString }} libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)' - commitId: $(OnnxRuntimeGitCommitHash) + - template: templates/component-governance-component-detection-steps.yml parameters : diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml index 60b5ff71852fe..4cc3d8674260c 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml @@ -11,6 +11,8 @@ jobs: pool: ${{ parameters.AgentPool }} dependsOn: - NuGet_Packaging_${{ parameters.ArtifactSuffix }} + # For downloading Linux CustomOp TestData + - Linux_C_API_Packaging_CPU_x64 condition: succeeded() variables: - name: OnnxRuntimeBuildDirectory @@ -26,7 +28,7 @@ jobs: - task: DownloadPipelineArtifact@0 displayName: 'Download Linux CustomOp TestData' inputs: - artifactName: 'drop-linux' + artifactName: 'onnxruntime-linux-x64' targetPath: '$(Build.BinariesDirectory)/testdata' - template: get-nuget-package-version-as-variable.yml diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml index c4c6333cf5125..07e8553ea6d41 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml @@ -23,7 +23,7 @@ jobs: - task: DownloadPipelineArtifact@0 displayName: 'Download OsX CustomOp test data' inputs: - artifactName: 'drop-osx' + artifactName: 'onnxruntime-osx' targetPath: '$(Build.BinariesDirectory)/testdata' - template: get-nuget-package-version-as-variable.yml @@ -31,6 +31,7 @@ jobs: packageFolder: '$(Build.BinariesDirectory)/nuget-artifact' - script: | + mv $(Build.BinariesDirectory)/testdata/x86_64/* $(Build.BinariesDirectory)/testdata $(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \ $(Build.BinariesDirectory)/nuget-artifact \ $(NuGetPackageVersionNumber) \ diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-posix.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-posix.yml index 88516bbc9d135..602510424c1a7 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-posix.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-posix.yml @@ -3,7 +3,7 @@ parameters: - name: buildConfig type: string - default: 'RelWithDebInfo' + default: 'Release' - name: artifactName type: string @@ -17,20 +17,12 @@ parameters: type: string default: 'libonnxruntime.so' -- name: commitId - type: string - default: '' - -- name: artifactCopyScript - type: string - default: 'tools/ci_build/github/linux/copy_strip_binary.sh' - steps: - task: ShellScript@2 displayName: 'Copy build artifacts for zipping' inputs: - scriptPath: '${{parameters.artifactCopyScript}}' - args: '-r $(Build.BinariesDirectory) -a ${{parameters.artifactName}} -l ${{parameters.libraryName}} -c ${{parameters.buildConfig}} -s $(Build.SourcesDirectory) -t ${{parameters.commitId}}' + scriptPath: 'tools/ci_build/github/linux/copy_strip_binary.sh' + args: '-r $(Build.BinariesDirectory) -a ${{parameters.artifactName}} -l ${{parameters.libraryName}} -c ${{parameters.buildConfig}} -s $(Build.SourcesDirectory) -t $(Build.SourceVersion)' workingDirectory: '$(Build.BinariesDirectory)/${{parameters.buildConfig}}' - task: ArchiveFiles@2 @@ -44,5 +36,5 @@ steps: - task: PublishPipelineArtifact@1 inputs: - targetPath: '$(Build.ArtifactStagingDirectory)/${{parameters.artifactName}}.tgz' + targetPath: '$(Build.ArtifactStagingDirectory)' artifactName: '${{parameters.artifactNameNoVersionString}}' diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index 7619a31d123ef..3cbf65bc0f0f1 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -47,25 +47,37 @@ jobs: - job: Linux_C_API_Packaging_CPU_x64 workspace: clean: all - timeoutInMinutes: 60 + timeoutInMinutes: 180 + strategy: + matrix: + ARCH_x86_64: + BASE_IMAGE: centos:7 + ONNXRUNTIME_ARCH: x64 + ONNXRUNTIME_CFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all' + ONNXRUNTIME_CXXFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all' + ARCH_AARCH64: + BASE_IMAGE: arm64v8/centos:7 + ONNXRUNTIME_ARCH: aarch64 + ONNXRUNTIME_CFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all' + ONNXRUNTIME_CXXFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all' pool: 'Linux-CPU' steps: - template: set-version-number-variables-step.yml - template: get-docker-image-steps.yml parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecpubuild + Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile + Context: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu + DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=$(BASE_IMAGE)" + Repository: onnxruntimecpubuildcentos7$(ONNXRUNTIME_ARCH) - task: CmdLine@2 inputs: script: | mkdir -p $HOME/.onnx - docker run --rm -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuild /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 \ + docker run --rm -e CFLAGS="$(ONNXRUNTIME_CFLAGS)" -e CXXFLAGS="$(ONNXRUNTIME_CXXFLAGS)" --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos7$(ONNXRUNTIME_ARCH) /bin/bash -c "python3 \ /onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \ - --skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-x64" + --skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-$(ONNXRUNTIME_ARCH)" workingDirectory: $(Build.SourcesDirectory) - script: | @@ -75,17 +87,11 @@ jobs: ls -al $(Build.ArtifactStagingDirectory) displayName: 'Create Artifacts' - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline Artifact' - inputs: - artifactName: 'drop-linux' - targetPath: '$(Build.ArtifactStagingDirectory)' - - template: java-api-artifacts-package-and-publish-steps-posix.yml parameters: - arch: 'linux-x64' + arch: 'linux-$(ONNXRUNTIME_ARCH)' buildConfig: 'Release' - artifactName: 'onnxruntime-java-linux-x64' + artifactName: 'onnxruntime-java-linux-$(ONNXRUNTIME_ARCH)' version: '$(OnnxRuntimeVersion)' libraryName: 'libonnxruntime.so' nativeLibraryName: 'libonnxruntime4j_jni.so' @@ -93,14 +99,14 @@ jobs: - template: c-api-artifacts-package-and-publish-steps-posix.yml parameters: buildConfig: 'Release' - artifactName: 'onnxruntime-linux-x64-$(OnnxRuntimeVersion)' - artifactNameNoVersionString: 'onnxruntime-linux-x64' + artifactName: 'onnxruntime-linux-$(ONNXRUNTIME_ARCH)-$(OnnxRuntimeVersion)' + artifactNameNoVersionString: 'onnxruntime-linux-$(ONNXRUNTIME_ARCH)' libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)' - commitId: $(OnnxRuntimeGitCommitHash) - template: component-governance-component-detection-steps.yml parameters : condition : 'succeeded' + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 displayName: 'Clean Agent Directories' condition: always() @@ -112,7 +118,7 @@ jobs: MACOSX_DEPLOYMENT_TARGET: '10.14' pool: vmImage: 'macOS-11' - timeoutInMinutes: 120 + timeoutInMinutes: 240 steps: - template: set-version-number-variables-step.yml @@ -127,46 +133,34 @@ jobs: export PATH=$(Build.BinariesDirectory)/protobuf/bin:$PATH export ONNX_ML=1 export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" - sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' + python3 -m pip install numpy==1.16.6 wheel + python3 -m pip install $(Build.SourcesDirectory)/cmake/external/onnx sudo xcode-select --switch /Applications/Xcode_12.4.app/Contents/Developer - brew install libomp - python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_java ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config Release displayName: 'Build and Test MacOS' - - template: c-api-artifacts-package-and-publish-steps-posix.yml - parameters: - buildConfig: 'Release' - artifactName: 'onnxruntime-osx-x64-$(OnnxRuntimeVersion)' - artifactNameNoVersionString: 'onnxruntime-osx-x64' - libraryName: 'libonnxruntime.$(OnnxRuntimeVersion).dylib' - commitId: $(OnnxRuntimeGitCommitHash) + + - template: mac-packaging.yml + parameters : + AdditionalBuildFlags : ${{ parameters.AdditionalBuildFlags }} --build_java --cmake_extra_defines CMAKE_OSX_ARCHITECTURES=arm64 + MacosArch: arm64 + + - template: mac-packaging.yml + parameters : + AdditionalBuildFlags : ${{ parameters.AdditionalBuildFlags }} --cmake_extra_defines CMAKE_OSX_ARCHITECTURES="arm64;x86_64" + MacosArch: universal2 - - script: | - set -e -x - mkdir $(Build.ArtifactStagingDirectory)/testdata - cp $(Build.BinariesDirectory)/Release/libcustom_op_library.dylib $(Build.ArtifactStagingDirectory)/testdata - displayName: 'Create Artifacts' + - template: mac-packaging.yml + parameters : + AdditionalBuildFlags : ${{ parameters.AdditionalBuildFlags }} --build_java + MacosArch: x86_64 - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline Artifact' + - task: PublishPipelineArtifact@1 inputs: - artifactName: 'drop-osx' targetPath: '$(Build.ArtifactStagingDirectory)' - - - template: java-api-artifacts-package-and-publish-steps-posix.yml - parameters: - arch: 'osx-x64' - buildConfig: 'Release' - artifactName: 'onnxruntime-java-osx-x64' - version: '$(OnnxRuntimeVersion)' - libraryName: 'libonnxruntime.dylib' - nativeLibraryName: 'libonnxruntime4j_jni.dylib' + artifactName: 'onnxruntime-osx' - template: component-governance-component-detection-steps.yml parameters : condition : 'succeeded' - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() - template: android-java-api-aar.yml parameters: @@ -316,9 +310,9 @@ jobs: displayName: 'Download Pipeline Artifact - MacOS x64' inputs: buildType: 'current' - artifactName: 'drop-onnxruntime-java-osx-x64' - targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-osx-x64' - + artifactName: 'drop-onnxruntime-java-osx-x86_64' + targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-osx-x86_64' + - task: PowerShell@2 displayName: 'PowerShell Script' inputs: @@ -328,8 +322,6 @@ jobs: showWarnings: true workingDirectory: '$(Build.BinariesDirectory)\java-artifact' - - - task: CopyFiles@2 displayName: 'Copy Java Files to Artifact Staging Directory' inputs: @@ -396,7 +388,7 @@ jobs: - task: DownloadPipelineArtifact@0 displayName: 'Download osx-x64 Pipeline Artifact' inputs: - artifactName: 'onnxruntime-osx-x64' + artifactName: 'onnxruntime-osx' targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - task: DownloadPipelineArtifact@0 @@ -404,6 +396,12 @@ jobs: inputs: artifactName: 'onnxruntime-linux-x64' targetPath: '$(Build.BinariesDirectory)/nuget-artifact' + + - task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact - NuGet' + inputs: + artifactName: 'onnxruntime-linux-aarch64' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - task: DownloadPipelineArtifact@2 displayName: 'Download iOS Pipeline Artifact' @@ -507,7 +505,7 @@ jobs: PackageType: 'nuget' PackagePath: '$(Build.ArtifactStagingDirectory)' PackageName: 'Microsoft.ML.OnnxRuntime.*nupkg' - PlatformsSupported: 'win-x64,win-x86,linux-x64,osx.10.14-x64' + PlatformsSupported: 'win-x64,win-x86,linux-x64,linux-aarch64,osx.10.14-x64' VerifyNugetSigning: false - task: PublishPipelineArtifact@0 @@ -563,7 +561,6 @@ jobs: workspace: clean: all variables: - - group: Dashboard_MySQL_Secret - name: GDN_CODESIGN_TARGETDIRECTORY value: '$(Build.BinariesDirectory)/nuget-artifact/final-package' pool: 'Win-CPU-2021' @@ -573,7 +570,8 @@ jobs: - NuGet_Test_Linux_CPU - NuGet_Test_MacOS steps: - + - checkout: self + submodules: false - template: set-version-number-variables-step.yml - task: DownloadPipelineArtifact@0 diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-packaging.yml b/tools/ci_build/github/azure-pipelines/templates/mac-packaging.yml new file mode 100644 index 0000000000000..301ed83c73609 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/mac-packaging.yml @@ -0,0 +1,52 @@ +parameters: +- name: AdditionalBuildFlags + displayName: Additional build flags for build.py + type: string + default: '' + +- name: MacosArch + type: string + default: 'x86_64' + +- name: BuildJava + type: boolean + default: true + +steps: + - script: | + rm -rf $(Build.BinariesDirectory)/Release + python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --update --build ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config Release + displayName: 'Build ${{ parameters.MacosArch }}' + + - task: ShellScript@2 + displayName: 'Copy build artifacts for zipping' + inputs: + scriptPath: 'tools/ci_build/github/linux/copy_strip_binary.sh' + args: '-r $(Build.BinariesDirectory) -a onnxruntime-osx-${{ parameters.MacosArch }}-$(OnnxRuntimeVersion) -l libonnxruntime.$(OnnxRuntimeVersion).dylib -c Release -s $(Build.SourcesDirectory) -t $(Build.SourceVersion)' + workingDirectory: '$(Build.BinariesDirectory)/Release' + + - task: ArchiveFiles@2 + inputs: + rootFolderOrFile: '$(Build.BinariesDirectory)/onnxruntime-osx-${{ parameters.MacosArch }}-$(OnnxRuntimeVersion)' + includeRootFolder: true + archiveType: 'tar' # Options: zip, 7z, tar, wim + tarCompression: 'gz' + archiveFile: '$(Build.ArtifactStagingDirectory)/onnxruntime-osx-${{ parameters.MacosArch }}-$(OnnxRuntimeVersion).tgz' + replaceExistingArchive: true + + - script: | + set -e -x + mkdir -p $(Build.ArtifactStagingDirectory)/testdata + cp $(Build.BinariesDirectory)/Release/libcustom_op_library.dylib $(Build.ArtifactStagingDirectory)/testdata + displayName: 'Copy libcustom_op_library.dylib to ArtifactStagingDirectory' + condition: and(succeeded(), eq('${{ parameters.MacosArch }}', 'x86_64')) + + - ${{ if and(ne(parameters.MacosArch, 'universal2'), eq(parameters.buildJava, true)) }}: + - template: java-api-artifacts-package-and-publish-steps-posix.yml + parameters: + arch: 'osx-${{ parameters.MacosArch }}' + buildConfig: 'Release' + artifactName: 'onnxruntime-java-osx-${{ parameters.MacosArch }}' + version: '$(OnnxRuntimeVersion)' + libraryName: 'libonnxruntime.dylib' + nativeLibraryName: 'libonnxruntime4j_jni.dylib' \ No newline at end of file diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile new file mode 100644 index 0000000000000..a8b0762af4788 --- /dev/null +++ b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile @@ -0,0 +1,20 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# This file is used by Zip-Nuget Packaging NoContribOps Pipeline,Zip-Nuget-Java Packaging Pipeline +ARG BASEIMAGE=centos:7 +FROM $BASEIMAGE + +ENV PATH /usr/local/gradle/bin:/opt/rh/devtoolset-10/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +ENV LANG=en_US.utf8 +ENV LC_ALL=en_US.utf8 + +ADD scripts /tmp/scripts +RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts + +ARG BUILD_UID=1001 +ARG BUILD_USER=onnxruntimedev +RUN adduser --uid $BUILD_UID $BUILD_USER +WORKDIR /home/$BUILD_USER +USER $BUILD_USER + diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh new file mode 100755 index 0000000000000..3847d86b78100 --- /dev/null +++ b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_centos.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -e -x + +os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1) +CPU_ARCH=`uname -m` +PACKAGES_TO_INSTALL="which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind aria2 bzip2 bzip2-devel java-11-openjdk-devel graphviz devtoolset-10-binutils devtoolset-10-gcc devtoolset-10-gcc-c++ devtoolset-10-gcc-gfortran python3 python3-devel python3-pip" +if ! rpm -q --quiet epel-release ; then + if [[ "$CPU_ARCH" = "x86_64" ]]; then + yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$os_major_version.noarch.rpm https://repo.ius.io/ius-release-el$os_major_version.rpm + PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL git224" + else + yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$os_major_version.noarch.rpm + PACKAGES_TO_INSTALL="$PACKAGES_TO_INSTALL git" + fi +fi + +echo "installing for os major version : $os_major_version" +yum install -y centos-release-scl-rh +yum install -y $PACKAGES_TO_INSTALL + +pip3 install --upgrade pip +localedef -i en_US -f UTF-8 en_US.UTF-8 \ No newline at end of file diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_deps.sh new file mode 100755 index 0000000000000..9a724e871164e --- /dev/null +++ b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/install_deps.sh @@ -0,0 +1,91 @@ +#!/bin/bash +set -e -x + +# Download a file from internet +function GetFile { + local uri=$1 + local path=$2 + local force=${3:-false} + local download_retries=${4:-5} + local retry_wait_time_seconds=${5:-30} + + if [[ -f $path ]]; then + if [[ $force = false ]]; then + echo "File '$path' already exists. Skipping download" + return 0 + else + rm -rf $path + fi + fi + + if [[ -f $uri ]]; then + echo "'$uri' is a file path, copying file to '$path'" + cp $uri $path + return $? + fi + + echo "Downloading $uri" + # Use aria2c if available, otherwise use curl + if command -v aria2c > /dev/null; then + aria2c -q -d $(dirname $path) -o $(basename $path) "$uri" + else + curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail + fi + + return $? +} + + +os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1) + +SYS_LONG_BIT=$(getconf LONG_BIT) +mkdir -p /tmp/src +GLIBC_VERSION=$(getconf GNU_LIBC_VERSION | cut -f 2 -d \.) + +DISTRIBUTOR=$(lsb_release -i -s) + +if [[ "$DISTRIBUTOR" = "CentOS" && $SYS_LONG_BIT = "64" ]]; then + LIBDIR="lib64" +else + LIBDIR="lib" +fi + +cd /tmp/src + +echo "Installing cmake" +GetFile https://github.com/Kitware/CMake/releases/download/v3.21.1/cmake-3.21.1-linux-`uname -m`.tar.gz /tmp/src/cmake-3.21.1-linux-`uname -m`.tar.gz +tar -zxf /tmp/src/cmake-3.21.1-linux-`uname -m`.tar.gz --strip=1 -C /usr + +echo "Installing Ninja" +GetFile https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz /tmp/src/ninja-linux.tar.gz +tar -zxf ninja-linux.tar.gz +cd ninja-1.10.0 +cmake -Bbuild-cmake -H. +cmake --build build-cmake +mv ./build-cmake/ninja /usr/bin +echo "Installing Node.js" +GetFile https://nodejs.org/dist/v12.16.3/node-v12.16.3-linux-x64.tar.gz /tmp/src/node-v12.16.3-linux-x64.tar.gz +tar --strip 1 -xf /tmp/src/node-v12.16.3-linux-x64.tar.gz -C /usr + +cd /tmp/src +GetFile https://downloads.gradle-dn.com/distributions/gradle-6.3-bin.zip /tmp/src/gradle-6.3-bin.zip +unzip /tmp/src/gradle-6.3-bin.zip +mv /tmp/src/gradle-6.3 /usr/local/gradle + +if ! [ -x "$(command -v protoc)" ]; then + GetFile https://github.com/protocolbuffers/protobuf/archive/v3.16.0.tar.gz /tmp/src/v3.16.0.tar.gz + tar -xf /tmp/src/v3.16.0.tar.gz -C /tmp/src + cd /tmp/src/protobuf-3.16.0 + cmake ./cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Relwithdebinfo -DCMAKE_INSTALL_LIBDIR=lib64 + make -j$(getconf _NPROCESSORS_ONLN) + make install +fi + +export ONNX_ML=1 +export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" + +pip3 install -r ${0/%install_deps\.sh/requirements\.txt} + + +cd / +rm -rf /tmp/src diff --git a/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/requirements.txt b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/requirements.txt new file mode 100644 index 0000000000000..14abf7c6be8fd --- /dev/null +++ b/tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/scripts/requirements.txt @@ -0,0 +1,4 @@ +numpy +git+http://github.com/onnx/onnx.git@1f63dcb7fcc3a8bf5c3c8e326867ecd6f5c43f35#egg=onnx +protobuf +flatbuffers diff --git a/tools/ci_build/github/linux/java_copy_strip_binary.sh b/tools/ci_build/github/linux/java_copy_strip_binary.sh index be530ca5e045f..bfc9fd886f128 100755 --- a/tools/ci_build/github/linux/java_copy_strip_binary.sh +++ b/tools/ci_build/github/linux/java_copy_strip_binary.sh @@ -9,7 +9,7 @@ a) ARTIFACT_NAME=${OPTARG};; c) BUILD_CONFIG=${OPTARG};; l) LIB_NAME=${OPTARG};; n) NATIVE_LIB_NAME=${OPTARG};; -h) ARCH=${OPTARG};; +h) ARCH=${OPTARG};; #must match the JAVA_OS_ARCH variable in onnxruntime_java.cmake v) VERSION_NUMBER=${OPTARG};; esac done @@ -19,6 +19,9 @@ EXIT_CODE=1 uname -a echo "Version: $VERSION_NUMBER" +if [[ $LIB_NAME == *.dylib ]] && [[ $ARCH == 'osx-x86_64' ]]; then + ARCH='osx-x64' +fi NATIVE_FOLDER=ai/onnxruntime/native/$ARCH mkdir -p $BINARY_DIR/$ARTIFACT_NAME/$NATIVE_FOLDER diff --git a/tools/ci_build/github/windows/jar_packaging.ps1 b/tools/ci_build/github/windows/jar_packaging.ps1 index ae86c5f95c067..793443dd3a09e 100644 --- a/tools/ci_build/github/windows/jar_packaging.ps1 +++ b/tools/ci_build/github/windows/jar_packaging.ps1 @@ -10,8 +10,8 @@ Write-Output "Run 7z" Remove-Item -Path libcustom_op_library.so 7z a $Env:BUILD_BINARIESDIRECTORY\java-artifact\onnxruntime-java-win-x64\onnxruntime-$Env:ONNXRUNTIMEVERSION.jar . popd -pushd onnxruntime-java-osx-x64 +pushd onnxruntime-java-osx-x86_64 7z a $Env:BUILD_BINARIESDIRECTORY\java-artifact\onnxruntime-java-win-x64\testing.jar libcustom_op_library.dylib Remove-Item -Path libcustom_op_library.dylib 7z a $Env:BUILD_BINARIESDIRECTORY\java-artifact\onnxruntime-java-win-x64\onnxruntime-$Env:ONNXRUNTIMEVERSION.jar . -popd \ No newline at end of file +popd diff --git a/tools/nuget/generate_nuspec_for_native_nuget.py b/tools/nuget/generate_nuspec_for_native_nuget.py index 37b21b05e6e41..24bfe140582ec 100644 --- a/tools/nuget/generate_nuspec_for_native_nuget.py +++ b/tools/nuget/generate_nuspec_for_native_nuget.py @@ -54,9 +54,11 @@ def generate_file_list_for_ep(nuget_artifacts_dir, ep, files_list): if child_file.suffix in ['.dll', '.pdb', '.lib'] and is_this_file_needed(ep, child_file.name): files_list.append('' % cpu_arch) - for cpu_arch in ['x64', 'arm64']: + for cpu_arch in ['x86_64', 'arm64']: if child.name == get_package_name('osx', cpu_arch, ep): child = child / 'lib' + if cpu_arch == 'x86_64': + cpu_arch = 'x64' for child_file in child.iterdir(): # Check if the file has digits like onnxruntime.1.8.0.dylib. We can skip such things is_versioned_dylib = re.match(r'.*[\.\d+]+\.dylib$', child_file.name)