Skip to content

Commit

Permalink
Add Linux/MacOS ARM64 support to nuget packaging pipeline (#9570)
Browse files Browse the repository at this point in the history
  • Loading branch information
snnn authored Oct 28, 2021
1 parent 2d44bd5 commit 87b1fdd
Show file tree
Hide file tree
Showing 14 changed files with 266 additions and 79 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,6 @@ jobs:
artifactName: 'onnxruntime-linux-x64-cuda-$(OnnxRuntimeVersion)'
artifactNameNoVersionString: 'onnxruntime-linux-x64-cuda'
libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'
commitId: $(OnnxRuntimeGitCommitHash)

- template: templates/component-governance-component-detection-steps.yml
parameters :
Expand Down Expand Up @@ -560,3 +559,4 @@ jobs:
AgentPool : Onnxruntime-Linux-GPU
ArtifactSuffix: 'GPU'
NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu'

Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ jobs:
artifactName: ${{ parameters.artifactName }}
artifactNameNoVersionString: ${{ parameters.artifactNameNoVersionString }}
libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'
commitId: $(OnnxRuntimeGitCommitHash)


- template: templates/component-governance-component-detection-steps.yml
parameters :
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ jobs:
pool: ${{ parameters.AgentPool }}
dependsOn:
- NuGet_Packaging_${{ parameters.ArtifactSuffix }}
# For downloading Linux CustomOp TestData
- Linux_C_API_Packaging_CPU_x64
condition: succeeded()
variables:
- name: OnnxRuntimeBuildDirectory
Expand All @@ -26,7 +28,7 @@ jobs:
- task: DownloadPipelineArtifact@0
displayName: 'Download Linux CustomOp TestData'
inputs:
artifactName: 'drop-linux'
artifactName: 'onnxruntime-linux-x64'
targetPath: '$(Build.BinariesDirectory)/testdata'

- template: get-nuget-package-version-as-variable.yml
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,14 +23,15 @@ jobs:
- task: DownloadPipelineArtifact@0
displayName: 'Download OsX CustomOp test data'
inputs:
artifactName: 'drop-osx'
artifactName: 'onnxruntime-osx'
targetPath: '$(Build.BinariesDirectory)/testdata'

- template: get-nuget-package-version-as-variable.yml
parameters:
packageFolder: '$(Build.BinariesDirectory)/nuget-artifact'

- script: |
mv $(Build.BinariesDirectory)/testdata/x86_64/* $(Build.BinariesDirectory)/testdata
$(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \
$(Build.BinariesDirectory)/nuget-artifact \
$(NuGetPackageVersionNumber) \
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
parameters:
- name: buildConfig
type: string
default: 'RelWithDebInfo'
default: 'Release'

- name: artifactName
type: string
Expand All @@ -17,20 +17,12 @@ parameters:
type: string
default: 'libonnxruntime.so'

- name: commitId
type: string
default: ''

- name: artifactCopyScript
type: string
default: 'tools/ci_build/github/linux/copy_strip_binary.sh'

steps:
- task: ShellScript@2
displayName: 'Copy build artifacts for zipping'
inputs:
scriptPath: '${{parameters.artifactCopyScript}}'
args: '-r $(Build.BinariesDirectory) -a ${{parameters.artifactName}} -l ${{parameters.libraryName}} -c ${{parameters.buildConfig}} -s $(Build.SourcesDirectory) -t ${{parameters.commitId}}'
scriptPath: 'tools/ci_build/github/linux/copy_strip_binary.sh'
args: '-r $(Build.BinariesDirectory) -a ${{parameters.artifactName}} -l ${{parameters.libraryName}} -c ${{parameters.buildConfig}} -s $(Build.SourcesDirectory) -t $(Build.SourceVersion)'
workingDirectory: '$(Build.BinariesDirectory)/${{parameters.buildConfig}}'

- task: ArchiveFiles@2
Expand All @@ -44,5 +36,5 @@ steps:

- task: PublishPipelineArtifact@1
inputs:
targetPath: '$(Build.ArtifactStagingDirectory)/${{parameters.artifactName}}.tgz'
targetPath: '$(Build.ArtifactStagingDirectory)'
artifactName: '${{parameters.artifactNameNoVersionString}}'
116 changes: 57 additions & 59 deletions tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml
Original file line number Diff line number Diff line change
Expand Up @@ -47,25 +47,37 @@ jobs:
- job: Linux_C_API_Packaging_CPU_x64
workspace:
clean: all
timeoutInMinutes: 60
timeoutInMinutes: 180
strategy:
matrix:
ARCH_x86_64:
BASE_IMAGE: centos:7
ONNXRUNTIME_ARCH: x64
ONNXRUNTIME_CFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all'
ONNXRUNTIME_CXXFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all'
ARCH_AARCH64:
BASE_IMAGE: arm64v8/centos:7
ONNXRUNTIME_ARCH: aarch64
ONNXRUNTIME_CFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all'
ONNXRUNTIME_CXXFLAGS: '-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -O3 -Wl,--strip-all'
pool: 'Linux-CPU'
steps:
- template: set-version-number-variables-step.yml
- template: get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
Repository: onnxruntimecpubuild
Dockerfile: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu/Dockerfile
Context: tools/ci_build/github/linux/docker/inference/x86_64/default/cpu
DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=$(BASE_IMAGE)"
Repository: onnxruntimecpubuildcentos7$(ONNXRUNTIME_ARCH)

- task: CmdLine@2
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --rm -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuild /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 \
docker run --rm -e CFLAGS="$(ONNXRUNTIME_CFLAGS)" -e CXXFLAGS="$(ONNXRUNTIME_CXXFLAGS)" --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos7$(ONNXRUNTIME_ARCH) /bin/bash -c "python3 \
/onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \
--skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-x64"
--skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-$(ONNXRUNTIME_ARCH)"
workingDirectory: $(Build.SourcesDirectory)

- script: |
Expand All @@ -75,32 +87,26 @@ jobs:
ls -al $(Build.ArtifactStagingDirectory)
displayName: 'Create Artifacts'
- task: PublishPipelineArtifact@0
displayName: 'Publish Pipeline Artifact'
inputs:
artifactName: 'drop-linux'
targetPath: '$(Build.ArtifactStagingDirectory)'

- template: java-api-artifacts-package-and-publish-steps-posix.yml
parameters:
arch: 'linux-x64'
arch: 'linux-$(ONNXRUNTIME_ARCH)'
buildConfig: 'Release'
artifactName: 'onnxruntime-java-linux-x64'
artifactName: 'onnxruntime-java-linux-$(ONNXRUNTIME_ARCH)'
version: '$(OnnxRuntimeVersion)'
libraryName: 'libonnxruntime.so'
nativeLibraryName: 'libonnxruntime4j_jni.so'

- template: c-api-artifacts-package-and-publish-steps-posix.yml
parameters:
buildConfig: 'Release'
artifactName: 'onnxruntime-linux-x64-$(OnnxRuntimeVersion)'
artifactNameNoVersionString: 'onnxruntime-linux-x64'
artifactName: 'onnxruntime-linux-$(ONNXRUNTIME_ARCH)-$(OnnxRuntimeVersion)'
artifactNameNoVersionString: 'onnxruntime-linux-$(ONNXRUNTIME_ARCH)'
libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'
commitId: $(OnnxRuntimeGitCommitHash)

- template: component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'

- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
Expand All @@ -112,7 +118,7 @@ jobs:
MACOSX_DEPLOYMENT_TARGET: '10.14'
pool:
vmImage: 'macOS-11'
timeoutInMinutes: 120
timeoutInMinutes: 240
steps:
- template: set-version-number-variables-step.yml

Expand All @@ -127,46 +133,34 @@ jobs:
export PATH=$(Build.BinariesDirectory)/protobuf/bin:$PATH
export ONNX_ML=1
export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF"
sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt'
python3 -m pip install numpy==1.16.6 wheel
python3 -m pip install $(Build.SourcesDirectory)/cmake/external/onnx
sudo xcode-select --switch /Applications/Xcode_12.4.app/Contents/Developer
brew install libomp
python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_java ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config Release
displayName: 'Build and Test MacOS'
- template: c-api-artifacts-package-and-publish-steps-posix.yml
parameters:
buildConfig: 'Release'
artifactName: 'onnxruntime-osx-x64-$(OnnxRuntimeVersion)'
artifactNameNoVersionString: 'onnxruntime-osx-x64'
libraryName: 'libonnxruntime.$(OnnxRuntimeVersion).dylib'
commitId: $(OnnxRuntimeGitCommitHash)
- template: mac-packaging.yml
parameters :
AdditionalBuildFlags : ${{ parameters.AdditionalBuildFlags }} --build_java --cmake_extra_defines CMAKE_OSX_ARCHITECTURES=arm64
MacosArch: arm64

- template: mac-packaging.yml
parameters :
AdditionalBuildFlags : ${{ parameters.AdditionalBuildFlags }} --cmake_extra_defines CMAKE_OSX_ARCHITECTURES="arm64;x86_64"
MacosArch: universal2

- script: |
set -e -x
mkdir $(Build.ArtifactStagingDirectory)/testdata
cp $(Build.BinariesDirectory)/Release/libcustom_op_library.dylib $(Build.ArtifactStagingDirectory)/testdata
displayName: 'Create Artifacts'
- template: mac-packaging.yml
parameters :
AdditionalBuildFlags : ${{ parameters.AdditionalBuildFlags }} --build_java
MacosArch: x86_64

- task: PublishPipelineArtifact@0
displayName: 'Publish Pipeline Artifact'
- task: PublishPipelineArtifact@1
inputs:
artifactName: 'drop-osx'
targetPath: '$(Build.ArtifactStagingDirectory)'

- template: java-api-artifacts-package-and-publish-steps-posix.yml
parameters:
arch: 'osx-x64'
buildConfig: 'Release'
artifactName: 'onnxruntime-java-osx-x64'
version: '$(OnnxRuntimeVersion)'
libraryName: 'libonnxruntime.dylib'
nativeLibraryName: 'libonnxruntime4j_jni.dylib'
artifactName: 'onnxruntime-osx'

- template: component-governance-component-detection-steps.yml
parameters :
condition : 'succeeded'
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()

- template: android-java-api-aar.yml
parameters:
Expand Down Expand Up @@ -316,9 +310,9 @@ jobs:
displayName: 'Download Pipeline Artifact - MacOS x64'
inputs:
buildType: 'current'
artifactName: 'drop-onnxruntime-java-osx-x64'
targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-osx-x64'

artifactName: 'drop-onnxruntime-java-osx-x86_64'
targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-osx-x86_64'
- task: PowerShell@2
displayName: 'PowerShell Script'
inputs:
Expand All @@ -328,8 +322,6 @@ jobs:
showWarnings: true
workingDirectory: '$(Build.BinariesDirectory)\java-artifact'



- task: CopyFiles@2
displayName: 'Copy Java Files to Artifact Staging Directory'
inputs:
Expand Down Expand Up @@ -396,14 +388,20 @@ jobs:
- task: DownloadPipelineArtifact@0
displayName: 'Download osx-x64 Pipeline Artifact'
inputs:
artifactName: 'onnxruntime-osx-x64'
artifactName: 'onnxruntime-osx'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact'

- task: DownloadPipelineArtifact@0
displayName: 'Download linux-x64 Pipeline Artifact'
inputs:
artifactName: 'onnxruntime-linux-x64'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact'

- task: DownloadPipelineArtifact@0
displayName: 'Download Pipeline Artifact - NuGet'
inputs:
artifactName: 'onnxruntime-linux-aarch64'
targetPath: '$(Build.BinariesDirectory)/nuget-artifact'

- task: DownloadPipelineArtifact@2
displayName: 'Download iOS Pipeline Artifact'
Expand Down Expand Up @@ -507,7 +505,7 @@ jobs:
PackageType: 'nuget'
PackagePath: '$(Build.ArtifactStagingDirectory)'
PackageName: 'Microsoft.ML.OnnxRuntime.*nupkg'
PlatformsSupported: 'win-x64,win-x86,linux-x64,osx.10.14-x64'
PlatformsSupported: 'win-x64,win-x86,linux-x64,linux-aarch64,osx.10.14-x64'
VerifyNugetSigning: false

- task: PublishPipelineArtifact@0
Expand Down Expand Up @@ -563,7 +561,6 @@ jobs:
workspace:
clean: all
variables:
- group: Dashboard_MySQL_Secret
- name: GDN_CODESIGN_TARGETDIRECTORY
value: '$(Build.BinariesDirectory)/nuget-artifact/final-package'
pool: 'Win-CPU-2021'
Expand All @@ -573,7 +570,8 @@ jobs:
- NuGet_Test_Linux_CPU
- NuGet_Test_MacOS
steps:

- checkout: self
submodules: false
- template: set-version-number-variables-step.yml

- task: DownloadPipelineArtifact@0
Expand Down
52 changes: 52 additions & 0 deletions tools/ci_build/github/azure-pipelines/templates/mac-packaging.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
parameters:
- name: AdditionalBuildFlags
displayName: Additional build flags for build.py
type: string
default: ''

- name: MacosArch
type: string
default: 'x86_64'

- name: BuildJava
type: boolean
default: true

steps:
- script: |
rm -rf $(Build.BinariesDirectory)/Release
python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --update --build ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config Release
displayName: 'Build ${{ parameters.MacosArch }}'
- task: ShellScript@2
displayName: 'Copy build artifacts for zipping'
inputs:
scriptPath: 'tools/ci_build/github/linux/copy_strip_binary.sh'
args: '-r $(Build.BinariesDirectory) -a onnxruntime-osx-${{ parameters.MacosArch }}-$(OnnxRuntimeVersion) -l libonnxruntime.$(OnnxRuntimeVersion).dylib -c Release -s $(Build.SourcesDirectory) -t $(Build.SourceVersion)'
workingDirectory: '$(Build.BinariesDirectory)/Release'

- task: ArchiveFiles@2
inputs:
rootFolderOrFile: '$(Build.BinariesDirectory)/onnxruntime-osx-${{ parameters.MacosArch }}-$(OnnxRuntimeVersion)'
includeRootFolder: true
archiveType: 'tar' # Options: zip, 7z, tar, wim
tarCompression: 'gz'
archiveFile: '$(Build.ArtifactStagingDirectory)/onnxruntime-osx-${{ parameters.MacosArch }}-$(OnnxRuntimeVersion).tgz'
replaceExistingArchive: true

- script: |
set -e -x
mkdir -p $(Build.ArtifactStagingDirectory)/testdata
cp $(Build.BinariesDirectory)/Release/libcustom_op_library.dylib $(Build.ArtifactStagingDirectory)/testdata
displayName: 'Copy libcustom_op_library.dylib to ArtifactStagingDirectory'
condition: and(succeeded(), eq('${{ parameters.MacosArch }}', 'x86_64'))
- ${{ if and(ne(parameters.MacosArch, 'universal2'), eq(parameters.buildJava, true)) }}:
- template: java-api-artifacts-package-and-publish-steps-posix.yml
parameters:
arch: 'osx-${{ parameters.MacosArch }}'
buildConfig: 'Release'
artifactName: 'onnxruntime-java-osx-${{ parameters.MacosArch }}'
version: '$(OnnxRuntimeVersion)'
libraryName: 'libonnxruntime.dylib'
nativeLibraryName: 'libonnxruntime4j_jni.dylib'
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.

# This file is used by Zip-Nuget Packaging NoContribOps Pipeline,Zip-Nuget-Java Packaging Pipeline
ARG BASEIMAGE=centos:7
FROM $BASEIMAGE

ENV PATH /usr/local/gradle/bin:/opt/rh/devtoolset-10/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
ENV LANG=en_US.utf8
ENV LC_ALL=en_US.utf8

ADD scripts /tmp/scripts
RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts

ARG BUILD_UID=1001
ARG BUILD_USER=onnxruntimedev
RUN adduser --uid $BUILD_UID $BUILD_USER
WORKDIR /home/$BUILD_USER
USER $BUILD_USER

Loading

0 comments on commit 87b1fdd

Please sign in to comment.