Update Aten pipeline's docker file to use UBI8 (#20856)

### Description Now it uses CentOS 7 which is EOL. This PR updates it to UBI8. ### Motivation and Context To deprecate CentOS 7 .
microsoft · May 30, 2024 · 65ef270 · 65ef270
1 parent 59b13b7
commit 65ef270
Show file tree

Hide file tree

Showing 8 changed files with 55 additions and 50 deletions.
diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt
@@ -652,6 +652,12 @@ else()
   check_cxx_compiler_flag(-Wunused-variable HAS_UNUSED_VARIABLE)
   check_cxx_compiler_flag(-Wuseless-cast HAS_USELESS_CAST)
   check_cxx_compiler_flag(-Wstringop-overflow HAS_STRINGOP_OVERFLOW)
+  if(onnxruntime_ENABLE_TRAINING_APIS)
+	check_cxx_compiler_flag(-Wdangling-reference HAS_DANGLING_REFERENCE)
+	if(HAS_DANGLING_REFERENCE)
+	  list(APPEND ORT_WARNING_FLAGS -Wno-dangling-reference)
+	endif()
+  endif()
   check_function_exists(reallocarray HAS_REALLOCARRAY)
   if (NOT APPLE AND NOT CMAKE_SYSTEM_NAME STREQUAL "Emscripten" AND onnxruntime_target_platform STREQUAL "aarch64")
    check_cxx_compiler_flag(-march=armv8.2-a+bf16 HAS_ARM64_BFLOAT16)

diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml
@@ -27,14 +27,6 @@ pr:
     - 'onnxruntime/core/providers/js'
 #### end trigger ####
 
-resources:
-  repositories:
-  - repository: manylinux
-    type: Github
-    endpoint: Microsoft
-    name: pypa/manylinux
-    ref: 5eda9aded5462201e6310105728d33016e637ea7
-
 jobs:
 - job: Linux_Build
   timeoutInMinutes: 120
@@ -55,7 +47,7 @@ jobs:
 
   - template: templates/get-docker-image-steps.yml
     parameters:
-      Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_aten_cpu
+      Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.aten_cpu
       Context: tools/ci_build/github/linux/docker
       DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
       Repository: 'onnxruntimecpubuildaten'

diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-eager-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-eager-pipeline.yml
@@ -27,14 +27,6 @@ pr:
     - 'onnxruntime/core/providers/js'
 #### end trigger ####
 
-resources:
-  repositories:
-  - repository: manylinux
-    type: Github
-    endpoint: Microsoft
-    name: pypa/manylinux
-    ref: 5eda9aded5462201e6310105728d33016e637ea7
-
 jobs:
 # This pipeline builds the latest PyTorch commit from source
 # and use it in ORT tests. See Dockerfile.manylinux2014_lort_cpu
@@ -59,7 +51,7 @@ jobs:
 
   - template: templates/get-docker-image-steps.yml
     parameters:
-      Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_lort_cpu
+      Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.lort_cpu
       Context: tools/ci_build/github/linux/docker
       DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
       Repository: onnxruntimecpubuildlort
@@ -75,7 +67,7 @@ jobs:
           -e NIGHTLY_BUILD \
           -e BUILD_BUILDNUMBER \
           onnxruntimecpubuildlort \
-            /opt/python/cp39-cp39/bin/python3.9 /onnxruntime_src/tools/ci_build/build.py \
+            python3 /onnxruntime_src/tools/ci_build/build.py \
               --build_dir /build --cmake_generator Ninja \
               --config Release \
               --skip_submodule_sync \
@@ -97,9 +89,9 @@ jobs:
           onnxruntimecpubuildlort \
             bash -c "
               export PYTHONPATH=/build/Release && \
-              /opt/python/cp39-cp39/bin/python3.9 -m pip install /build/Release/dist/*.whl && \
-              /opt/python/cp39-cp39/bin/python3.9 /onnxruntime_src/orttraining/orttraining/test/python/orttraining_test_dort.py && \
-              cd /build/Release && /opt/python/cp39-cp39/bin/python3.9 /onnxruntime_src/orttraining/orttraining/test/python/orttraining_test_dort_custom_ops.py"
+              python3 -m pip install /build/Release/dist/*.whl && \
+              python3 /onnxruntime_src/orttraining/orttraining/test/python/orttraining_test_dort.py && \
+              cd /build/Release && python3 /onnxruntime_src/orttraining/orttraining/test/python/orttraining_test_dort_custom_ops.py"
       workingDirectory: $(Build.SourcesDirectory)
       condition: succeededOrFailed()
 

diff --git a/.../docker/Dockerfile.manylinux2014_aten_cpu → ...d/github/linux/docker/Dockerfile.aten_cpu b/.../docker/Dockerfile.manylinux2014_aten_cpu → ...d/github/linux/docker/Dockerfile.aten_cpu
@@ -1,4 +1,4 @@
-FROM quay.io/pypa/manylinux2014_x86_64:latest
+FROM onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cpu_x64_ubi8_gcc12:20240523.1
 
 ADD scripts /tmp/scripts
 RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh && /tmp/scripts/manylinux/install_deps_aten.sh && rm -rf /tmp/scripts
@@ -7,4 +7,4 @@ ARG BUILD_UID=1001
 ARG BUILD_USER=onnxruntimedev
 RUN adduser --uid $BUILD_UID $BUILD_USER
 WORKDIR /home/$BUILD_USER
-USER $BUILD_USER
+USER $BUILD_USER
diff --git a/.../docker/Dockerfile.manylinux2014_lort_cpu → ...d/github/linux/docker/Dockerfile.lort_cpu b/.../docker/Dockerfile.manylinux2014_lort_cpu → ...d/github/linux/docker/Dockerfile.lort_cpu
@@ -1,8 +1,8 @@
-FROM quay.io/pypa/manylinux2014_x86_64:latest
+FROM registry.access.redhat.com/ubi8/ubi
 
 ADD scripts /tmp/scripts
 RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh && /tmp/scripts/manylinux/install_deps_lort.sh && rm -rf /tmp/scripts
-
+ENV PATH /opt/rh/gcc-toolset-13/root/usr/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 ARG BUILD_UID=1002
 ARG BUILD_USER=onnxruntimedev
 RUN adduser --uid $BUILD_UID $BUILD_USER

diff --git a/tools/ci_build/github/linux/docker/scripts/lort/requirements.txt b/tools/ci_build/github/linux/docker/scripts/lort/requirements.txt
@@ -0,0 +1,27 @@
+transformers
+beartype==0.15.0
+flatbuffers
+cerberus
+h5py
+onnx==1.16.1
+# Python dependencies required for pytorch development
+astunparse
+expecttest!=0.2.0
+hypothesis
+numpy
+psutil
+pyyaml
+requests
+setuptools
+types-dataclasses
+typing-extensions>=4.8.0
+sympy
+filelock
+networkx
+jinja2
+fsspec
+lintrunner
+# setuptools was removed from default python install
+setuptools ; python_version >= "3.12"
+packaging
+optree>=0.11.0
diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh
@@ -2,7 +2,7 @@
 set -e -x
 
 # Development tools and libraries
-yum -y install \
+dnf -y install \
     graphviz
 
 if [ ! -d "/opt/conda/bin" ]; then
@@ -11,11 +11,9 @@ else
     PYTHON_EXES=("/opt/conda/bin/python")
 fi
 
-os_major_version=$(tr -dc '0-9.' < /etc/redhat-release |cut -d \. -f1)
 
 SYS_LONG_BIT=$(getconf LONG_BIT)
 mkdir -p /tmp/src
-GLIBC_VERSION=$(getconf GNU_LIBC_VERSION | cut -f 2 -d \.)
 
 DISTRIBUTOR=$(lsb_release -i -s)
 
@@ -47,13 +45,5 @@ do
   fi
 done
 
-cd /tmp/src
-GetFile 'https://sourceware.org/pub/valgrind/valgrind-3.16.1.tar.bz2' /tmp/src/valgrind-3.16.1.tar.bz2
-tar -jxvf valgrind-3.16.1.tar.bz2
-cd valgrind-3.16.1
-./configure --prefix=/usr --libdir=/usr/lib64 --enable-only64bit --enable-tls
-make -j$(getconf _NPROCESSORS_ONLN)
-make install
-
 cd /
 rm -rf /tmp/src
diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_lort.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_lort.sh
@@ -2,9 +2,9 @@
 set -e -x
 
 # Development tools and libraries
-yum -y install \
-    graphviz
-
+dnf -y install \
+    graphviz xz gcc-toolset-13-gcc-c++ gcc-toolset-13-gcc gcc-toolset-13-libstdc++-devel cmake python39-devel git
+source /opt/rh/gcc-toolset-13/enable
 mkdir -p /tmp/src
 
 cd /tmp/src
@@ -18,32 +18,30 @@ fi
 
 export ONNX_ML=1
 export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF"
+PYTHON_EXE=/usr/bin/python3.9
 
+echo "Installing Pytorch requirements"
 # This may install PyTorch, which will be overrided by the PyTorch local build below.
-/opt/python/cp39-cp39/bin/python3.9 -m pip install transformers
-
 # beartype is installed here so that onnxscript installation step won't
 # install a version PyTorch doesn't like. Once beartype fixes this problem.
 # We can remove this line.
-/opt/python/cp39-cp39/bin/python3.9 -m pip install beartype==0.15.0
+$PYTHON_EXE -m pip install -r /tmp/scripts/lort/requirements.txt
 
 cd /usr/local/
 echo "Cloning ONNX Script"
 git clone --recursive https://github.com/microsoft/onnxscript.git
 cd onnxscript
-/opt/python/cp39-cp39/bin/python3.9 -m pip install .
-cd ~ && /opt/python/cp39-cp39/bin/python3.9 -c "import onnxscript; print(f'Installed ONNX Script: {onnxscript.__version__}')"
+$PYTHON_EXE -m pip install .
+cd ~ && $PYTHON_EXE -c "import onnxscript; print(f'Installed ONNX Script: {onnxscript.__version__}')"
 
 cd /usr/local
 echo "Cloning Pytorch"
 git clone --recursive https://github.com/pytorch/pytorch.git
 cd pytorch
-echo "Installing Pytorch requirements"
-/opt/python/cp39-cp39/bin/python3.9 -m pip install -r requirements.txt
-/opt/python/cp39-cp39/bin/python3.9 -m pip install flatbuffers cerberus h5py onnx
+
 echo "Building and installing Pytorch"
-VERBOSE=1 BUILD_LAZY_TS_BACKEND=1 /opt/python/cp39-cp39/bin/python3.9 setup.py install
-cd ~ && /opt/python/cp39-cp39/bin/python3.9 -c "import torch; print(f'Installed Pytorch: {torch.__version__}')"
+VERBOSE=1 BUILD_LAZY_TS_BACKEND=1 $PYTHON_EXE setup.py install
+cd ~ && $PYTHON_EXE -c "import torch; print(f'Installed Pytorch: {torch.__version__}')"
 
 cd /
 rm -rf /tmp/src