apacheGH-33804: [Python] Add support for manylinux_2_28 wheel (apache…

…#34818) Closes apache#33804 ### Rationale for this change At some point, it would be useful to support the new C++ ABI `_GLIBCXX_USE_CXX11_ABI=1` in pyarrow wheels, especially when moving to C++17: - apache#32415 I wanted to create a pyarrow wheel that supported the above ABI and adapted the existing CENTOS 7 manylinux2014 Dockerfile/wheel to produce a AlmaLinux 8 manylinux_2_28 Dockerfile/wheel. Publishing wheels with a new ABI needs [careful consideration](https://pypackaging-native.github.io/key-issues/native-dependencies/cpp_deps/) so I think this is low priority, but I thought I'd provide this manylinux_2_28 implementation in case it was useful for current/future adoption. ### What changes are included in this PR? A manylinux_2_28 Dockerfile, adopted from the existing manylinux2014 Dockerfile ### Are these changes tested? Manually tested at present ### Are there any user-facing changes? Yes, there's a major ABI change, as pyarrow will be compiled with `_GLIBCXX_USE_CXX11_ABI=1` * Closes: apache#33804 Supercedes: * apache#33805 * Closes: apache#33804 Lead-authored-by: Simon Perkins <[email protected]> Co-authored-by: Sutou Kouhei <[email protected]> Signed-off-by: Sutou Kouhei <[email protected]>
NetEase · May 3, 2023 · 42d42b1 · 42d42b1
1 parent 205ceb9
commit 42d42b1
Show file tree

Hide file tree

Showing 7 changed files with 57 additions and 56 deletions.
diff --git a/.env b/.env
@@ -96,8 +96,8 @@ DEVTOOLSET_VERSION=
 # Please also update the crossbow configuration in order to keep the github
 # actions cache up to date for the macOS wheels:
 #   https://github.com/ursacomputing/crossbow/blob/master/.github/workflows/cache_vcpkg.yml
-# vcpkg minimum version "09adfdc8cdad76345b7cc7f3305899e1cbd66297" due to CVE-2022-3786
-VCPKG="2871ddd918cecb9cb642bcb9c56897f397283192"
+# TODO: Update to an official release tag https://github.com/apache/arrow/issues/35099
+VCPKG="b619a233fbf7b2c9765fb4458f3ecb05bd3166e3"    # 2023.04.03
 
 # This must be updated when we update
 # ci/docker/python-wheel-windows-vs2017.dockerfile.

diff --git a/...er/python-wheel-manylinux-201x.dockerfile → ci/docker/python-wheel-manylinux.dockerfile b/...er/python-wheel-manylinux-201x.dockerfile → ci/docker/python-wheel-manylinux.dockerfile
@@ -24,8 +24,19 @@ ARG manylinux
 
 ENV MANYLINUX_VERSION=${manylinux}
 
+# Ensure dnf is installed, especially for the manylinux2014 base
+RUN yum install -y dnf
+
 # Install basic dependencies
-RUN yum install -y git flex curl autoconf zip perl-IPC-Cmd wget 
+RUN dnf install -y git flex curl autoconf zip perl-IPC-Cmd wget kernel-headers
+
+# A system Python is required for ninja and vcpkg in this Dockerfile.
+# On manylinux2014 base images, system Python is 2.7.5, while
+# on manylinux_2_28, no system python is installed.
+# We therefore override the PATH with Python 3.8 in /opt/python
+# so that we have a consistent Python version across base images.
+ENV CPYTHON_VERSION=cp38
+ENV PATH=/opt/python/${CPYTHON_VERSION}-${CPYTHON_VERSION}/bin:${PATH}
 
 # Install CMake
 # AWS SDK doesn't work with CMake=3.22 due to https://gitlab.kitware.com/cmake/cmake/-/issues/22524
@@ -45,12 +56,10 @@ RUN /arrow/ci/scripts/install_ccache.sh ${ccache} /usr/local
 
 # Install vcpkg
 ARG vcpkg
-ARG glibc=2.18
 COPY ci/vcpkg/*.patch \
      ci/vcpkg/*linux*.cmake \
      arrow/ci/vcpkg/
 COPY ci/scripts/install_vcpkg.sh \
-     ci/scripts/install_glibc.sh \
      arrow/ci/scripts/
 ENV VCPKG_ROOT=/opt/vcpkg
 RUN arrow/ci/scripts/install_vcpkg.sh ${VCPKG_ROOT} ${vcpkg}
@@ -77,6 +86,7 @@ RUN vcpkg install \
         --x-feature=json \
         --x-feature=parquet
 
+# Configure Python for applications running in the bash shell of this Dockerfile
 ARG python=3.8
 ENV PYTHON_VERSION=${python}
 RUN PYTHON_ROOT=$(find /opt/python -name cp${PYTHON_VERSION/./}-*) && \

diff --git a/ci/scripts/install_glibc.sh b/ci/scripts/install_glibc.sh
diff --git a/dev/release/verify-release-candidate.sh b/dev/release/verify-release-candidate.sh
@@ -1015,7 +1015,7 @@ test_linux_wheels() {
   fi
 
   local python_versions="${TEST_PYTHON_VERSIONS:-3.7m 3.8 3.9 3.10 3.11}"
-  local platform_tags="manylinux_2_17_${arch}.manylinux2014_${arch}"
+  local platform_tags="${TEST_WHEEL_PLATFORM_TAGS:-manylinux_2_17_${arch}.manylinux2014_${arch} manylinux_2_28_${arch}}"
 
   for python in ${python_versions}; do
     local pyver=${python/m}

diff --git a/dev/tasks/python-wheels/github.linux.yml b/dev/tasks/python-wheels/github.linux.yml
@@ -68,6 +68,7 @@ jobs:
             -e TEST_DEFAULT=0 \
             -e TEST_PYARROW_VERSION={{ arrow.no_rc_version }} \
             -e TEST_PYTHON_VERSIONS={{ python_version }} \
+            -e TEST_WHEEL_PLATFORM_TAGS={{ wheel_platform_tag }} \
             -e TEST_WHEELS=1 \
             almalinux-verify-rc
 
@@ -82,6 +83,7 @@ jobs:
             -e TEST_DEFAULT=0 \
             -e TEST_PYARROW_VERSION={{ arrow.no_rc_version }} \
             -e TEST_PYTHON_VERSIONS={{ python_version }} \
+            -e TEST_WHEEL_PLATFORM_TAGS={{ wheel_platform_tag }} \
             -e TEST_WHEELS=1 \
             ubuntu-verify-rc
 
@@ -96,6 +98,7 @@ jobs:
             -e TEST_DEFAULT=0 \
             -e TEST_PYARROW_VERSION={{ arrow.no_rc_version }} \
             -e TEST_PYTHON_VERSIONS={{ python_version }} \
+            -e TEST_WHEEL_PLATFORM_TAGS={{ wheel_platform_tag }} \
             -e TEST_WHEELS=1 \
             ubuntu-verify-rc
 

diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
@@ -416,17 +416,21 @@ tasks:
 
 {############################## Wheel Linux ##################################}
 
-{% for arch, arch_alias, x_y, manylinux in [("amd64", "x86_64", "2_17", "2014"),
-                                            ("arm64", "aarch64", "2_17", "2014")] %}
-  wheel-manylinux{{ manylinux }}-{{ python_tag }}-{{ arch }}:
+
+{% for arch, manylinux_version, platform_tag in [("amd64", "2014", "manylinux_2_17_x86_64.manylinux2014_x86_64"),
+                                                 ("amd64", "2-28", "manylinux_2_28_x86_64"),
+                                                 ("arm64", "2014", "manylinux_2_17_aarch64.manylinux2014_aarch64"),
+                                                 ("arm64", "2-28", "manylinux_2_28_aarch64")] %}
+  wheel-manylinux-{{ manylinux_version }}-{{ python_tag }}-{{ arch }}:
     ci: github
     template: python-wheels/github.linux.yml
     params:
       arch: "{{ arch }}"
       python_version: "{{ python_version }}"
-      manylinux_version: "{{ manylinux }}"
+      manylinux_version: "{{ manylinux_version }}"
+      wheel_platform_tag: "{{ platform_tag }}"
     artifacts:
-      - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-manylinux_{{ x_y }}_{{ arch_alias }}.manylinux{{ manylinux }}_{{ arch_alias }}.whl
+      - pyarrow-{no_rc_version}-{{ python_tag }}-{{ abi_tag }}-{{ platform_tag }}.whl
 {% endfor %}
 
 {############################## Wheel OSX ####################################}
@@ -1438,13 +1442,13 @@ tasks:
       - docs.tar.gz
 
   ############################## CUDA tests #################################
-  
+
   test-cuda-cpp:
     ci: github
     template: docker-tests/github.cuda.yml
     params:
       image: ubuntu-cuda-cpp
-  
+
   test-cuda-python:
     ci: github
     template: docker-tests/github.cuda.yml

diff --git a/docker-compose.yml b/docker-compose.yml
@@ -161,6 +161,7 @@ x-hierarchy:
   - postgres
   - python-wheel-manylinux-2014:
     - java-jni-manylinux-2014
+  - python-wheel-manylinux-2-28
   - python-wheel-manylinux-test-imports
   - python-wheel-manylinux-test-unittests
   - python-wheel-windows-vs2017
@@ -183,6 +184,8 @@ volumes:
     name: maven-cache
   python-wheel-manylinux2014-ccache:
     name: python-wheel-manylinux2014-ccache
+  python-wheel-manylinux-2-28-ccache:
+    name: python-wheel-manylinux-2-28-ccache
   python-wheel-windows-clcache:
     name: python-wheel-windows-clcache
   ubuntu-ccache:
@@ -1019,7 +1022,7 @@ services:
         python: ${PYTHON}
         manylinux: 2014
       context: .
-      dockerfile: ci/docker/python-wheel-manylinux-201x.dockerfile
+      dockerfile: ci/docker/python-wheel-manylinux.dockerfile
       cache_from:
         - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG}
     environment:
@@ -1029,6 +1032,29 @@ services:
       - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated
     command: /arrow/ci/scripts/python_wheel_manylinux_build.sh
 
+  # See available versions at:
+  #    https://quay.io/repository/pypa/manylinux_2_28_x86_64?tab=tags
+  python-wheel-manylinux-2-28:
+    image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2-28-vcpkg-${VCPKG}
+    build:
+      args:
+        arch: ${ARCH}
+        arch_short: ${ARCH_SHORT}
+        base: quay.io/pypa/manylinux_2_28_${ARCH_ALIAS}:2023-01-14-103cb93
+        vcpkg: ${VCPKG}
+        python: ${PYTHON}
+        manylinux: 2_28
+      context: .
+      dockerfile: ci/docker/python-wheel-manylinux.dockerfile
+      cache_from:
+        - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2-28-vcpkg-${VCPKG}
+    environment:
+      <<: *ccache
+    volumes:
+      - .:/arrow:delegated
+      - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux-2-28-ccache:/ccache:delegated
+    command: /arrow/ci/scripts/python_wheel_manylinux_build.sh
+
   python-wheel-manylinux-test-imports:
     image: ${ARCH}/python:${PYTHON}
     shm_size: 2G