ARROW-12626: [C++] Support toolchain xsimd, update toolchain version …

…to version 8.1.0 (apache#13244)

This also updates pinned vcpkg to use xsimd 8.1.0.

This also implements auto python-wheel-windows-vs2017 image update mechanism.
We have a problem of "docker build" on Windows. "docker build" doesn't reuse pulled
image as caches. "docker build" always rebuilds an image. This implements manual
reuse mechanism like the following:

    if ! docker pull; then
       docker build # build only when built images don't exist
    fi
    docker run

But this doesn't work when ci/docker/python-wheel-windows-vs2017.dockerfile is updated
but pinned vcpkg revision isn't changed. In the case, "docker build" isn't run because
"docker pull" is succeeded. 

To work this mechanism, this introduces "PYTHON_WHEEL_WINDOWS_IMAGE_REVISION".
We must bump it manually when we update ci/docker/python-wheel-windows-vs2017.dockerfile.
"PYTHON_WHEEL_WINDOWS_IMAGE_REVISION" is used in tag name. So "docker pull"
is failed with new "PYTHON_WHEEL_WINDOWS_IMAGE_REVISION" and then "docker build"
is used.

Lead-authored-by: Sutou Kouhei <[email protected]>
Co-authored-by: Yibo Cai <[email protected]>
Co-authored-by: Wes McKinney <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>

.env

@@ -93,12 +93,13 @@ DEVTOOLSET_VERSION=-1
 # Please also update the crossbow configuration in order to keep the github
 # actions cache up to date for the macOS wheels:
 #   https://github.com/ursacomputing/crossbow/blob/master/.github/workflows/cache_vcpkg.yml
-#
-# After commit 89295c9 openssl is only available as 3.0.2 which is
-# incompatible with aws-sdk-cpp 1.8.3 see
-# https://github.com/aws/aws-sdk-cpp/issues/1582 and causes issues on
-# manylinux2010
-VCPKG="89295c9"
+VCPKG="38bb87c"
+
+# This must be updated when we update
+# ci/docker/python-wheel-windows-vs2017.dockerfile.
+# This is a workaround for our CI problem that "archery docker build" doesn't
+# use pulled built images in dev/tasks/python-wheels/github.windows.yml.
+PYTHON_WHEEL_WINDOWS_IMAGE_REVISION=2022-06-12
 
 # Use conanio/${CONAN} for "docker-compose run --rm conan". See
 # https://github.com/conan-io/conan-docker-tools#readme for available

ci/conan/all/conanfile.py

@@ -295,11 +295,7 @@ def requirements(self):
         if tools.Version(self.version) >= "6.0.0" and \
             self.options.get_safe("simd_level") != None or \
             self.options.get_safe("runtime_simd_level") != None:
-            if tools.Version(self.version) >= "8.0.0":
-                # TODO: Requires xsimd/master
-                pass
-            else:
-                self.requires("xsimd/8.0.3")
+            self.requires("xsimd/8.1.0")
         if self.options.with_zlib:
             self.requires("zlib/1.2.12")
         if self.options.with_zstd:
@@ -581,11 +577,7 @@ def package_info(self):
         if self.options.with_snappy:
             self.cpp_info.components["libarrow"].requires.append("snappy::snappy")
         if self.options.get_safe("simd_level") != None or self.options.get_safe("runtime_simd_level") != None:
-            if tools.Version(self.version) >= "8.0.0":
-                # Requires xsimd/master
-                pass
-            else:
-                self.cpp_info.components["libarrow"].requires.append("xsimd::xsimd")
+            self.cpp_info.components["libarrow"].requires.append("xsimd::xsimd")
         if self.options.with_zlib:
             self.cpp_info.components["libarrow"].requires.append("zlib::zlib")
         if self.options.with_zstd:

ci/conda_env_cpp.txt

@@ -43,6 +43,7 @@ rapidjson
 re2
 snappy
 thrift-cpp>=0.11.0
+xsimd
 zlib
 zstd
 flatbuffers

ci/docker/debian-10-cpp.dockerfile

@@ -103,4 +103,5 @@ ENV ARROW_BUILD_TESTS=ON \
     ORC_SOURCE=BUNDLED \
     PATH=/usr/lib/ccache/:$PATH \
     Protobuf_SOURCE=BUNDLED \
+    xsimd_SOURCE=BUNDLED \
     zstd_SOURCE=BUNDLED

ci/docker/debian-11-cpp.dockerfile

@@ -103,4 +103,5 @@ ENV ARROW_BUILD_TESTS=ON \
     google_cloud_cpp_storage_SOURCE=BUNDLED \
     ORC_SOURCE=BUNDLED \
     PATH=/usr/lib/ccache/:$PATH \
-    Protobuf_SOURCE=BUNDLED
+    Protobuf_SOURCE=BUNDLED \
+    xsimd_SOURCE=BUNDLED

ci/docker/fedora-35-cpp.dockerfile

@@ -95,6 +95,7 @@ ENV ARROW_BUILD_TESTS=ON \
     CXX=g++ \
     google_cloud_cpp_storage_SOURCE=BUNDLED \
     ORC_SOURCE=BUNDLED \
-    PARQUET_BUILD_EXECUTABLES=ON \
     PARQUET_BUILD_EXAMPLES=ON \
-    PATH=/usr/lib/ccache/:$PATH
+    PARQUET_BUILD_EXECUTABLES=ON \
+    PATH=/usr/lib/ccache/:$PATH \
+    xsimd_SOURCE=BUNDLED

ci/docker/python-wheel-manylinux-201x.dockerfile

@@ -81,7 +81,8 @@ RUN vcpkg install \
         --x-feature=flight \
         --x-feature=gcs \
         --x-feature=json \
-        --x-feature=parquet
+        --x-feature=parquet \
+        --x-feature=s3
 
 ARG python=3.8
 ENV PYTHON_VERSION=${python}

ci/docker/python-wheel-windows-vs2017.dockerfile

@@ -15,6 +15,9 @@
 # specific language governing permissions and limitations
 # under the License.
 
+# NOTE: You must update PYTHON_WHEEL_WINDOWS_IMAGE_REVISION in .env
+# when you update this file.
+
 # based on mcr.microsoft.com/windows/servercore:ltsc2019
 # contains choco and vs2017 preinstalled
 FROM abrarov/msvc-2017:2.11.0

ci/docker/ubuntu-18.04-cpp.dockerfile

@@ -129,4 +129,5 @@ ENV ARROW_BUILD_TESTS=ON \
     PATH=/usr/lib/ccache/:$PATH \
     Thrift_SOURCE=BUNDLED \
     utf8proc_SOURCE=BUNDLED \
+    xsimd_SOURCE=BUNDLED \
     zstd_SOURCE=BUNDLED

ci/docker/ubuntu-20.04-cpp.dockerfile

@@ -150,11 +150,12 @@ ENV ARROW_BUILD_TESTS=ON \
     ASAN_SYMBOLIZER_PATH=/usr/lib/llvm-${llvm}/bin/llvm-symbolizer \
     AWSSDK_SOURCE=BUNDLED \
     google_cloud_cpp_storage_SOURCE=BUNDLED \
-    GTest_SOURCE=BUNDLED \
     gRPC_SOURCE=BUNDLED \
+    GTest_SOURCE=BUNDLED \
     ORC_SOURCE=BUNDLED \
     PARQUET_BUILD_EXAMPLES=ON \
     PARQUET_BUILD_EXECUTABLES=ON \
-    PATH=/usr/lib/ccache/:$PATH \
     Protobuf_SOURCE=BUNDLED \
-    PYTHON=python3
+    PATH=/usr/lib/ccache/:$PATH \
+    PYTHON=python3 \
+    xsimd_SOURCE=BUNDLED

ci/docker/ubuntu-22.04-cpp.dockerfile

@@ -181,6 +181,7 @@ ENV ARROW_BUILD_TESTS=ON \
     ORC_SOURCE=BUNDLED \
     PARQUET_BUILD_EXAMPLES=ON \
     PARQUET_BUILD_EXECUTABLES=ON \
-    Protobuf_SOURCE=BUNDLED \
     PATH=/usr/lib/ccache/:$PATH \
-    PYTHON=python3
+    Protobuf_SOURCE=BUNDLED \
+    PYTHON=python3 \
+    xsimd_SOURCE=BUNDLED

ci/scripts/cpp_build.sh

@@ -161,6 +161,7 @@ cmake \
   -DThrift_SOURCE=${Thrift_SOURCE:-} \
   -Dutf8proc_SOURCE=${utf8proc_SOURCE:-} \
   -Dzstd_SOURCE=${zstd_SOURCE:-} \
+  -Dxsimd_SOURCE=${xsimd_SOURCE:-} \
   -G "${CMAKE_GENERATOR:-Ninja}" \
   ${CMAKE_ARGS} \
   ${source_dir}

ci/scripts/python_wheel_macos_build.sh

@@ -123,7 +123,6 @@ cmake \
     -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
     -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
     -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
-    -DAWSSDK_SOURCE=BUNDLED \
     -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 \
     -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
     -DCMAKE_INSTALL_LIBDIR=lib \

ci/scripts/python_wheel_manylinux_build.sh

@@ -117,7 +117,6 @@ cmake \
     -DARROW_WITH_SNAPPY=${ARROW_WITH_SNAPPY} \
     -DARROW_WITH_ZLIB=${ARROW_WITH_ZLIB} \
     -DARROW_WITH_ZSTD=${ARROW_WITH_ZSTD} \
-    -DAWSSDK_SOURCE=BUNDLED \
     -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} \
     -DCMAKE_INSTALL_LIBDIR=lib \
     -DCMAKE_INSTALL_PREFIX=/tmp/arrow-dist \

ci/scripts/python_wheel_windows_build.bat

@@ -47,6 +47,9 @@ set ARROW_WITH_LZ4=ON
 set ARROW_WITH_SNAPPY=ON
 set ARROW_WITH_ZLIB=ON
 set ARROW_WITH_ZSTD=ON
+@rem Workaround for https://github.com/aws/aws-sdk-cpp/issues/1809 .
+@rem Use (old) bundled AWS SDK C++ instead of (newer) AWS SDK C++.
+set AWSSDK_SOURCE=BUNDLED
 set CMAKE_UNITY_BUILD=ON
 set CMAKE_GENERATOR=Visual Studio 15 2017 Win64
 set VCPKG_ROOT=C:\vcpkg
@@ -81,7 +84,7 @@ cmake ^
     -DARROW_WITH_SNAPPY=%ARROW_WITH_SNAPPY% ^
     -DARROW_WITH_ZLIB=%ARROW_WITH_ZLIB% ^
     -DARROW_WITH_ZSTD=%ARROW_WITH_ZSTD% ^
-    -DAWSSDK_SOURCE=BUNDLED ^
+    -DAWSSDK_SOURCE=%AWSSDK_SOURCE% ^
     -DCMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE% ^
     -DCMAKE_CXX_COMPILER=clcache ^
     -DCMAKE_INSTALL_PREFIX=C:\arrow-dist ^

ci/vcpkg/ports.patch

@@ -71,44 +71,6 @@ index 1289eed36a..b010a69f13 100644
  )
 
  vcpkg_check_features(OUT_FEATURE_OPTIONS FEATURE_OPTIONS
-diff --git a/ports/aws-sdk-cpp/fix-find-crypto.patch b/ports/aws-sdk-cpp/fix-find-crypto.patch
-new file mode 100644
-index 0000000000..aafc631e93
---- /dev/null
-+++ b/ports/aws-sdk-cpp/fix-find-crypto.patch
-@@ -0,0 +1,20 @@
-+diff --git a/cmake/setup_cmake_find_module.cmake b/cmake/setup_cmake_find_module.cmake
-+index f5f147a0f..4561b8c39 100644
-+--- a/cmake/setup_cmake_find_module.cmake
-++++ b/cmake/setup_cmake_find_module.cmake
-+@@ -30,15 +30,6 @@ file(WRITE
-+     "set(AWSSDK_INSTALL_BINDIR ${BINARY_DIRECTORY})\n"
-+     "set(AWSSDK_INSTALL_INCLUDEDIR ${INCLUDE_DIRECTORY})\n"
-+     "set(AWSSDK_INSTALL_ARCHIVEDIR ${ARCHIVE_DIRECTORY})\n"
-+-    "if (NOT LibCrypto_INCLUDE_DIR)\n"
-+-    "    set(LibCrypto_INCLUDE_DIR ${OPENSSL_INCLUDE_DIR} CACHE INTERNAL \"The OpenSSL include directory\")\n"
-+-    "endif()\n"
-+-    "if (NOT LibCrypto_STATIC_LIBRARY)\n"
-+-    "    set(LibCrypto_STATIC_LIBRARY ${OPENSSL_CRYPTO_LIBRARY} CACHE INTERNAL \"The OpenSSL crypto static library\")\n"
-+-    "endif()\n"
-+-    "if (NOT LibCrypto_SHARED_LIBRARY)\n"
-+-    "    set(LibCrypto_SHARED_LIBRARY ${OPENSSL_CRYPTO_LIBRARY} CACHE INTERNAL \"The OpenSSL crypto shared library\")\n"
-+-    "endif()\n"
-+     )
-+
-+ if (NOT SIMPLE_INSTALL)
-diff --git a/ports/aws-sdk-cpp/portfile.cmake b/ports/aws-sdk-cpp/portfile.cmake
-index b520a17ae3..ea9e770cf5 100644
---- a/ports/aws-sdk-cpp/portfile.cmake
-+++ b/ports/aws-sdk-cpp/portfile.cmake
-@@ -8,6 +8,7 @@ vcpkg_from_github(
-     PATCHES
-         patch-relocatable-rpath.patch
-         fix-aws-root.patch
-+        fix-find-crypto.patch
- )
-
- string(COMPARE EQUAL "${VCPKG_CRT_LINKAGE}" "dynamic" FORCE_SHARED_CRT)
 diff --git a/ports/curl/portfile.cmake b/ports/curl/portfile.cmake
 index f81d0c491d..e5ea9cef57 100644
 --- a/ports/curl/portfile.cmake

ci/vcpkg/vcpkg.json

@@ -25,6 +25,7 @@
     "re2",
     "snappy",
     "utf8proc",
+    "xsimd",
     "zlib",
     "zstd",
     {

cpp/cmake_modules/ThirdpartyToolchain.cmake

@@ -2204,16 +2204,20 @@ if((NOT ARROW_SIMD_LEVEL STREQUAL "NONE") OR (NOT ARROW_RUNTIME_SIMD_LEVEL STREQ
 else()
   set(ARROW_USE_XSIMD FALSE)
 endif()
+
 if(ARROW_USE_XSIMD)
-  set(xsimd_SOURCE "BUNDLED")
-  resolve_dependency(xsimd)
+  resolve_dependency(xsimd REQUIRED_VERSION "8.1.0")
 
-  add_library(xsimd INTERFACE IMPORTED)
-  if(CMAKE_VERSION VERSION_LESS 3.11)
-    set_target_properties(xsimd PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
-                                           "${XSIMD_INCLUDE_DIR}")
+  if(xsimd_SOURCE STREQUAL "BUNDLED")
+    add_library(xsimd INTERFACE IMPORTED)
+    if(CMAKE_VERSION VERSION_LESS 3.11)
+      set_target_properties(xsimd PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+                                             "${XSIMD_INCLUDE_DIR}")
+    else()
+      target_include_directories(xsimd INTERFACE "${XSIMD_INCLUDE_DIR}")
+    endif()
   else()
-    target_include_directories(xsimd INTERFACE "${XSIMD_INCLUDE_DIR}")
+    message(STATUS "xsimd found. Headers: ${xsimd_INCLUDE_DIRS}")
   endif()
 endif()
 
@@ -4437,6 +4441,14 @@ macro(build_awssdk)
       -DENABLE_UNITY_BUILD=ON
       "-DCMAKE_INSTALL_PREFIX=${AWSSDK_PREFIX}"
       "-DCMAKE_PREFIX_PATH=${AWSSDK_PREFIX}")
+  if(NOT MSVC)
+    list(APPEND
+         AWSSDK_COMMON_CMAKE_ARGS
+         # Workaround for https://github.com/aws/aws-sdk-cpp/issues/1582
+         "-DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS} -Wno-error=deprecated-declarations"
+         "-DCMAKE_CXX_FLAGS_${UPPERCASE_BUILD_TYPE}=${EP_CXX_FLAGS} -Wno-error=deprecated-declarations"
+    )
+  endif()
 
   # provide hint for AWS SDK to link with the already located openssl
   get_filename_component(OPENSSL_ROOT_HINT "${OPENSSL_INCLUDE_DIR}" DIRECTORY)

cpp/thirdparty/versions.txt

@@ -93,8 +93,8 @@ ARROW_UCX_BUILD_VERSION=1.12.1
 ARROW_UCX_BUILD_SHA256_CHECKSUM=9bef31aed0e28bf1973d28d74d9ac4f8926c43ca3b7010bd22a084e164e31b71
 ARROW_UTF8PROC_BUILD_VERSION=v2.7.0
 ARROW_UTF8PROC_BUILD_SHA256_CHECKSUM=4bb121e297293c0fd55f08f83afab6d35d48f0af4ecc07523ad8ec99aa2b12a1
-ARROW_XSIMD_BUILD_VERSION=7d1778c3b38d63db7cec7145d939f40bc5d859d1
-ARROW_XSIMD_BUILD_SHA256_CHECKSUM=d861f4f3034550cdc62ad93fd60e8b0f2413a6ea49081c8698922b7b4f043ec6
+ARROW_XSIMD_BUILD_VERSION=8.1.0
+ARROW_XSIMD_BUILD_SHA256_CHECKSUM=d52551360d37709675237d2a0418e28f70995b5b7cdad7c674626bcfbbf48328
 ARROW_ZLIB_BUILD_VERSION=1.2.12
 ARROW_ZLIB_BUILD_SHA256_CHECKSUM=91844808532e5ce316b3c010929493c0244f3d37593afd6de04f71821d5136d9
 ARROW_ZSTD_BUILD_VERSION=v1.5.1

cpp/vcpkg.json

@@ -47,6 +47,7 @@
     "sqlite3",
     "thrift",
     "utf8proc",
+    "xsimd",
     "zlib",
     "zstd"
   ],

dev/archery/archery/docker/cli.py

@@ -79,6 +79,36 @@ def check_config(obj):
     # during the configuration loading
 
 
+@docker.command('pull')
+@click.argument('image')
+@click.option('--using-docker-cli', default=False, is_flag=True,
+              envvar='ARCHERY_USE_DOCKER_CLI',
+              help="Use docker CLI directly for pulling instead of calling "
+                   "docker-compose. This may help to reuse cached layers.")
+@click.option('--pull-leaf/--no-leaf', default=True,
+              help="Whether to pull leaf images too.")
+@click.option('--ignore-pull-failures/--no-ignore-pull-failures', default=True,
+              help="Whether to ignore pull failures.")
+@click.pass_obj
+def docker_pull(obj, image, *, using_docker_cli, pull_leaf,
+                ignore_pull_failures):
+    """
+    Execute docker-compose pull.
+    """
+    compose = obj['compose']
+
+    try:
+        compose.pull(image, pull_leaf=pull_leaf, using_docker=using_docker_cli,
+                     ignore_pull_failures=ignore_pull_failures)
+    except UndefinedImage as e:
+        raise click.ClickException(
+            "There is no service/image defined in docker-compose.yml with "
+            "name: {}".format(str(e))
+        )
+    except RuntimeError as e:
+        raise click.ClickException(str(e))
+
+
 @docker.command('build')
 @click.argument('image')
 @click.option('--force-pull/--no-pull', default=True,

dev/archery/archery/docker/core.py

@@ -219,7 +219,8 @@ def _execute_docker(self, *args, **kwargs):
                 )
             )
 
-    def pull(self, service_name, pull_leaf=True, using_docker=False):
+    def pull(self, service_name, pull_leaf=True, using_docker=False,
+             ignore_pull_failures=True):
         def _pull(service):
             args = ['pull']
             if service['image'] in self.pull_memory:
@@ -229,10 +230,14 @@ def _pull(service):
                 try:
                     self._execute_docker(*args, service['image'])
                 except Exception as e:
-                    # better --ignore-pull-failures handling
-                    print(e)
+                    if ignore_pull_failures:
+                        # better --ignore-pull-failures handling
+                        print(e)
+                    else:
+                        raise
             else:
-                args.append('--ignore-pull-failures')
+                if ignore_pull_failures:
+                    args.append('--ignore-pull-failures')
                 self._execute_compose(*args, service['name'])
 
             self.pull_memory.add(service['image'])

dev/tasks/conda-recipes/arrow-cpp/meta.yaml

@@ -90,6 +90,7 @@ outputs:
         - re2
         - snappy
         - thrift-cpp
+        - xsimd
         - zlib
         - zstd
       run:

dev/tasks/conda-recipes/azure.osx.yml

@@ -57,6 +57,7 @@ jobs:
 
   - script: |
       source activate base
+      set -e
       set +x
       if [[ "${CONFIG}" == osx_arm* ]]; then
         EXTRA_CB_OPTIONS="${EXTRA_CB_OPTIONS:-} --no-test"

dev/tasks/macros.jinja

@@ -38,16 +38,19 @@ on:
 
 {%- macro github_login_dockerhub() -%}
   - name: Login to Dockerhub
-    uses: docker/login-action@v1
+    uses: docker/login-action@v2
     with:
       username: {{ '${{ secrets.DOCKERHUB_USER }}' }}
       password: {{ '${{ secrets.DOCKERHUB_TOKEN }}' }}
 {% endmacro %}
 
 {%- macro github_login_ghcr() -%}
   - name: Login to GitHub Container Registry
-    shell: bash
-    run: docker login ghcr.io -u {{ '${{ github.repository_owner }}' }} -p {{ '${{ secrets.CROSSBOW_GHCR_TOKEN }}' }}
+    uses: docker/login-action@v2
+    with:
+      registry: ghcr.io
+      username: {{ '${{ github.actor }}' }}
+      password: {{ '${{ secrets.GITHUB_TOKEN }}' }}
 {% endmacro %}
 
 {%- macro github_install_archery() -%}
@@ -292,4 +295,4 @@ on:
       # getwd() is necessary as this macro is used within jobs using a docker container
       tools::write_PACKAGES(file.path(getwd(), "/repo/src/contrib", fsep = "/"), type = "source", verbose = TRUE)
   - run: ls -R repo
-{% endmacro %}
+{% endmacro %}