Skip to content

Commit

Permalink
ARROW-14171: [C++][Python][Packaging] Upgrade VCPKG version and add g…
Browse files Browse the repository at this point in the history
…oogle-cloud-cpp dependency

Closes apache#11569 from coryan/ARROW-14171-add-google-cloud-cpp-package

Lead-authored-by: Krisztián Szűcs <[email protected]>
Co-authored-by: Carlos O'Ryan <[email protected]>
Signed-off-by: Krisztián Szűcs <[email protected]>
  • Loading branch information
kszucs and coryan committed Dec 1, 2021
1 parent 736a69e commit 2233ac5
Show file tree
Hide file tree
Showing 26 changed files with 516 additions and 526 deletions.
11 changes: 8 additions & 3 deletions .env
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ BUILDKIT_INLINE_CACHE=1
# different architecture notations
ARCH=amd64
ARCH_ALIAS=x86_64
ARCH_SHORT_ALIAS=x64
ARCH_SHORT=amd64

ULIMIT_CORE=-1
REPO=apache/arrow-dev
Expand Down Expand Up @@ -70,6 +70,11 @@ TZ=UTC
# -1 does not attempt to install a devtoolset version, any positive integer will install devtoolset-n
DEVTOOLSET_VERSION=-1

# Used for the manylinux and windows wheels, please update the crossbow configuration on update:
# Used through docker-compose.yml and serves as the default version for the
# ci/scripts/install_vcpkg.sh script. Prefer to use short SHAs to keep the
# docker tags more readable.
#
# Please also update the crossbow configuration in order to keep the github
# actions cache up to date for the macOS wheels:
# https://github.com/ursacomputing/crossbow/blob/master/.github/workflows/cache_vcpkg.yml
VCPKG="2021.04.30"
VCPKG="30465138e"
61 changes: 29 additions & 32 deletions ci/docker/python-wheel-manylinux-201x.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -18,65 +18,61 @@
ARG base
FROM ${base}

ARG arch_alias
ARG arch_short_alias
ARG arch
ARG arch_short
ARG manylinux

ENV MANYLINUX_VERSION=${manylinux}

# Install basic dependencies
RUN yum install -y git flex curl autoconf zip wget

# Install CMake
ARG cmake=3.19.3
RUN wget -q https://github.com/Kitware/CMake/releases/download/v${cmake}/cmake-${cmake}-Linux-${arch_alias}.tar.gz -O - | \
tar -xzf - --directory /usr/local --strip-components=1
# AWS SDK doesn't work with CMake=3.22 due to https://gitlab.kitware.com/cmake/cmake/-/issues/22524
ARG cmake=3.21.4
COPY ci/scripts/install_cmake.sh arrow/ci/scripts/
RUN /arrow/ci/scripts/install_cmake.sh ${arch} linux ${cmake} /usr/local

# Install Ninja
ARG ninja=1.10.2
RUN mkdir /tmp/ninja && \
wget -q https://github.com/ninja-build/ninja/archive/v${ninja}.tar.gz -O - | \
tar -xzf - --directory /tmp/ninja --strip-components=1 && \
cd /tmp/ninja && \
./configure.py --bootstrap && \
mv ninja /usr/local/bin && \
rm -rf /tmp/ninja
COPY ci/scripts/install_ninja.sh arrow/ci/scripts/
RUN /arrow/ci/scripts/install_ninja.sh ${ninja} /usr/local

# Install ccache
ARG ccache=4.1
RUN mkdir /tmp/ccache && \
wget -q https://github.com/ccache/ccache/archive/v${ccache}.tar.gz -O - | \
tar -xzf - --directory /tmp/ccache --strip-components=1 && \
cd /tmp/ccache && \
mkdir build && \
cd build && \
cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DZSTD_FROM_INTERNET=ON .. && \
ninja install && \
rm -rf /tmp/ccache
COPY ci/scripts/install_ccache.sh arrow/ci/scripts/
RUN /arrow/ci/scripts/install_ccache.sh ${ccache} /usr/local

# Install vcpkg
# Install vcpkg and in case of manylinux2010 install a more recent glibc>2.15
# for the prebuilt vcpkg binary
ARG vcpkg
RUN git clone https://github.com/microsoft/vcpkg /opt/vcpkg && \
git -C /opt/vcpkg checkout ${vcpkg} && \
/opt/vcpkg/bootstrap-vcpkg.sh -useSystemBinaries -disableMetrics && \
ln -s /opt/vcpkg/vcpkg /usr/bin/vcpkg

# Patch ports files as needed
ARG glibc=2.18
COPY ci/vcpkg/*.patch \
ci/vcpkg/*linux*.cmake \
arrow/ci/vcpkg/
RUN cd /opt/vcpkg && git apply --ignore-whitespace /arrow/ci/vcpkg/ports.patch
COPY ci/scripts/install_vcpkg.sh \
ci/scripts/install_glibc.sh \
arrow/ci/scripts/
RUN arrow/ci/scripts/install_vcpkg.sh /opt/vcpkg ${vcpkg} && \
if [ "${manylinux}" == "2010" ]; then \
arrow/ci/scripts/install_glibc.sh ${glibc} /opt/glibc-${glibc} && \
patchelf --set-interpreter /opt/glibc-2.18/lib/ld-linux-x86-64.so.2 /opt/vcpkg/vcpkg && \
patchelf --set-rpath /opt/glibc-2.18/lib:/usr/lib64 /opt/vcpkg/vcpkg; \
fi
ENV PATH="/opt/vcpkg:${PATH}"

ARG build_type=release
ENV CMAKE_BUILD_TYPE=${build_type} \
VCPKG_FORCE_SYSTEM_BINARIES=1 \
VCPKG_OVERLAY_TRIPLETS=/arrow/ci/vcpkg \
VCPKG_DEFAULT_TRIPLET=${arch_short_alias}-linux-static-${build_type} \
VCPKG_DEFAULT_TRIPLET=${arch_short}-linux-static-${build_type} \
VCPKG_FEATURE_FLAGS=-manifests

# Need to install the boost-build prior installing the boost packages, otherwise
# vcpkg will raise an error.
# TODO(kszucs): factor out the package enumeration to a text file and reuse it
# from the windows image and potentially in a future macos wheel build
RUN vcpkg install --clean-after-build \
boost-build:${arch_short_alias}-linux && \
vcpkg install --clean-after-build \
abseil \
aws-sdk-cpp[config,cognito-identity,core,identity-management,s3,sts,transfer] \
boost-filesystem \
Expand All @@ -87,6 +83,7 @@ RUN vcpkg install --clean-after-build \
flatbuffers \
gflags \
glog \
google-cloud-cpp[core,storage] \
grpc \
lz4 \
openssl \
Expand Down
15 changes: 6 additions & 9 deletions ci/docker/python-wheel-windows-vs2017.dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

# based on mcr.microsoft.com/windows/servercore:ltsc2019
# contains choco and vs2017 preinstalled
FROM abrarov/msvc-2017:2.10.0
FROM abrarov/msvc-2017:2.12.1

# Install CMake and Ninja
RUN choco install --no-progress -r -y cmake --installargs 'ADD_CMAKE_TO_PATH=System' && \
Expand All @@ -31,16 +31,12 @@ RUN setx path "%path%;C:\Program Files\Git\usr\bin"
# Compiling vcpkg itself from a git tag doesn't work anymore since vcpkg has
# started to ship precompiled binaries for the vcpkg-tool.
ARG vcpkg
RUN git clone https://github.com/Microsoft/vcpkg && \
vcpkg\bootstrap-vcpkg.bat -disableMetrics && \
setx PATH "%PATH%;C:\vcpkg" && \
git -C vcpkg checkout %vcpkg%

# Patch ports files as needed
COPY ci/vcpkg/*.patch \
ci/vcpkg/*windows*.cmake \
arrow/ci/vcpkg/
RUN cd vcpkg && git apply --ignore-whitespace C:/arrow/ci/vcpkg/ports.patch
COPY ci/scripts/install_vcpkg.sh arrow/ci/scripts/
RUN bash arrow/ci/scripts/install_vcpkg.sh /c/vcpkg %vcpkg% && \
setx PATH "%PATH%;C:\vcpkg"

# Configure vcpkg and install dependencies
# NOTE: use windows batch environment notation for build arguments in RUN
Expand All @@ -50,7 +46,7 @@ RUN cd vcpkg && git apply --ignore-whitespace C:/arrow/ci/vcpkg/ports.patch
ARG build_type=release
ENV CMAKE_BUILD_TYPE=${build_type} \
VCPKG_OVERLAY_TRIPLETS=C:\\arrow\\ci\\vcpkg \
VCPKG_DEFAULT_TRIPLET=x64-windows-static-md-${build_type} \
VCPKG_DEFAULT_TRIPLET=amd64-windows-static-md-${build_type} \
VCPKG_FEATURE_FLAGS=-manifests

RUN vcpkg install --clean-after-build \
Expand All @@ -66,6 +62,7 @@ RUN vcpkg install --clean-after-build \
flatbuffers \
gflags \
glog \
google-cloud-cpp[core,storage] \
grpc \
lz4 \
openssl \
Expand Down
45 changes: 45 additions & 0 deletions ci/scripts/install_ccache.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -e

if [ "$#" -ne 2 ]; then
echo "Usage: $0 <version> <prefix>"
exit 1
fi

version=$1
prefix=$2
url="https://github.com/ccache/ccache/archive/v${version}.tar.gz"

mkdir /tmp/ccache
wget -q ${url} -O - | tar -xzf - --directory /tmp/ccache --strip-components=1

mkdir /tmp/ccache/build
pushd /tmp/ccache/build
cmake \
-GNinja \
-DCMAKE_BUILD_TYPE=Release \
-DCMAKE_INSTALL_PREFIX=${prefix} \
-DZSTD_FROM_INTERNET=ON \
..
ninja install
popd

rm -rf /tmp/ccache
42 changes: 42 additions & 0 deletions ci/scripts/install_cmake.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -e

declare -A archs
archs=([amd64]=x86_64
[arm64v8]=aarch64)

declare -A platforms
platforms=([linux]=linux
[macos]=macos
[windows]=windows)

if [ "$#" -ne 4 ]; then
echo "Usage: $0 <architecture> <platform> <version> <prefix>"
exit 1
fi

arch=${archs[$1]}
platform=${platforms[$2]}
version=$3
prefix=$4

url="https://github.com/Kitware/CMake/releases/download/v${version}/cmake-${version}-${platform}-${arch}.tar.gz"
wget -q ${url} -O - | tar -xzf - --directory ${prefix} --strip-components=1
42 changes: 42 additions & 0 deletions ci/scripts/install_glibc.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -e

if [ "$#" -ne 2 ]; then
echo "Usage: $0 <version> <prefix>"
exit 1
fi

version=$1
prefix=$2

url="http://ftp.gnu.org/gnu/glibc/glibc-${version}.tar.gz"

mkdir /tmp/glibc
wget -q ${url} -O - | tar -xzf - --directory /tmp/glibc --strip-components=1

mkdir /tmp/glibc/build
pushd /tmp/glibc/build
../configure --prefix=${prefix}
make -j$(nproc)
make install
popd

rm -rf /tmp/glibc
40 changes: 40 additions & 0 deletions ci/scripts/install_ninja.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

set -e

if [ "$#" -ne 2 ]; then
echo "Usage: $0 <version> <prefix>"
exit 1
fi

version=$1
prefix=$2

url="https://github.com/ninja-build/ninja/archive/v${version}.tar.gz"

mkdir /tmp/ninja
wget -q ${url} -O - | tar -xzf - --directory /tmp/ninja --strip-components=1

pushd /tmp/ninja
./configure.py --bootstrap
mv ninja ${prefix}/bin
popd

rm -rf /tmp/ninja
32 changes: 23 additions & 9 deletions ci/scripts/install_vcpkg.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,21 +19,35 @@

set -e

if [ "$#" -ne 2 ]; then
echo "Usage: $0 <vcpkg version> <target directory>"
if [ "$#" -lt 1 ]; then
echo "Usage: $0 ``<target-directory> [<vcpkg-version> [<vcpkg-ports-patch>]]"
exit 1
fi

vcpkg_version=$1
vcpkg_destination=$2
vcpkg_patch=$(realpath $(dirname "${0}")/../vcpkg/ports.patch)
arrow_dir=$(cd -- "$(dirname -- "$0")/../.." && pwd -P)
default_vcpkg_version=$(cat "${arrow_dir}/.env" | grep "VCPKG" | cut -d "=" -f2 | tr -d '"')
default_vcpkg_ports_patch="${arrow_dir}/ci/vcpkg/ports.patch"

git clone --depth 1 --branch ${vcpkg_version} https://github.com/microsoft/vcpkg ${vcpkg_destination}
vcpkg_destination=$1
vcpkg_version=${2:-$default_vcpkg_version}
vcpkg_ports_patch=${3:-$default_vcpkg_ports_patch}

# reduce the fetched data using a shallow clone
git clone --shallow-since=2021-04-01 https://github.com/microsoft/vcpkg ${vcpkg_destination}

pushd ${vcpkg_destination}

./bootstrap-vcpkg.sh -useSystemBinaries -disableMetrics
git apply --ignore-whitespace ${vcpkg_patch}
echo "Patch successfully applied!"
git checkout "${vcpkg_version}"

if [[ "$OSTYPE" == "msys" ]]; then
./bootstrap-vcpkg.bat -disableMetrics
else
./bootstrap-vcpkg.sh -disableMetrics
fi

if [ -f "${vcpkg_ports_patch}" ]; then
git apply --verbose --ignore-whitespace ${vcpkg_ports_patch}
echo "Patch successfully applied to the VCPKG port files!"
fi

popd
Loading

0 comments on commit 2233ac5

Please sign in to comment.