Skip to content

Commit

Permalink
ARROW-14064: [CI] Use Debian 11
Browse files Browse the repository at this point in the history
Debian 11 is the current stable release.

Closes apache#11209 from kou/ci-debian-11

Authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Sutou Kouhei <[email protected]>
  • Loading branch information
kou committed Sep 22, 2021
1 parent 3317f83 commit 075c6c6
Show file tree
Hide file tree
Showing 9 changed files with 212 additions and 24 deletions.
2 changes: 1 addition & 1 deletion .env
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ ARCH_SHORT_ALIAS=x64
ULIMIT_CORE=-1
REPO=apache/arrow-dev
CUDA=9.1
DEBIAN=10
DEBIAN=11
UBUNTU=20.04
FEDORA=33
PYTHON=3.6
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ env:
jobs:

docker:
name: AMD64 Debian 10 Go ${{ matrix.go }}
name: AMD64 Debian 11 Go ${{ matrix.go }}
runs-on: ubuntu-latest
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
timeout-minutes: 15
Expand Down Expand Up @@ -76,7 +76,7 @@ jobs:
run: archery docker push debian-go

docker_cgo_python:
name: AMD64 Debian 10 GO ${{ matrix.go }} - CGO Python
name: AMD64 Debian 11 GO ${{ matrix.go }} - CGO Python
runs-on: ubuntu-latest
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
timeout-minutes: 15
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/js.yml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ env:
jobs:

docker:
name: AMD64 Debian 10 NodeJS 14
name: AMD64 Debian 11 NodeJS 14
runs-on: ubuntu-latest
if: ${{ !contains(github.event.pull_request.title, 'WIP') }}
timeout-minutes: 60
Expand Down
98 changes: 98 additions & 0 deletions ci/docker/debian-11-cpp.dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

ARG arch=amd64
FROM ${arch}/debian:11
ARG arch

ENV DEBIAN_FRONTEND noninteractive

ARG llvm
RUN apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
apt-transport-https \
ca-certificates \
gnupg \
wget && \
wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
echo "deb https://apt.llvm.org/bullseye/ llvm-toolchain-bullseye-${llvm} main" > \
/etc/apt/sources.list.d/llvm.list && \
apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
autoconf \
ccache \
clang-${llvm} \
cmake \
g++ \
gcc \
gdb \
git \
libbenchmark-dev \
libboost-all-dev \
libbrotli-dev \
libbz2-dev \
libc-ares-dev \
libcurl4-openssl-dev \
libgflags-dev \
libgmock-dev \
libgoogle-glog-dev \
libgrpc++-dev \
liblz4-dev \
libre2-dev \
libsnappy-dev \
libssl-dev \
libthrift-dev \
libutf8proc-dev \
libzstd-dev \
llvm-${llvm}-dev \
make \
ninja-build \
pkg-config \
protobuf-compiler-grpc \
rapidjson-dev \
tzdata \
zlib1g-dev && \
apt-get clean && \
rm -rf /var/lib/apt/lists/*

COPY ci/scripts/install_minio.sh \
/arrow/ci/scripts/
RUN /arrow/ci/scripts/install_minio.sh ${arch} linux latest /usr/local

ENV ARROW_BUILD_TESTS=ON \
ARROW_DATASET=ON \
ARROW_DEPENDENCY_SOURCE=SYSTEM \
ARROW_FLIGHT=ON \
ARROW_GANDIVA=ON \
ARROW_HOME=/usr/local \
ARROW_ORC=ON \
ARROW_PARQUET=ON \
ARROW_PLASMA=ON \
ARROW_S3=ON \
ARROW_USE_CCACHE=ON \
ARROW_WITH_BROTLI=ON \
ARROW_WITH_BZ2=ON \
ARROW_WITH_LZ4=ON \
ARROW_WITH_SNAPPY=ON \
ARROW_WITH_ZLIB=ON \
ARROW_WITH_ZSTD=ON \
AWSSDK_SOURCE=BUNDLED \
CC=gcc \
CXX=g++ \
ORC_SOURCE=BUNDLED \
PATH=/usr/lib/ccache/:$PATH \
Protobuf_SOURCE=BUNDLED
36 changes: 36 additions & 0 deletions ci/docker/debian-11-go-cgo-python.dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

ARG base
FROM ${base}

ENV DEBIAN_FRONTEND noninteractive

# Install python3 and pip so we can install pyarrow to test the C data interface.
RUN apt-get update -y -q && \
apt-get install -y -q --no-install-recommends \
python3 \
python3-pip && \
apt-get clean

RUN ln -s /usr/bin/python3 /usr/local/bin/python && \
ln -s /usr/bin/pip3 /usr/local/bin/pip

# Need a newer pip than Debian's to install manylinux201x wheels
RUN pip install -U pip

RUN pip install pyarrow cffi --only-binary pyarrow
26 changes: 26 additions & 0 deletions ci/docker/debian-11-go.dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

ARG arch=amd64
ARG go=1.15
FROM ${arch}/golang:${go}-buster


# TODO(kszucs):
# 1. add the files required to install the dependencies to .dockerignore
# 2. copy these files to their appropriate path
# 3. download and compile the dependencies
27 changes: 27 additions & 0 deletions ci/docker/debian-11-js.dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

ARG arch=amd64
ARG node=14
FROM ${arch}/node:${node}

ENV NODE_NO_WARNINGS=1

# TODO(kszucs):
# 1. add the files required to install the dependencies to .dockerignore
# 2. copy these files to their appropriate path
# 3. download and compile the dependencies
11 changes: 6 additions & 5 deletions dev/tasks/tasks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -875,12 +875,12 @@ tasks:
image: ubuntu-cpp
{% endfor %}

test-debian-10-cpp:
test-debian-11-cpp:
ci: github
template: docker-tests/github.linux.yml
params:
env:
DEBIAN: 10
DEBIAN: 11
image: debian-cpp

test-fedora-33-cpp:
Expand Down Expand Up @@ -954,12 +954,12 @@ tasks:
PYTEST_ARGS: "-m hypothesis"
image: conda-python-pandas

test-debian-10-python-3:
test-debian-11-python-3:
ci: azure
template: docker-tests/azure.linux.yml
params:
env:
DEBIAN: 10
DEBIAN: 11
run: debian-python

test-ubuntu-18.04-python-3:
Expand Down Expand Up @@ -1129,11 +1129,12 @@ tasks:
ci: github
template: r/github.linux.revdepcheck.yml

test-debian-10-go-1.15:
test-debian-11-go-1.15:
ci: azure
template: docker-tests/azure.linux.yml
params:
env:
DEBIAN: 11
GO: 1.15
run: debian-go

Expand Down
30 changes: 15 additions & 15 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,7 @@ services:
# docker-compose run --rm debian-cpp
# Parameters:
# ARCH: amd64, arm64v8, ...
# DEBIAN: 9, 10
# DEBIAN: 10, 11
image: ${REPO}:${ARCH}-debian-${DEBIAN}-cpp
build:
context: .
Expand Down Expand Up @@ -465,7 +465,7 @@ services:
# docker-compose run --rm debian-c-glib
# Parameters:
# ARCH: amd64, arm64v8, ...
# DEBIAN: 9, 10
# DEBIAN: 10, 11
image: ${REPO}:${ARCH}-debian-${DEBIAN}-c-glib
build:
context: .
Expand Down Expand Up @@ -524,7 +524,7 @@ services:
# docker-compose run --rm debian-ruby
# Parameters:
# ARCH: amd64, arm64v8, ...
# DEBIAN: 9, 10
# DEBIAN: 10, 11
image: ${REPO}:${ARCH}-debian-${DEBIAN}-ruby
build:
context: .
Expand Down Expand Up @@ -633,7 +633,7 @@ services:
# docker-compose run --rm debian-python
# Parameters:
# ARCH: amd64, arm64v8, ...
# DEBIAN: 9, 10
# DEBIAN: 10, 11
image: ${REPO}:${ARCH}-debian-${DEBIAN}-python-3
build:
context: .
Expand Down Expand Up @@ -1183,12 +1183,12 @@ services:
# Usage:
# docker-compose build debian-go
# docker-compose run debian-go
image: ${REPO}:${ARCH}-debian-10-go-${GO}
image: ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO}
build:
context: .
dockerfile: ci/docker/debian-10-go.dockerfile
dockerfile: ci/docker/debian-${DEBIAN}-go.dockerfile
cache_from:
- ${REPO}:${ARCH}-debian-10-go-${GO}
- ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO}
args:
arch: ${ARCH}
go: ${GO}
Expand All @@ -1203,32 +1203,32 @@ services:
# Usage:
# docker-compose build debian-go-cgo-python
# docker-compose run debian-go-cgo-python
image: ${REPO}:${ARCH}-debian-10-go-${GO}-cgo-python
image: ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO}-cgo-python
build:
context: .
dockerfile: ci/docker/debian-10-go-cgo-python.dockerfile
dockerfile: ci/docker/debian-${DEBIAN}-go-cgo-python.dockerfile
cache_from:
- ${REPO}:${ARCH}-debian-10-go-${GO}-cgo-python
- ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO}-cgo-python
args:
base: ${REPO}:${ARCH}-debian-10-go-${GO}
base: ${REPO}:${ARCH}-debian-${DEBIAN}-go-${GO}
shm_size: *shm-size
volumes: *debian-volumes
command: &go-cgo-python-command >
/bin/bash -c "
/arrow/ci/scripts/go_cgo_python_test.sh /arrow"

############################# JavaScript ####################################

debian-js:
# Usage:
# docker-compose build debian-js
# docker-compose run debian-js
image: ${REPO}:${ARCH}-debian-10-js-${NODE}
image: ${REPO}:${ARCH}-debian-${DEBIAN}-js-${NODE}
build:
context: .
dockerfile: ci/docker/debian-10-js.dockerfile
dockerfile: ci/docker/debian-${DEBIAN}-js.dockerfile
cache_from:
- ${REPO}:${ARCH}-debian-10-js-${NODE}
- ${REPO}:${ARCH}-debian-${DEBIAN}-js-${NODE}
args:
arch: ${ARCH}
node: ${NODE}
Expand Down

0 comments on commit 075c6c6

Please sign in to comment.