ci/docker: Update run_envoy_docker.sh to use docker compose (#40453)

phlax · phlax · commit c83e0dc72c96 · 2025-08-02T22:40:49.000+01:00
also removes no longer used/tested windows container support

Signed-off-by: Ryan Northey &lt;ryan@synca.io&gt;
diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml
@@ -0,0 +1,112 @@
+x-envoy-build-base: &envoy-build-base
+  image: >-
+    ${ENVOY_BUILD_IMAGE:-envoyproxy/envoy-build-ubuntu:f4a881a1205e8e6db1a57162faf3df7aed88eae8@sha256:b10346fe2eee41733dbab0e02322c47a538bf3938d093a5daebad9699860b814}
+  user: root:root
+  working_dir: ${ENVOY_DOCKER_SOURCE_DIR:-/source}
+  stdin_open: true
+  tty: true
+  platform: ${ENVOY_DOCKER_PLATFORM:-}
+  environment:
+  # Core build environment
+  - BUILD_DIR=/build
+  - ENVOY_DOCKER_SOURCE_DIR=${ENVOY_DOCKER_SOURCE_DIR:-/source}
+  - ENVOY_DOCKER_BUILD_DIR="${ENVOY_DOCKER_BUILD_DIR:-/tmp/envoy-docker-build}"
+
+  # Proxy settings
+  - HTTP_PROXY
+  - HTTPS_PROXY
+  - NO_PROXY
+  - GOPROXY
+
+  # Bazel configuration
+  - BAZEL_STARTUP_OPTIONS
+  - BAZEL_BUILD_EXTRA_OPTIONS
+  - BAZEL_EXTRA_TEST_OPTIONS
+  - BAZEL_REMOTE_CACHE
+  - BAZEL_STARTUP_EXTRA_OPTIONS
+  - BAZEL_REMOTE_INSTANCE
+  - BAZELISK_BASE_URL
+
+  # CI/CD variables
+  - CI_BRANCH
+  - CI_SHA1
+  - CI_TARGET_BRANCH
+  - BUILD_REASON
+  - GITHUB_REF_NAME
+  - GITHUB_REF_TYPE
+  - GITHUB_TOKEN
+  - GITHUB_APP_ID
+  - GITHUB_INSTALL_ID
+
+  # Build configuration
+  - NUM_CPUS
+  - ENVOY_BRANCH
+  - ENVOY_RBE
+  - ENVOY_BUILD_IMAGE
+  - ENVOY_SRCDIR
+  - ENVOY_BUILD_TARGET
+  - ENVOY_BUILD_DEBUG_INFORMATION
+  - ENVOY_BUILD_FILTER_EXAMPLE
+  - ENVOY_COMMIT
+  - ENVOY_HEAD_REF
+  - ENVOY_REPO
+  - ENVOY_BUILD_ARCH
+  - ENVOY_GEN_COMPDB_OPTIONS
+
+  # Publishing and artifacts
+  - DOCKERHUB_USERNAME
+  - DOCKERHUB_PASSWORD
+  - ENVOY_DOCKER_SAVE_IMAGE
+  - ENVOY_PUBLISH_DRY_RUN
+  - ENVOY_TARBALL_DIR
+  - GCS_ARTIFACT_BUCKET
+  - GCS_REDIRECT_PATH
+  - GCP_SERVICE_ACCOUNT_KEY
+  - GCP_SERVICE_ACCOUNT_KEY_PATH
+
+  - MOBILE_DOCS_CHECKOUT_DIR
+  - SYSTEM_STAGEDISPLAYNAME
+  - SYSTEM_JOBDISPLAYNAME
+  - SSH_AUTH_SOCK
+
+  entrypoint:
+  - "/bin/bash"
+  - "-c"
+  - |
+    groupadd --gid ${DOCKER_GID:-${USER_GID:-$(id -g)}} -f envoygroup
+    useradd -o \
+        --uid ${USER_UID:-$(id -u)} \
+        --gid ${DOCKER_GID:-${USER_GID:-$(id -g)}} \
+        --no-create-home \
+        -s /bin/bash \
+        --home-dir /build envoybuild
+    usermod -a -G pcap envoybuild
+    chown envoybuild:envoygroup /build
+    chown envoybuild /proc/self/fd/2 2>/dev/null || true
+    [[ -e /entrypoint-extra.sh ]] && /entrypoint-extra.sh
+    sudo -EHs -u envoybuild bash -c 'cd ${ENVOY_DOCKER_SOURCE_DIR:-/source} && exec ${DOCKER_COMMAND:-bash}'
+
+services:
+  envoy-build:
+    <<: *envoy-build-base
+    volumes:
+    - ${ENVOY_DOCKER_BUILD_DIR:-/tmp/envoy-docker-build}:/build
+    - ${SOURCE_DIR:-..}:/source
+    - ${SHARED_TMP_DIR:-/tmp/bazel-shared}:${SHARED_TMP_DIR:-/tmp/bazel-shared}
+
+  envoy-build-gpg:
+    <<: *envoy-build-base
+    volumes:
+    - ${ENVOY_DOCKER_BUILD_DIR:-/tmp/envoy-docker-build}:/build
+    - ${SOURCE_DIR:-..}:/source
+    - ${ENVOY_GPG_DIR-${HOME}/.gnupg}:/build/.gnupg
+    - ${SHARED_TMP_DIR:-/tmp/bazel-shared}:${SHARED_TMP_DIR:-/tmp/bazel-shared}
+
+  envoy-build-dind:
+    privileged: true
+    <<: *envoy-build-base
+    volumes:
+    - ${ENVOY_DOCKER_BUILD_DIR:-/tmp/envoy-docker-build}:/build
+    - ${SOURCE_DIR:-..}:/source
+    - /var/run/docker.sock:/var/run/docker.sock
+    - ${SHARED_TMP_DIR:-/tmp/bazel-shared}:${SHARED_TMP_DIR:-/tmp/bazel-shared}
diff --git a/ci/run_envoy_docker.sh b/ci/run_envoy_docker.sh
@@ -2,172 +2,56 @@
 
 set -e
 
-CURRENT_SCRIPT_DIR="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"
+# TODO(phlax): Add a check that a usable version of docker compose is available
 
-# shellcheck source=ci/envoy_build_sha.sh
-. "${CURRENT_SCRIPT_DIR}"/envoy_build_sha.sh
 
-function is_windows() {
-  [[ "$(uname -s)" == *NT* ]]
-}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Source build SHA information
+# shellcheck source=ci/envoy_build_sha.sh
+source "${SCRIPT_DIR}/envoy_build_sha.sh"
 
-read -ra ENVOY_DOCKER_OPTIONS <<< "${ENVOY_DOCKER_OPTIONS:-}"
+# User/group IDs
+USER_UID="$(id -u)"
+USER_GID="$(id -g)"
+export USER_UID
+export USER_GID
 
+# These should probably go in users .env as docker compose will pick that up
 export HTTP_PROXY="${HTTP_PROXY:-${http_proxy:-}}"
 export HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy:-}}"
 export NO_PROXY="${NO_PROXY:-${no_proxy:-}}"
 export GOPROXY="${GOPROXY:-${go_proxy:-}}"
 
-if is_windows; then
-  [[ -z "${IMAGE_NAME}" ]] && IMAGE_NAME="envoyproxy/envoy-build-windows2019"
-  # TODO(sunjayBhatia): Currently ENVOY_DOCKER_OPTIONS is ignored on Windows because
-  # CI sets it to a Linux-specific value. Undo this once https://github.com/envoyproxy/envoy/issues/13272
-  # is resolved.
-  ENVOY_DOCKER_OPTIONS=()
-  # Replace MSYS style drive letter (/c/) with Windows drive letter designation (C:/)
-  DEFAULT_ENVOY_DOCKER_BUILD_DIR=$(echo "${TEMP}" | sed -E "s#^/([a-zA-Z])/#\1:/#")/envoy-docker-build
-  BUILD_DIR_MOUNT_DEST=C:/build
-  SOURCE_DIR=$(echo "${PWD}" | sed -E "s#^/([a-zA-Z])/#\1:/#")
-  SOURCE_DIR_MOUNT_DEST=C:/source
-  START_COMMAND=("bash" "-c" "cd /c/source && export HOME=/c/build && $*")
-else
-  [[ -z "${IMAGE_NAME}" ]] && IMAGE_NAME="envoyproxy/envoy-build-ubuntu"
-  # We run as root and later drop permissions. This is required to setup the USER
-  # in useradd below, which is need for correct Python execution in the Docker
-  # environment.
-  ENVOY_DOCKER_OPTIONS+=(-u root:root)
-  DOCKER_USER_ARGS=()
-  DOCKER_GROUP_ARGS=()
-  DEFAULT_ENVOY_DOCKER_BUILD_DIR=/tmp/envoy-docker-build
-  USER_UID="$(id -u)"
-  USER_GID="$(id -g)"
-  if [[ -n "$ENVOY_DOCKER_IN_DOCKER" ]]; then
-      ENVOY_DOCKER_OPTIONS+=(-v /var/run/docker.sock:/var/run/docker.sock)
-      DOCKER_GID="$(stat -c %g /var/run/docker.sock 2>/dev/null || stat -f %g /var/run/docker.sock)"
-      DOCKER_USER_ARGS=(--gid "${DOCKER_GID}")
-      DOCKER_GROUP_ARGS=(--gid "${DOCKER_GID}")
-  else
-      DOCKER_GROUP_ARGS+=(--gid "${USER_GID}")
-      DOCKER_USER_ARGS+=(--gid "${USER_GID}")
-  fi
-  BUILD_DIR_MOUNT_DEST=/build
-  SOURCE_DIR="${PWD}"
-  SOURCE_DIR_MOUNT_DEST=/source
-  ENVOY_DOCKER_SOURCE_DIR="${ENVOY_DOCKER_SOURCE_DIR:-${SOURCE_DIR_MOUNT_DEST}}"
-  START_COMMAND=(
-      "/bin/bash"
-      "-lc"
-      "groupadd ${DOCKER_GROUP_ARGS[*]} -f envoygroup \
-          && useradd -o --uid ${USER_UID} ${DOCKER_USER_ARGS[*]} --no-create-home --home-dir /build envoybuild \
-          && usermod -a -G pcap envoybuild \
-          && chown envoybuild:envoygroup /build \
-          && chown envoybuild /proc/self/fd/2 \
-          && sudo -EHs -u envoybuild bash -c 'cd ${ENVOY_DOCKER_SOURCE_DIR} && $*'")
+# Docker-in-Docker handling
+if [[ -n "$ENVOY_DOCKER_IN_DOCKER" ]]; then
+    DOCKER_GID="$(stat -c %g /var/run/docker.sock 2>/dev/null || echo "$USER_GID")"
+    export DOCKER_GID
+fi
+
+if [[ -n "$ENVOY_DOCKER_IN_DOCKER" || -n "$ENVOY_SHARED_TMP_DIR" ]]; then
+    export SHARED_TMP_DIR="${ENVOY_SHARED_TMP_DIR:-/tmp/bazel-shared}"
+    mkdir -p "${SHARED_TMP_DIR}"
+    chmod 777 "${SHARED_TMP_DIR}"
 fi
 
 if [[ -n "$ENVOY_DOCKER_PLATFORM" ]]; then
     echo "Setting Docker platform: ${ENVOY_DOCKER_PLATFORM}"
     docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
-    ENVOY_DOCKER_OPTIONS+=(--platform "$ENVOY_DOCKER_PLATFORM")
 fi
 
-# The IMAGE_ID defaults to the CI hash but can be set to an arbitrary image ID (found with 'docker
-# images').
-if [[ -z "${IMAGE_ID}" ]]; then
-    IMAGE_ID="${ENVOY_BUILD_SHA}"
-    if ! is_windows && [[ -n "$ENVOY_BUILD_CONTAINER_SHA" ]]; then
-        IMAGE_ID="${ENVOY_BUILD_SHA}@sha256:${ENVOY_BUILD_CONTAINER_SHA}"
-    fi
-fi
-[[ -z "${ENVOY_DOCKER_BUILD_DIR}" ]] && ENVOY_DOCKER_BUILD_DIR="${DEFAULT_ENVOY_DOCKER_BUILD_DIR}"
-# Replace backslash with forward slash for Windows style paths
-ENVOY_DOCKER_BUILD_DIR="${ENVOY_DOCKER_BUILD_DIR//\\//}"
-mkdir -p "${ENVOY_DOCKER_BUILD_DIR}"
-
-[[ -t 1 ]] && ENVOY_DOCKER_OPTIONS+=("-it")
-[[ -f .git ]] && [[ ! -d .git ]] && ENVOY_DOCKER_OPTIONS+=(-v "$(git rev-parse --git-common-dir):$(git rev-parse --git-common-dir)")
-[[ -n "${SSH_AUTH_SOCK}" ]] && ENVOY_DOCKER_OPTIONS+=(-v "${SSH_AUTH_SOCK}:${SSH_AUTH_SOCK}" -e SSH_AUTH_SOCK)
-
-export ENVOY_BUILD_IMAGE="${IMAGE_NAME}:${IMAGE_ID}"
-
-VOLUMES=(
-    -v "${ENVOY_DOCKER_BUILD_DIR}":"${BUILD_DIR_MOUNT_DEST}"
-    -v "${SOURCE_DIR}":"${SOURCE_DIR_MOUNT_DEST}")
+export DOCKER_COMMAND="${*:-bash}"
 
+COMPOSE_SERVICE="envoy-build"
 if [[ -n "$MOUNT_GPG_HOME" ]]; then
-    VOLUMES+=(
-        -v "${HOME}/.gnupg:${BUILD_DIR_MOUNT_DEST}/.gnupg")
-fi
-
-if ! is_windows; then
-    export BUILD_DIR="${BUILD_DIR_MOUNT_DEST}"
-fi
-
-if [[ -n "$ENVOY_DOCKER_IN_DOCKER" || -n "$ENVOY_SHARED_TMP_DIR" ]]; then
-    # Create a "shared" directory that has the same path in/outside the container
-    # This allows the host docker engine to see artefacts using a temporary path created inside the container,
-    # at the same path.
-    # For example, a directory created with `mktemp -d --tmpdir /tmp/bazel-shared` can be mounted as a volume
-    # from within the build container.
-    SHARED_TMP_DIR="${ENVOY_SHARED_TMP_DIR:-/tmp/bazel-shared}"
-    mkdir -p "${SHARED_TMP_DIR}"
-    chmod +rwx "${SHARED_TMP_DIR}"
-    VOLUMES+=(-v "${SHARED_TMP_DIR}":"${SHARED_TMP_DIR}")
-fi
-
-if [[ -n "${ENVOY_DOCKER_PULL}" ]]; then
-    time docker pull "${ENVOY_BUILD_IMAGE}"
+    COMPOSE_SERVICE="envoy-build-gpg"
+elif [[ -n "$ENVOY_DOCKER_IN_DOCKER" ]]; then
+    COMPOSE_SERVICE="envoy-build-dind"
 fi
 
-# Since we specify an explicit hash, docker-run will pull from the remote repo if missing.
-docker run --rm \
-       "${ENVOY_DOCKER_OPTIONS[@]}" \
-       "${VOLUMES[@]}" \
-       -e BUILD_DIR \
-       -e HTTP_PROXY \
-       -e HTTPS_PROXY \
-       -e NO_PROXY \
-       -e GOPROXY \
-       -e BAZEL_STARTUP_OPTIONS \
-       -e BAZEL_BUILD_EXTRA_OPTIONS \
-       -e BAZEL_EXTRA_TEST_OPTIONS \
-       -e BAZEL_REMOTE_CACHE \
-       -e BAZEL_STARTUP_EXTRA_OPTIONS \
-       -e CI_BRANCH \
-       -e CI_SHA1 \
-       -e CI_TARGET_BRANCH \
-       -e DOCKERHUB_USERNAME \
-       -e DOCKERHUB_PASSWORD \
-       -e ENVOY_DOCKER_SAVE_IMAGE \
-       -e BUILD_REASON \
-       -e BAZEL_REMOTE_INSTANCE \
-       -e GCP_SERVICE_ACCOUNT_KEY \
-       -e GCP_SERVICE_ACCOUNT_KEY_PATH \
-       -e NUM_CPUS \
-       -e ENVOY_BRANCH \
-       -e ENVOY_RBE \
-       -e ENVOY_BUILD_IMAGE \
-       -e ENVOY_SRCDIR \
-       -e ENVOY_BUILD_TARGET \
-       -e ENVOY_BUILD_DEBUG_INFORMATION \
-       -e ENVOY_BUILD_FILTER_EXAMPLE \
-       -e ENVOY_COMMIT \
-       -e ENVOY_HEAD_REF \
-       -e ENVOY_PUBLISH_DRY_RUN \
-       -e ENVOY_REPO \
-       -e ENVOY_TARBALL_DIR \
-       -e ENVOY_GEN_COMPDB_OPTIONS \
-       -e GCS_ARTIFACT_BUCKET \
-       -e GCS_REDIRECT_PATH \
-       -e GITHUB_REF_NAME \
-       -e GITHUB_REF_TYPE \
-       -e GITHUB_TOKEN \
-       -e GITHUB_APP_ID \
-       -e GITHUB_INSTALL_ID \
-       -e MOBILE_DOCS_CHECKOUT_DIR \
-       -e BAZELISK_BASE_URL \
-       -e ENVOY_BUILD_ARCH \
-       -e SYSTEM_STAGEDISPLAYNAME \
-       -e SYSTEM_JOBDISPLAYNAME \
-       "${ENVOY_BUILD_IMAGE}" \
-       "${START_COMMAND[@]}"
+exec docker compose \
+    -f "${SCRIPT_DIR}/docker-compose.yml" \
+    ${ENVOY_DOCKER_PLATFORM:+-p "$ENVOY_DOCKER_PLATFORM"} \
+    run \
+    --rm \
+    "${COMPOSE_SERVICE}"
