docker/ci: Build oci images natively (#40557)

Signed-off-by: Ryan Northey <[email protected]>

Author: phlax

.github/config.yml

@@ -86,7 +86,7 @@ checks:
     name: >-
       Envoy/Publish and verify
     on-run:
-    - publish
+    - release
     - verify
     required: true
 
@@ -132,7 +132,7 @@ run:
   precheck-publish:
     paths:
     - "**/*"
-  publish:
+  release:
     paths:
     - .bazelrc
     - .bazelversion
@@ -141,6 +141,7 @@ run:
     - bazel/**/*
     - ci/**/*
     - contrib/**/*
+    - distribution/**/*
     - envoy/**/*
     - examples/**/*
     - source/**/*
@@ -155,6 +156,7 @@ run:
     - bazel/**/*
     - ci/**/*
     - contrib/**/*
+    - distribution/**/*
     - envoy/**/*
     - examples/**/*
     - source/**/*

.github/workflows/_publish_build.yml

@@ -6,15 +6,16 @@ permissions:
 on:
   workflow_call:
     secrets:
-      dockerhub-password:
-        required: false
       gcs-cache-key:
         required: true
       gpg-key:
         required: true
       gpg-key-password:
         required: true
     inputs:
+      arch:
+        type: string
+        required: true
       gcs-cache-bucket:
         type: string
         required: true
@@ -31,7 +32,7 @@ concurrency:
     ${{ github.actor != 'trigger-release-envoy[bot]'
         && github.event.inputs.head_ref
         || github.run_id
-    }}-${{ github.event.workflow.id }}-publish
+    }}-${{ inputs.arch }}-${{ github.event.workflow.id }}-publish
   cancel-in-progress: true
 
 
@@ -42,152 +43,86 @@ jobs:
     permissions:
       contents: read
       packages: read
-    name: ${{ matrix.name || matrix.target }}
+    name: Binary
     uses: ./.github/workflows/_run.yml
     with:
-      arch: ${{ matrix.arch }}
-      bazel-extra: ${{ matrix.bazel-extra }}
-      target: ${{ matrix.target }}
-      target-suffix: ${{ matrix.arch }}
-      cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }}
-      cache-build-image-key-suffix: ${{ matrix.arch == 'arm64' && format('-{0}', matrix.arch) || '' }}
-      concurrency-suffix: -${{ matrix.arch }}
-      gcs-cache-bucket: ${{ inputs.gcs-cache-bucket }}
-      rbe: ${{ matrix.rbe }}
-      request: ${{ inputs.request }}
-      runs-on: ${{ matrix.runs-on }}
-      timeout-minutes: 120
-      trusted: ${{ inputs.trusted }}
-      upload-name: release.${{ matrix.arch }}
-      upload-path: envoy/${{ matrix.arch }}/bin/
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-        - target: release.server_only
-          name: Release (x64)
-          arch: x64
-          bazel-extra: >-
-            --config=remote-envoy-engflow
-          rbe: true
-        - target: release.server_only
-          name: Release (arm64)
-          arch: arm64
-          bazel-extra: >-
-            --config=remote-envoy-engflow
-          rbe: true
-          runs-on: ${{ vars.ENVOY_ARM_VM || 'ubuntu-24.04-arm' }}
-
-  distribution:
-    permissions:
-      contents: read
-      packages: read
-    secrets:
-      gcs-cache-key: ${{ secrets.gcs-cache-key }}
-      gpg-key: ${{ secrets.gpg-key }}
-      gpg-key-password: ${{ secrets.gpg-key-password }}
-    name: ${{ matrix.name || matrix.target }}
-    needs:
-    - binary
-    uses: ./.github/workflows/_run.yml
-    with:
-      arch: ${{ matrix.arch }}
+      arch: ${{ inputs.arch }}
       bazel-extra: >-
-        --config=remote-cache-envoy-engflow
-      downloads: |
-        release.${{ matrix.arch }}: release/${{ matrix.arch }}/bin/
-      target: ${{ matrix.target }}
-      target-suffix: ${{ matrix.arch }}
+        --config=remote-envoy-engflow
+      target: release.server_only
+      target-suffix: ${{ inputs.arch }}
       cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }}
-      cache-build-image-key-suffix: ${{ matrix.cache-build-image-key-suffix }}
-      concurrency-suffix: -${{ matrix.arch }}
+      cache-build-image-key-suffix: ${{ inputs.arch == 'arm64' && '-arm64' || '' }}
+      concurrency-suffix: -${{ inputs.arch }}
       gcs-cache-bucket: ${{ inputs.gcs-cache-bucket }}
-      import-gpg: true
-      rbe: false
+      rbe: true
       request: ${{ inputs.request }}
-      runs-on: ${{ matrix.runs-on }}
+      runs-on: ${{ inputs.arch == 'arm64' && (vars.ENVOY_ARM_VM || 'ubuntu-24.04-arm') || null }}
+      timeout-minutes: 120
       trusted: ${{ inputs.trusted }}
-      upload-name: packages.${{ matrix.arch }}
-      upload-path: envoy/${{ matrix.arch }}
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-        - target: distribution
-          name: Package debs (x64)
-          arch: x64
-        - target: distribution
-          name: Package debs (arm64)
-          arch: arm64
-          cache-build-image-key-suffix: -arm64
-          runs-on: ${{ vars.ENVOY_ARM_VM || 'ubuntu-24.04-arm' }}
+      upload-name: release.${{ inputs.arch }}
+      upload-path: envoy/${{ inputs.arch }}/bin/
 
   docker:
     permissions:
       contents: read
       packages: read
-    secrets:
-      dockerhub-password: ${{ secrets.dockerhub-password }}
-    name: ${{ matrix.name || matrix.target }}
+    name: Docker OCI
     needs:
     - binary
     uses: ./.github/workflows/_run.yml
     with:
-      target: ${{ matrix.target }}
+      arch: ${{ inputs.arch }}
+      target: docker
+      target-suffix: ${{ inputs.arch }}
       cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }}
+      cache-build-image-key-suffix: ${{ inputs.arch == 'arm64' && '-arm64' || '' }}
+      concurrency-suffix: -${{ inputs.arch }}
       downloads: |
-        release.arm64: envoy/arm64/bin/
-        release.x64: envoy/x64/bin/
+        release.${{ inputs.arch }}: envoy/${{ inputs.arch }}/bin/
       request: ${{ inputs.request }}
       source: |
         export NO_BUILD_SETUP=1
         export ENVOY_DOCKER_IN_DOCKER=1
+        export ENVOY_DOCKER_SAVE_IMAGE=true
+        export ENVOY_OCI_DIR=build_images
+
+        # export DOCKER_BUILD_PLATFORM=${{ inputs.arch == 'x64' && 'linux/amd64' || 'linux/arm64' }}
+        # export DOCKER_LOAD_IMAGES=true
+        # export DOCKER_FORCE_OCI_OUTPUT=true
       trusted: ${{ inputs.trusted }}
-      upload-name: docker
-      upload-path: build_images
-    strategy:
-      fail-fast: false
-      matrix:
-        include:
-        - target: docker
-          name: Docker (Linux multiarch)
+      upload-name: oci.${{ inputs.arch }}
+      upload-path: envoy/${{ inputs.arch }}/build_images
+      runs-on: ${{ inputs.arch == 'arm64' && (vars.ENVOY_ARM_VM || 'ubuntu-24.04-arm') || null }}
 
-  sign:
+  distribution:
     permissions:
       contents: read
       packages: read
     secrets:
       gcs-cache-key: ${{ secrets.gcs-cache-key }}
       gpg-key: ${{ secrets.gpg-key }}
       gpg-key-password: ${{ secrets.gpg-key-password }}
-    name: ${{ matrix.name || matrix.target }}
+    name: Packages
     needs:
-    - distribution
+    - binary
     uses: ./.github/workflows/_run.yml
     with:
-      target: release.signed
+      arch: ${{ inputs.arch }}
       bazel-extra: >-
-        --//distribution:x64-packages=//distribution:custom/x64/packages.x64.tar.gz
-        --//distribution:arm64-packages=//distribution:custom/arm64/packages.arm64.tar.gz
-        --//distribution:x64-release=//distribution:custom/x64/bin/release.tar.zst
-        --//distribution:arm64-release=//distribution:custom/arm64/bin/release.tar.zst
-      cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }}
-      diskspace-hack: true
+        --config=remote-cache-envoy-engflow
       downloads: |
-        packages.arm64: envoy/arm64/
-        packages.x64: envoy/x64/
-        release.arm64: envoy/arm64/bin/
-        release.x64: envoy/x64/bin/
+        release.${{ inputs.arch }}: release/${{ inputs.arch }}/bin/
+      target: distribution
+      target-suffix: ${{ inputs.arch }}
+      cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }}
+      cache-build-image-key-suffix: ${{ inputs.arch == 'arm64' && '-arm64' || '' }}
+      concurrency-suffix: -${{ inputs.arch }}
       gcs-cache-bucket: ${{ inputs.gcs-cache-bucket }}
       import-gpg: true
+      rbe: false
       request: ${{ inputs.request }}
-      source: |
-        export NO_BUILD_SETUP=1
+      runs-on: ${{ inputs.arch == 'arm64' && (vars.ENVOY_ARM_VM || 'ubuntu-24.04-arm') || null }}
       trusted: ${{ inputs.trusted }}
-      upload-name: release.signed
-      upload-path: envoy/release.signed.tar.zst
-      steps-pre: |
-        - run: |
-            mkdir distribution/custom
-            cp -a %{{ runner.temp }}/envoy/x64 %{{ runner.temp }}/envoy/arm64 distribution/custom
-          shell: bash
+      upload-name: packages.${{ inputs.arch }}
+      upload-path: envoy/${{ inputs.arch }}

.github/workflows/_publish_publish.yml renamed to .github/workflows/_publish_release.yml

@@ -6,12 +6,18 @@ permissions:
 on:
   workflow_call:
     secrets:
+      dockerhub-password:
+      dockerhub-username:
       ENVOY_CI_SYNC_APP_ID:
       ENVOY_CI_SYNC_APP_KEY:
       ENVOY_CI_PUBLISH_APP_ID:
       ENVOY_CI_PUBLISH_APP_KEY:
       gcs-cache-key:
         required: true
+      gpg-key:
+        required: true
+      gpg-key-password:
+        required: true
     inputs:
       gcs-cache-bucket:
         type: string
@@ -33,14 +39,75 @@ concurrency:
 
 
 jobs:
-  publish:
+  sign:
+    permissions:
+      contents: read
+      packages: read
+    secrets:
+      gcs-cache-key: ${{ secrets.gcs-cache-key }}
+      gpg-key: ${{ secrets.gpg-key }}
+      gpg-key-password: ${{ secrets.gpg-key-password }}
+    if: ${{ github.repository == 'envoyproxy/envoy-ci-staging' }}
+    name: Sign packages
+    uses: ./.github/workflows/_run.yml
+    with:
+      target: release.signed
+      bazel-extra: >-
+        --//distribution:x64-packages=//distribution:custom/x64/packages.x64.tar.gz
+        --//distribution:arm64-packages=//distribution:custom/arm64/packages.arm64.tar.gz
+        --//distribution:x64-release=//distribution:custom/x64/bin/release.tar.zst
+        --//distribution:arm64-release=//distribution:custom/arm64/bin/release.tar.zst
+      cache-build-image: ${{ fromJSON(inputs.request).request.build-image.default }}
+      diskspace-hack: true
+      downloads: |
+        packages.arm64: envoy/arm64/
+        packages.x64: envoy/x64/
+        release.arm64: envoy/arm64/bin/
+        release.x64: envoy/x64/bin/
+      gcs-cache-bucket: ${{ inputs.gcs-cache-bucket }}
+      import-gpg: true
+      request: ${{ inputs.request }}
+      source: |
+        export NO_BUILD_SETUP=1
+      trusted: ${{ inputs.trusted }}
+      upload-name: release.signed
+      upload-path: envoy/release.signed.tar.zst
+      steps-pre: |
+        - run: |
+            mkdir distribution/custom
+            cp -a %{{ runner.temp }}/envoy/x64 %{{ runner.temp }}/envoy/arm64 distribution/custom
+          shell: bash
+
+  container:
+    secrets:
+      dockerhub-username: ${{ secrets.dockerhub-username }}
+      dockerhub-password: ${{ secrets.dockerhub-password }}
+    permissions:
+      contents: read
+      packages: read
+    name: Publish container images
+    uses: ./.github/workflows/_publish_release_container.yml
+    with:
+      dockerhub-repo: ${{ vars.DOCKERHUB_REPO || 'envoy' }}
+      dev: ${{ fromJSON(inputs.request).request.version.dev }}
+      sha: ${{ fromJSON(inputs.request).request.sha }}
+      target-branch: ${{ fromJSON(inputs.request).request.target-branch }}
+      trusted: ${{ inputs.trusted }}
+      version-major: ${{ fromJSON(inputs.request).request.version.major }}
+      version-minor: ${{ fromJSON(inputs.request).request.version.minor }}
+      version-patch: ${{ fromJSON(inputs.request).request.version.patch }}
+
+  release:
     secrets:
       app-id: ${{ inputs.trusted && secrets.ENVOY_CI_PUBLISH_APP_ID || '' }}
       app-key: ${{ inputs.trusted && secrets.ENVOY_CI_PUBLISH_APP_KEY || '' }}
       gcs-cache-key: ${{ secrets.gcs-cache-key }}
     permissions:
       contents: read
       packages: read
+    needs:
+    - container
+    - sign
     name: ${{ matrix.name || matrix.target }}
     uses: ./.github/workflows/_run.yml
     with:
@@ -65,7 +132,7 @@ jobs:
             export ENVOY_REPO=${{ github.repository }}
             export ENVOY_PUBLISH_DRY_RUN=${{ (fromJSON(inputs.request).request.version.dev || ! inputs.trusted) && 1 || '' }}
 
-  publish_docs:
+  docs:
     # For normal commits to Envoy main this will trigger an update in the website repo,
     # which will update its envoy dep shas, and rebuild the website for the latest docs
     #
@@ -76,7 +143,7 @@ jobs:
     if: ${{ inputs.trusted && github.repository == 'envoyproxy/envoy' }}
     runs-on: ${{ fromJSON(inputs.request).config.ci.agent-ubuntu }}
     needs:
-    - publish
+    - release
     steps:
     - uses: envoyproxy/toolshed/gh-actions/[email protected]
       id: appauth