One python version per container

.github/workflows/build-and-publish-images.yml

@@ -0,0 +1,103 @@
+name: build and publish imgs workflow
+
+on:
+  workflow_call:
+    inputs:
+      build_type:
+        required: true
+        type: string
+
+defaults:
+  run:
+    shell: bash
+
+jobs:
+  compute-matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      MATRIX: ${{ steps.compute-matrix.outputs.MATRIX }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Compute matrix
+        id: compute-matrix
+        run: |
+          MATRIX=$(ci/compute-mx.sh)
+          echo "MATRIX=${MATRIX}" | tee -a ${GITHUB_OUTPUT}
+  docker:
+    needs: compute-matrix
+    strategy:
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
+      fail-fast: false
+    secrets: inherit
+    uses: ./.github/workflows/build-image.yml
+    with:
+      ARCHES: ${{ toJSON(matrix.ARCHES) }}
+      CUDA_VER: ${{ matrix.CUDA_VER }}
+      LINUX_VER: ${{ matrix.LINUX_VER }}
+      PYTHON_VER: ${{ matrix.PYTHON_VER }}
+      IMAGE_REPO: ${{ matrix.IMAGE_REPO }}
+      TAG: rapidsai/${{ matrix.IMAGE_REPO }}:cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PYTHON_VER }}
+      BUILD_TYPE: ${{ inputs.build_type }}
+  build-multiarch-manifest:
+    # if: inputs.build_type == 'branch'
+    needs: [docker, compute-matrix]
+    strategy:
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
+          password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
+      - name: Create multiarch manifest
+        run: |
+          LATEST_CUDA_VER=$(yq '.CUDA_VER | sort | .[-1]' axis.yaml)
+          LATEST_PYTHON_VER=$(yq -o json '.PYTHON_VER' axis.yaml | jq -r 'max_by(split(".") | map(tonumber))')
+          LATEST_UBUNTU_VER=$(yq '.LINUX_VER | map(select(. == "*ubuntu*")) | sort | .[-1]' axis.yaml)
+
+          source_tags=()
+          tag="rapidsai/${{ matrix.IMAGE_REPO }}:cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PYTHON_VER }}"
+          for arch in $(echo '${{ toJSON(matrix.ARCHES) }}' | jq .[] -r); do
+            source_tags+=("${tag}-${arch}")
+          done
+
+          docker manifest create ${tag} ${source_tags[@]}
+          docker manifest push ${tag}
+          if [[
+            "${LATEST_UBUNTU_VER}" == "${{ matrix.LINUX_VER }}" &&
+            "${LATEST_CUDA_VER}" == "${{ matrix.CUDA_VER }}" &&
+            "${LATEST_PYTHON_VER}" == "${{ matrix.PYTHON_VER }}"
+          ]]; then
+            docker manifest create rapidsai/${{ matrix.IMAGE_REPO }}:latest ${source_tags[@]}
+            docker manifest push rapidsai/${{ matrix.IMAGE_REPO }}:latest
+          fi
+  delete-temp-images:
+    # if: inputs.build_type == 'branch'
+    needs: [compute-matrix, build-multiarch-manifest]
+    strategy:
+      matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Remove temporary images
+        run: |
+          HUB_TOKEN=$(
+            curl -s -H "Content-Type: application/json" \
+              -X POST \
+              -d "{\"username\": \"${{ secrets.GPUCIBOT_DOCKERHUB_USER }}\", \"password\": \"${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}\"}" \
+              https://hub.docker.com/v2/users/login/ | jq -r .token \
+          )
+
+          org="rapidsai"
+          repo="${{ matrix.IMAGE_REPO }}"
+          tag="cuda${{ matrix.CUDA_VER }}-${{ matrix.LINUX_VER }}-py${{ matrix.PYTHON_VER }}"
+
+          for arch in $(echo '${{ toJSON(matrix.ARCHES) }}' | jq .[] -r); do
+            curl -i -X DELETE \
+              -H "Accept: application/json" \
+              -H "Authorization: JWT $HUB_TOKEN" \
+              "https://hub.docker.com/v2/repositories/$org/$repo/tags/$tag-$arch/"
+          done

.github/workflows/build-image.yml

@@ -0,0 +1,81 @@
+name: Build and push image variant
+
+on:
+  workflow_call:
+    inputs:
+      ARCHES:
+        required: true
+        type: string
+      CUDA_VER:
+        required: true
+        type: string
+      LINUX_VER:
+        required: true
+        type: string
+      PYTHON_VER:
+        required: true
+        type: string
+      IMAGE_REPO:
+        required: true
+        type: string
+      TAG:
+        required: true
+        type: string
+      BUILD_TYPE:
+        required: true
+        type: string
+
+jobs:
+  docker-build:
+    strategy:
+      matrix:
+        ARCH: ${{ fromJSON(inputs.ARCHES) }}
+        CUDA_VER: ["${{ inputs.CUDA_VER }}"]
+        LINUX_VER: ["${{ inputs.LINUX_VER }}"]
+        PYTHON_VER: ["${{ inputs.PYTHON_VER }}"]
+      fail-fast: false
+    runs-on: "linux-${{ matrix.ARCH }}-cpu4"
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.GPUCIBOT_DOCKERHUB_USER }}
+          password: ${{ secrets.GPUCIBOT_DOCKERHUB_TOKEN }}
+      - name: Set up Docker Context for Buildx
+        id: buildx-context
+        run: |
+          docker context create builders
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          driver: docker
+          endpoint: builders
+      - name: Get Real Arch and Manylinux Version
+        id: get-real-arch
+        run: |
+          REAL_ARCH=$(arch)
+          echo "REAL_ARCH=${REAL_ARCH}" >> $GITHUB_OUTPUT
+          if [[ "${{ inputs.LINUX_VER }}" == "ubuntu20.04" || "${{ inputs.LINUX_VER }}" == "ubuntu18.04" ]]; then
+            echo "MANYLINUX_VER=manylinux_2_31" >> $GITHUB_OUTPUT
+          else
+            echo "MANYLINUX_VER=manylinux_2_17" >> $GITHUB_OUTPUT
+          fi
+      - name: Build image
+        if: 
+        uses: docker/build-push-action@v4
+        with:
+          context: ./${{ inputs.IMAGE_REPO }}
+          file: ./${{ inputs.IMAGE_REPO }}/Dockerfile
+          # push: ${{ inputs.BUILD_TYPE == 'branch' }}
+          push: true
+          pull: true
+          build-args: |
+            CUDA_VER=${{ inputs.CUDA_VER }}
+            LINUX_VER=${{ inputs.LINUX_VER }}
+            PYTHON_VER=${{ inputs.PYTHON_VER }}
+            CPU_ARCH=${{ matrix.ARCH }}
+            REAL_ARCH=${{ steps.get-real-arch.outputs.REAL_ARCH }}
+            MANYLINUX_VER=${{ steps.get-real-arch.outputs.MANYLINUX_VER }}
+          tags: ${{ inputs.TAG }}-${{ matrix.ARCH }}