Merge pull request #594 from opendatahub-io/sync__main__2024a__202406…

…27194556

Sync `2024a` branch with `main` branch

.github/workflows/build-notebooks-TEMPLATE.yaml

@@ -34,6 +34,38 @@ jobs:
           username: ${{ github.actor }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Free up additional disk space
+        # https://docs.github.com/en/actions/learn-github-actions/expressions
+        if: "${{ contains(inputs.target, 'amd') || contains(inputs.target, 'cuda') || contains(inputs.target, 'intel') ||
+         contains(inputs.target, 'pytorch') || contains(inputs.target, 'tensorflow') }}"
+        run: |
+          set -x
+
+          df -h
+
+          sudo rm -rf /usr/local/lib/android &
+          sudo rm -rf /usr/local/share/boost &
+          sudo rm -rf /usr/local/lib/node_modules &
+          sudo rm -rf /usr/share/dotnet &
+          sudo rm -rf /opt/ghc &
+          sudo rm -rf /opt/hostedtoolcache/CodeQL &
+
+          sudo docker image prune --all --force &
+
+          wait
+
+          df -h
+
+      - name: Mount lvm overlay for podman builds
+        run: |
+          df -h
+          free -h
+
+          bash ./ci/cached-builds/gha_lvm_overlay.bash
+
+          df -h
+          free -h
+
       # https://github.com/containers/buildah/issues/2521#issuecomment-884779112
       - name: Workaround https://github.com/containers/podman/issues/22152#issuecomment-2027705598
         run: sudo apt-get -qq remove podman crun
@@ -58,12 +90,10 @@ jobs:
           mkdir -p $HOME/.config/containers/
           cp ci/cached-builds/containers.conf $HOME/.config/containers/containers.conf
           cp ci/cached-builds/storage.conf $HOME/.config/containers/storage.conf
+
           # should at least reset storage when touching storage.conf
-          sudo mkdir -p /mnt/containers/
-          sudo chown -R $USER:$USER /mnt/containers
           podman system reset --force
-          # podman bug? need to create this _after_ doing the reset
-          mkdir -p /mnt/containers/tmp
+          mkdir -p $HOME/.local/share/containers/storage/tmp
 
       # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#push
       - name: "push: make ${{ inputs.target }}"

.github/workflows/build-notebooks-pr.yaml

@@ -1,28 +1,40 @@
 ---
 "name": "Build Notebooks"
-"permissions":
-  "packages": "read"
 "on":
   "pull_request":
 
+permissions:
+  contents: read
+  packages: read
+  pull-requests: read
+
 jobs:
   gen:
     name: Generate job matrix
     runs-on: ubuntu-latest
     outputs:
       matrix: ${{ steps.gen.outputs.matrix }}
+      has_jobs: ${{ steps.gen.outputs.has_jobs }}
     steps:
       - uses: actions/checkout@v4
-      - run: python3 ci/cached-builds/gen_gha_matrix_jobs.py
+
+      - run: |
+          python3 ci/cached-builds/gen_gha_matrix_jobs.py \
+            --owner=${{ github.repository_owner }} \
+            --repo=${{ github.event.pull_request.base.repo.name }} \
+            --pr-number=${{ github.event.pull_request.number }} \
+            --skip-unchanged
         id: gen
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
-  # base images
   build:
     needs: ["gen"]
     strategy:
       fail-fast: false
       matrix: "${{ fromJson(needs.gen.outputs.matrix) }}"
     uses: ./.github/workflows/build-notebooks-TEMPLATE.yaml
+    if: ${{ fromJson(needs.gen.outputs.has_jobs) }}
     with:
       target: "${{ matrix.target }}"
       github: "${{ toJSON(github) }}"

.github/workflows/notebooks-digest-updater-upstream.yaml

@@ -67,57 +67,61 @@ jobs:
 
       - name: Update the param.env file
         run: |
-              echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N}}
-              IMAGES=("odh-minimal-notebook-image-n"
-               "odh-minimal-gpu-notebook-image-n"
-               "odh-pytorch-gpu-notebook-image-n"
-               "odh-generic-data-science-notebook-image-n"
-               "odh-tensorflow-gpu-notebook-image-n"
-               "odh-trustyai-notebook-image-n"
-               "odh-codeserver-notebook-image-n"
-               "odh-rstudio-notebook-image-n"
-               "odh-rstudio-gpu-notebook-image-n")
-
-              for ((i=0;i<${#IMAGES[@]};++i)); do
-                image=${IMAGES[$i]}
-                echo "CHECKING: " $image
-                img=$(cat manifests/base/params.env | grep -E "${image}=" | cut -d '=' -f2)
-                registry=$(echo $img | cut -d '@' -f1)
-                src_tag=$(skopeo inspect docker://$img | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
-                regex="$src_tag-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}"
-                latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
-                digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')
-                output=$registry@$digest
-                echo "NEW:" $output
-                sed -i "s|${image}=.*|${image}=$output|" manifests/base/params.env
+              PARAMS_ENV_PATH="manifests/base/params.env"
+
+              echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N }}
+
+              # Get the complete list of images N-version to update
+              IMAGES=$(cat "${PARAMS_ENV_PATH}" | grep "\-n=" | cut -d "=" -f 1)
+
+              for image in ${IMAGES}; do
+                echo "CHECKING: '${image}'"
+                img=$(grep -E "${image}=" "${PARAMS_ENV_PATH}" | cut -d '=' -f2)
+                registry=$(echo "${img}" | cut -d '@' -f1)
+
+                skopeo_metadata=$(skopeo inspect --retry-times 3 "docker://${img}")
+
+                src_tag=$(echo "${skopeo_metadata}" | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
+                regex="^$src_tag-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}\$"
+                latest_tag=$(echo "${skopeo_metadata}" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
+                # use `--no-tags` for skopeo once available in newer version
+                digest=$(skopeo inspect --retry-times 3 "docker://${registry}:${latest_tag}" | jq .Digest | tr -d '"')
+                output="${registry}@${digest}"
+                echo "NEW: ${output}"
+                sed -i "s|${image}=.*|${image}=${output}|" "${PARAMS_ENV_PATH}"
               done
+
               if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
-                git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
+                git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && \
+                  git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && \
+                  git add "${PARAMS_ENV_PATH}" && \
+                  git commit -m "Update images for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
+                  git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
               else
-                echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
+                echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N }}"
               fi
 
       - name: Update the commit.env file
         run: |
-            echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N}}
-            COMMIT=("odh-minimal-notebook-image-commit-n"
-             "odh-minimal-gpu-notebook-image-commit-n"
-             "odh-pytorch-gpu-notebook-image-commit-n"
-             "odh-generic-data-science-notebook-image-commit-n"
-             "odh-tensorflow-gpu-notebook-image-commit-n"
-             "odh-trustyai-notebook-image-commit-n"
-             "odh-codeserver-notebook-image-commit-n"
-             "odh-rstudio-notebook-image-commit-n"
-             "odh-rstudio-gpu-notebook-image-commit-n")
-
-            for val in "${COMMIT[@]}"; do
-              echo $val
-              sed -i "s|${val}=.*|${val}=${{ steps.hash-n.outputs.HASH_N }}|" manifests/base/commit.env
+            COMMIT_ENV_PATH="manifests/base/commit.env"
+
+            echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N }}
+            # Get the complete list of images N-1-version to update
+            COMMIT=$(grep "\-n=" "${COMMIT_ENV_PATH}" | cut -d "=" -f 1)
+
+            for val in ${COMMIT}; do
+              echo "${val}"
+              sed -i "s|${val}=.*|${val}=${{ steps.hash-n.outputs.HASH_N }}|" "${COMMIT_ENV_PATH}"
             done
+
             if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
-              git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
+              git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
+                git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && \
+                git add "${COMMIT_ENV_PATH}" && \
+                git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
+                git push origin ${{ env.DIGEST_UPDATER_BRANCH }}
             else
-              echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
+              echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N }}"
             fi
 
   update-n-1-version:
@@ -147,56 +151,61 @@ jobs:
 
       - name: Update the param.env file
         run: |
-              echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1}}
-              IMAGES=("odh-minimal-notebook-image-n-1"
-               "odh-minimal-gpu-notebook-image-n-1"
-               "odh-pytorch-gpu-notebook-image-n-1"
-               "odh-generic-data-science-notebook-image-n-1"
-               "odh-tensorflow-gpu-notebook-image-n-1"
-               "odh-trustyai-notebook-image-n-1"
-               "odh-codeserver-notebook-image-n-1"
-               "odh-rstudio-notebook-image-n-1"
-               "odh-rstudio-gpu-notebook-image-n-1")
-
-              for ((i=0;i<${#IMAGES[@]};++i)); do
-                image=${IMAGES[$i]}
-                echo "CHECKING: " $image
-                img=$(cat manifests/base/params.env | grep -E "${image}=" | cut -d '=' -f2)
-                registry=$(echo $img | cut -d '@' -f1)
-                src_tag=$(skopeo inspect docker://$img | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
-                regex="$src_tag-${{ env.RELEASE_VERSION_N_1}}-\d+-${{ steps.hash-n-1.outputs.HASH_N_1 }}"
-                latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
-                digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')
-                output=$registry@$digest
-                echo "NEW:" $output
-                sed -i "s|${image}=.*|${image}=$output|" manifests/base/params.env
+              PARAMS_ENV_PATH="manifests/base/params.env"
+
+              echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1 }}
+
+              # Get the complete list of images N-1-version to update
+              IMAGES=$(cat "${PARAMS_ENV_PATH}" | grep "\-n-1=" | cut -d "=" -f 1)
+
+              for image in ${IMAGES}; do
+                echo "CHECKING: '${image}'"
+                img=$(grep -E "${image}=" "${PARAMS_ENV_PATH}" | cut -d '=' -f2)
+                registry=$(echo "${img}" | cut -d '@' -f1)
+
+                skopeo_metadata=$(skopeo inspect --retry-times 3 "docker://${img}")
+
+                src_tag=$(echo "${skopeo_metadata}" | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
+                regex="^$src_tag-${{ env.RELEASE_VERSION_N_1}}-\d+-${{ steps.hash-n-1.outputs.HASH_N_1 }}\$"
+                latest_tag=$(echo "${skopeo_metadata}" | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
+                # use `--no-tags` for skopeo once available in newer version
+                digest=$(skopeo inspect --retry-times 3 "docker://${registry}:${latest_tag}" | jq .Digest | tr -d '"')
+                output="${registry}@${digest}"
+                echo "NEW: ${output}"
+                sed -i "s|${image}=.*|${image}=${output}|" "${PARAMS_ENV_PATH}"
               done
+
               if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
-                git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
+                git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && \
+                  git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && \
+                  git add "${PARAMS_ENV_PATH}" && \
+                  git commit -m "Update images for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
+                  git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
               else
-                  echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N_1}}"
+                  echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N_1 }}"
               fi
+
       - name: Update the commit.env file
         run: |
-              echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1}}
-              COMMIT=("odh-minimal-notebook-image-commit-n-1"
-               "odh-minimal-gpu-notebook-image-commit-n-1"
-               "odh-pytorch-gpu-notebook-image-commit-n-1"
-               "odh-generic-data-science-notebook-image-commit-n-1"
-               "odh-tensorflow-gpu-notebook-image-commit-n-1"
-               "odh-trustyai-notebook-image-commit-n-1"
-               "odh-codeserver-notebook-image-commit-n-1"
-               "odh-rstudio-notebook-image-commit-n-1"
-               "odh-rstudio-gpu-notebook-image-commit-n-1")
-
-              for val in "${COMMIT[@]}"; do
-                echo $val
-                sed -i "s|${val}=.*|${val}=${{ steps.hash-n-1.outputs.HASH_N_1 }}|" manifests/base/commit.env
+              COMMIT_ENV_PATH="manifests/base/commit.env"
+
+              echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1 }}
+              # Get the complete list of images N-1-version to update
+              COMMIT=$(grep "\-n-1=" "${COMMIT_ENV_PATH}" | cut -d "=" -f 1)
+
+              for val in ${COMMIT}; do
+                echo "${val}"
+                sed -i "s|${val}=.*|${val}=${{ steps.hash-n-1.outputs.HASH_N_1 }}|" "${COMMIT_ENV_PATH}"
               done
+
               if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then
-                git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
+                git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && \
+                  git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && \
+                  git add "${COMMIT_ENV_PATH}" && \
+                  git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && \
+                  git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
               else
-                echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}"
+                echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N }}"
               fi
 
   open-pull-request:

.github/workflows/runtimes-digest-updater-upstream.yaml

@@ -92,7 +92,7 @@ jobs:
                   name="minimal-$name"
               fi
               registry=$(echo $img | cut -d '@' -f1)
-              regex="runtime-$name-$py_version-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}"
+              regex="^runtime-$name-$py_version-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}\$"
               echo "CHECKING: "  $regex
               latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
               digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')

.github/workflows/sync-branches-through-pr.yml

@@ -0,0 +1,47 @@
+---
+name: Sync branches through Pull Request
+
+on:  # yamllint disable-line rule:truthy
+  workflow_dispatch:
+    inputs:
+      source:
+        description: Source branch
+        required: true
+      target:
+        description: Target branch
+        required: true
+
+jobs:
+  sync:
+    permissions:
+      contents: write
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.target }}
+          fetch-depth: 0
+
+      - name: Prepare sync branch
+        id: prepare
+        run: |
+          git fetch origin ${{ github.event.inputs.source }}
+          git reset --hard origin/${{ github.event.inputs.source }}
+
+          TIMESTAMP=$(date +'%Y%m%d%H%M%S')
+          SYNC_BRANCH=sync__${{ github.event.inputs.source }}__${{ github.event.inputs.target }}__${TIMESTAMP}
+          echo "branch=$SYNC_BRANCH" >> $GITHUB_OUTPUT
+
+      - name: Create pull request
+        uses: peter-evans/create-pull-request@c5a7806660adbe173f04e3e038b0ccdcd758773c  # v6.1.0
+        with:
+          branch: ${{ steps.prepare.outputs.branch }}
+          title: "Sync `${{ github.event.inputs.target }}` branch with `${{ github.event.inputs.source }}` branch"
+          body: |
+            :robot: This is an automated Pull Request created by `/.github/workflows/sync-branches-through-pr.yml`.
+
+            It merges all commits from `${{ github.event.inputs.source }}` branch into `${{ github.event.inputs.target }}` branch.
+
+            :warning: **IMPORTANT NOTE**: Remember to delete the `${{ steps.prepare.outputs.branch }}` branch after merging the changes.