Trigger Nightly FBC Build #76
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Trigger Nightly FBC Build | |
| run-name: Trigger Nightly FBC Build | |
| on: | |
| workflow_dispatch: | |
| push: | |
| paths: | |
| - schedule/catalog-github-trigger.txt | |
| branches: | |
| - 'rhoai-2.1[6-9]+' | |
| permissions: | |
| contents: write | |
| env: | |
| GITHUB_ORG: red-hat-data-services | |
| GITHUB_RKA_ORG: rhoai-rhtap | |
| jobs: | |
| process-fbc: | |
| if: ${{ github.ref_name != 'main' }} | |
| runs-on: ubuntu-latest | |
| container: | |
| image: quay.io/rhoai/rhoai-task-toolset:latest | |
| options: --privileged | |
| steps: | |
| - name: Get Current branch name | |
| shell: bash | |
| run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT | |
| id: get_branch | |
| - name: Git checkout RBC main | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: main | |
| path: main | |
| sparse-checkout: | | |
| pcc | |
| catalog/${{ steps.get_branch.outputs.branch }} | |
| sparse-checkout-cone-mode: false | |
| - name: Git checkout RBC release | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ steps.get_branch.outputs.branch }} | |
| path: ${{ steps.get_branch.outputs.branch }} | |
| - name: Git checkout utils | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ env.GITHUB_RKA_ORG }}/RHOAI-Konflux-Automation | |
| ref: main | |
| path: utils | |
| - name: Install dependencies | |
| env: | |
| RHOAI_QUAY_RO_USERNAME: ${{ secrets.RHOAI_QUAY_RO_USERNAME }} | |
| RHOAI_QUAY_RO_TOKEN: ${{ secrets.RHOAI_QUAY_RO_TOKEN }} | |
| run: | | |
| os="$(uname -s | tr '[:upper:]' '[:lower:]')" | |
| arch="$(uname -m | sed 's/x86_64/amd64/')" | |
| yq_version="v4.44.3" | |
| yq_filename="yq-$yq_version" | |
| echo "-> Downloading yq" >&2 | |
| curl -sSfLo "$yq_filename" "https://github.com/mikefarah/yq/releases/download/$yq_version/yq_${os}_${arch}" | |
| chmod +x $yq_filename | |
| ln -s $yq_filename yq | |
| cp $yq_filename /usr/local/bin/yq | |
| opm_version="v1.47.0" | |
| opm_filename="opm-$opm_version" | |
| echo "-> Downloading opm" >&2 | |
| curl -sSfLo "$opm_filename" "https://github.com/operator-framework/operator-registry/releases/download/$opm_version/$os-$arch-opm" | |
| chmod +x "$opm_filename" | |
| ln -fs "$opm_filename" opm | |
| cp "$opm_filename" /usr/local/bin/opm | |
| pip install -r utils/utils/fbc-processor/requirements.txt | |
| microdnf install -y skopeo && \ | |
| microdnf clean all && rm -rf /var/cache/dnf/* | |
| skopeo login -u "${RHOAI_QUAY_RO_USERNAME}" -p "${RHOAI_QUAY_RO_TOKEN}" quay.io/rhoai | |
| - name: Check if PCC Cache Valid | |
| id: check-if-pcc-cache-valid | |
| env: | |
| RHOAI_CATALOG_SA_USERNAME: ${{ secrets.RHOAI_CATALOG_SA_USERNAME }} | |
| RHOAI_CATALOG_SA_TOKEN: ${{ secrets.RHOAI_CATALOG_SA_TOKEN }} | |
| run: | | |
| microdnf install -y coreutils-single && \ | |
| microdnf clean all && rm -rf /var/cache/dnf/* | |
| PCC_CACHE_VALID=YES | |
| skopeo login registry.redhat.io -u "${RHOAI_CATALOG_SA_USERNAME}" -p "${RHOAI_CATALOG_SA_TOKEN}" | |
| skopeo list-tags docker://registry.redhat.io/rhoai/odh-operator-bundle | jq -r '.Tags | .[] | select(. | startswith ("v"))' | sort > latest_shipped_rhoai_versions.txt | |
| echo "latest_shipped_rhoai_versions = " | |
| cat latest_shipped_rhoai_versions.txt | |
| echo "shipped_rhoai_versions = " | |
| cat main/pcc/shipped_rhoai_versions.txt | |
| #diff=$(cmp --silent latest_shipped_rhoai_versions.txt main/pcc/shipped_rhoai_versions.txt || echo "files are different") | |
| diff=$(python -c 'print(list(set(open("latest_shipped_rhoai_versions.txt").readlines()) - set(open("main/pcc/shipped_rhoai_versions.txt").readlines())).__len__())') | |
| if [[ $diff -gt 0 ]] | |
| then | |
| diff=$(python -c 'print(list(set(open("latest_shipped_rhoai_versions.txt").readlines()) - set(open("main/pcc/shipped_rhoai_versions.txt").readlines())))') | |
| echo "following new versions are shipped - $diff" | |
| cp latest_shipped_rhoai_versions.txt main/pcc/shipped_rhoai_versions.txt | |
| PCC_CACHE_VALID=NO | |
| fi | |
| echo "PCC_CACHE_VALID=${PCC_CACHE_VALID}" >> $GITHUB_OUTPUT | |
| - name: Regenerate PCC Cache | |
| id: regenerate-pcc-cache | |
| if: ${{ steps.check-if-pcc-cache-valid.outputs.PCC_CACHE_VALID == 'NO' }} | |
| env: | |
| BRANCH: ${{ github.event.inputs.release_branch }} | |
| RHOAI_VERSION: ${{ github.event.inputs.rhoai_version }} | |
| run: | | |
| #install opm cli | |
| os="$(uname -s | tr '[:upper:]' '[:lower:]')" | |
| arch="$(uname -m | sed 's/x86_64/amd64/')" | |
| opm_version="v1.47.0" | |
| opm_filename="opm-$opm_version" | |
| echo "-> Downloading opm" >&2 | |
| curl -sSfLo "$opm_filename" "https://github.com/operator-framework/operator-registry/releases/download/$opm_version/$os-$arch-opm" | |
| chmod +x "$opm_filename" | |
| ln -fs "$opm_filename" opm | |
| cp "$opm_filename" /usr/local/bin/opm | |
| microdnf install -y findutils && \ | |
| microdnf clean all && rm -rf /var/cache/dnf/* | |
| #Declare basic variables | |
| COMPONENT_SUFFIX=${RHOAI_VERSION/./-} | |
| OPERATOR_BUNDLE_COMPONENT_NAME=odh-operator-bundle | |
| #Declare FBC processing variables | |
| BUILD_CONFIG_PATH=${BRANCH}/config/build-config.yaml | |
| PATCH_YAML_PATH=${BRANCH}/catalog/catalog-patch.yaml | |
| CATALOG_GENERATION_REF_OCP_VERSION=v4.17 | |
| CSV_META_MIN_OCP_VERSION=417 | |
| BUNDLE_OBJECT_CATALOG_YAML_PATH=main/pcc/bundle_object_catalog.yaml | |
| CSV_META_CATALOG_YAML_PATH=main/pcc/csv_meta_catalog.yaml | |
| opm migrate registry.redhat.io/redhat/redhat-operator-index:${CATALOG_GENERATION_REF_OCP_VERSION} ./catalog-migrate | |
| opm alpha convert-template basic catalog-migrate/rhods-operator/catalog.json -o yaml > catalog-template.yaml | |
| opm alpha render-template basic catalog-template.yaml -o yaml > ${BUNDLE_OBJECT_CATALOG_YAML_PATH} | |
| opm alpha render-template basic catalog-template.yaml --migrate-level=bundle-object-to-csv-metadata -o yaml > ${CSV_META_CATALOG_YAML_PATH} | |
| #while IFS= read -r value; | |
| #do | |
| # OPENSHIFT_VERSION=$value | |
| # NUMERIC_OCP_VERSION=${OPENSHIFT_VERSION/v4./4} | |
| # echo "OPENSHIFT_VERSION=$OPENSHIFT_VERSION" | |
| # | |
| # CATALOG_YAML_PATH=${BUNDLE_OBJECT_CATALOG_YAML_PATH} | |
| # CATALOG_DIR=main/pcc/${OPENSHIFT_VERSION}/rhods-operator | |
| # mkdir -p ${CATALOG_DIR} | |
| # | |
| # if [[ $NUMERIC_OCP_VERSION -ge $CSV_META_MIN_OCP_VERSION ]] | |
| # then | |
| # CATALOG_YAML_PATH=${CSV_META_CATALOG_YAML_PATH} | |
| # fi | |
| # cp ${CATALOG_YAML_PATH} ${CATALOG_DIR}/catalog.yaml | |
| #done < <(find main/pcc/ -maxdepth 1 -mindepth 1 -type d -printf '%f\n') | |
| ls -l main/pcc/ | |
| - name: Push latest PCC Cache | |
| if: ${{ steps.check-if-pcc-cache-valid.outputs.PCC_CACHE_VALID == 'NO' }} | |
| uses: actions-js/push@master | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| branch: main | |
| message: "Regeneratd the PCC Cache" | |
| repository: ${{ env.GITHUB_ORG }}/RHOAI-Build-Config | |
| directory: main | |
| author_name: Openshift-AI DevOps | |
| author_email: [email protected] | |
| - name: Process FBC Fragment | |
| env: | |
| BRANCH: ${{ steps.get_branch.outputs.branch }} | |
| OC_TOKEN: ${{ secrets.KONFLUX_INTERNAL_OC_TOKEN }} | |
| CLUSTER: p02 | |
| RHOAI_QUAY_RO_USERNAME: ${{ secrets.RHOAI_QUAY_RO_USERNAME }} | |
| RHOAI_QUAY_RO_TOKEN: ${{ secrets.RHOAI_QUAY_RO_TOKEN }} | |
| RHOAI_QUAY_API_TOKEN: ${{ secrets.RHOAI_QUAY_API_TOKEN }} | |
| run: | | |
| #Declare basic variables | |
| BUILD_CONFIG_PATH=${BRANCH}/config/build-config.yaml | |
| RHOAI_VERSION=v${BRANCH/rhoai-/} | |
| COMPONENT_SUFFIX=${RHOAI_VERSION/./-} | |
| #OPERATOR_BUNDLE_COMPONENT_NAME=odh-operator-bundle-${COMPONENT_SUFFIX} | |
| OPERATOR_BUNDLE_COMPONENT_NAME=odh-operator-bundle | |
| PATCH_YAML_PATH=${BRANCH}/catalog/catalog-patch.yaml | |
| OC_TOKEN=$(echo $OC_TOKEN | awk '{$1=$1};1' | tr -d '\n') | |
| OPERATOR_BUNDLE_IMAGE_NAME=RELATED_IMAGE_ODH_OPERATOR_BUNDLE_IMAGE | |
| echo "OPERATOR_BUNDLE_IMAGE_NAME = $OPERATOR_BUNDLE_IMAGE_NAME" | |
| #Generate the catalog for the latest bundle | |
| #podman login quay.io -u ${RHOAI_QUAY_RO_USERNAME} -p ${RHOAI_QUAY_RO_TOKEN} | |
| BASE64_AUTH=$(echo -n "${RHOAI_QUAY_RO_USERNAME}:${RHOAI_QUAY_RO_TOKEN}" | base64 -w 0) | |
| mkdir ${HOME}/.docker | |
| echo '{"auths":{"quay.io/rhoai/odh-operator-bundle":{"username":"'"${RHOAI_QUAY_RO_USERNAME}"'","password":"'"${RHOAI_QUAY_RO_TOKEN}"'","email":"","auth":"'"${BASE64_AUTH}"'"}}}' > ${HOME}/.docker/config.json | |
| APPLICATION_NAME=rhoai-${COMPONENT_SUFFIX} | |
| echo "APPLICATION_NAME = $APPLICATION_NAME" | |
| #Invoke the FBC processor to extract the snapshot images | |
| CATALOG_BUILD_ARGS_FILE_PATH=${BRANCH}/catalog/catalog_build_args.map | |
| python3 utils/utils/fbc-processor/fbc-processor.py -op extract-snapshot-images -o utils/snapshot_images.json -v ${BRANCH} -f ${OPERATOR_BUNDLE_COMPONENT_NAME} -b ${BUILD_CONFIG_PATH} --catalog-build-args-file-path ${CATALOG_BUILD_ARGS_FILE_PATH} --build-type nightly | |
| LATEST_BUNDLE_IMAGE=$(jq --arg OPERATOR_BUNDLE_IMAGE_NAME "$OPERATOR_BUNDLE_IMAGE_NAME" -r '.[] | select(.name == $OPERATOR_BUNDLE_IMAGE_NAME) | .value' utils/snapshot_images.json) | |
| echo "LATEST_BUNDLE_IMAGE = $LATEST_BUNDLE_IMAGE" | |
| yq e -i '.stable.bundles[0].image = "$LATEST_BUNDLE_IMAGE"' utils/utils/fbc-processor/fbc-semver-template.yaml | |
| LATEST_BUNDLE_IMAGE="$LATEST_BUNDLE_IMAGE" yq e -i '.stable.bundles[0].image = env(LATEST_BUNDLE_IMAGE)' utils/utils/fbc-processor/fbc-semver-template.yaml | |
| #Generate the single bundle catalog === sbc | |
| CSV_META_MIN_OCP_VERSION=417 | |
| WORK_DIR=utils/utils/fbc | |
| mkdir -p ${WORK_DIR} | |
| BUNDLE_OBJECT_SINGLE_BUNDLE_PATH=${WORK_DIR}/bundle_object_sbc_semver.yaml | |
| CSV_META_SINGLE_BUNDLE_PATH=${WORK_DIR}/csv_meta_sbc_semver.yaml | |
| CSV_META_OPM_FLAG="--migrate-level=bundle-object-to-csv-metadata" | |
| DOCKER_CONFIG=${HOME}/.docker/ opm alpha render-template semver -o yaml utils/utils/fbc-processor/fbc-semver-template.yaml > ${BUNDLE_OBJECT_SINGLE_BUNDLE_PATH} | |
| DOCKER_CONFIG=${HOME}/.docker/ opm alpha render-template semver ${CSV_META_OPM_FLAG} -o yaml utils/utils/fbc-processor/fbc-semver-template.yaml > ${CSV_META_SINGLE_BUNDLE_PATH} | |
| PCC_BUNDLE_OBJECT_CATALOG_YAML_PATH=main/pcc/bundle_object_catalog.yaml | |
| PCC_CSV_META_CATALOG_YAML_PATH=main/pcc/csv_meta_catalog.yaml | |
| BUNDLE_OBJECT_CATALOG_YAML_PATH=${WORK_DIR}/bundle_object_catalog.yaml | |
| CSV_META_CATALOG_YAML_PATH=${WORK_DIR}/csv_meta_catalog.yaml | |
| PUSH_PIPELINE_PATH=${BRANCH}/.tekton/rhoai-fbc-fragment-${COMPONENT_SUFFIX}-push.yaml | |
| #Invoke FBC processor to patch the catalog | |
| python3 utils/utils/fbc-processor/fbc-processor.py -op catalog-patch -b ${BUILD_CONFIG_PATH} -c ${PCC_BUNDLE_OBJECT_CATALOG_YAML_PATH} -p ${PATCH_YAML_PATH} -s ${BUNDLE_OBJECT_SINGLE_BUNDLE_PATH} -o ${BUNDLE_OBJECT_CATALOG_YAML_PATH} --push-pipeline-yaml-path ${PUSH_PIPELINE_PATH} --push-pipeline-operation disable | |
| python3 utils/utils/fbc-processor/fbc-processor.py -op catalog-patch -b ${BUILD_CONFIG_PATH} -c ${PCC_CSV_META_CATALOG_YAML_PATH} -p ${PATCH_YAML_PATH} -s ${CSV_META_SINGLE_BUNDLE_PATH} -o ${CSV_META_CATALOG_YAML_PATH} --push-pipeline-yaml-path ${PUSH_PIPELINE_PATH} --push-pipeline-operation disable | |
| while IFS= read -r value; | |
| do | |
| #Declare FBC processing variables | |
| OPENSHIFT_VERSION=$value | |
| NUMERIC_OCP_VERSION=${OPENSHIFT_VERSION/v4./4} | |
| echo "OPENSHIFT_VERSION=$OPENSHIFT_VERSION" | |
| OUTPUT_CATALOG_PATH=${BRANCH}/catalog/${OPENSHIFT_VERSION}/rhods-operator/catalog.yaml | |
| SOURCE_CATALOG=${BUNDLE_OBJECT_CATALOG_YAML_PATH} | |
| if [[ $NUMERIC_OCP_VERSION -ge $CSV_META_MIN_OCP_VERSION ]] | |
| then | |
| SOURCE_CATALOG=${CSV_META_CATALOG_YAML_PATH} | |
| fi | |
| cp ${SOURCE_CATALOG} ${OUTPUT_CATALOG_PATH} | |
| #cat ${OUTPUT_CATALOG_PATH} | |
| done < <(yq eval '.config.supported-ocp-versions.build[].name' $BUILD_CONFIG_PATH) | |
| # Update the schedule file to trigger the full fbc build | |
| echo $(date +'%d-%m-%Y %H:%M:%S:%3N') > ${BRANCH}/schedule/catalog-tekton-trigger.txt | |
| - name: Commit and push the changes to release branch | |
| uses: actions-js/push@master | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| branch: ${{ steps.get_branch.outputs.branch }} | |
| message: "Updating the catalog.yaml with latest images and patches" | |
| repository: ${{ env.GITHUB_ORG }}/RHOAI-Build-Config | |
| directory: ${{ steps.get_branch.outputs.branch }} | |
| author_name: Openshift-AI DevOps | |
| author_email: [email protected] | |