From d149f7ee7c8cc33d358b3c6717f1ecf6af2ca132 Mon Sep 17 00:00:00 2001 From: jiajliu Date: Fri, 3 Jan 2025 12:14:10 +0800 Subject: [PATCH] mirror images with oc-mirror in upgrade job --- ...-stable-4.18-upgrade-from-stable-4.17.yaml | 74 ++++++++++ ...t-verification-tests-master-periodics.yaml | 74 ++++++++++ .../upgrade/mirror-images/by-oc-mirror/OWNERS | 10 ++ ...ade-mirror-images-by-oc-mirror-commands.sh | 126 ++++++++++++++++++ ...rror-images-by-oc-mirror-ref.metadata.json | 17 +++ ...pgrade-mirror-images-by-oc-mirror-ref.yaml | 24 ++++ 6 files changed, 325 insertions(+) create mode 100644 ci-operator/config/openshift/verification-tests/openshift-verification-tests-master__ota-amd64-stable-4.18-upgrade-from-stable-4.17.yaml create mode 100644 ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/OWNERS create mode 100644 ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/cucushift-upgrade-mirror-images-by-oc-mirror-commands.sh create mode 100644 ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/cucushift-upgrade-mirror-images-by-oc-mirror-ref.metadata.json create mode 100644 ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/cucushift-upgrade-mirror-images-by-oc-mirror-ref.yaml diff --git a/ci-operator/config/openshift/verification-tests/openshift-verification-tests-master__ota-amd64-stable-4.18-upgrade-from-stable-4.17.yaml b/ci-operator/config/openshift/verification-tests/openshift-verification-tests-master__ota-amd64-stable-4.18-upgrade-from-stable-4.17.yaml new file mode 100644 index 0000000000000..8b3eae4cad8ed --- /dev/null +++ b/ci-operator/config/openshift/verification-tests/openshift-verification-tests-master__ota-amd64-stable-4.18-upgrade-from-stable-4.17.yaml @@ -0,0 +1,74 @@ +base_images: + ansible: + name: "4.17" + namespace: ocp + tag: ansible + cli: + name: "4.17" + namespace: ocp + tag: cli + dev-scripts: + name: test + namespace: ocp-kni + tag: dev-scripts + openstack-installer: + name: "4.17" + namespace: ocp + tag: openstack-installer + tests-private-postupg: + name: tests-private + namespace: ci + tag: "4.18" + tests-private-preupg: + name: tests-private + namespace: ci + tag: "4.17" + tools: + name: "4.17" + namespace: ocp + tag: tools + upi-installer: + name: "4.17" + namespace: ocp + tag: upi-installer + verification-tests: + name: verification-tests + namespace: ci + tag: latest +releases: + latest: + release: + architecture: amd64 + channel: fast + version: "4.17" + target: + release: + architecture: amd64 + channel: candidate + version: "4.18" +resources: + '*': + requests: + cpu: 100m + memory: 200Mi +tests: +- as: azure-ipi-disc-oidc-oc-mirror-f28 + cron: 51 13 12 * * + steps: + cluster_profile: azure-qe + env: + BASE_DOMAIN: qe.azure.devcluster.openshift.com + EXTRACT_MANIFEST_INCLUDED: "true" + MIRROR_BIN: oc-mirror + MIRROR_GRAPH_DATA: "true" + test: + - ref: cucushift-upgrade-mirror-images-by-oc-mirror + - ref: cucushift-upgrade-prehealthcheck + - ref: cucushift-upgrade-toimage + - ref: cucushift-upgrade-healthcheck + workflow: cucushift-installer-rehearse-azure-ipi-disconnected-cco-manual-workload-identity +zz_generated_metadata: + branch: master + org: openshift + repo: verification-tests + variant: ota-amd64-stable-4.18-upgrade-from-stable-4.17 diff --git a/ci-operator/jobs/openshift/verification-tests/openshift-verification-tests-master-periodics.yaml b/ci-operator/jobs/openshift/verification-tests/openshift-verification-tests-master-periodics.yaml index 29a47b5266999..f7e215ce6e504 100644 --- a/ci-operator/jobs/openshift/verification-tests/openshift-verification-tests-master-periodics.yaml +++ b/ci-operator/jobs/openshift/verification-tests/openshift-verification-tests-master-periodics.yaml @@ -20261,6 +20261,80 @@ periodics: - name: result-aggregator secret: secretName: result-aggregator +- agent: kubernetes + cluster: build01 + cron: 51 13 12 * * + decorate: true + decoration_config: + skip_cloning: true + extra_refs: + - base_ref: master + org: openshift + repo: verification-tests + labels: + ci-operator.openshift.io/cloud: azure4 + ci-operator.openshift.io/cloud-cluster-profile: azure-qe + ci-operator.openshift.io/variant: ota-amd64-stable-4.18-upgrade-from-stable-4.17 + ci.openshift.io/generator: prowgen + pj-rehearse.openshift.io/can-be-rehearsed: "true" + name: periodic-ci-openshift-verification-tests-master-ota-amd64-stable-4.18-upgrade-from-stable-4.17-azure-ipi-disc-oidc-oc-mirror-f28 + spec: + containers: + - args: + - --gcs-upload-secret=/secrets/gcs/service-account.json + - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson + - --lease-server-credentials-file=/etc/boskos/credentials + - --report-credentials-file=/etc/report/credentials + - --secret-dir=/secrets/ci-pull-credentials + - --target=azure-ipi-disc-oidc-oc-mirror-f28 + - --variant=ota-amd64-stable-4.18-upgrade-from-stable-4.17 + command: + - ci-operator + image: ci-operator:latest + imagePullPolicy: Always + name: "" + resources: + requests: + cpu: 10m + volumeMounts: + - mountPath: /etc/boskos + name: boskos + readOnly: true + - mountPath: /secrets/ci-pull-credentials + name: ci-pull-credentials + readOnly: true + - mountPath: /secrets/gcs + name: gcs-credentials + readOnly: true + - mountPath: /secrets/manifest-tool + name: manifest-tool-local-pusher + readOnly: true + - mountPath: /etc/pull-secret + name: pull-secret + readOnly: true + - mountPath: /etc/report + name: result-aggregator + readOnly: true + serviceAccountName: ci-operator + volumes: + - name: boskos + secret: + items: + - key: credentials + path: credentials + secretName: boskos-credentials + - name: ci-pull-credentials + secret: + secretName: ci-pull-credentials + - name: manifest-tool-local-pusher + secret: + secretName: manifest-tool-local-pusher + - name: pull-secret + secret: + secretName: registry-pull-credentials + - name: result-aggregator + secret: + secretName: result-aggregator - agent: kubernetes cluster: build05 cron: 43 3 16 * * diff --git a/ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/OWNERS b/ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/OWNERS new file mode 100644 index 0000000000000..1b2f24a36b737 --- /dev/null +++ b/ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/OWNERS @@ -0,0 +1,10 @@ +approvers: + - jianlinliu + - jiajliu + - shellyyang1989 + - jhou1 +reviewers: + - jiajliu + - jianlinliu + - shellyyang1989 + - jhou1 diff --git a/ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/cucushift-upgrade-mirror-images-by-oc-mirror-commands.sh b/ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/cucushift-upgrade-mirror-images-by-oc-mirror-commands.sh new file mode 100644 index 0000000000000..073de94959a11 --- /dev/null +++ b/ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/cucushift-upgrade-mirror-images-by-oc-mirror-commands.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +set -o nounset +set -o errexit +set -o pipefail + +trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wait; fi' TERM +# save the exit code for junit xml file generated in step gather-must-gather +# pre configuration steps before running installation, exit code 100 if failed, +# save to install-pre-config-status.txt +# post check steps after cluster installation, exit code 101 if failed, +# save to install-post-check-status.txt +EXIT_CODE=100 +trap 'if [[ "$?" == 0 ]]; then EXIT_CODE=0; fi; echo "${EXIT_CODE}" > "${SHARED_DIR}/install-pre-config-status.txt"' EXIT TERM + +export HOME="${HOME:-/tmp/home}" +export XDG_RUNTIME_DIR="${HOME}/run" +export REGISTRY_AUTH_PREFERENCE=podman # TODO: remove later, used for migrating oc from docker to podman +mkdir -p "${XDG_RUNTIME_DIR}" + +function run_command() { + local CMD="$1" + echo "Running command: ${CMD}" + eval "${CMD}" +} + +# private mirror registry host +# : +MIRROR_REGISTRY_HOST=$(head -n 1 "${SHARED_DIR}/mirror_registry_url") +echo "MIRROR_REGISTRY_HOST: $MIRROR_REGISTRY_HOST" +echo "OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE: ${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE}" + +# target release +target_release_image="${MIRROR_REGISTRY_HOST}/${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE#*/}" +target_release_image_repo="${target_release_image%:*}" +target_release_image_repo="${target_release_image_repo%@sha256*}" +echo "target_release_image_repo: $target_release_image_repo" + +# since ci-operator gives steps KUBECONFIG pointing to cluster under test under some circumstances, +# unset KUBECONFIG to ensure this step always interact with the build farm. +unset KUBECONFIG +oc registry login + +run_command "which oc" +run_command "oc version --client" +oc_mirror_dir=$(mktemp -d) +pushd "${oc_mirror_dir}" +new_pull_secret="${oc_mirror_dir}/new_pull_secret" + +# combine custom registry credential and default pull secret +registry_cred=$(head -n 1 "/var/run/vault/mirror-registry/registry_creds" | base64 -w 0) +cat "${CLUSTER_PROFILE_DIR}/pull-secret" | python3 -c 'import json,sys;j=json.load(sys.stdin);a=j["auths"];a["'${MIRROR_REGISTRY_HOST}'"]={"auth":"'${registry_cred}'"};j["auths"]=a;print(json.dumps(j))' > "${new_pull_secret}" + +#Because user does not have permission to update subgid and subuid file, so use another workaround. +ocp_version=$(oc adm release info ${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE} -ojsonpath='{.metadata.version}' | cut -d. -f 1,2) +ocp_minor_version=$(echo "${ocp_version}" | awk --field-separator=. '{print $2}') +if ((ocp_minor_version > 17)); then + echo "export TEST_E2E=true to workaournd OCPBUGS-43986" + export TEST_E2E=true +fi + +oc_mirror_bin="oc-mirror" +run_command "'${oc_mirror_bin}' version --output=yaml" + +# set the imagesetconfigure +image_set_config="image_set_config.yaml" +cat < ${XDG_RUNTIME_DIR}/containers/auth.json +mkdir -p "${XDG_RUNTIME_DIR}/containers/" +cp -rf "${new_pull_secret}" "${XDG_RUNTIME_DIR}/containers/auth.json" + +unset REGISTRY_AUTH_PREFERENCE + +# execute the oc-mirror command +run_command "'${oc_mirror_bin}' -c ${image_set_config} docker://${target_release_image_repo} --dest-tls-verify=false --v2 --workspace file://${oc_mirror_dir}" + +# Save output from oc-mirror +result_folder="${oc_mirror_dir}/working-dir" +idms_file="${result_folder}/cluster-resources/idms-oc-mirror.yaml" +itms_file="${result_folder}/cluster-resources/itms-oc-mirror.yaml" + +if [ ! -s "${idms_file}" ]; then + echo "${idms_file} not found, exit..." + exit 1 +else + run_command "cat '${idms_file}'" + run_command "cp -rf '${idms_file}' ${SHARED_DIR}" +fi + +if [ -s "${itms_file}" ]; then + echo "${itms_file} found" + run_command "cat '${itms_file}'" + run_command "cp -rf '${itms_file}' ${SHARED_DIR}" +fi + +if [[ "${MIRROR_GRAPH_DATA}" == "true" ]]; then + export KUBECONFIG=${SHARED_DIR}/kubeconfig + us_file="${result_folder}/cluster-resources/updateService.yaml" + if [ ! -s "${us_file}" ]; then + echo "${us_file} not found, exit..." + exit 1 + else + run_command "cat '${us_file}'" + run_command "cp -rf '${us_file}' ${SHARED_DIR}" + fi + run_command "ls '${result_folder}'" + sig_folder="${result_folder}/signatures" + if [[ -z "${sig_folder}" ]]; then + echo "signatures not found, exit..." + exit 1 + fi + run_command "ls '${sig_folder}'" + oc apply -f "${sig_folder}" +fi + +# Ending +rm -f "${new_pull_secret}" diff --git a/ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/cucushift-upgrade-mirror-images-by-oc-mirror-ref.metadata.json b/ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/cucushift-upgrade-mirror-images-by-oc-mirror-ref.metadata.json new file mode 100644 index 0000000000000..de9b172e464f9 --- /dev/null +++ b/ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/cucushift-upgrade-mirror-images-by-oc-mirror-ref.metadata.json @@ -0,0 +1,17 @@ +{ + "path": "cucushift/upgrade/mirror-images/by-oc-mirror/cucushift-upgrade-mirror-images-by-oc-mirror-ref.yaml", + "owners": { + "approvers": [ + "jianlinliu", + "jiajliu", + "shellyyang1989", + "jhou1" + ], + "reviewers": [ + "jiajliu", + "jianlinliu", + "shellyyang1989", + "jhou1" + ] + } +} diff --git a/ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/cucushift-upgrade-mirror-images-by-oc-mirror-ref.yaml b/ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/cucushift-upgrade-mirror-images-by-oc-mirror-ref.yaml new file mode 100644 index 0000000000000..b0ac0383242c7 --- /dev/null +++ b/ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/cucushift-upgrade-mirror-images-by-oc-mirror-ref.yaml @@ -0,0 +1,24 @@ +ref: + as: cucushift-upgrade-mirror-images-by-oc-mirror + from: oc-mirror + cli: target + grace_period: 10m + commands: cucushift-upgrade-mirror-images-by-oc-mirror-commands.sh + resources: + requests: + cpu: 10m + memory: 100Mi + env: + - name: MIRROR_GRAPH_DATA + default: "false" + documentation: |- + Determin if generate osus resource and mirror graph-data image + credentials: + - namespace: test-credentials + name: openshift-custom-mirror-registry + mount_path: /var/run/vault/mirror-registry + dependencies: + - name: "release:target" + env: "OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE" + documentation: |- + Mirror release image to local image registry.