mirror images with oc-mirror in upgrade job

ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.19__amd64-nightly-4.19-upgrade-from-stable-4.18.yaml

@@ -444,6 +444,21 @@ tests:
     test:
     - chain: openshift-upgrade-qe-test-disconnected
     workflow: cucushift-installer-rehearse-azure-ipi-disconnected-cco-manual-workload-identity
+- as: azure-ipi-disc-oidc-oc-mirror-f28-ota
+  cron: 5 19 1 * *
+  steps:
+    cluster_profile: azure-qe
+    env:
+      BASE_DOMAIN: qe.azure.devcluster.openshift.com
+      EXTRACT_MANIFEST_INCLUDED: "true"
+      MIRROR_BIN: oc-mirror
+      MIRROR_GRAPH_DATA: "true"
+    test:
+    - ref: cucushift-upgrade-mirror-images-by-oc-mirror
+    - ref: cucushift-upgrade-prehealthcheck
+    - ref: cucushift-upgrade-toimage
+    - ref: cucushift-upgrade-healthcheck
+    workflow: cucushift-installer-rehearse-azure-ipi-disconnected-cco-manual-workload-identity
 - as: azure-ipi-marketplace-mini-perm-f28
   cron: 26 6 3 * *
   steps:

ci-operator/config/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.19__amd64-stable-4.19-upgrade-from-stable-4.18.yaml

@@ -121,6 +121,21 @@ tests:
     test:
     - chain: openshift-upgrade-qe-sanity-disconnected
     workflow: cucushift-installer-rehearse-azure-ipi-disconnected-fullyprivate
+- as: azure-ipi-disc-oidc-oc-mirror-f28-ota
+  cron: 17 9 3 * *
+  steps:
+    cluster_profile: azure-qe
+    env:
+      BASE_DOMAIN: qe.azure.devcluster.openshift.com
+      EXTRACT_MANIFEST_INCLUDED: "true"
+      MIRROR_BIN: oc-mirror
+      MIRROR_GRAPH_DATA: "true"
+    test:
+    - ref: cucushift-upgrade-mirror-images-by-oc-mirror
+    - ref: cucushift-upgrade-prehealthcheck
+    - ref: cucushift-upgrade-toimage
+    - ref: cucushift-upgrade-healthcheck
+    workflow: cucushift-installer-rehearse-azure-ipi-disconnected-cco-manual-workload-identity
 - as: azure-mag-ipi-fips-f28
   cron: 56 20 30 * *
   steps:

ci-operator/jobs/openshift/openshift-tests-private/openshift-openshift-tests-private-release-4.19-periodics.yaml

@@ -3566,6 +3566,87 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build09
+  cron: 5 19 1 * *
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: release-4.19
+    org: openshift
+    repo: openshift-tests-private
+  labels:
+    ci-operator.openshift.io/cloud: azure4
+    ci-operator.openshift.io/cloud-cluster-profile: azure-qe
+    ci-operator.openshift.io/variant: amd64-nightly-4.19-upgrade-from-stable-4.18
+    ci.openshift.io/generator: prowgen
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-openshift-openshift-tests-private-release-4.19-amd64-nightly-4.19-upgrade-from-stable-4.18-azure-ipi-disc-oidc-oc-mirror-f28-ota
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --oauth-token-path=/usr/local/github-credentials/oauth
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --target=azure-ipi-disc-oidc-oc-mirror-f28-ota
+      - --variant=amd64-nightly-4.19-upgrade-from-stable-4.18
+      command:
+      - ci-operator
+      image: ci-operator:latest
+      imagePullPolicy: Always
+      name: ""
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /usr/local/github-credentials
+        name: github-credentials-openshift-ci-robot-private-git-cloner
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: github-credentials-openshift-ci-robot-private-git-cloner
+      secret:
+        secretName: github-credentials-openshift-ci-robot-private-git-cloner
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build09
   cron: 26 6 3 * *
@@ -46176,6 +46257,87 @@ periodics:
     - name: result-aggregator
       secret:
         secretName: result-aggregator
+- agent: kubernetes
+  cluster: build09
+  cron: 17 9 3 * *
+  decorate: true
+  decoration_config:
+    skip_cloning: true
+  extra_refs:
+  - base_ref: release-4.19
+    org: openshift
+    repo: openshift-tests-private
+  labels:
+    ci-operator.openshift.io/cloud: azure4
+    ci-operator.openshift.io/cloud-cluster-profile: azure-qe
+    ci-operator.openshift.io/variant: amd64-stable-4.19-upgrade-from-stable-4.18
+    ci.openshift.io/generator: prowgen
+    pj-rehearse.openshift.io/can-be-rehearsed: "true"
+  name: periodic-ci-openshift-openshift-tests-private-release-4.19-amd64-stable-4.19-upgrade-from-stable-4.18-azure-ipi-disc-oidc-oc-mirror-f28-ota
+  spec:
+    containers:
+    - args:
+      - --gcs-upload-secret=/secrets/gcs/service-account.json
+      - --image-import-pull-secret=/etc/pull-secret/.dockerconfigjson
+      - --lease-server-credentials-file=/etc/boskos/credentials
+      - --oauth-token-path=/usr/local/github-credentials/oauth
+      - --report-credentials-file=/etc/report/credentials
+      - --secret-dir=/secrets/ci-pull-credentials
+      - --target=azure-ipi-disc-oidc-oc-mirror-f28-ota
+      - --variant=amd64-stable-4.19-upgrade-from-stable-4.18
+      command:
+      - ci-operator
+      image: ci-operator:latest
+      imagePullPolicy: Always
+      name: ""
+      resources:
+        requests:
+          cpu: 10m
+      volumeMounts:
+      - mountPath: /etc/boskos
+        name: boskos
+        readOnly: true
+      - mountPath: /secrets/ci-pull-credentials
+        name: ci-pull-credentials
+        readOnly: true
+      - mountPath: /secrets/gcs
+        name: gcs-credentials
+        readOnly: true
+      - mountPath: /usr/local/github-credentials
+        name: github-credentials-openshift-ci-robot-private-git-cloner
+        readOnly: true
+      - mountPath: /secrets/manifest-tool
+        name: manifest-tool-local-pusher
+        readOnly: true
+      - mountPath: /etc/pull-secret
+        name: pull-secret
+        readOnly: true
+      - mountPath: /etc/report
+        name: result-aggregator
+        readOnly: true
+    serviceAccountName: ci-operator
+    volumes:
+    - name: boskos
+      secret:
+        items:
+        - key: credentials
+          path: credentials
+        secretName: boskos-credentials
+    - name: ci-pull-credentials
+      secret:
+        secretName: ci-pull-credentials
+    - name: github-credentials-openshift-ci-robot-private-git-cloner
+      secret:
+        secretName: github-credentials-openshift-ci-robot-private-git-cloner
+    - name: manifest-tool-local-pusher
+      secret:
+        secretName: manifest-tool-local-pusher
+    - name: pull-secret
+      secret:
+        secretName: registry-pull-credentials
+    - name: result-aggregator
+      secret:
+        secretName: result-aggregator
 - agent: kubernetes
   cluster: build09
   cron: 56 20 30 * *

ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/OWNERS

@@ -0,0 +1,10 @@
+approvers:
+  - jianlinliu
+  - jiajliu
+  - shellyyang1989
+  - jhou1
+reviewers:
+  - jiajliu
+  - jianlinliu
+  - shellyyang1989
+  - jhou1

ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/cucushift-upgrade-mirror-images-by-oc-mirror-commands.sh

@@ -0,0 +1,146 @@
+#!/bin/bash
+
+set -o nounset
+set -o errexit
+set -o pipefail
+
+trap 'CHILDREN=$(jobs -p); if test -n "${CHILDREN}"; then kill ${CHILDREN} && wait; fi' TERM
+
+export HOME="${HOME:-/tmp/home}"
+export XDG_RUNTIME_DIR="${HOME}/run"
+export REGISTRY_AUTH_PREFERENCE=podman # TODO: remove later, used for migrating oc from docker to podman
+mkdir -p "${XDG_RUNTIME_DIR}"
+
+function run_command() {
+    local CMD="$1"
+    echo "Running command: ${CMD}"
+    eval "${CMD}"
+}
+
+function check_signed() {
+    local digest algorithm hash_value response try max_retries
+    if [[ "${TARGET}" =~ "@sha256:" ]]; then
+        digest="$(echo "${TARGET}" | cut -f2 -d@)"
+        echo "The target image is using digest pullspec, its digest is ${digest}"
+    else
+        digest="$(oc image info "${TARGET}" -o json | jq -r ".digest")"
+        echo "The target image is using tagname pullspec, its digest is ${digest}"
+    fi
+    algorithm="$(echo "${digest}" | cut -f1 -d:)"
+    hash_value="$(echo "${digest}" | cut -f2 -d:)"
+    try=0
+    max_retries=3
+    response=0
+    while (( try < max_retries && response != 200 )); do
+        echo "Trying #${try}"
+        response=$(https_proxy="" HTTPS_PROXY="" curl -L --silent --output /dev/null --write-out %"{http_code}" "https://mirror.openshift.com/pub/openshift-v4/signatures/openshift/release/${algorithm}=${hash_value}/signature-1")
+        (( try += 1 ))
+        sleep 60
+    done
+    if (( response == 200 )); then
+        echo "${TARGET} is signed" && return 0
+    else
+        echo "Seem like ${TARGET} is not signed" && return 1
+    fi
+}
+
+# private mirror registry host
+# <public_dns>:<port>
+MIRROR_REGISTRY_HOST=$(head -n 1 "${SHARED_DIR}/mirror_registry_url")
+echo "MIRROR_REGISTRY_HOST: $MIRROR_REGISTRY_HOST"
+echo "OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE: ${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE}"
+
+# target release
+target_release_image="${MIRROR_REGISTRY_HOST}/${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE#*/}"
+target_release_image_repo="${target_release_image%:*}"
+target_release_image_repo="${target_release_image_repo%@sha256*}"
+echo "target_release_image_repo: $target_release_image_repo"
+
+# since ci-operator gives steps KUBECONFIG pointing to cluster under test under some circumstances,
+# unset KUBECONFIG to ensure this step always interact with the build farm.
+unset KUBECONFIG
+oc registry login
+
+run_command "which oc"
+run_command "oc version --client"
+oc_mirror_dir=$(mktemp -d)
+pushd "${oc_mirror_dir}"
+new_pull_secret="${oc_mirror_dir}/new_pull_secret"
+
+# combine custom registry credential and default pull secret
+registry_cred=$(head -n 1 "/var/run/vault/mirror-registry/registry_creds" | base64 -w 0)
+cat "${CLUSTER_PROFILE_DIR}/pull-secret" | python3 -c 'import json,sys;j=json.load(sys.stdin);a=j["auths"];a["'${MIRROR_REGISTRY_HOST}'"]={"auth":"'${registry_cred}'"};j["auths"]=a;print(json.dumps(j))' > "${new_pull_secret}"
+
+oc_mirror_bin="oc-mirror"
+run_command "'${oc_mirror_bin}' version --output=yaml"
+
+# set the imagesetconfigure
+image_set_config="image_set_config.yaml"
+cat <<END | tee "${image_set_config}"
+kind: ImageSetConfiguration
+apiVersion: mirror.openshift.io/v2alpha1
+mirror:
+  platform:
+    release: ${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE}
+    graph: ${MIRROR_GRAPH_DATA}
+END
+
+# https://github.com/openshift/oc-mirror/blob/main/docs/usage.md#authentication
+# oc-mirror only respect ~/.docker/config.json -> ${XDG_RUNTIME_DIR}/containers/auth.json
+mkdir -p "${XDG_RUNTIME_DIR}/containers/"
+cp -rf "${new_pull_secret}" "${XDG_RUNTIME_DIR}/containers/auth.json"
+
+unset REGISTRY_AUTH_PREFERENCE
+
+# execute the oc-mirror command
+run_command "'${oc_mirror_bin}' -c ${image_set_config} docker://${target_release_image_repo} --dest-tls-verify=false --v2 --workspace file://${oc_mirror_dir}"
+
+# Save output from oc-mirror
+result_folder="${oc_mirror_dir}/working-dir"
+idms_file="${result_folder}/cluster-resources/idms-oc-mirror.yaml"
+itms_file="${result_folder}/cluster-resources/itms-oc-mirror.yaml"
+
+if [ ! -s "${idms_file}" ]; then
+    echo "${idms_file} not found, exit..."
+    exit 1
+else
+    run_command "cat '${idms_file}'"
+    run_command "cp -rf '${idms_file}' ${SHARED_DIR}"
+fi
+
+if [ -s "${itms_file}" ]; then
+    echo "${itms_file} found"
+    run_command "cat '${itms_file}'"
+    run_command "cp -rf '${itms_file}' ${SHARED_DIR}"
+fi
+
+if [[ "${MIRROR_GRAPH_DATA}" == "true" ]]; then
+    us_file="${result_folder}/cluster-resources/updateService.yaml"
+    if [ ! -s "${us_file}" ]; then
+        echo "${us_file} not found, exit..."
+        exit 1
+    else
+        run_command "cat '${us_file}'"
+        run_command "cp -rf '${us_file}' ${SHARED_DIR}"
+    fi
+    run_command "ls '${result_folder}'"
+
+    export TARGET="${OPENSHIFT_UPGRADE_RELEASE_IMAGE_OVERRIDE}"
+    if ! check_signed; then
+        echo "You're mirroring an unsigned images, don't apply signature"
+    else
+        echo "You're mirroring a signed images, will apply signature"
+        # oc-mirror v2 support mirror with signatures from 4.18
+        sig_folder="${result_folder}/signatures"
+        if [[ ! -d "${sig_folder}" || -z "$(ls -A ${sig_folder})" ]]; then
+            echo "signatures not found, exit..."
+            exit 1
+        fi
+        run_command "ls '${sig_folder}'"
+        export KUBECONFIG=${SHARED_DIR}/kubeconfig
+        oc apply -f "${sig_folder}"
+    fi
+fi
+
+# Ending
+rm -f "${new_pull_secret}"

ci-operator/step-registry/cucushift/upgrade/mirror-images/by-oc-mirror/cucushift-upgrade-mirror-images-by-oc-mirror-ref.metadata.json

@@ -0,0 +1,17 @@
+{
+	"path": "cucushift/upgrade/mirror-images/by-oc-mirror/cucushift-upgrade-mirror-images-by-oc-mirror-ref.yaml",
+	"owners": {
+		"approvers": [
+			"jianlinliu",
+			"jiajliu",
+			"shellyyang1989",
+			"jhou1"
+		],
+		"reviewers": [
+			"jiajliu",
+			"jianlinliu",
+			"shellyyang1989",
+			"jhou1"
+		]
+	}
+}