From 2f255e9e49806c7a2ef25fe9cf08a32a0608341f Mon Sep 17 00:00:00 2001 From: gandalf-at-lerian Date: Mon, 20 Jul 2026 15:19:09 -0300 Subject: [PATCH 1/8] fix(release): run post-release backmerge after changelog commit Move the post-release backmerge (backmerge_source -> backmerge_target) out of the publish_release matrix job into a dedicated 'backmerge' job that depends on generate_changelog. Previously the backmerge ran inline in publish_release, which completes before the separate generate_changelog job commits CHANGELOG.md to main. As a result main->develop was backmerged without the changelog, leaving develop missing it. The backmerge-sync composite fetches origin/ fresh, so ordering the job after the changelog commit guarantees the changelog is carried into the target branch. Also wires the new job into the notify job's failed-job reporting. --- .github/workflows/release.yml | 89 ++++++++++++++++++++++++++++------- 1 file changed, 71 insertions(+), 18 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0e797c6a..eafbdb2c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -351,22 +351,13 @@ jobs: NPM_TOKEN: ${{ secrets.NPM_TOKEN }} # ----------------- Backmerge ----------------- - - name: Backmerge ${{ inputs.backmerge_source }} → ${{ inputs.backmerge_target }} - if: | - inputs.backmerge_enabled && - steps.semantic.outputs.new_release_published == 'true' && - github.ref_name == inputs.backmerge_source - uses: LerianStudio/github-actions-shared-workflows/src/config/backmerge-sync@v1 - with: - github-token: ${{ steps.app-token.outputs.token }} - source-branch: ${{ inputs.backmerge_source }} - target-branch: ${{ inputs.backmerge_target }} - mode: ${{ inputs.backmerge_mode }} - dry-run: ${{ inputs.dry_run }} - commit-message: "chore(release): backmerge ${source} into ${target} [skip ci]" - pr-title: "chore(release): backmerge ${source} → ${target} (v${{ steps.semantic.outputs.new_release_version }})" - git-user-name: ${{ secrets.LERIAN_CI_CD_USER_NAME }} - git-user-email: ${{ secrets.LERIAN_CI_CD_USER_EMAIL }} + # The post-release backmerge (backmerge_source → backmerge_target) is NOT + # done here anymore. It now runs in the dedicated `backmerge` job below, + # which depends on generate_changelog so the changelog commit is already + # on backmerge_source (main) before it is merged into backmerge_target + # (develop). Running it inline here completes before generate_changelog, + # so the target branch never received the changelog. The pre-version + # prerelease sync above stays inline — it must happen before version calc. # ----------------- Per-leg release publish marker ----------------- # Matrix job outputs are last-writer-wins, so we persist each leg's @@ -496,6 +487,67 @@ jobs: openai-model: ${{ inputs.openai_model }} bot-ignore-list: ${{ inputs.changelog_bot_ignore_list }} + # ----------------- Backmerge (post-changelog) ----------------- + # Runs the post-release backmerge (backmerge_source → backmerge_target) only + # AFTER generate_changelog has committed the CHANGELOG to backmerge_source + # (main). The backmerge-sync composite fetches origin/ fresh, so + # ordering this job after the changelog commit is what guarantees the + # changelog is carried into backmerge_target (develop). This used to be a + # step inside publish_release, which finishes before generate_changelog and + # therefore backmerged main WITHOUT the changelog. + backmerge: + name: Backmerge ${{ inputs.backmerge_source }} → ${{ inputs.backmerge_target }} + needs: [publish_release, publish_release_status, update_major_tag, generate_changelog] + if: >- + always() && + inputs.backmerge_enabled && + needs.publish_release_status.outputs.release_published == 'true' && + github.ref_name == inputs.backmerge_source && + (needs.update_major_tag.result == 'success' || needs.update_major_tag.result == 'skipped') && + (needs.generate_changelog.result == 'success' || needs.generate_changelog.result == 'skipped') + runs-on: ${{ inputs.runner_type }} + environment: + name: create_release + permissions: + contents: write + pull-requests: write + steps: + - uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + id: app-token + with: + client-id: ${{ secrets.LERIAN_STUDIO_MIDAZ_PUSH_BOT_APP_ID }} + private-key: ${{ secrets.LERIAN_STUDIO_MIDAZ_PUSH_BOT_PRIVATE_KEY }} + + - name: Checkout repository + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + fetch-depth: 0 + token: ${{ steps.app-token.outputs.token }} + + - name: Import GPG key + uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7 + with: + gpg_private_key: ${{ secrets.LERIAN_CI_CD_USER_GPG_KEY }} + passphrase: ${{ secrets.LERIAN_CI_CD_USER_GPG_KEY_PASSWORD }} + git_committer_name: ${{ secrets.LERIAN_CI_CD_USER_NAME }} + git_committer_email: ${{ secrets.LERIAN_CI_CD_USER_EMAIL }} + git_config_global: true + git_user_signingkey: true + git_commit_gpgsign: true + + - name: Backmerge ${{ inputs.backmerge_source }} → ${{ inputs.backmerge_target }} + uses: LerianStudio/github-actions-shared-workflows/src/config/backmerge-sync@v1 + with: + github-token: ${{ steps.app-token.outputs.token }} + source-branch: ${{ inputs.backmerge_source }} + target-branch: ${{ inputs.backmerge_target }} + mode: ${{ inputs.backmerge_mode }} + dry-run: ${{ inputs.dry_run }} + commit-message: "chore(release): backmerge ${source} into ${target} [skip ci]" + pr-title: "chore(release): backmerge ${source} → ${target} (v${{ needs.publish_release_status.outputs.release_version }})" + git-user-name: ${{ secrets.LERIAN_CI_CD_USER_NAME }} + git-user-email: ${{ secrets.LERIAN_CI_CD_USER_EMAIL }} + # ----------------- Major Tag Update ----------------- update_major_tag: name: Update Major Tag @@ -557,7 +609,7 @@ jobs: # Slack notification notify: name: Notify - needs: [prepare, publish_release, publish_release_status, generate_changelog, update_major_tag, enforce_latest] + needs: [prepare, publish_release, publish_release_status, generate_changelog, backmerge, update_major_tag, enforce_latest] if: always() && needs.prepare.outputs.has_changes == 'true' uses: ./.github/workflows/slack-notify.yml with: @@ -566,9 +618,10 @@ jobs: && 'failure' || 'success' }} workflow_name: "Release" failed_jobs: >- - ${{ format('{0}{1}{2}{3}', + ${{ format('{0}{1}{2}{3}{4}', (needs.publish_release.result == 'failure' && 'Publish Release' || ''), (needs.generate_changelog.result == 'failure' && ' / Generate Changelog' || ''), + (needs.backmerge.result == 'failure' && ' / Backmerge' || ''), (needs.update_major_tag.result == 'failure' && ' / Update Major Tag' || ''), (needs.enforce_latest.result == 'failure' && ' / Enforce Latest' || '') ) }} From c4fe894e560d55c015ee93d8051c73a944a36151 Mon Sep 17 00:00:00 2001 From: Lucas Bedatty Date: Wed, 22 Jul 2026 10:18:22 -0300 Subject: [PATCH 2/8] fix(build): retry cosign signing when on_existing_tag skips the build When on_existing_tag: skip skips build/push because the tag already exists, the cosign steps were also skipped since they depended on steps.build-push.outputs.digest and should_build != 'false'. A transient cosign/Rekor failure on an already-pushed tag then had no retry path short of a full rebuild or a brand new release. Add a 'Resolve digest for existing tag' step that resolves the digest from the existing registry tag via docker buildx imagetools inspect when the build was skipped, and gate the cosign steps on enable_cosign_sign alone, feeding steps.build-push.outputs.digest || steps.existing-digest.outputs.digest. Closes #559 --- .github/workflows/build.yml | 32 +++++++++++++++++++++++++++----- docs/build-workflow.md | 2 +- 2 files changed, 28 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index d6d486f7..8e37d16f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -471,11 +471,33 @@ jobs: github_token=${{ secrets.MANAGE_TOKEN }} # ----------------- Cosign Image Signing ----------------- + # Lets `on_existing_tag: skip` retry just the signing step for an already-pushed + # tag (e.g. after a transient cosign/Rekor failure) without a full rebuild. + - name: Resolve digest for existing tag + if: inputs.enable_cosign_sign && steps.preflight.outputs.should_build == 'false' + id: existing-digest + env: + APP_NAME: ${{ matrix.app.name }} + VERSION: ${{ steps.version.outputs.version }} + ENABLE_DOCKERHUB: ${{ inputs.enable_dockerhub }} + DOCKERHUB_ORG: ${{ inputs.dockerhub_org }} + GHCR_ORG: ${{ steps.image-names.outputs.ghcr_org }} + run: | + set -euo pipefail + TAG="${VERSION#v}" + if [ "$ENABLE_DOCKERHUB" == "true" ]; then + REF="${DOCKERHUB_ORG}/${APP_NAME}:${TAG}" + else + REF="ghcr.io/${GHCR_ORG}/${APP_NAME}:${TAG}" + fi + DIGEST=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' | jq -r '.digest') + echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" + - name: Build cosign image references - if: inputs.enable_cosign_sign && steps.preflight.outputs.should_build != 'false' + if: inputs.enable_cosign_sign id: cosign-refs env: - DIGEST: ${{ steps.build-push.outputs.digest }} + DIGEST: ${{ steps.build-push.outputs.digest || steps.existing-digest.outputs.digest }} ENABLE_DOCKERHUB: ${{ inputs.enable_dockerhub }} ENABLE_GHCR: ${{ inputs.enable_ghcr }} DOCKERHUB_ORG: ${{ inputs.dockerhub_org }} @@ -500,7 +522,7 @@ jobs: } >> "$GITHUB_OUTPUT" - name: Sign container images with cosign - if: inputs.enable_cosign_sign && steps.preflight.outputs.should_build != 'false' + if: inputs.enable_cosign_sign id: cosign-sign continue-on-error: ${{ inputs.continue_gitops_on_signing_failure || false }} uses: LerianStudio/github-actions-shared-workflows/src/security/cosign-sign@v1 @@ -511,10 +533,10 @@ jobs: max-delay: ${{ inputs.cosign_max_delay }} - name: Report cosign signing status - if: inputs.enable_cosign_sign && steps.preflight.outputs.should_build != 'false' && steps.cosign-sign.outcome == 'failure' + if: inputs.enable_cosign_sign && steps.cosign-sign.outcome == 'failure' env: REFS: ${{ steps.cosign-refs.outputs.refs }} - DIGEST: ${{ steps.build-push.outputs.digest }} + DIGEST: ${{ steps.build-push.outputs.digest || steps.existing-digest.outputs.digest }} APP_NAME: ${{ matrix.app.name }} run: | echo "::warning::Cosign signing FAILED for ${APP_NAME} after all retries — image is unsigned in the registry." diff --git a/docs/build-workflow.md b/docs/build-workflow.md index deebe56c..1fbd8de5 100644 --- a/docs/build-workflow.md +++ b/docs/build-workflow.md @@ -132,7 +132,7 @@ Uses `secrets: inherit` pattern. Required secrets: Before building, the workflow checks whether the target image tag already exists in each enabled registry (via `docker manifest inspect`, reusing the registry logins). This avoids a full rebuild that would only fail at push time on registries with tag immutability enabled. Behaviour is controlled by `on_existing_tag`: - **`fail`** (default): abort early with a clear error instead of rebuilding then failing at push. -- **`skip`**: skip the build/push but still emit the GitOps tag artifacts (from the version), so a re-run remains idempotent for the downstream GitOps update. +- **`skip`**: skip the build/push but still emit the GitOps tag artifacts (from the version), so a re-run remains idempotent for the downstream GitOps update. Cosign signing (if enabled) also retries in this mode: the digest is resolved from the existing registry tag via `docker buildx imagetools inspect` instead of the build/push step, so a transient signing failure on an already-pushed tag can be recovered with a plain re-run instead of cutting a new release. - **`warn`**: emit a warning and build anyway (push may still fail on immutable registries). A non-existent tag (or a check that errors out, e.g. transient registry issues) is treated as "not present" so the check never blocks a legitimate build. From e158f99f160a7559aa6180a4c0779241455264dc Mon Sep 17 00:00:00 2001 From: Lucas Bedatty Date: Wed, 22 Jul 2026 10:31:04 -0300 Subject: [PATCH 3/8] fix(build): resolve cosign digest per registry independently Address CodeRabbit finding on PR #603: when both DockerHub and GHCR are enabled, the existing-tag digest resolution always picked DockerHub (or GHCR alone) and reused that single digest for both registries' refs. If a tag existed in only one registry, this could construct a nonexistent ref for the other registry using the wrong digest. Resolve each enabled registry's digest independently and only emit a ref for a registry when its own digest was actually found. --- .github/workflows/build.yml | 46 +++++++++++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8e37d16f..7337e36b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -480,24 +480,42 @@ jobs: APP_NAME: ${{ matrix.app.name }} VERSION: ${{ steps.version.outputs.version }} ENABLE_DOCKERHUB: ${{ inputs.enable_dockerhub }} + ENABLE_GHCR: ${{ inputs.enable_ghcr }} DOCKERHUB_ORG: ${{ inputs.dockerhub_org }} GHCR_ORG: ${{ steps.image-names.outputs.ghcr_org }} run: | - set -euo pipefail + set -uo pipefail TAG="${VERSION#v}" + + # Each registry is inspected independently: a tag can be immutable-blocked + # in one registry but absent from another (e.g. a prior partial push), so + # reusing one registry's digest for the other's ref would sign a + # nonexistent (or wrong) reference. if [ "$ENABLE_DOCKERHUB" == "true" ]; then REF="${DOCKERHUB_ORG}/${APP_NAME}:${TAG}" - else + DIGEST="" + if OUTPUT=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' 2>/dev/null); then + DIGEST=$(echo "$OUTPUT" | jq -r '.digest // empty') + fi + echo "dockerhub_digest=$DIGEST" >> "$GITHUB_OUTPUT" + fi + + if [ "$ENABLE_GHCR" == "true" ]; then REF="ghcr.io/${GHCR_ORG}/${APP_NAME}:${TAG}" + DIGEST="" + if OUTPUT=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' 2>/dev/null); then + DIGEST=$(echo "$OUTPUT" | jq -r '.digest // empty') + fi + echo "ghcr_digest=$DIGEST" >> "$GITHUB_OUTPUT" fi - DIGEST=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' | jq -r '.digest') - echo "digest=$DIGEST" >> "$GITHUB_OUTPUT" - name: Build cosign image references if: inputs.enable_cosign_sign id: cosign-refs env: - DIGEST: ${{ steps.build-push.outputs.digest || steps.existing-digest.outputs.digest }} + DIGEST: ${{ steps.build-push.outputs.digest }} + DOCKERHUB_DIGEST: ${{ steps.existing-digest.outputs.dockerhub_digest }} + GHCR_DIGEST: ${{ steps.existing-digest.outputs.ghcr_digest }} ENABLE_DOCKERHUB: ${{ inputs.enable_dockerhub }} ENABLE_GHCR: ${{ inputs.enable_ghcr }} DOCKERHUB_ORG: ${{ inputs.dockerhub_org }} @@ -506,13 +524,21 @@ jobs: run: | REFS="" - if [ "$ENABLE_DOCKERHUB" == "true" ]; then - REFS="docker.io/${DOCKERHUB_ORG}/${APP_NAME}@${DIGEST}" + # A single build-push pushes the identical content to every enabled + # registry, so DIGEST (from build-push) applies to both. When the build + # was skipped (existing tag), fall back to each registry's own resolved + # digest instead, and only emit a ref when that registry's digest was + # actually found. + DOCKERHUB_FINAL="${DIGEST:-$DOCKERHUB_DIGEST}" + GHCR_FINAL="${DIGEST:-$GHCR_DIGEST}" + + if [ "$ENABLE_DOCKERHUB" == "true" ] && [ -n "$DOCKERHUB_FINAL" ]; then + REFS="docker.io/${DOCKERHUB_ORG}/${APP_NAME}@${DOCKERHUB_FINAL}" fi - if [ "$ENABLE_GHCR" == "true" ]; then + if [ "$ENABLE_GHCR" == "true" ] && [ -n "$GHCR_FINAL" ]; then [ -n "$REFS" ] && REFS="${REFS}"$'\n' - REFS="${REFS}ghcr.io/${GHCR_ORG}/${APP_NAME}@${DIGEST}" + REFS="${REFS}ghcr.io/${GHCR_ORG}/${APP_NAME}@${GHCR_FINAL}" fi { @@ -536,7 +562,7 @@ jobs: if: inputs.enable_cosign_sign && steps.cosign-sign.outcome == 'failure' env: REFS: ${{ steps.cosign-refs.outputs.refs }} - DIGEST: ${{ steps.build-push.outputs.digest || steps.existing-digest.outputs.digest }} + DIGEST: ${{ steps.build-push.outputs.digest || steps.existing-digest.outputs.dockerhub_digest || steps.existing-digest.outputs.ghcr_digest }} APP_NAME: ${{ matrix.app.name }} run: | echo "::warning::Cosign signing FAILED for ${APP_NAME} after all retries — image is unsigned in the registry." From 9d71835da44a30e7e9d97ae97366cc5b9975291c Mon Sep 17 00:00:00 2001 From: Lucas Bedatty Date: Wed, 22 Jul 2026 10:47:26 -0300 Subject: [PATCH 4/8] fix(build): surface digest inspection failures, stop collapsing report digests Address CodeRabbit findings on PR #603: - Resolve digest for existing tag: distinguish a genuine 'tag not found' from an inspect/lookup failure (transient network error, auth issue) by emitting a ::warning:: with the actual error in the latter case, instead of silently treating both as 'no digest'. Kept fail-open (skip that registry, don't abort the job) to match this file's existing convention for the same tradeoff in the pre-flight tag-existence check just above. - Report cosign signing status: stop collapsing to a single DIGEST value that discarded one registry's digest when both were resolved; REFS already lists each registry's fully-qualified ref with its own digest, so the redundant/lossy single-digest line was removed instead of duplicating that same registry-qualified data. --- .github/workflows/build.yml | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7337e36b..92eda9da 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -494,8 +494,11 @@ jobs: if [ "$ENABLE_DOCKERHUB" == "true" ]; then REF="${DOCKERHUB_ORG}/${APP_NAME}:${TAG}" DIGEST="" - if OUTPUT=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' 2>/dev/null); then + if OUTPUT=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' 2>&1); then DIGEST=$(echo "$OUTPUT" | jq -r '.digest // empty') + [ -z "$DIGEST" ] && echo "::warning::docker buildx imagetools inspect for ${REF} returned no digest — cosign will skip this registry." + else + echo "::warning::Failed to inspect ${REF} (registry lookup error, possibly transient) — cosign will skip signing for this registry: ${OUTPUT}" fi echo "dockerhub_digest=$DIGEST" >> "$GITHUB_OUTPUT" fi @@ -503,8 +506,11 @@ jobs: if [ "$ENABLE_GHCR" == "true" ]; then REF="ghcr.io/${GHCR_ORG}/${APP_NAME}:${TAG}" DIGEST="" - if OUTPUT=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' 2>/dev/null); then + if OUTPUT=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' 2>&1); then DIGEST=$(echo "$OUTPUT" | jq -r '.digest // empty') + [ -z "$DIGEST" ] && echo "::warning::docker buildx imagetools inspect for ${REF} returned no digest — cosign will skip this registry." + else + echo "::warning::Failed to inspect ${REF} (registry lookup error, possibly transient) — cosign will skip signing for this registry: ${OUTPUT}" fi echo "ghcr_digest=$DIGEST" >> "$GITHUB_OUTPUT" fi @@ -562,7 +568,6 @@ jobs: if: inputs.enable_cosign_sign && steps.cosign-sign.outcome == 'failure' env: REFS: ${{ steps.cosign-refs.outputs.refs }} - DIGEST: ${{ steps.build-push.outputs.digest || steps.existing-digest.outputs.dockerhub_digest || steps.existing-digest.outputs.ghcr_digest }} APP_NAME: ${{ matrix.app.name }} run: | echo "::warning::Cosign signing FAILED for ${APP_NAME} after all retries — image is unsigned in the registry." @@ -570,14 +575,13 @@ jobs: echo "### ⚠️ Unsigned image — manual action required" echo "" echo "- **App:** \`${APP_NAME}\`" - echo "- **Digest:** \`${DIGEST}\`" - echo "- **Refs:**" + echo "- **Refs (each already includes its own digest):**" echo '```' echo "${REFS}" echo '```' echo "" echo "GitOps artifact upload was allowed to continue (\`continue_gitops_on_signing_failure: true\`)." - echo "The image is immutable and present in the registry but **not signed**. Run \`cosign sign\` manually against the digest above before promoting to production." + echo "The image(s) above are immutable and present in the registry but **not signed**. Run \`cosign sign\` manually against each ref above before promoting to production." } >> "$GITHUB_STEP_SUMMARY" # ----------------- GitOps Artifacts ----------------- From 71fe9a402e5dbf6ebb72c1a8527228321a512955 Mon Sep 17 00:00:00 2001 From: Lucas Bedatty Date: Wed, 22 Jul 2026 10:59:53 -0300 Subject: [PATCH 5/8] fix(build): escape workflow-command output, soften partial-failure wording Address CodeRabbit findings on PR #603: - Resolve digest for existing tag: percent-encode the captured 'docker buildx imagetools inspect' output (%, CR, LF) before embedding it in a ::warning:: line, so it can't corrupt or spoof a workflow command annotation. - Report cosign signing status: cosign-sign signs refs one by one, so a failure partway through the list doesn't mean every ref is unsigned. Reworded the summary to reflect that signing didn't complete for every ref instead of asserting all refs are unsigned. --- .github/workflows/build.yml | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 92eda9da..1e13c033 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -487,6 +487,19 @@ jobs: set -uo pipefail TAG="${VERSION#v}" + # GitHub Actions parses every line of step output for workflow commands + # (::warning::, ::error::, etc.). Registry inspect output is not + # attacker-controlled here, but it could still contain newlines or a + # stray "::" that would corrupt or spoof an annotation, so it's + # percent-encoded into a single line before being embedded in one. + escape_for_warning() { + local s="$1" + s="${s//%/%25}" + s="${s//$'\r'/%0D}" + s="${s//$'\n'/%0A}" + printf '%s' "$s" + } + # Each registry is inspected independently: a tag can be immutable-blocked # in one registry but absent from another (e.g. a prior partial push), so # reusing one registry's digest for the other's ref would sign a @@ -498,7 +511,7 @@ jobs: DIGEST=$(echo "$OUTPUT" | jq -r '.digest // empty') [ -z "$DIGEST" ] && echo "::warning::docker buildx imagetools inspect for ${REF} returned no digest — cosign will skip this registry." else - echo "::warning::Failed to inspect ${REF} (registry lookup error, possibly transient) — cosign will skip signing for this registry: ${OUTPUT}" + echo "::warning::Failed to inspect ${REF} (registry lookup error, possibly transient) — cosign will skip signing for this registry: $(escape_for_warning "$OUTPUT")" fi echo "dockerhub_digest=$DIGEST" >> "$GITHUB_OUTPUT" fi @@ -510,7 +523,7 @@ jobs: DIGEST=$(echo "$OUTPUT" | jq -r '.digest // empty') [ -z "$DIGEST" ] && echo "::warning::docker buildx imagetools inspect for ${REF} returned no digest — cosign will skip this registry." else - echo "::warning::Failed to inspect ${REF} (registry lookup error, possibly transient) — cosign will skip signing for this registry: ${OUTPUT}" + echo "::warning::Failed to inspect ${REF} (registry lookup error, possibly transient) — cosign will skip signing for this registry: $(escape_for_warning "$OUTPUT")" fi echo "ghcr_digest=$DIGEST" >> "$GITHUB_OUTPUT" fi @@ -570,18 +583,18 @@ jobs: REFS: ${{ steps.cosign-refs.outputs.refs }} APP_NAME: ${{ matrix.app.name }} run: | - echo "::warning::Cosign signing FAILED for ${APP_NAME} after all retries — image is unsigned in the registry." + echo "::warning::Cosign signing did not complete for every ref for ${APP_NAME} — one or more of the refs below may still be unsigned." { - echo "### ⚠️ Unsigned image — manual action required" + echo "### ⚠️ Signing did not complete for every ref — manual action required" echo "" echo "- **App:** \`${APP_NAME}\`" - echo "- **Refs (each already includes its own digest):**" + echo "- **Requested refs (each already includes its own digest):**" echo '```' echo "${REFS}" echo '```' echo "" echo "GitOps artifact upload was allowed to continue (\`continue_gitops_on_signing_failure: true\`)." - echo "The image(s) above are immutable and present in the registry but **not signed**. Run \`cosign sign\` manually against each ref above before promoting to production." + echo "The image(s) above are immutable and already present in the registry, but signing failed after all retries for at least one ref — some refs above may already be signed, others may not be. Verify with \`cosign verify\` and run \`cosign sign\` manually against any ref that isn't before promoting to production." } >> "$GITHUB_STEP_SUMMARY" # ----------------- GitOps Artifacts ----------------- From c073d032b58409b11974207ca0f247b7be33a8b3 Mon Sep 17 00:00:00 2001 From: Lucas Bedatty Date: Wed, 22 Jul 2026 12:09:10 -0300 Subject: [PATCH 6/8] fix(build): stop merging stderr into digest inspect stdout Address CodeRabbit finding on PR #604: 'docker buildx imagetools inspect ... 2>&1' merged stderr into the same variable later fed to jq on the success path. Any stderr chatter on an otherwise-successful call (deprecation notice, credential-helper output) would corrupt the JSON and produce a false 'no digest' warning. Extracted the duplicated DockerHub/GHCR inspect logic into a resolve_digest() helper that captures stdout and stderr separately (stderr to a temp file, consumed only for the warning message on failure), so a successful call is never at risk of stderr noise breaking the digest parse. --- .github/workflows/build.yml | 39 ++++++++++++++++++++----------------- 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1e13c033..3bcb21ad 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -500,32 +500,35 @@ jobs: printf '%s' "$s" } + # Captures stdout and stderr separately (not merged) so any stderr + # chatter on an otherwise-successful call (deprecation notices, + # credential-helper output) can't corrupt the JSON fed to jq. + resolve_digest() { + local ref="$1" + local out err digest="" + local err_file + err_file="$(mktemp)" + if out=$(docker buildx imagetools inspect "$ref" --format '{{json .Manifest}}' 2>"$err_file"); then + digest=$(echo "$out" | jq -r '.digest // empty') + [ -z "$digest" ] && echo "::warning::docker buildx imagetools inspect for ${ref} returned no digest — cosign will skip this registry." >&2 + else + err=$(cat "$err_file") + echo "::warning::Failed to inspect ${ref} (registry lookup error, possibly transient) — cosign will skip signing for this registry: $(escape_for_warning "$err")" >&2 + fi + rm -f "$err_file" + printf '%s' "$digest" + } + # Each registry is inspected independently: a tag can be immutable-blocked # in one registry but absent from another (e.g. a prior partial push), so # reusing one registry's digest for the other's ref would sign a # nonexistent (or wrong) reference. if [ "$ENABLE_DOCKERHUB" == "true" ]; then - REF="${DOCKERHUB_ORG}/${APP_NAME}:${TAG}" - DIGEST="" - if OUTPUT=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' 2>&1); then - DIGEST=$(echo "$OUTPUT" | jq -r '.digest // empty') - [ -z "$DIGEST" ] && echo "::warning::docker buildx imagetools inspect for ${REF} returned no digest — cosign will skip this registry." - else - echo "::warning::Failed to inspect ${REF} (registry lookup error, possibly transient) — cosign will skip signing for this registry: $(escape_for_warning "$OUTPUT")" - fi - echo "dockerhub_digest=$DIGEST" >> "$GITHUB_OUTPUT" + echo "dockerhub_digest=$(resolve_digest "${DOCKERHUB_ORG}/${APP_NAME}:${TAG}")" >> "$GITHUB_OUTPUT" fi if [ "$ENABLE_GHCR" == "true" ]; then - REF="ghcr.io/${GHCR_ORG}/${APP_NAME}:${TAG}" - DIGEST="" - if OUTPUT=$(docker buildx imagetools inspect "$REF" --format '{{json .Manifest}}' 2>&1); then - DIGEST=$(echo "$OUTPUT" | jq -r '.digest // empty') - [ -z "$DIGEST" ] && echo "::warning::docker buildx imagetools inspect for ${REF} returned no digest — cosign will skip this registry." - else - echo "::warning::Failed to inspect ${REF} (registry lookup error, possibly transient) — cosign will skip signing for this registry: $(escape_for_warning "$OUTPUT")" - fi - echo "ghcr_digest=$DIGEST" >> "$GITHUB_OUTPUT" + echo "ghcr_digest=$(resolve_digest "ghcr.io/${GHCR_ORG}/${APP_NAME}:${TAG}")" >> "$GITHUB_OUTPUT" fi - name: Build cosign image references From 07c4d0d7435346228aba221f8597e1abf5078647 Mon Sep 17 00:00:00 2001 From: Lucas Bedatty Date: Wed, 22 Jul 2026 13:28:46 -0300 Subject: [PATCH 7/8] fix(build): surface registry lookup failures even when signing succeeds Address CodeRabbit finding on PR #604: when on_existing_tag: skip retries signing across multiple registries, a genuine registry lookup failure (as opposed to a confirmed-absent tag) collapsed to an empty digest just like a normal 'not found', so that registry was silently excluded from signing. If the OTHER enabled registry resolved fine, cosign-sign succeeded for it, and the failure report step (gated only on cosign-sign's own outcome) never ran -- leaving one registry's already-published image unsigned with no visible failure signal beyond a log warning. resolve_digest() now also reports whether the lookup itself failed (vs. the tag being genuinely absent). Build cosign image references aggregates that into a has_lookup_error output, and Report cosign signing status now fires on either cosign-sign failing OR a lookup error, so a partially-unverified multi-registry signing run is always visibly reported instead of looking like a clean success. --- .github/workflows/build.yml | 58 +++++++++++++++++++++++++++++-------- 1 file changed, 46 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 3bcb21ad..7a29b339 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -503,20 +503,29 @@ jobs: # Captures stdout and stderr separately (not merged) so any stderr # chatter on an otherwise-successful call (deprecation notices, # credential-helper output) can't corrupt the JSON fed to jq. + # + # Emits digest then "true"/"false" (lookup failed) on separate lines. + # An empty digest is ambiguous by itself: it means either "tag + # genuinely absent here" (fine, that registry is just skipped) or + # "the lookup itself errored" (ambiguous — the tag may well exist + # and just went unsigned). The caller needs that distinction to + # avoid silently reporting success when a registry couldn't be + # verified at all. resolve_digest() { local ref="$1" - local out err digest="" + local out err digest="" failed="false" local err_file err_file="$(mktemp)" if out=$(docker buildx imagetools inspect "$ref" --format '{{json .Manifest}}' 2>"$err_file"); then digest=$(echo "$out" | jq -r '.digest // empty') [ -z "$digest" ] && echo "::warning::docker buildx imagetools inspect for ${ref} returned no digest — cosign will skip this registry." >&2 else + failed="true" err=$(cat "$err_file") echo "::warning::Failed to inspect ${ref} (registry lookup error, possibly transient) — cosign will skip signing for this registry: $(escape_for_warning "$err")" >&2 fi rm -f "$err_file" - printf '%s' "$digest" + printf '%s\n%s\n' "$digest" "$failed" } # Each registry is inspected independently: a tag can be immutable-blocked @@ -524,11 +533,15 @@ jobs: # reusing one registry's digest for the other's ref would sign a # nonexistent (or wrong) reference. if [ "$ENABLE_DOCKERHUB" == "true" ]; then - echo "dockerhub_digest=$(resolve_digest "${DOCKERHUB_ORG}/${APP_NAME}:${TAG}")" >> "$GITHUB_OUTPUT" + { read -r DH_DIGEST; read -r DH_FAILED; } < <(resolve_digest "${DOCKERHUB_ORG}/${APP_NAME}:${TAG}") + echo "dockerhub_digest=$DH_DIGEST" >> "$GITHUB_OUTPUT" + echo "dockerhub_lookup_failed=$DH_FAILED" >> "$GITHUB_OUTPUT" fi if [ "$ENABLE_GHCR" == "true" ]; then - echo "ghcr_digest=$(resolve_digest "ghcr.io/${GHCR_ORG}/${APP_NAME}:${TAG}")" >> "$GITHUB_OUTPUT" + { read -r GHCR_DIGEST_OUT; read -r GHCR_FAILED; } < <(resolve_digest "ghcr.io/${GHCR_ORG}/${APP_NAME}:${TAG}") + echo "ghcr_digest=$GHCR_DIGEST_OUT" >> "$GITHUB_OUTPUT" + echo "ghcr_lookup_failed=$GHCR_FAILED" >> "$GITHUB_OUTPUT" fi - name: Build cosign image references @@ -538,6 +551,8 @@ jobs: DIGEST: ${{ steps.build-push.outputs.digest }} DOCKERHUB_DIGEST: ${{ steps.existing-digest.outputs.dockerhub_digest }} GHCR_DIGEST: ${{ steps.existing-digest.outputs.ghcr_digest }} + DOCKERHUB_LOOKUP_FAILED: ${{ steps.existing-digest.outputs.dockerhub_lookup_failed }} + GHCR_LOOKUP_FAILED: ${{ steps.existing-digest.outputs.ghcr_lookup_failed }} ENABLE_DOCKERHUB: ${{ inputs.enable_dockerhub }} ENABLE_GHCR: ${{ inputs.enable_ghcr }} DOCKERHUB_ORG: ${{ inputs.dockerhub_org }} @@ -569,6 +584,16 @@ jobs: echo "EOF" } >> "$GITHUB_OUTPUT" + # A registry lookup error (as opposed to a confirmed-absent tag) means + # that registry's image was never actually verified for signing, even + # if the other enabled registry resolves fine and cosign succeeds for + # it. Surface that as its own signal so the run isn't reported as a + # clean success while one registry's image silently stayed unsigned. + HAS_LOOKUP_ERROR="false" + [ "$ENABLE_DOCKERHUB" == "true" ] && [ "$DOCKERHUB_LOOKUP_FAILED" == "true" ] && HAS_LOOKUP_ERROR="true" + [ "$ENABLE_GHCR" == "true" ] && [ "$GHCR_LOOKUP_FAILED" == "true" ] && HAS_LOOKUP_ERROR="true" + echo "has_lookup_error=$HAS_LOOKUP_ERROR" >> "$GITHUB_OUTPUT" + - name: Sign container images with cosign if: inputs.enable_cosign_sign id: cosign-sign @@ -581,23 +606,32 @@ jobs: max-delay: ${{ inputs.cosign_max_delay }} - name: Report cosign signing status - if: inputs.enable_cosign_sign && steps.cosign-sign.outcome == 'failure' + if: >- + inputs.enable_cosign_sign && + (steps.cosign-sign.outcome == 'failure' || steps.cosign-refs.outputs.has_lookup_error == 'true') env: REFS: ${{ steps.cosign-refs.outputs.refs }} APP_NAME: ${{ matrix.app.name }} + SIGNING_FAILED: ${{ steps.cosign-sign.outcome == 'failure' }} + HAS_LOOKUP_ERROR: ${{ steps.cosign-refs.outputs.has_lookup_error }} run: | - echo "::warning::Cosign signing did not complete for every ref for ${APP_NAME} — one or more of the refs below may still be unsigned." + echo "::warning::Cosign signing did not complete for every image of ${APP_NAME} — one or more images may still be unsigned." { - echo "### ⚠️ Signing did not complete for every ref — manual action required" + echo "### ⚠️ Signing did not complete for every image — manual action required" echo "" echo "- **App:** \`${APP_NAME}\`" - echo "- **Requested refs (each already includes its own digest):**" - echo '```' - echo "${REFS}" - echo '```' + if [ "$HAS_LOOKUP_ERROR" == "true" ]; then + echo "- **Note:** a registry lookup failed while resolving the existing tag's digest (see the ::warning:: above) — that registry's image was never verified or signed, even if signing succeeded for the other one." + fi + if [ "$SIGNING_FAILED" == "true" ]; then + echo "- **Requested refs (each already includes its own digest):**" + echo '```' + echo "${REFS}" + echo '```' + fi echo "" echo "GitOps artifact upload was allowed to continue (\`continue_gitops_on_signing_failure: true\`)." - echo "The image(s) above are immutable and already present in the registry, but signing failed after all retries for at least one ref — some refs above may already be signed, others may not be. Verify with \`cosign verify\` and run \`cosign sign\` manually against any ref that isn't before promoting to production." + echo "The image(s) above are immutable and already present in the registry, but signing did not complete for every one — some may already be signed, others may not be. Verify with \`cosign verify\` and run \`cosign sign\` manually against any ref that isn't before promoting to production." } >> "$GITHUB_STEP_SUMMARY" # ----------------- GitOps Artifacts ----------------- From f77e0c0ee71c92ffac9bca0a40968c9b74f02a9e Mon Sep 17 00:00:00 2001 From: Lucas Bedatty Date: Wed, 22 Jul 2026 13:34:43 -0300 Subject: [PATCH 8/8] fix(build): add !cancelled() so the cosign report step can actually run Address CodeRabbit finding on PR #604: the 'Report cosign signing status' step's if: had no explicit status-check function (success()/always()/cancelled()/failure()), so GitHub Actions implicitly ANDs success() onto it. When cosign-sign fails without continue-on-error (the default, since continue_gitops_on_signing_failure defaults to false), that implicit success() check is false, so this report step -- whose entire purpose is to fire on that exact failure -- was silently skipped in the default configuration. Adding !cancelled() removes the implicit success() requirement while still skipping on a cancelled run, letting the step run whenever cosign-sign actually failed or a registry lookup error occurred. --- .github/workflows/build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7a29b339..60d4349f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -607,6 +607,7 @@ jobs: - name: Report cosign signing status if: >- + !cancelled() && inputs.enable_cosign_sign && (steps.cosign-sign.outcome == 'failure' || steps.cosign-refs.outputs.has_lookup_error == 'true') env: