docker-build.yml

# =============================================================================
# iSpy Agent DVR Docker Image Builder
# =============================================================================
# Consolidated workflow for automated and manual Docker image builds
#
# Features:
#   - Automatic release detection (every 30 minutes - time-based)
#   - Multi-platform builds (amd64, arm64, arm/v7)
#   - Dual registry support (Docker Hub + ghcr.io)
#   - Registry synchronization
#   - Manual version builds (range or comma-separated)
#   - Image promotion (latest/stable tags)
#   - 5-day stable promotion
#   - ZSTD compression + OCI media types
#   - Vulnerability scanning (non-blocking)
#   - Build provenance and SBOM
#   - Version validation with warnings (missing versions don't fail build)
#
# Required Secrets:
#   - DOCKERHUB_USERNAME: Docker Hub username
#   - DOCKERHUB_TOKEN: Docker Hub access token (with push permissions)
#
# Note: GITHUB_TOKEN is automatically provided and has packages:write scope
#
# =============================================================================

name: Docker Build & Publish

on:
  # =========================================================================
  # EXTERNAL TRIGGER: For reliable scheduling via cron-job.org or similar
  # =========================================================================
  repository_dispatch:
    types: [Docker Build & Publish - Cron Scheduled, release-published]

  # =========================================================================
  # DAILY SAFETY NET: catches anything the release-published dispatch missed
  # (failed dispatch, token problems), ages :latest into :stable after 5 days,
  # and picks up package-refresh rebuilds. GitHub cron can run minutes late;
  # that's fine for a daily sweep.
  # =========================================================================
  schedule:
    - cron: '17 3 * * *'

  # =========================================================================
  # MANUAL TRIGGER: With detailed input options
  # =========================================================================
  workflow_dispatch:
    inputs:
      # --- Action Selection ---
      action:
        description: 'Select the action to perform'
        required: true
        type: choice
        options:
          - 'auto-check'
          - 'build-versions'
          - 'promote-image'
          - 'force-promote-latest'
          - 'promote-stable'
          - 'force-promote-stable'
          - 'sync-registries'
        default: 'auto-check'

      # --- Version Build Options ---
      versions:
        description: |
          Versions to build (build-versions) or promote (force-promote-latest).
          Minimum buildable version: 7.9.4.0 (older releases lack bundled FFmpeg; see MIN_BUILD_VERSION)
          Examples:
            - Single: 7.9.4.0
            - Multiple: 7.9.4.0,7.9.5.0
            - Range: 7.9.4.0-7.9.6.0
        required: false
        type: string
        default: ''

      # --- Base Image Selection ---
      base_image:
        description: 'Base image for Docker build'
        required: false
        type: choice
        options:
          - 'default (latest)'
          - 'custom'
        default: 'default (latest)'

      custom_base_image:
        description: 'Custom base image URL (only when base_image=custom)'
        required: false
        type: string
        default: ''

      # --- Image Promotion Options ---
      promote_version:
        description: 'Source version to promote (for promote-image action)'
        required: false
        type: string
        default: ''

      promote_tag:
        description: 'Target tag for promotion'
        required: false
        type: choice
        options:
          - 'latest'
          - 'stable'
          - 'beta'
        default: 'latest'

      # --- Build Behavior Options (Checkboxes) ---
      force_build:
        description: 'Force rebuild even if image already exists in registries'
        required: false
        type: boolean
        default: false

      skip_existing:
        description: 'Skip versions that already exist in registries'
        required: false
        type: boolean
        default: true

      tag_as_latest:
        description: 'Also tag the built image(s) as latest'
        required: false
        type: boolean
        default: true

      run_security_scan:
        description: 'Run Trivy security vulnerability scan'
        required: false
        type: boolean
        default: true

      push_to_dockerhub:
        description: 'Push to Docker Hub registry'
        required: false
        type: boolean
        default: true

      push_to_ghcr:
        description: 'Push to GitHub Container Registry (ghcr.io)'
        required: false
        type: boolean
        default: true

      # --- Base Image Update Options ---
      update_base_image:
        description: 'Rebuild if base image has updates (checks digest)'
        required: false
        type: boolean
        default: false

      # --- Platform Options ---
      platforms:
        description: 'Target platforms for multi-arch build'
        required: false
        type: choice
        options:
          - 'linux/amd64,linux/arm64,linux/arm/v7'
          - 'linux/amd64,linux/arm64'
          - 'linux/amd64'
        default: 'linux/amd64,linux/arm64,linux/arm/v7'

      # --- Compression Options ---
      compression_level:
        description: 'ZSTD compression level (1-22, higher = smaller but slower)'
        required: false
        type: choice
        options:
          - '22'
          - '19'
          - '15'
          - '10'
          - '5'
          - '3'
        default: '22'

# Cancel in-progress runs for the same trigger/action context.
# This avoids unrelated manual actions sharing the same concurrency key.
concurrency:
  group: docker-build-${{ github.event_name }}-${{ github.event_name == 'workflow_dispatch' && github.event.inputs.action || 'auto-check' }}-${{ github.event_name == 'workflow_dispatch' && github.event.inputs.versions || 'default' }}
  cancel-in-progress: false

env:
  # Configuration sourced from repository variables (Settings → Secrets and variables → Actions → Variables)
  # Fallback values used if variables are not set
  TZ: ${{ vars.TZ || 'UTC' }}
  DOCKERHUB_REPO: ${{ vars.DOCKERHUB_REPO || 'ispysoftware/agentdvr' }}
  GHCR_REPO: ${{ vars.GHCR_REPO || 'ghcr.io/ispysoftware/agentdvr' }}
  DEFAULT_PLATFORMS: ${{ vars.DEFAULT_PLATFORMS || 'linux/amd64,linux/arm64,linux/arm/v7' }}
  BINARY_BASE_URL: ${{ vars.BINARY_BASE_URL || 'https://files.ispyconnect.com/downloads' }}
  RELEASE_URL: ${{ vars.RELEASE_URL || 'https://www.ispyconnect.com/producthistory?productid=27' }}
  # Primary version source: reads dbo.Products directly with no origin or edge caching,
  # so a release-published dispatch fired seconds after UpdateVersion sees the new
  # version immediately. The download-API URLs below are the (5-minute-cached) fallback.
  LIVE_VERSION_URL: ${{ vars.LIVE_VERSION_URL || 'https://www.ispyconnect.com/liveversion?productid=27' }}
  # Beta channel lives outside the release-history page: iSpy stopped tagging betas with
  # "(Beta):" there after 7.7.5.0 (24 Jul 2026) and now only exposes them via this API
  # (same endpoint the official linux_setup2.sh uses for USE_BETA=true).
  BETA_API_URL: ${{ vars.BETA_API_URL || 'https://www.ispyconnect.com/api/Agent/DownloadLocation5?platform=Linux64&useVersion=0&useBeta=True' }}
  # Same endpoint on the stable channel. Used as a safety net: if the release-history
  # page moves or changes markup again, this still yields the current release so the
  # newest version never goes unbuilt (a page redirect cost 3 weeks of builds in Jun 2026).
  STABLE_API_URL: ${{ vars.STABLE_API_URL || 'https://www.ispyconnect.com/api/Agent/DownloadLocation5?platform=Linux64&useVersion=0&useBeta=False' }}
  # NOTE: deliberately does NOT read github.event.client_payload.force_build.
  # The scheduled cron payload has carried force_build:"true" for months while
  # github.event.inputs was empty for repository_dispatch, so it never took effect.
  # Honouring it turned every 30-minute poll into a full multi-arch rebuild.
  # Use the package_refresh knobs below for scheduled rebuilds, or workflow_dispatch
  # for a genuine one-off force.
  FORCE_BUILD: ${{ github.event.inputs.force_build || 'false' }}
  # Rebuild the latest version once its published image ages past this, so Debian
  # package updates in the base image actually reach the shipped image.
  # Resolution order: client_payload > repo variable > default. Resolved in shell,
  # since `x != null && x || y` collapses an explicit `false` back to `y`.
  #
  #   {"event_type":"Docker Build & Publish - Cron Scheduled",
  #    "client_payload":{"package_refresh":false,"max_image_age_days":14}}
  CP_PACKAGE_REFRESH: ${{ github.event.client_payload.package_refresh }}
  CP_MAX_IMAGE_AGE_DAYS: ${{ github.event.client_payload.max_image_age_days }}
  VAR_PACKAGE_REFRESH: ${{ vars.PACKAGE_REFRESH_ENABLED }}
  VAR_MAX_IMAGE_AGE_DAYS: ${{ vars.MAX_IMAGE_AGE_DAYS }}
  DEFAULT_BASE_IMAGE: ${{ vars.DEFAULT_BASE_IMAGE || 'ispysoftware/agentdvr-base-image:latest' }}
  # Base image update check: set to 'true' to enable automatic rebuilds when base image changes
  UPDATE_BASE_IMAGE_ENABLED: ${{ vars.UPDATE_BASE_IMAGE_ENABLED || 'false' }}
  # Comma-separated list of versions to exclude from builds (e.g. "7.0.5.0,7.0.6.0")
  EXCLUDE_VERSIONS: ${{ vars.EXCLUDE_VERSIONS || '' }}
  # Oldest version this pipeline may build. Releases before 7.9.4.0 do not bundle
  # FFmpeg, and the org base image no longer provides one — building them would
  # produce images with no FFmpeg at all. Applied at the source (the release list),
  # so backfills, manual build requests and stable promotion all respect it.
  MIN_BUILD_VERSION: ${{ vars.MIN_BUILD_VERSION || '7.9.4.0' }}
  # How long the latest version must remain current before stable promotion (e.g. "5d", "7d", "30d", "12h")
  STABLE_PROMOTION_AGE: ${{ vars.STABLE_PROMOTION_AGE || '5d' }}
  # Number of recent upstream versions to scan for missing builds
  VERSION_CHECK_COUNT: ${{ vars.VERSION_CHECK_COUNT || '10' }}
  # BuildKit concurrency and log tuning
  EXPORT_CACHE_CONCURRENCY: '4'
  EXPORT_LAYERS_CONCURRENCY: '4'
  BUILDKIT_STEP_LOG_MAX_SIZE: '50000000'
  BUILDKIT_STEP_LOG_MAX_SPEED: '100000000'
  BUILDKIT_PROGRESS: 'plain'

permissions:
  contents: write
  packages: write
  security-events: write

jobs:
  # ===========================================================================
  # Job 0: Quick state check — skip pipeline if nothing changed
  # ===========================================================================
  quick-check:
    timeout-minutes: 15
    name: Quick state check
    runs-on: ${{ vars.ACTION_RUNNER_VERSION || 'ubuntu-latest' }}
    outputs:
      should_continue: ${{ steps.state.outputs.should_continue }}
    steps:
      - name: Checkout
        uses: actions/checkout@v7
        with:
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Check pipeline state
        id: state
        env:
          RELEASE_URL: ${{ env.RELEASE_URL }}
        run: |
          FORCE="${{ env.FORCE_BUILD }}"
          ACTION="${{ github.event.inputs.action || 'auto-check' }}"

          # Non-auto-check actions always continue
          if [[ "$ACTION" != "auto-check" ]]; then
            echo "should_continue=true" >> "$GITHUB_OUTPUT"
            echo "Action '$ACTION' always runs"
            exit 0
          fi

          # Force build always continues
          if [[ "$FORCE" == "true" ]]; then
            echo "should_continue=true" >> "$GITHUB_OUTPUT"
            echo "Force build requested"
            exit 0
          fi

          # Fetch current latest version from upstream (lightweight curl + grep)
          CURRENT_VERSION=""
          if CURRENT_VERSION=$(curl -sfL "$RELEASE_URL" | grep -oP '<td[^>]*>\d+\.\d+\.\d+\.\d+</td>' | head -1 | sed 's/<[^>]*>//g'); then
            echo "Current upstream version: $CURRENT_VERSION"
          else
            echo "::warning::Could not fetch upstream version -- continuing pipeline"
            echo "should_continue=true" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          # Always continue -- auto-check now scans all 20 versions for missing builds,
          # not just the latest. The full registry check happens in check-releases.
          echo "should_continue=true" >> "$GITHUB_OUTPUT"
          echo "Current upstream latest: $CURRENT_VERSION -- continuing to full scan"

  # ===========================================================================
  # Job 1: Check for new releases and validate requested versions
  # ===========================================================================
  check-releases:
    timeout-minutes: 15
    name: Check Releases & Validate Versions
    needs: quick-check
    if: needs.quick-check.outputs.should_continue == 'true'
    runs-on: ${{ vars.ACTION_RUNNER_VERSION || 'ubuntu-latest' }}
    outputs:
      versions_to_build: ${{ steps.filter-excluded.outputs.versions || steps.determine-versions.outputs.versions }}
      versions_json: ${{ steps.filter-excluded.outputs.versions_json || steps.determine-versions.outputs.versions_json }}
      latest_version: ${{ steps.fetch-releases.outputs.latest_version }}
      latest_is_beta: ${{ steps.fetch-releases.outputs.latest_is_beta }}
      latest_beta_version: ${{ steps.fetch-releases.outputs.latest_beta_version }}
      beta_versions: ${{ steps.fetch-releases.outputs.beta_versions }}
      available_versions: ${{ steps.fetch-releases.outputs.available_versions }}
      should_build: ${{ steps.filter-excluded.outputs.should_build || steps.determine-versions.outputs.should_build }}
      should_mark_stable: ${{ steps.check-stable.outputs.should_mark_stable }}
      stable_version: ${{ steps.check-stable.outputs.stable_version }}
      action_type: ${{ steps.set-action.outputs.action }}
      missing_versions: ${{ steps.validate-versions.outputs.missing_versions }}
      base_image_selection: ${{ steps.resolve-base-image.outputs.selection }}
      custom_base_image: ${{ steps.resolve-base-image.outputs.custom }}
      base_image_updated: ${{ steps.check-base-image.outputs.updated }}
      platforms: ${{ steps.set-platforms.outputs.platforms }}
      compression_level: ${{ steps.set-compression.outputs.level }}
      build_matrix: ${{ steps.generate-matrix.outputs.matrix }}
      latest_needs_promote: ${{ steps.determine-versions.outputs.latest_needs_promote }}
      stable_needs_promote: ${{ steps.determine-versions.outputs.stable_needs_promote }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v7

      - name: Set action type
        id: set-action
        run: |
          if [ "${{ github.event_name }}" == "repository_dispatch" ] || [ "${{ github.event_name }}" == "schedule" ]; then
            echo "action=auto-check" >> $GITHUB_OUTPUT
            echo "Running scheduled auto-check (trigger: ${{ github.event_name }})..."
          else
            ACTION="${{ github.event.inputs.action }}"
            VERSIONS_INPUT="${{ github.event.inputs.versions }}"
            # If specific versions are passed with the default auto-check action,
            # honor them: build exactly those instead of sweeping the last N
            # releases. Prevents an unintended mass rebuild when a version is
            # typed but the action dropdown is left on auto-check.
            if [ "$ACTION" == "auto-check" ] && [ -n "$VERSIONS_INPUT" ]; then
              echo "versions input ('$VERSIONS_INPUT') provided with auto-check -- routing to build-versions"
              ACTION="build-versions"
            fi
            echo "action=$ACTION" >> $GITHUB_OUTPUT
            echo "Running manual action: $ACTION"
          fi

      - name: Set platforms
        id: set-platforms
        run: |
          if [ -n "${{ github.event.inputs.platforms }}" ]; then
            echo "platforms=${{ github.event.inputs.platforms }}" >> $GITHUB_OUTPUT
          else
            echo "platforms=${{ env.DEFAULT_PLATFORMS }}" >> $GITHUB_OUTPUT
          fi

      - name: Set compression level
        id: set-compression
        run: |
          # Default to level 22 (maximum compression) if not specified
          if [ -n "${{ github.event.inputs.compression_level }}" ]; then
            echo "level=${{ github.event.inputs.compression_level }}" >> $GITHUB_OUTPUT
          else
            echo "level=22" >> $GITHUB_OUTPUT
          fi
          echo "ZSTD compression level: ${{ github.event.inputs.compression_level || '22' }}"

      - name: Resolve base image selection
        id: resolve-base-image
        run: |
          INPUT_BASE="${{ github.event.inputs.base_image }}"
          CUSTOM_BASE="${{ github.event.inputs.custom_base_image }}"

          # Output the selection key (not the resolved URL to avoid secret masking)
          # The actual URL will be resolved in each build job
          if [ -z "$INPUT_BASE" ] || [ "$INPUT_BASE" == "default (latest)" ]; then
            echo "selection=default" >> $GITHUB_OUTPUT
          else
            echo "selection=$INPUT_BASE" >> $GITHUB_OUTPUT
          fi

          # Pass custom base image if specified
          if [ "$INPUT_BASE" == "custom" ] && [ -n "$CUSTOM_BASE" ]; then
            echo "custom=$CUSTOM_BASE" >> $GITHUB_OUTPUT
          else
            echo "custom=" >> $GITHUB_OUTPUT
          fi

          echo "Base image selection: ${INPUT_BASE:-default}"

      - name: Install dependencies
        run: |
          # jq and curl ship with the runner image; only skopeo needs apt, and it is
          # merely a fallback for crane -- a stalled mirror here must not hold the
          # concurrency group. A hung apt once blocked this job for hours.
          sudo rm -f /etc/apt/sources.list.d/google-chrome.list
          if ! sudo timeout 120 apt-get update -qq; then
            echo "::warning::apt-get update failed -- continuing with crane only"
            exit 0
          fi
          sudo timeout 180 apt-get install -y -qq skopeo \
            || echo "::warning::skopeo unavailable -- crane only"

      - name: Setup build environment
        uses: ./.github/actions/setup-build-env
        with:
          do_checkout: 'false'
          setup_qemu: 'false'
          setup_buildx: 'false'
          login_dockerhub: 'true'
          login_ghcr: 'true'
          dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
          ghcr_actor: ${{ github.actor }}
          ghcr_token: ${{ secrets.GITHUB_TOKEN }}

      - name: Fetch and parse release history
        id: fetch-releases
        run: |
          echo "Fetching release history from iSpy..."

          # Fetch the page (follow redirects; iSpy dropped .aspx URLs with a 301 in Jun 2026)
          curl -sfL "${{ env.RELEASE_URL }}" -o /tmp/releases.html

          # Parse the HTML table to extract versions and dates
          # Note: HTML uses <td valign="top"> so we match full td tag and strip with sed
          VERSION_LIMIT="${{ env.VERSION_CHECK_COUNT }}"
          grep -oP '<td[^>]*>\d+\.\d+\.\d+\.\d+</td>' /tmp/releases.html | sed 's/<[^>]*>//g' | head -"$VERSION_LIMIT" > /tmp/versions.txt

          # Resolve both release channels. Primary source is /liveversion, which reads
          # the database directly with no origin or edge caching — a release-published
          # dispatch fires seconds after UpdateVersion, inside the 5-minute window every
          # other version surface caches on, so only this endpoint is guaranteed fresh.
          # Falls back to the download API (5-minute-aligned cache) when unavailable.
          STABLE_CHANNEL_VERSION=""
          BETA_CHANNEL_VERSION=""
          if LIVE_JSON=$(curl -sfL --max-time 20 "${{ env.LIVE_VERSION_URL }}"); then
            STABLE_CHANNEL_VERSION=$(echo "$LIVE_JSON" | jq -r '.version // empty' 2>/dev/null | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' || true)
            BETA_CHANNEL_VERSION=$(echo "$LIVE_JSON" | jq -r '.betaVersion // empty' 2>/dev/null | grep -E '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' || true)
          fi

          # Fallback: the URLs iSpy's own installer uses, so they answer even when the
          # release-history page does not. Prints e.g. .../Agent_Linux64_7_9_3_0.zip.
          # Returns empty on any curl failure (4xx/5xx/network) so callers skip the splice.
          # Quotes are stripped in-shell rather than by a pipe, so curl's exit status is
          # the command's exit status and the failure branch actually fires.
          channel_version() {
            local url=$1 dl=""
            dl=$(curl -sfL "$url") || return 0
            echo "${dl//\"/}" | sed -nE 's#.*_([0-9]+)_([0-9]+)_([0-9]+)_([0-9]+)\.zip$#\1.\2.\3.\4#p'
          }
          if [ -z "$STABLE_CHANNEL_VERSION" ]; then
            echo "::warning::/liveversion unavailable or unparseable -- falling back to the download API (5-minute aligned cache)"
            # Bridge for sites without /liveversion: a release-published dispatch fires
            # seconds after UpdateVersion, inside the stale window of the download API's
            # cache. That cache expires at 5-minute wall-clock boundaries, so waiting
            # just past the next boundary guarantees a fresh read. Skipped entirely
            # once /liveversion answers above.
            if [ "${{ github.event.action }}" == "release-published" ]; then
              NOW=$(date +%s)
              WAIT=$(( 300 - NOW % 300 + 15 ))
              echo "release-published dispatch: waiting ${WAIT}s for the aligned caches to roll over"
              sleep "$WAIT"
            fi
            STABLE_CHANNEL_VERSION=$(channel_version "${{ env.STABLE_API_URL }}")
            BETA_CHANNEL_VERSION=$(channel_version "${{ env.BETA_API_URL }}")
          fi
          echo "Version channels -- stable: ${STABLE_CHANNEL_VERSION:-<unresolved>}, beta: ${BETA_CHANNEL_VERSION:-<unresolved>}"

          # Hard-fail only when every source is dead. A silently empty version list
          # disabled all builds for 3 weeks when the release URL started redirecting;
          # now a broken page degrades to API-only instead of stopping the pipeline.
          if [ ! -s /tmp/versions.txt ]; then
            if [ -z "$STABLE_CHANNEL_VERSION" ] && [ -z "$BETA_CHANNEL_VERSION" ]; then
              echo "::error::No versions parsed from $RELEASE_URL and the download API returned nothing"
              exit 1
            fi
            echo "::warning::No versions parsed from $RELEASE_URL -- page moved or markup changed. Falling back to the download API alone; backfill scanning and :stable promotion are degraded until the page parses again."
          fi
          grep -oP '<td[^>]*>\d{1,2}/\d{1,2}/\d{4}</td>' /tmp/releases.html | sed 's/<[^>]*>//g' | head -"$VERSION_LIMIT" > /tmp/dates.txt

          # Parse beta status: HTML spans multiple lines, so convert to single-line first
          # Then extract each row and check if update info contains (Beta):
          tr '\n' ' ' < /tmp/releases.html | sed 's/  */ /g' > /tmp/releases_oneline.html

          > /tmp/beta_status.txt
          while IFS= read -r version; do
            # Extract the row for this version and check for (Beta): in the update info
            ROW=$(grep -oP "<tr><td[^>]*>${version}</td><td[^>]*>.*?</td><td[^>]*>\d{1,2}/\d{1,2}/\d{4}</td></tr>" /tmp/releases_oneline.html | head -1)
            if echo "$ROW" | grep -q '(Beta):'; then
              echo "$version|true" >> /tmp/beta_status.txt
            else
              echo "$version|false" >> /tmp/beta_status.txt
            fi
          done < /tmp/versions.txt

          # Preserve the release-history view of "newest" before splicing anything in.
          HISTORY_LATEST_IS_BETA=$(head -1 /tmp/beta_status.txt | cut -d'|' -f2)

          # --- Splice in versions the release-history page does not list ---
          # iSpy stopped marking betas with "(Beta):" on that page after 7.7.5.0 (24 Jul 2026),
          # and the page can lag or break for stable releases too. Anything the download API
          # reports but the page omits is prepended as the newest entry so it still gets built.
          # Date is "-" (unknown): these entries never feed the :stable promotion clock.
          splice_version() {
            local version=$1 is_beta=$2 label=$3
            if [ -z "$version" ]; then
              echo "::warning::Could not resolve a $label version from the download API"
              return 0
            fi
            if grep -qx "$version" /tmp/versions.txt; then
              echo "$label channel version $version already in release history -- no splice"
              return 0
            fi
            echo "$label channel exposes $version (absent from release history) -- splicing in"
            # Prepend via a temp file: "sed -i 1i" is a no-op on an empty file, which is
            # exactly the case here when the release-history page failed to parse.
            { echo "$version"; cat /tmp/versions.txt; } > /tmp/versions.new && mv /tmp/versions.new /tmp/versions.txt
            { echo '-'; cat /tmp/dates.txt; } > /tmp/dates.new && mv /tmp/dates.new /tmp/dates.txt
            { echo "${version}|${is_beta}"; cat /tmp/beta_status.txt; } > /tmp/beta_status.new && mv /tmp/beta_status.new /tmp/beta_status.txt
          }
          # Stable first so that, if both splice, the beta still ends up on top.
          splice_version "$STABLE_CHANNEL_VERSION" "false" "Stable"
          splice_version "$BETA_CHANNEL_VERSION" "true" "Beta"

          # Enforce MIN_BUILD_VERSION: drop anything older from the release list so no
          # downstream path (auto-check backfill, build-versions validation, stable
          # promotion) can select a version the current base image cannot support.
          # The three files are line-aligned, so they are filtered by the same indices.
          MIN_VER="${{ env.MIN_BUILD_VERSION }}"
          if [ -n "$MIN_VER" ]; then
            : > /tmp/versions.new; : > /tmp/dates.new; : > /tmp/beta_status.new
            i=0
            while IFS= read -r v; do
              i=$((i+1))
              if [ "$(printf '%s\n%s\n' "$v" "$MIN_VER" | sort -V | head -1)" = "$MIN_VER" ]; then
                echo "$v" >> /tmp/versions.new
                sed -n "${i}p" /tmp/dates.txt >> /tmp/dates.new
                sed -n "${i}p" /tmp/beta_status.txt >> /tmp/beta_status.new
              else
                echo "Skipping $v -- below version floor $MIN_VER (no bundled FFmpeg)"
              fi
            done < /tmp/versions.txt
            mv /tmp/versions.new /tmp/versions.txt
            mv /tmp/dates.new /tmp/dates.txt
            mv /tmp/beta_status.new /tmp/beta_status.txt
            if [ ! -s /tmp/versions.txt ]; then
              echo "::warning::No versions at or above the $MIN_VER floor -- nothing is eligible to build yet"
            fi
          fi

          # Create combined output (version|date|is_beta)
          paste -d'|' /tmp/versions.txt /tmp/dates.txt > /tmp/releases_temp.txt
          paste -d'|' /tmp/releases_temp.txt <(cut -d'|' -f2 /tmp/beta_status.txt) > /tmp/releases.txt

          # Get latest version and its beta status
          LATEST_VERSION=$(awk -F'|' '$3 != "true" { print $1; exit }' /tmp/releases.txt)
          LATEST_IS_BETA="$HISTORY_LATEST_IS_BETA"
          echo "latest_version=$LATEST_VERSION" >> $GITHUB_OUTPUT
          echo "latest_is_beta=$LATEST_IS_BETA" >> $GITHUB_OUTPUT
          # :beta tracks a beta only while it is actually ahead of stable. Once a beta is
          # promoted, the newest "(Beta):" row still inside the scan window is historical --
          # letting it stay the rolling target would drag :beta backwards if registry drift
          # ever marked that old version for rebuild.
          LATEST_BETA_VERSION=$(grep '|true$' /tmp/beta_status.txt | head -1 | cut -d'|' -f1)
          if [ -n "$LATEST_BETA_VERSION" ] && [ -n "$LATEST_VERSION" ] && \
             [ "$(printf '%s\n%s\n' "$LATEST_BETA_VERSION" "$LATEST_VERSION" | sort -V | tail -1)" = "$LATEST_VERSION" ]; then
            echo "Newest beta $LATEST_BETA_VERSION is not ahead of stable $LATEST_VERSION -- no live beta channel"
            LATEST_BETA_VERSION=""
          fi
          echo "latest_beta_version=$LATEST_BETA_VERSION" >> $GITHUB_OUTPUT

          # Create JSON object mapping version -> beta status
          BETA_VERSIONS=$(awk -F'|' '{printf "\"%s\":%s,", $1, $2}' /tmp/beta_status.txt | sed 's/,$//' | sed 's/^/{/' | sed 's/$/}/')
          echo "beta_versions=$BETA_VERSIONS" >> $GITHUB_OUTPUT

          # Create JSON array of available versions
          AVAILABLE_VERSIONS=$(cat /tmp/versions.txt | jq -R -s -c 'split("\n") | map(select(length > 0))')
          echo "available_versions=$AVAILABLE_VERSIONS" >> $GITHUB_OUTPUT

          echo "Latest version: $LATEST_VERSION (beta: $LATEST_IS_BETA)"
          echo "Found $(wc -l < /tmp/versions.txt) versions in release history"
          echo "Beta versions:"
          grep '|true$' /tmp/beta_status.txt | cut -d'|' -f1 || echo "  (none in top 20)"

          # Save for other steps
          cp /tmp/versions.txt /tmp/available_versions.txt
          cp /tmp/beta_status.txt /tmp/beta_status_available.txt

      - name: Validate requested versions against available versions
        id: validate-versions
        run: |
          INPUT_VERSIONS="${{ github.event.inputs.versions }}"
          AVAILABLE_FILE="/tmp/available_versions.txt"

          # Function to check if version exists
          version_exists() {
            grep -q "^$1$" "$AVAILABLE_FILE"
          }

          # Function to expand version range (bidirectional - handles both orderings)
          expand_range() {
            local start=$1
            local end=$2
            local found_versions=""
            local in_range=false
            local start_line=0
            local end_line=0
            local line_num=0

            # First pass: find line numbers of start and end versions
            while IFS= read -r version; do
              line_num=$((line_num + 1))
              if [ "$version" == "$start" ]; then
                start_line=$line_num
              fi
              if [ "$version" == "$end" ]; then
                end_line=$line_num
              fi
            done < "$AVAILABLE_FILE"

            # Determine direction: if start_line > end_line, versions are newest-first
            # Swap if needed so we always iterate from first_line to last_line
            local first_line=$start_line
            local last_line=$end_line
            if [ $start_line -gt $end_line ] && [ $end_line -gt 0 ]; then
              first_line=$end_line
              last_line=$start_line
            fi

            # Second pass: collect versions in range
            line_num=0
            while IFS= read -r version; do
              line_num=$((line_num + 1))
              if [ $line_num -ge $first_line ] && [ $line_num -le $last_line ]; then
                if [ -z "$found_versions" ]; then
                  found_versions="$version"
                else
                  found_versions="$found_versions,$version"
                fi
              fi
            done < "$AVAILABLE_FILE"

            echo "$found_versions"
          }

          MISSING_VERSIONS=""
          VALID_VERSIONS=""

          if [ -n "$INPUT_VERSIONS" ]; then
            # Check if it's a range format (contains - but not at start, and no comma)
            if [[ "$INPUT_VERSIONS" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+-[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
              # Range format: 7.0.5.0-7.0.9.0
              START_VER=$(echo "$INPUT_VERSIONS" | cut -d'-' -f1)
              END_VER=$(echo "$INPUT_VERSIONS" | cut -d'-' -f2)

              echo "Processing version range: $START_VER to $END_VER"

              # Check if start and end versions exist
              if ! version_exists "$START_VER"; then
                echo "::warning::Start version $START_VER not found in release history"
                MISSING_VERSIONS="$START_VER"
              fi
              if ! version_exists "$END_VER"; then
                echo "::warning::End version $END_VER not found in release history"
                if [ -z "$MISSING_VERSIONS" ]; then
                  MISSING_VERSIONS="$END_VER"
                else
                  MISSING_VERSIONS="$MISSING_VERSIONS,$END_VER"
                fi
              fi

              # Expand range to get all versions
              VALID_VERSIONS=$(expand_range "$START_VER" "$END_VER")

              if [ -z "$VALID_VERSIONS" ]; then
                echo "::warning::No versions found in range $START_VER to $END_VER"
              fi

            else
              # Comma-separated or single version
              IFS=',' read -ra VERSION_ARRAY <<< "$INPUT_VERSIONS"

              for version in "${VERSION_ARRAY[@]}"; do
                version=$(echo "$version" | xargs)  # Trim whitespace
                if version_exists "$version"; then
                  if [ -z "$VALID_VERSIONS" ]; then
                    VALID_VERSIONS="$version"
                  else
                    VALID_VERSIONS="$VALID_VERSIONS,$version"
                  fi
                else
                  echo "::warning::Version $version not found in release history - will be skipped"
                  if [ -z "$MISSING_VERSIONS" ]; then
                    MISSING_VERSIONS="$version"
                  else
                    MISSING_VERSIONS="$MISSING_VERSIONS,$version"
                  fi
                fi
              done
            fi
          fi

          echo "valid_versions=$VALID_VERSIONS" >> $GITHUB_OUTPUT
          echo "missing_versions=$MISSING_VERSIONS" >> $GITHUB_OUTPUT

          if [ -n "$MISSING_VERSIONS" ]; then
            echo "::warning::The following versions were not found and will be skipped: $MISSING_VERSIONS"
          fi

          echo "Valid versions to process: $VALID_VERSIONS"

      - name: Check base image for updates
        id: check-base-image
        env:
          DEFAULT_IMAGE: ${{ env.DEFAULT_BASE_IMAGE }}
        run: |
          source "${{ github.workspace }}/.github/scripts/lib-retry.sh"

          # Resolve base image from selection (same logic as build-platform job)
          BASE_SELECTION="${{ steps.resolve-base-image.outputs.selection }}"
          CUSTOM_BASE="${{ steps.resolve-base-image.outputs.custom }}"

          case "$BASE_SELECTION" in
            "custom")
              BASE_IMAGE="${CUSTOM_BASE:-$DEFAULT_IMAGE}"
              ;;
            *)
              BASE_IMAGE="$DEFAULT_IMAGE"
              ;;
          esac

          LATEST_VERSION="${{ steps.fetch-releases.outputs.latest_version }}"
          UPDATE_ENABLED="${{ github.event.inputs.update_base_image || env.UPDATE_BASE_IMAGE_ENABLED }}"

          echo "Base image update check enabled: $UPDATE_ENABLED"

          if [ "$UPDATE_ENABLED" != "true" ]; then
            echo "updated=false" >> $GITHUB_OUTPUT
            echo "Base image update check is disabled"
            exit 0
          fi

          echo "Checking if base image has updates..."

          # Get base image digest (crane digest with retry, falls back to skopeo)
          BASE_DIGEST=$(run_with_retry_output crane digest "${BASE_IMAGE}" 2>/dev/null || run_with_retry_output skopeo inspect "docker://${BASE_IMAGE}" 2>/dev/null | jq -r '.Digest' || echo "unknown")
          echo "Base image digest: $BASE_DIGEST"

          if [ "$BASE_DIGEST" == "unknown" ] || [ -z "$BASE_DIGEST" ]; then
            echo "::warning::Could not fetch base image digest"
            echo "updated=false" >> $GITHUB_OUTPUT
            exit 0
          fi

          # Check if our latest image exists and get its base layer info
          # We store base image digest in labels during build
          CURRENT_IMAGE="${{ env.DOCKERHUB_REPO }}:${LATEST_VERSION}"
          CURRENT_BASE_LABEL=$(run_with_retry_output skopeo inspect "docker://${CURRENT_IMAGE}" 2>/dev/null | jq -r '.Labels["org.opencontainers.image.base.digest"] // empty' || echo "")

          if [ -z "$CURRENT_BASE_LABEL" ]; then
            # Fallback: check image creation time vs base image creation time
            CURRENT_CREATED=$(run_with_retry_output skopeo inspect "docker://${CURRENT_IMAGE}" 2>/dev/null | jq -r '.Created' || echo "")
            BASE_CREATED=$(run_with_retry_output skopeo inspect "docker://${BASE_IMAGE}" 2>/dev/null | jq -r '.Created' || echo "")

            if [ -n "$CURRENT_CREATED" ] && [ -n "$BASE_CREATED" ]; then
              CURRENT_EPOCH=$(date -d "$CURRENT_CREATED" +%s 2>/dev/null || echo "0")
              BASE_EPOCH=$(date -d "$BASE_CREATED" +%s 2>/dev/null || echo "0")

              if [ "$BASE_EPOCH" -gt "$CURRENT_EPOCH" ]; then
                echo "updated=true" >> $GITHUB_OUTPUT
                echo "Base image is newer than current build - rebuild recommended"
                exit 0
              fi
            fi
            echo "updated=false" >> $GITHUB_OUTPUT
            echo "No base image update detected (no label, using timestamp comparison)"
          else
            if [ "$CURRENT_BASE_LABEL" != "$BASE_DIGEST" ]; then
              echo "updated=true" >> $GITHUB_OUTPUT
              echo "Base image digest changed: $CURRENT_BASE_LABEL -> $BASE_DIGEST"
            else
              echo "updated=false" >> $GITHUB_OUTPUT
              echo "Base image unchanged"
            fi
          fi

      - name: Check for 5-day stable promotion
        id: check-stable
        run: |
          ACTION="${{ steps.set-action.outputs.action }}"
          FORCE_PROMOTE_STABLE="false"
          if [[ "$ACTION" == "force-promote-stable" ]]; then
            FORCE_PROMOTE_STABLE="true"
          fi

          # Find latest non-beta release candidate (do not reset when latest release is beta)
          CANDIDATE_VERSION=""
          CANDIDATE_DATE=""
          while IFS='|' read -r VERSION DATE IS_BETA; do
            if [ "$IS_BETA" != "true" ] && [ -n "$VERSION" ]; then
              CANDIDATE_VERSION="$VERSION"
              CANDIDATE_DATE="$DATE"
              break
            fi
          done < /tmp/releases.txt

          # A candidate spliced in from the download API has no release date yet. Hold the
          # promotion rather than aging an older build into :stable ahead of schedule --
          # the real date arrives once the release-history page lists it.
          if [ -n "$CANDIDATE_VERSION" ] && ! echo "$CANDIDATE_DATE" | grep -qE '^[0-9]{1,2}/[0-9]{1,2}/[0-9]{4}$'; then
            echo "::warning::$CANDIDATE_VERSION has no release date yet (not on the release-history page) -- holding :stable promotion"
            echo "should_mark_stable=false" >> $GITHUB_OUTPUT
            echo "stable_version=" >> $GITHUB_OUTPUT
            exit 0
          fi

          if [ -z "$CANDIDATE_VERSION" ] || [ -z "$CANDIDATE_DATE" ]; then
            echo "::warning::Could not determine a non-beta stable candidate"
            echo "should_mark_stable=false" >> $GITHUB_OUTPUT
            echo "stable_version=" >> $GITHUB_OUTPUT
            exit 0
          fi

          # Convert date (M/D/YYYY) to epoch
          # Handle both 1/1/2025 and 12/27/2025 formats
          MONTH=$(echo "$CANDIDATE_DATE" | cut -d'/' -f1)
          DAY=$(echo "$CANDIDATE_DATE" | cut -d'/' -f2)
          YEAR=$(echo "$CANDIDATE_DATE" | cut -d'/' -f3)

          LATEST_EPOCH=$(date -d "$YEAR-$MONTH-$DAY" +%s 2>/dev/null || echo "0")
          CURRENT_EPOCH=$(date +%s)

          # Parse STABLE_PROMOTION_AGE (e.g. "5d", "7d", "30d", "12h"; empty = disabled)
          PROMO_AGE="${{ env.STABLE_PROMOTION_AGE }}"
          if [[ -z "$PROMO_AGE" ]]; then
            echo "::warning::STABLE_PROMOTION_AGE is empty -- stable promotion is disabled. Set it to a value like '5d', '12h', or '30m' to enable."
            echo "should_mark_stable=false" >> $GITHUB_OUTPUT
            echo "stable_version=" >> $GITHUB_OUTPUT
            exit 0
          fi
          PROMO_VALUE=$(echo "$PROMO_AGE" | grep -oP '^\d+')
          PROMO_UNIT=$(echo "$PROMO_AGE" | grep -oP '[a-z]+$')
          case "$PROMO_UNIT" in
            h) PROMO_SECONDS=$(( PROMO_VALUE * 3600 )) ;;
            d) PROMO_SECONDS=$(( PROMO_VALUE * 86400 )) ;;
            *) PROMO_SECONDS=$(( PROMO_VALUE * 86400 )) ;; # default to days
          esac

          AGE_SECONDS=$(( CURRENT_EPOCH - LATEST_EPOCH ))
          DAYS_OLD=$(( AGE_SECONDS / 86400 ))
          HOURS_OLD=$(( AGE_SECONDS / 3600 ))

          echo "Latest non-beta release ($CANDIDATE_VERSION) is $DAYS_OLD days old ($HOURS_OLD hours)"
          echo "Stable promotion threshold: $PROMO_AGE ($PROMO_SECONDS seconds)"

          if [ "$FORCE_PROMOTE_STABLE" == "true" ]; then
            echo "should_mark_stable=true" >> $GITHUB_OUTPUT
            echo "stable_version=$CANDIDATE_VERSION" >> $GITHUB_OUTPUT
            echo "Force-promote enabled, selecting $CANDIDATE_VERSION for stable"
          elif [ $LATEST_EPOCH -gt 0 ] && [ $AGE_SECONDS -ge $PROMO_SECONDS ]; then
            echo "should_mark_stable=true" >> $GITHUB_OUTPUT
            echo "stable_version=$CANDIDATE_VERSION" >> $GITHUB_OUTPUT
            echo "Version $CANDIDATE_VERSION is $DAYS_OLD days old, eligible for stable (threshold: $PROMO_AGE)"
          else
            echo "should_mark_stable=false" >> $GITHUB_OUTPUT
            echo "stable_version=" >> $GITHUB_OUTPUT
            echo "Version $CANDIDATE_VERSION not old enough for stable yet (age: ${HOURS_OLD}h, threshold: $PROMO_AGE)"
          fi

      - name: Determine versions to build
        id: determine-versions
        run: |
          ACTION="${{ steps.set-action.outputs.action }}"
          FORCE_BUILD="${{ env.FORCE_BUILD }}"
          SKIP_EXISTING="${{ github.event.inputs.skip_existing || github.event.client_payload.skip_existing }}"
          LATEST_VERSION="${{ steps.fetch-releases.outputs.latest_version }}"
          BETA_VERSIONS='${{ steps.fetch-releases.outputs.beta_versions }}'

          # Check a single version against both registries using digest comparison (background job)
          # A version is "exists" only if both registries have the tag AND digests match.
          # If digests differ, it's flagged as missing so sync/rebuild can fix it.
          # SHA-256 of empty input — used to detect failed skopeo fallback
          EMPTY_SHA="sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"

          get_digest() {
            local ref=$1 digest=""
            digest=$(crane digest "$ref" 2>/dev/null) && [[ -n "$digest" ]] && echo "$digest" && return 0
            local raw; raw=$(skopeo inspect --raw "docker://$ref" 2>/dev/null)
            [[ -n "$raw" ]] || return 1
            digest=$(echo "$raw" | sha256sum | awk '{print "sha256:"$1}')
            [[ "$digest" != "$EMPTY_SHA" ]] && echo "$digest" && return 0
            return 1
          }

          check_version_exists() {
            local version=$1 check_tag=$2 result_dir=$3
            local dh_digest="" ghcr_digest=""

            dh_digest=$(get_digest "${{ env.DOCKERHUB_REPO }}:${check_tag}") || dh_digest=""
            ghcr_digest=$(get_digest "${{ env.GHCR_REPO }}:${check_tag}") || ghcr_digest=""

            if [[ -n "$dh_digest" && -n "$ghcr_digest" && "$dh_digest" == "$ghcr_digest" ]]; then
              echo "exists" > "${result_dir}/${version}"
              echo "Skipping version $version (tag: $check_tag) - digest match in both registries ($dh_digest)" >&2
            else
              echo "missing" > "${result_dir}/${version}"
              if [[ -n "$dh_digest" && -n "$ghcr_digest" ]]; then
                echo "Version $version (tag: $check_tag) - digest MISMATCH (DH: $dh_digest, GHCR: $ghcr_digest)" >&2
              fi
            fi
          }

          filter_existing_versions() {
            local versions=$1
            local filtered=""

            if [ "$SKIP_EXISTING" != "true" ] || [ "$FORCE_BUILD" == "true" ]; then
              echo "$versions"
              return 0
            fi

            local result_dir
            result_dir="$(mktemp -d)"

            # Launch all version checks in parallel
            IFS=',' read -ra VERSION_ARRAY <<< "$versions"
            for version in "${VERSION_ARRAY[@]}"; do
              version=$(echo "$version" | xargs)
              IS_BETA=$(echo "$BETA_VERSIONS" | jq -r --arg v "$version" '.[$v] // false')
              if [ "$IS_BETA" == "true" ]; then
                CHECK_TAG="${version}-beta"
              else
                CHECK_TAG="${version}"
              fi
              check_version_exists "$version" "$CHECK_TAG" "$result_dir" &
            done
            wait

            # Collect results in original order
            for version in "${VERSION_ARRAY[@]}"; do
              version=$(echo "$version" | xargs)
              if [[ -f "${result_dir}/${version}" && "$(cat "${result_dir}/${version}")" == "missing" ]]; then
                if [ -z "$filtered" ]; then
                  filtered="$version"
                else
                  filtered="$filtered,$version"
                fi
              fi
            done

            rm -rf "$result_dir"
            echo "$filtered"
          }

          # Check if base image update should trigger rebuild
          BASE_IMAGE_UPDATED="${{ steps.check-base-image.outputs.updated }}"
          UPDATE_BASE_IMAGE="${{ github.event.inputs.update_base_image || env.UPDATE_BASE_IMAGE_ENABLED }}"

          case "$ACTION" in
            "auto-check")
              # Scan all available versions for missing builds, oldest first
              ALL_VERSIONS=$(tac /tmp/available_versions.txt | paste -sd ',' -)

              # Pre-filter excluded versions before registry scan to avoid wasted checks
              EXCLUDE_LIST="${{ env.EXCLUDE_VERSIONS }}"
              if [[ -n "$EXCLUDE_LIST" ]]; then
                FILTERED_ALL=""
                IFS=',' read -ra ALL_ARR <<< "$ALL_VERSIONS"
                for v in "${ALL_ARR[@]}"; do
                  v=$(echo "$v" | xargs)
                  if echo ",$EXCLUDE_LIST," | grep -q ",$v,"; then
                    echo "Pre-excluding version $v (in EXCLUDE_VERSIONS)"
                  else
                    FILTERED_ALL="${FILTERED_ALL:+$FILTERED_ALL,}$v"
                  fi
                done
                ALL_VERSIONS="$FILTERED_ALL"
              fi

              echo "Scanning versions for missing builds: $ALL_VERSIONS"

              # Use filter_existing_versions to find which ones are missing
              # Temporarily force skip_existing behavior for auto-check
              ORIG_SKIP="$SKIP_EXISTING"
              SKIP_EXISTING="true"
              MISSING_VERSIONS=$(filter_existing_versions "$ALL_VERSIONS")
              SKIP_EXISTING="$ORIG_SKIP"

              # Also rebuild latest if base image updated
              if [ "$BASE_IMAGE_UPDATED" == "true" ] && [ "$UPDATE_BASE_IMAGE" == "true" ]; then
                echo "Base image updated -- ensuring latest version is included"
                if ! echo ",$MISSING_VERSIONS," | grep -q ",$LATEST_VERSION,"; then
                  if [ -z "$MISSING_VERSIONS" ]; then
                    MISSING_VERSIONS="$LATEST_VERSION"
                  else
                    MISSING_VERSIONS="$LATEST_VERSION,$MISSING_VERSIONS"
                  fi
                fi
              fi

              # Package refresh: this image layers on ispyagentdvr-base-image, so Debian
              # updates only arrive on a rebuild. Latest version only -- older static
              # version tags are historical and must keep their original contents.
              PACKAGE_REFRESH="${CP_PACKAGE_REFRESH:-${VAR_PACKAGE_REFRESH:-true}}"
              MAX_IMAGE_AGE_DAYS="${CP_MAX_IMAGE_AGE_DAYS:-${VAR_MAX_IMAGE_AGE_DAYS:-30}}"
              if [ "$PACKAGE_REFRESH" == "true" ] && [ "$MAX_IMAGE_AGE_DAYS" != "0" ] \
                 && ! echo ",$MISSING_VERSIONS," | grep -q ",$LATEST_VERSION,"; then
                CREATED=$(crane config "${{ env.GHCR_REPO }}:${LATEST_VERSION}" 2>/dev/null | jq -r '.created // empty' || echo "")
                if [ -n "$CREATED" ]; then
                  CREATED_EPOCH=$(date -d "$CREATED" +%s 2>/dev/null || echo 0)
                  if [ "$CREATED_EPOCH" -gt 0 ]; then
                    AGE_DAYS=$(( ( $(date +%s) - CREATED_EPOCH ) / 86400 ))
                    if [ "$AGE_DAYS" -ge "$MAX_IMAGE_AGE_DAYS" ]; then
                      echo "Latest version $LATEST_VERSION is ${AGE_DAYS}d old (>= ${MAX_IMAGE_AGE_DAYS}d) -- rebuilding for package updates"
                      MISSING_VERSIONS="${LATEST_VERSION}${MISSING_VERSIONS:+,$MISSING_VERSIONS}"
                    else
                      echo "Latest version $LATEST_VERSION is ${AGE_DAYS}d old (< ${MAX_IMAGE_AGE_DAYS}d) -- no package-refresh rebuild"
                    fi
                  fi
                else
                  echo "::warning::Could not read the creation date of ${LATEST_VERSION} -- skipping the image age check"
                fi
              elif [ "$PACKAGE_REFRESH" != "true" ] || [ "$MAX_IMAGE_AGE_DAYS" == "0" ]; then
                echo "Package-refresh rebuild disabled (package_refresh=$PACKAGE_REFRESH, max_image_age_days=$MAX_IMAGE_AGE_DAYS)"
              fi

              # --- Digest check: :latest tag should match latest version tag ---
              LATEST_IS_BETA="${{ steps.fetch-releases.outputs.latest_is_beta }}"
              if [ "$LATEST_IS_BETA" == "true" ]; then
                ROLLING_TAG="beta"
                LATEST_VER_TAG="${LATEST_VERSION}-beta"
              else
                ROLLING_TAG="latest"
                LATEST_VER_TAG="${LATEST_VERSION}"
              fi

              GHCR_ROLLING=$(crane digest "${{ env.GHCR_REPO }}:${ROLLING_TAG}" 2>/dev/null || echo "")
              GHCR_LATEST_VER=$(crane digest "${{ env.GHCR_REPO }}:${LATEST_VER_TAG}" 2>/dev/null || echo "")
              DH_ROLLING=$(crane digest "${{ env.DOCKERHUB_REPO }}:${ROLLING_TAG}" 2>/dev/null || echo "")

              if [[ -z "$GHCR_LATEST_VER" ]]; then
                # Latest version not in GHCR yet — promote after build
                echo "latest_needs_promote=true" >> $GITHUB_OUTPUT
                echo ":${ROLLING_TAG} -- latest version ${LATEST_VER_TAG} not yet in GHCR"
              elif [[ "$GHCR_ROLLING" != "$GHCR_LATEST_VER" ]]; then
                echo "latest_needs_promote=true" >> $GITHUB_OUTPUT
                echo ":${ROLLING_TAG} digest ($GHCR_ROLLING) != ${LATEST_VER_TAG} digest ($GHCR_LATEST_VER) -- needs promotion"
              elif [[ "$DH_ROLLING" != "$GHCR_ROLLING" ]]; then
                echo "latest_needs_promote=true" >> $GITHUB_OUTPUT
                echo ":${ROLLING_TAG} DH/GHCR out of sync (DH: $DH_ROLLING, GHCR: $GHCR_ROLLING)"
              else
                echo "latest_needs_promote=false" >> $GITHUB_OUTPUT
                echo ":${ROLLING_TAG} tag matches ${LATEST_VER_TAG} and is in sync across registries"
              fi

              # --- Digest check: :stable tag should match stable version (only if criteria met) ---
              SHOULD_MARK_STABLE="${{ steps.check-stable.outputs.should_mark_stable }}"
              STABLE_VERSION="${{ steps.check-stable.outputs.stable_version }}"

              if [[ "$SHOULD_MARK_STABLE" == "true" && -n "$STABLE_VERSION" ]]; then
                GHCR_STABLE=$(crane digest "${{ env.GHCR_REPO }}:stable" 2>/dev/null || echo "")
                GHCR_STABLE_VER=$(crane digest "${{ env.GHCR_REPO }}:${STABLE_VERSION}" 2>/dev/null || echo "")
                DH_STABLE=$(crane digest "${{ env.DOCKERHUB_REPO }}:stable" 2>/dev/null || echo "")

                if [[ -z "$GHCR_STABLE_VER" ]]; then
                  echo "stable_needs_promote=true" >> $GITHUB_OUTPUT
                  echo ":stable -- version ${STABLE_VERSION} not yet in GHCR"
                elif [[ "$GHCR_STABLE" != "$GHCR_STABLE_VER" ]]; then
                  echo "stable_needs_promote=true" >> $GITHUB_OUTPUT
                  echo ":stable digest ($GHCR_STABLE) != ${STABLE_VERSION} digest ($GHCR_STABLE_VER) -- needs promotion"
                elif [[ "$DH_STABLE" != "$GHCR_STABLE" ]]; then
                  echo "stable_needs_promote=true" >> $GITHUB_OUTPUT
                  echo ":stable DH/GHCR out of sync (DH: $DH_STABLE, GHCR: $GHCR_STABLE)"
                else
                  echo "stable_needs_promote=false" >> $GITHUB_OUTPUT
                  echo ":stable tag matches ${STABLE_VERSION} and is in sync -- already promoted"
                fi
              else
                echo "stable_needs_promote=false" >> $GITHUB_OUTPUT
                echo ":stable -- promotion criteria not met (should_mark_stable=$SHOULD_MARK_STABLE), skipping check"
              fi

              if [ -n "$MISSING_VERSIONS" ] || [ "$FORCE_BUILD" == "true" ]; then
                if [ "$FORCE_BUILD" == "true" ] && [ -z "$MISSING_VERSIONS" ]; then
                  MISSING_VERSIONS="$LATEST_VERSION"
                fi
                JSON_ARRAY=$(echo "$MISSING_VERSIONS" | sed 's/,/","/g' | sed 's/^/["/' | sed 's/$/"]/')
                echo "versions=$MISSING_VERSIONS" >> $GITHUB_OUTPUT
                echo "versions_json=$JSON_ARRAY" >> $GITHUB_OUTPUT
                echo "should_build=true" >> $GITHUB_OUTPUT
                echo "Will build missing versions: $MISSING_VERSIONS"
              else
                echo "versions=" >> $GITHUB_OUTPUT
                echo "versions_json=[]" >> $GITHUB_OUTPUT
                echo "should_build=false" >> $GITHUB_OUTPUT
                echo "All versions exist in both registries (base image unchanged)"
              fi
              ;;

            "build-versions")
              VALID_VERSIONS="${{ steps.validate-versions.outputs.valid_versions }}"

              if [ -z "$VALID_VERSIONS" ]; then
                echo "::warning::No valid versions to build"
                echo "versions=" >> $GITHUB_OUTPUT
                echo "versions_json=[]" >> $GITHUB_OUTPUT
                echo "should_build=false" >> $GITHUB_OUTPUT
              else
                # Filter out existing versions if skip_existing is true
                FILTERED_VERSIONS=$(filter_existing_versions "$VALID_VERSIONS")

                if [ -z "$FILTERED_VERSIONS" ]; then
                  echo "::notice::All requested versions already exist in registries"
                  echo "versions=" >> $GITHUB_OUTPUT
                  echo "versions_json=[]" >> $GITHUB_OUTPUT
                  echo "should_build=false" >> $GITHUB_OUTPUT
                else
                  echo "versions=$FILTERED_VERSIONS" >> $GITHUB_OUTPUT
                  # Convert to JSON array
                  JSON_ARRAY=$(echo "$FILTERED_VERSIONS" | sed 's/,/","/g' | sed 's/^/["/' | sed 's/$/"]/')
                  echo "versions_json=$JSON_ARRAY" >> $GITHUB_OUTPUT
                  echo "should_build=true" >> $GITHUB_OUTPUT
                  echo "Will build versions: $FILTERED_VERSIONS"
                fi
              fi
              ;;

            "promote-image"|"force-promote-latest"|"promote-stable"|"force-promote-stable"|"sync-registries")
              echo "versions=" >> $GITHUB_OUTPUT
              echo "versions_json=[]" >> $GITHUB_OUTPUT
              echo "should_build=false" >> $GITHUB_OUTPUT
              ;;
          esac

      - name: Filter excluded versions
        id: filter-excluded
        if: env.EXCLUDE_VERSIONS != ''
        run: |
          VERSIONS_CSV="${{ steps.determine-versions.outputs.versions }}"
          SHOULD_BUILD="${{ steps.determine-versions.outputs.should_build }}"

          # Nothing to filter if no versions or not building
          if [[ -z "$VERSIONS_CSV" || "$SHOULD_BUILD" != "true" ]]; then
            exit 0
          fi

          # Build exclusion regex from comma-separated EXCLUDE_VERSIONS
          EXCLUDE_PATTERN="$(echo "$EXCLUDE_VERSIONS" | tr ',' '\n' | sed 's/^ *//; s/ *$//' | sed '/^$/d' | paste -sd '|')"

          FILTERED=""
          EXCLUDED_COUNT=0
          IFS=',' read -ra ARR <<< "$VERSIONS_CSV"
          for ver in "${ARR[@]}"; do
            ver="$(echo "$ver" | xargs)"
            if echo "$ver" | grep -Exq "$EXCLUDE_PATTERN"; then
              echo "Excluding version: $ver"
              (( EXCLUDED_COUNT++ )) || true
            else
              FILTERED="${FILTERED:+$FILTERED,}$ver"
            fi
          done

          if [[ "$EXCLUDED_COUNT" -gt 0 ]]; then
            echo "Excluded $EXCLUDED_COUNT version(s): $EXCLUDE_VERSIONS"
          fi

          if [[ -z "$FILTERED" ]]; then
            echo "All versions excluded — nothing to build"
            echo "versions=" >> "$GITHUB_OUTPUT"
            echo "versions_json=[]" >> "$GITHUB_OUTPUT"
            echo "should_build=false" >> "$GITHUB_OUTPUT"
            exit 0
          fi

          # Rebuild JSON array
          JSON_ARRAY=$(echo "$FILTERED" | sed 's/,/","/g; s/^/["/; s/$/"]/')
          echo "versions=$FILTERED" >> "$GITHUB_OUTPUT"
          echo "versions_json=$JSON_ARRAY" >> "$GITHUB_OUTPUT"
          echo "should_build=true" >> "$GITHUB_OUTPUT"
          echo "Versions after exclusion filter: $FILTERED"

      - name: Generate build matrix
        id: generate-matrix
        run: |
          VERSIONS='${{ steps.filter-excluded.outputs.versions_json || steps.determine-versions.outputs.versions_json }}'
          BETA_VERSIONS='${{ steps.fetch-releases.outputs.beta_versions }}'
          PLATFORMS='${{ steps.set-platforms.outputs.platforms }}'
          
          # If no versions to build, output empty matrix
          if [ "$VERSIONS" == "[]" ] || [ -z "$VERSIONS" ]; then
            echo "matrix=[]" >> $GITHUB_OUTPUT
            echo "No versions to build, empty matrix"
            exit 0
          fi
          
          # Generate combined matrix: [{version, platform, platform_slug, is_beta, runner}, ...]
          # Runner affinity lookup table: maps each Docker platform to a runner suffix
          ACTION_RUNNER="${{ vars.ACTION_RUNNER_VERSION || 'ubuntu-24.04' }}"
          runner_suffix_for() {
            case "$1" in
              linux/amd64)    echo "" ;;        # native amd64
              linux/arm64)    echo "-arm" ;;    # native arm64
              linux/arm/v7)   echo "-arm" ;;    # arm64 runner, native ARM32 compat
              linux/arm/v6)   echo "-arm" ;;    # arm64 runner, native ARM32 compat
              linux/riscv64)  echo "-riscv" ;;  # native riscv64
              linux/386)      echo "" ;;        # amd64 runner, native i386 compat
              linux/s390x)    echo "" ;;        # amd64 runner, QEMU
              linux/ppc64le)  echo "" ;;        # amd64 runner, QEMU
              linux/mips64le) echo "" ;;        # amd64 runner, QEMU
              *)              echo "" ;;        # fallback to amd64 runner
            esac
          }

          # Build the JSON matrix using the lookup table
          MATRIX="["
          FIRST=true
          for v in $(echo "$VERSIONS" | jq -r '.[]'); do
            IS_BETA=$(echo "$BETA_VERSIONS" | jq -r --arg v "$v" 'if .[$v] == true then "true" else "false" end')
            IFS=',' read -ra PLAT_ARRAY <<< "$(echo "$PLATFORMS" | tr -d ' ')"
            for p in "${PLAT_ARRAY[@]}"; do
              [ -z "$p" ] && continue
              SUFFIX=$(runner_suffix_for "$p")
              RUNNER="${ACTION_RUNNER}${SUFFIX}"
              SLUG=$(echo "$p" | tr '/' '-')
              $FIRST || MATRIX+=","
              FIRST=false
              MATRIX+=$(jq -nc --arg v "$v" --arg p "$p" --arg s "$SLUG" --arg b "$IS_BETA" --arg r "$RUNNER" \
                '{version:$v, platform:$p, platform_slug:$s, is_beta:$b, runner:$r}')
            done
          done
          MATRIX+="]"
          
          echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
          echo "Generated build matrix with $(echo $MATRIX | jq length) entries"
          echo "$MATRIX" | jq .

      - name: Summary
        run: |
          echo "## Release Check Summary" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY
          echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY
          echo "| Action | ${{ steps.set-action.outputs.action }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Latest Version | ${{ steps.fetch-releases.outputs.latest_version }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Versions to Build | ${{ steps.filter-excluded.outputs.versions || steps.determine-versions.outputs.versions || 'None' }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Should Build | ${{ steps.filter-excluded.outputs.should_build || steps.determine-versions.outputs.should_build }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Excluded Versions | ${{ env.EXCLUDE_VERSIONS || 'None' }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Missing Versions | ${{ steps.validate-versions.outputs.missing_versions || 'None' }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Eligible for Stable | ${{ steps.check-stable.outputs.should_mark_stable }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Base Image Selection | ${{ steps.resolve-base-image.outputs.selection }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Base Image Updated | ${{ steps.check-base-image.outputs.updated }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Platforms | ${{ steps.set-platforms.outputs.platforms }} |" >> $GITHUB_STEP_SUMMARY
          echo "| ZSTD Compression | Level ${{ steps.set-compression.outputs.level }} |" >> $GITHUB_STEP_SUMMARY

  # ===========================================================================
  # Job 2: Build Docker images per platform (Split Build Strategy)
  # ===========================================================================
  build-platform:
    timeout-minutes: 120
    name: Build ${{ matrix.version }} (${{ matrix.platform_slug }})
    needs: check-releases
    if: needs.check-releases.outputs.should_build == 'true' && needs.check-releases.outputs.build_matrix != ''
    runs-on: ${{ matrix.runner }}
    permissions:
      contents: read
      packages: write
    # Prevent building the same version+platform concurrently
    concurrency:
      group: build-${{ matrix.version }}-${{ matrix.platform_slug }}
      cancel-in-progress: false
    strategy:
      fail-fast: false
      max-parallel: 20
      matrix:
        include: ${{ fromJson(needs.check-releases.outputs.build_matrix || '[{"skip":"true","version":"none","platform":"none","platform_slug":"skip","is_beta":"false","runner":"ubuntu-24.04"}]') }}

    steps:
      - name: Checkout repository
        uses: actions/checkout@v7

      - name: Free disk space
        run: |
          echo "Freeing up disk space..."
          sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc
          sudo apt-get clean
          df -h

      - name: Setup build environment
        uses: ./.github/actions/setup-build-env
        with:
          do_checkout: 'false'
          setup_qemu: 'true'
          setup_buildx: 'true'
          login_dockerhub: 'true'
          login_ghcr: 'true'
          dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
          ghcr_actor: ${{ github.actor }}
          ghcr_token: ${{ secrets.GITHUB_TOKEN }}

      - name: Prepare build context
        id: prepare
        env:
          DEFAULT_IMAGE: ${{ env.DEFAULT_BASE_IMAGE }}
        run: |
          VERSION="${{ matrix.version }}"
          VERSION_UNDERSCORE="${VERSION//./_}"
          BASE_SELECTION="${{ needs.check-releases.outputs.base_image_selection }}"
          CUSTOM_BASE="${{ needs.check-releases.outputs.custom_base_image }}"

          # Resolve base image from selection key (avoids GitHub Actions secret masking)
          case "$BASE_SELECTION" in
            "custom")
              if [ -n "$CUSTOM_BASE" ]; then
                BASE_IMAGE="$CUSTOM_BASE"
              else
                BASE_IMAGE="$DEFAULT_IMAGE"
              fi
              ;;
            *)
              BASE_IMAGE="$DEFAULT_IMAGE"
              ;;
          esac

          # Final fallback
          if [ -z "$BASE_IMAGE" ]; then
            BASE_IMAGE="$DEFAULT_IMAGE"
          fi

          # Create build_data directory
          mkdir -p resources/build_data

          # Write required files
          echo "$BASE_IMAGE" > resources/build_data/base-image
          echo "$VERSION_UNDERSCORE" > resources/build_data/version
          echo "${{ env.BINARY_BASE_URL }}" > resources/build_data/binary_server_url
          # NOTE: Do NOT create 'publication' file - its existence skips setup in DockerfileModifier.sh

          # Create build timestamp (will be copied to /AgentDVR/build-timestamp in container)
          BUILD_TIME=$(TZ="${{ env.TZ }}" date +"%Y-%m-%d %H:%M:%S")
          echo "Image built on: $BUILD_TIME Bangladesh Standard Time | Version: $VERSION | Platform: ${{ matrix.platform }}" > resources/build-timestamp

          echo "=== Build Configuration ==="
          echo "Version: $VERSION"
          echo "Version (underscore): $VERSION_UNDERSCORE"
          echo "Platform: ${{ matrix.platform }}"
          echo "Base Image: $BASE_IMAGE"
          echo "Binary URL: ${{ env.BINARY_BASE_URL }}"

          # Output validated base image for subsequent steps
          echo "base_image=$BASE_IMAGE" >> $GITHUB_OUTPUT

          # Determine which registries to push to (used by build step outputs)
          PUSH_DH="${{ github.event.inputs.push_to_dockerhub || 'true' }}"
          PUSH_GHCR="${{ github.event.inputs.push_to_ghcr || 'true' }}"
          REGISTRY_NAMES=""
          if [ "$PUSH_DH" != "false" ]; then
            REGISTRY_NAMES="${{ env.DOCKERHUB_REPO }}"
          fi
          if [ "$PUSH_GHCR" != "false" ]; then
            if [ -n "$REGISTRY_NAMES" ]; then
              REGISTRY_NAMES="${REGISTRY_NAMES},${{ env.GHCR_REPO }}"
            else
              REGISTRY_NAMES="${{ env.GHCR_REPO }}"
            fi
          fi
          echo "registry_names=$REGISTRY_NAMES" >> $GITHUB_OUTPUT
          echo "Push targets: $REGISTRY_NAMES"

          # Generate Dockerfile
          bash DockerfileModifier.sh

          echo ""
          echo "=== Generated Dockerfile ==="
          cat Dockerfile.ispyagentdvr-docker

      - name: Get base image digest
        id: base-digest
        run: |
          # Use the base image resolved in prepare step
          BASE_IMAGE="${{ steps.prepare.outputs.base_image }}"
          # crane digest: single HEAD request, falls back to skopeo
          BASE_DIGEST=$(crane digest "${BASE_IMAGE}" 2>/dev/null || skopeo inspect "docker://${BASE_IMAGE}" 2>/dev/null | jq -r '.Digest' || echo "unknown")
          echo "digest=$BASE_DIGEST" >> $GITHUB_OUTPUT
          echo "Base image digest: $BASE_DIGEST"

      - name: Determine cache reference
        id: cache-ref
        run: |
          # Map platform to cache tag
          case "${{ matrix.platform }}" in
            "linux/amd64") CACHE_TAG="buildcache-amd64" ;;
            "linux/arm64") CACHE_TAG="buildcache-arm64" ;;
            "linux/arm/v7") CACHE_TAG="buildcache-armv7" ;;
            *) CACHE_TAG="buildcache-${{ matrix.platform_slug }}" ;;
          esac
          echo "cache_tag=$CACHE_TAG" >> $GITHUB_OUTPUT
          echo "Using cache reference: $CACHE_TAG"

      - name: Generate Docker metadata
        id: meta
        uses: docker/metadata-action@v6
        with:
          images: |
            ${{ env.DOCKERHUB_REPO }}
            ${{ env.GHCR_REPO }}
          labels: |
            org.opencontainers.image.title=iSpy Agent DVR
            org.opencontainers.image.description=Agent DVR - Professional video surveillance software for Linux, Windows and macOS
            org.opencontainers.image.vendor=iSpy
            org.opencontainers.image.version=${{ matrix.version }}
            org.opencontainers.image.source=https://github.com/${{ github.repository }}
            org.opencontainers.image.documentation=https://www.ispyconnect.com/docs/agent/about
            org.opencontainers.image.base.name=${{ steps.prepare.outputs.base_image }}
            org.opencontainers.image.base.digest=${{ steps.base-digest.outputs.digest }}
            org.opencontainers.image.revision=${{ github.sha }}
            ispyagentdvr.release.beta=${{ matrix.is_beta }}

      - name: Build and push by digest
        id: build-push
        uses: ./.github/actions/build-push-retry
        with:
          context: .
          file: ./Dockerfile.ispyagentdvr-docker
          platforms: ${{ matrix.platform }}
          labels: ${{ steps.meta.outputs.labels }}
          # Per-architecture cache configuration
          cache-from: |
            type=registry,ref=${{ env.GHCR_REPO }}:${{ steps.cache-ref.outputs.cache_tag }}
            type=gha,scope=build-${{ matrix.version }}-${{ matrix.platform_slug }}
          cache-to: |
            type=gha,scope=build-${{ matrix.version }}-${{ matrix.platform_slug }},mode=max,compression=zstd,compression-level=${{ needs.check-releases.outputs.compression_level }}
          # Security attestations
          provenance: mode=max
          sbom: true
          # OCI annotations
          annotations: |
            org.opencontainers.image.source=https://github.com/${{ github.repository }}
            org.opencontainers.image.description=iSpy AgentDVR Docker
            org.opencontainers.image.licenses=GPL-3.0
            org.opencontainers.image.title=ispyagentdvr-docker
            org.opencontainers.image.url=https://github.com/${{ github.repository }}
            org.opencontainers.image.authors=iSpy (https://www.ispyconnect.com)
            org.opencontainers.image.vendor=iSpy
          # Build arguments
          build-args: |
            BASE_IMAGE=${{ steps.prepare.outputs.base_image }}
            AGENTDVR_VERSION=${{ matrix.version }}
            BUILDKIT_INLINE_CACHE=1
          # Push by digest to enabled registries - no tags, outputs digest for manifest creation
          outputs: type=image,"name=${{ steps.prepare.outputs.registry_names }}",push-by-digest=true,name-canonical=true,push=true,oci-mediatypes=true,compression=zstd,compression-level=${{ needs.check-releases.outputs.compression_level }},force-compression=true
          # Allow network access during build
          allow: network.host
          # Use BuildKit features
          build-contexts: |
            resources=./resources

      - name: Export digest
        run: |
          mkdir -p /tmp/digests
          digest="${{ steps.build-push.outputs.digest }}"
          touch "/tmp/digests/${digest#sha256:}"
          echo "Exported digest: $digest"

      - name: Upload digest artifact
        uses: actions/upload-artifact@v7
        with:
          name: digest-${{ matrix.version }}-${{ matrix.platform_slug }}
          path: /tmp/digests/*
          if-no-files-found: error
          retention-days: 1

      - name: Platform build summary
        run: |
          echo "## 🔧 Platform Build: ${{ matrix.version }} (${{ matrix.platform }})" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY
          echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY
          echo "| Version | ${{ matrix.version }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Platform | ${{ matrix.platform }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Digest | \`${{ steps.build-push.outputs.digest }}\` |" >> $GITHUB_STEP_SUMMARY
          echo "| Image ID | ${{ steps.build-push.outputs.imageid }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Base Image | ${{ steps.prepare.outputs.base_image }} |" >> $GITHUB_STEP_SUMMARY
          echo "| ZSTD Compression | Level ${{ needs.check-releases.outputs.compression_level }} |" >> $GITHUB_STEP_SUMMARY

  # ===========================================================================
  # Job 2b: Merge platform manifests into multi-arch images
  # ===========================================================================
  merge-manifest:
    timeout-minutes: 120
    name: Merge Manifest ${{ matrix.version }}
    needs: [check-releases, build-platform]
    if: ${{ !cancelled() && needs.check-releases.result == 'success' && needs.check-releases.outputs.should_build == 'true' }}
    runs-on: ${{ vars.ACTION_RUNNER_VERSION || 'ubuntu-latest' }}
    permissions:
      contents: read
      packages: write
      security-events: write
    strategy:
      fail-fast: false
      matrix:
        version: ${{ fromJson(needs.check-releases.outputs.versions_json || '["_skip_"]') }}

    steps:
      - name: Download all digests for version
        id: download-digests
        continue-on-error: true
        uses: actions/download-artifact@v8
        with:
          path: /tmp/digests
          pattern: digest-${{ matrix.version }}-*
          merge-multiple: true

      - name: Check if merge is needed
        id: check-digests
        shell: bash
        run: |
          if [[ -d /tmp/digests ]] && ls /tmp/digests/* >/dev/null 2>&1; then
            echo "has_digests=true" >> "$GITHUB_OUTPUT"
            echo "Found digests to merge:"
            ls -la /tmp/digests/
          else
            echo "has_digests=false" >> "$GITHUB_OUTPUT"
            echo "No digests found — all builds were skipped (tags already exist)"
          fi

      - name: Checkout
        if: steps.check-digests.outputs.has_digests == 'true'
        uses: actions/checkout@v7

      - name: Setup build environment
        if: steps.check-digests.outputs.has_digests == 'true'
        uses: ./.github/actions/setup-build-env
        with:
          do_checkout: 'false'
          setup_qemu: 'false'
          setup_buildx: 'true'
          login_dockerhub: 'true'
          login_ghcr: 'true'
          dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
          ghcr_actor: ${{ github.actor }}
          ghcr_token: ${{ secrets.GITHUB_TOKEN }}

      - name: List downloaded digests
        if: steps.check-digests.outputs.has_digests == 'true'
        run: |
          echo "Downloaded digests for ${{ matrix.version }}:"
          ls -la /tmp/digests/
          echo "Digest files:"
          for f in /tmp/digests/*; do
            echo "  - sha256:$(basename $f)"
          done

      - name: Determine image tags
        if: steps.check-digests.outputs.has_digests == 'true'
        id: tags
        run: |
          VERSION="${{ matrix.version }}"
          LATEST_VERSION="${{ needs.check-releases.outputs.latest_version }}"
          BETA_VERSIONS='${{ needs.check-releases.outputs.beta_versions }}'
          TAG_AS_LATEST="${{ github.event.inputs.tag_as_latest }}"
          PUSH_DOCKERHUB="${{ github.event.inputs.push_to_dockerhub }}"
          PUSH_GHCR="${{ github.event.inputs.push_to_ghcr }}"

          # Check if current version is beta
          IS_BETA=$(echo "$BETA_VERSIONS" | jq -r --arg v "$VERSION" '.[$v] // false')
          echo "Version $VERSION is beta: $IS_BETA"

          # Generate date tag in Bangladesh time (UTC+6) - format: DDMMYYYY
          DATE_TAG=$(TZ=Asia/Dhaka date +"%d%m%Y")

          # Generate tags based on beta status. The rolling tag target differs per track:
          # :beta follows the newest beta, :latest follows the newest stable release.
          if [ "$IS_BETA" == "true" ]; then
            VERSION_TAG="${VERSION}-beta"
            VERSION_DATE_TAG="${VERSION}-beta-${DATE_TAG}"
            ROLLING_TAG="beta"
            ROLLING_TARGET="${{ needs.check-releases.outputs.latest_beta_version }}"
          else
            VERSION_TAG="${VERSION}"
            VERSION_DATE_TAG="${VERSION}-${DATE_TAG}"
            ROLLING_TAG="latest"
            ROLLING_TARGET="$LATEST_VERSION"
          fi

          # Build tag lists for imagetools
          DOCKERHUB_TAGS=""
          GHCR_TAGS=""

          if [ "$PUSH_DOCKERHUB" != "false" ]; then
            DOCKERHUB_TAGS="-t ${{ env.DOCKERHUB_REPO }}:${VERSION_TAG}"
            DOCKERHUB_TAGS="$DOCKERHUB_TAGS -t ${{ env.DOCKERHUB_REPO }}:${VERSION_DATE_TAG}"
            if [ "$VERSION" == "$ROLLING_TARGET" ] && [ "$TAG_AS_LATEST" != "false" ]; then
              DOCKERHUB_TAGS="$DOCKERHUB_TAGS -t ${{ env.DOCKERHUB_REPO }}:${ROLLING_TAG}"
            fi
          fi

          if [ "$PUSH_GHCR" != "false" ]; then
            GHCR_TAGS="-t ${{ env.GHCR_REPO }}:${VERSION_TAG}"
            GHCR_TAGS="$GHCR_TAGS -t ${{ env.GHCR_REPO }}:${VERSION_DATE_TAG}"
            if [ "$VERSION" == "$ROLLING_TARGET" ] && [ "$TAG_AS_LATEST" != "false" ]; then
              GHCR_TAGS="$GHCR_TAGS -t ${{ env.GHCR_REPO }}:${ROLLING_TAG}"
            fi
          fi

          echo "dockerhub_tags=$DOCKERHUB_TAGS" >> $GITHUB_OUTPUT
          echo "ghcr_tags=$GHCR_TAGS" >> $GITHUB_OUTPUT
          echo "date_tag=$DATE_TAG" >> $GITHUB_OUTPUT
          echo "is_beta=$IS_BETA" >> $GITHUB_OUTPUT
          echo "version_tag=$VERSION_TAG" >> $GITHUB_OUTPUT
          echo "rolling_tag=$ROLLING_TAG" >> $GITHUB_OUTPUT

          echo "Docker Hub tags: $DOCKERHUB_TAGS"
          echo "GHCR tags: $GHCR_TAGS"

      - name: Create and push multi-arch manifest (GHCR)
        if: steps.check-digests.outputs.has_digests == 'true' && github.event.inputs.push_to_ghcr != 'false'
        run: |
          # Rate-limit aware retry helper
          imagetools_create_rl() {
            local desc="$1"; shift
            local err_file; err_file="$(mktemp)"
            for attempt in 1 2 3 4 5; do
              echo "[attempt $attempt/5] $desc"
              if docker buildx imagetools create "$@" 2>"$err_file"; then
                rm -f "$err_file"; return 0
              fi
              if grep -qiE '429|toomanyrequests|rate limit' "$err_file"; then
                local wait=$(( attempt * 30 ))
                echo "::warning::Rate limited -- backing off ${wait}s"
              else
                local wait=$(( attempt * 10 ))
                cat "$err_file" >&2
              fi
              sleep "$wait"
            done
            rm -f "$err_file"; return 1
          }

          SOURCES=""
          for digest_file in /tmp/digests/*; do
            [[ -f "$digest_file" ]] || continue
            digest="sha256:$(basename $digest_file)"
            SOURCES="$SOURCES ${{ env.GHCR_REPO }}@${digest}"
          done

          ANNO_ARGS=()
          ANNO_ARGS+=(--annotation "index:org.opencontainers.image.source=https://github.com/${{ github.repository }}")
          ANNO_ARGS+=(--annotation "index:org.opencontainers.image.description=iSpy AgentDVR Docker")
          ANNO_ARGS+=(--annotation "index:org.opencontainers.image.licenses=GPL-3.0")
          ANNO_ARGS+=(--annotation "index:org.opencontainers.image.title=ispyagentdvr-docker")
          ANNO_ARGS+=(--annotation "index:org.opencontainers.image.url=https://github.com/${{ github.repository }}")
          ANNO_ARGS+=(--annotation "index:org.opencontainers.image.authors=iSpy (https://www.ispyconnect.com)")
          ANNO_ARGS+=(--annotation "index:org.opencontainers.image.vendor=iSpy")
          echo "Creating multi-arch manifest for GHCR..."
          imagetools_create_rl "GHCR manifest" \
            "${ANNO_ARGS[@]}" \
            ${{ steps.tags.outputs.ghcr_tags }} \
            $SOURCES

      - name: Create and push multi-arch manifest (Docker Hub)
        if: steps.check-digests.outputs.has_digests == 'true' && github.event.inputs.push_to_dockerhub != 'false'
        run: |
          imagetools_create_rl() {
            local desc="$1"; shift
            local err_file; err_file="$(mktemp)"
            for attempt in 1 2 3 4 5; do
              echo "[attempt $attempt/5] $desc"
              if docker buildx imagetools create "$@" 2>"$err_file"; then
                rm -f "$err_file"; return 0
              fi
              if grep -qiE '429|toomanyrequests|rate limit' "$err_file"; then
                local wait=$(( attempt * 30 ))
                echo "::warning::Rate limited -- backing off ${wait}s"
              else
                local wait=$(( attempt * 10 ))
                cat "$err_file" >&2
              fi
              sleep "$wait"
            done
            rm -f "$err_file"; return 1
          }

          # Prefer GHCR as source to avoid Docker Hub pull rate limits
          if [ "${{ github.event.inputs.push_to_ghcr }}" == "false" ]; then
            SOURCE_REPO="${{ env.DOCKERHUB_REPO }}"
          else
            SOURCE_REPO="${{ env.GHCR_REPO }}"
          fi

          SOURCES=""
          for digest_file in /tmp/digests/*; do
            [[ -f "$digest_file" ]] || continue
            digest="sha256:$(basename $digest_file)"
            SOURCES="$SOURCES ${SOURCE_REPO}@${digest}"
          done

          ANNO_ARGS=()
          ANNO_ARGS+=(--annotation "index:org.opencontainers.image.source=https://github.com/${{ github.repository }}")
          ANNO_ARGS+=(--annotation "index:org.opencontainers.image.description=iSpy AgentDVR Docker")
          ANNO_ARGS+=(--annotation "index:org.opencontainers.image.licenses=GPL-3.0")
          ANNO_ARGS+=(--annotation "index:org.opencontainers.image.title=ispyagentdvr-docker")
          ANNO_ARGS+=(--annotation "index:org.opencontainers.image.url=https://github.com/${{ github.repository }}")
          ANNO_ARGS+=(--annotation "index:org.opencontainers.image.authors=iSpy (https://www.ispyconnect.com)")
          ANNO_ARGS+=(--annotation "index:org.opencontainers.image.vendor=iSpy")
          echo "Creating multi-arch manifest for Docker Hub (sourcing from GHCR)..."
          imagetools_create_rl "Docker Hub manifest" \
            "${ANNO_ARGS[@]}" \
            ${{ steps.tags.outputs.dockerhub_tags }} \
            $SOURCES

      - name: Run Trivy vulnerability scanner
        if: steps.check-digests.outputs.has_digests == 'true' && github.event.inputs.run_security_scan != 'false'
        uses: aquasecurity/trivy-action@v0.35.0
        with:
          image-ref: '${{ env.DOCKERHUB_REPO }}:${{ steps.tags.outputs.version_tag }}'
          version: 'v0.71.0'
          format: 'sarif'
          output: 'trivy-results-${{ matrix.version }}.sarif'
          severity: 'CRITICAL,HIGH'
          timeout: '10m'
        continue-on-error: true

      - name: Upload Trivy scan results
        if: steps.check-digests.outputs.has_digests == 'true' && github.event.inputs.run_security_scan != 'false'
        uses: github/codeql-action/upload-sarif@v4
        with:
          sarif_file: 'trivy-results-${{ matrix.version }}.sarif'
          category: 'trivy-${{ matrix.version }}'
        continue-on-error: true

      - name: Build summary
        run: |
          BETA_STATUS="${{ steps.tags.outputs.is_beta }}"
          if [ "$BETA_STATUS" == "true" ]; then
            echo "## 🧪 Build Complete: ${{ matrix.version }} (BETA)" >> $GITHUB_STEP_SUMMARY
          else
            echo "## ✅ Build Complete: ${{ matrix.version }}" >> $GITHUB_STEP_SUMMARY
          fi
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "| Property | Value |" >> $GITHUB_STEP_SUMMARY
          echo "|----------|-------|" >> $GITHUB_STEP_SUMMARY
          echo "| Version | ${{ matrix.version }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Release Type | ${{ steps.tags.outputs.is_beta == 'true' && '🧪 Beta' || '📦 Stable' }} |" >> $GITHUB_STEP_SUMMARY
          echo "| Platforms | ${{ needs.check-releases.outputs.platforms }} |" >> $GITHUB_STEP_SUMMARY
          echo "| ZSTD Compression | Level ${{ needs.check-releases.outputs.compression_level }} |" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "### Tags Applied" >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          echo "${{ steps.tags.outputs.dockerhub_tags }} ${{ steps.tags.outputs.ghcr_tags }}" | tr ' ' '\n' | grep -v '^-t$' | grep -v '^$' >> $GITHUB_STEP_SUMMARY
          echo '```' >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "### Platform Digests" >> $GITHUB_STEP_SUMMARY
          for f in /tmp/digests/*; do
            echo "- \`sha256:$(basename $f)\`" >> $GITHUB_STEP_SUMMARY
          done

      - name: Upload build artifacts
        uses: actions/upload-artifact@v7
        with:
          name: build-info-${{ matrix.version }}
          path: |
            trivy-results-${{ matrix.version }}.sarif
          retention-days: 30
          if-no-files-found: ignore

  # ===========================================================================
  # Job 3: Sync registries
  # ===========================================================================
  sync-registries:
    timeout-minutes: 120
    name: Sync Registries
    needs: [check-releases, merge-manifest]
    if: |
      always() &&
      (needs.check-releases.outputs.action_type == 'sync-registries' ||
       (needs.check-releases.outputs.action_type == 'auto-check' &&
        (needs.merge-manifest.result == 'success' || needs.check-releases.outputs.should_build == 'false')))
    runs-on: ${{ vars.ACTION_RUNNER_VERSION || 'ubuntu-latest' }}

    steps:
      - name: Checkout
        uses: actions/checkout@v7

      - name: Setup build environment
        uses: ./.github/actions/setup-build-env
        with:
          do_checkout: 'false'
          setup_qemu: 'false'
          setup_buildx: 'false'
          login_dockerhub: 'true'
          login_ghcr: 'true'
          dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
          ghcr_actor: ${{ github.actor }}
          ghcr_token: ${{ secrets.GITHUB_TOKEN }}

      - name: Sync images between registries
        uses: ./.github/actions/registry-sync
        with:
          dockerhub_repo: ${{ env.DOCKERHUB_REPO }}
          ghcr_repo: ${{ env.GHCR_REPO }}
          tags: ${{ needs.check-releases.outputs.latest_version }},latest,stable

  # ===========================================================================
  # Job 4: Promote image to latest or stable
  # ===========================================================================
  promote-image:
    timeout-minutes: 120
    name: Promote Image
    needs: [check-releases, merge-manifest]
    if: |
      always() &&
      needs.check-releases.result == 'success' &&
      (
        needs.check-releases.outputs.action_type == 'promote-image' ||
        needs.check-releases.outputs.action_type == 'force-promote-latest' ||
        (needs.check-releases.outputs.action_type == 'auto-check' && needs.merge-manifest.result != 'failure' &&
         needs.check-releases.outputs.latest_needs_promote == 'true' &&
         needs.check-releases.outputs.latest_is_beta != 'true')
      )
    runs-on: ${{ vars.ACTION_RUNNER_VERSION || 'ubuntu-latest' }}
    concurrency:
      group: docker-build-${{ github.event.inputs.promote_tag == 'stable' && 'stable-promotion' || 'promote-image' }}
      cancel-in-progress: false
    outputs:
      promoted_version: ${{ steps.promote.outputs.promoted_version }}
      promoted_digest: ${{ steps.promote.outputs.promoted_digest }}
      promoted_at: ${{ steps.promote.outputs.promoted_at }}

    steps:
      - name: Validate inputs
        run: |
          ACTION="${{ needs.check-releases.outputs.action_type }}"
          # For manual promote-image, require promote_version input
          if [[ "$ACTION" == "promote-image" ]] && [ -z "${{ github.event.inputs.promote_version }}" ]; then
            echo "::error::Source version is required for image promotion"
            exit 1
          fi

      - name: Checkout
        uses: actions/checkout@v7

      - name: Setup build environment
        uses: ./.github/actions/setup-build-env
        with:
          do_checkout: 'false'
          setup_qemu: 'false'
          setup_buildx: 'false'
          login_dockerhub: 'true'
          login_ghcr: 'true'
          dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
          ghcr_actor: ${{ github.actor }}
          ghcr_token: ${{ secrets.GITHUB_TOKEN }}

      - name: Promote image
        id: promote
        run: |
          ACTION="${{ needs.check-releases.outputs.action_type }}"
          FORCE_LATEST="false"

          # Determine source and target based on action type
          if [[ "$ACTION" == "force-promote-latest" ]]; then
            # force-promote-latest: use versions input or latest_version, always target :latest
            SOURCE="${{ github.event.inputs.versions }}"
            if [[ -z "$SOURCE" ]]; then
              SOURCE="${{ needs.check-releases.outputs.latest_version }}"
            fi
            TARGET="latest"
            FORCE_LATEST="true"
          elif [[ "$ACTION" == "auto-check" ]]; then
            # auto-check: promote latest_version to :latest
            # Skip when upstream latest is beta — merge-manifest already pushes :beta rolling tag
            if [[ "${{ needs.check-releases.outputs.latest_is_beta }}" == "true" ]]; then
              echo "Upstream latest (${{ needs.check-releases.outputs.latest_version }}) is beta -- :latest promotion skipped"
              exit 0
            fi
            SOURCE="${{ needs.check-releases.outputs.latest_version }}"
            TARGET="latest"
          else
            # Manual promote-image
            SOURCE="${{ github.event.inputs.promote_version }}"
            TARGET="${{ github.event.inputs.promote_tag }}"
            # Beta builds are tagged VERSION-beta, so resolve the source tag
            # when promoting to :beta. A bare version (7.5.3.0) or an explicit
            # 7.5.3.0-beta both work.
            if [[ "$TARGET" == "beta" && -n "$SOURCE" && "$SOURCE" != *-beta ]]; then
              SOURCE="${SOURCE}-beta"
            fi
          fi

          if [[ -z "$SOURCE" ]]; then
            echo "No version to promote -- skipping"
            exit 0
          fi

          echo "Promoting version $SOURCE to $TARGET (action=$ACTION, force=$FORCE_LATEST)..."

          # -- Digest cache (crane digest: single HEAD request) --------
          declare -A _DIGEST_CACHE=()
          cached_digest() {
            local ref="$1"
            if [[ -n "${_DIGEST_CACHE[$ref]+x}" ]]; then
              echo "${_DIGEST_CACHE[$ref]}"
              return 0
            fi
            local result
            if result=$(crane digest "${ref}" 2>/dev/null); then
              _DIGEST_CACHE[$ref]="$result"
              echo "$result"
              return 0
            fi
            # Fallback to skopeo
            if result=$(skopeo inspect "docker://${ref}" 2>/dev/null | jq -r '.Digest'); then
              [[ -n "$result" && "$result" != "null" ]] || return 1
              _DIGEST_CACHE[$ref]="$result"
              echo "$result"
              return 0
            fi
            return 1
          }

          # -- Rate-limit tolerant copy (crane copy: server-side, OCI mount API) --------
          crane_copy_rl() {
            local desc="$1" src="$2" dst="$3"
            local err_file; err_file="$(mktemp)"
            for attempt in 1 2 3 4 5; do
              echo "  [attempt $attempt/5] $desc"
              if crane copy "${src}" "${dst}" 2>"$err_file"; then
                rm -f "$err_file"; return 0
              fi
              if grep -qiE '429|toomanyrequests|rate limit' "$err_file"; then
                local wait=$(( attempt * 30 ))
                echo "  ::warning::Rate limited -- backing off ${wait}s"
              else
                # Fallback to skopeo on crane failure
                echo "  crane failed, trying skopeo..."
                if skopeo copy --all "docker://${src}" "docker://${dst}" 2>"$err_file"; then
                  rm -f "$err_file"; return 0
                fi
                local wait=$(( attempt * 10 ))
                cat "$err_file" >&2
              fi
              sleep "$wait"
            done
            rm -f "$err_file"; return 1
          }

          # Check source exists in primary registry (GHCR)
          SOURCE_DIGEST=$(cached_digest "${{ env.GHCR_REPO }}:${SOURCE}") || {
            echo "::error::Source version $SOURCE not found in GHCR"
            exit 1
          }

          # Early-exit: check if target already points to same digest on GHCR
          TARGET_DIGEST=""
          TARGET_DIGEST=$(cached_digest "${{ env.GHCR_REPO }}:${TARGET}" 2>/dev/null) || true

          if [[ "$SOURCE_DIGEST" == "$TARGET_DIGEST" && -n "$TARGET_DIGEST" && "$FORCE_LATEST" != "true" ]]; then
            echo "GHCR :${TARGET} already points to $SOURCE (digest: $SOURCE_DIGEST)"

            # Ensure Docker Hub is also in sync
            DH_DIGEST=""
            DH_DIGEST=$(cached_digest "${{ env.DOCKERHUB_REPO }}:${TARGET}" 2>/dev/null) || true
            if [[ "$DH_DIGEST" != "$SOURCE_DIGEST" ]]; then
              echo "Docker Hub out of sync -- syncing from GHCR..."
              crane_copy_rl "$TARGET: GHCR->DH" "${{ env.GHCR_REPO }}:${TARGET}" "${{ env.DOCKERHUB_REPO }}:${TARGET}" || \
                echo "::warning::Could not sync $TARGET to Docker Hub (will retry on next run)"
            fi
          else
            # Promote on GHCR first
            echo "Promoting on GHCR..."
            crane_copy_rl "$SOURCE->$TARGET (GHCR)" "${{ env.GHCR_REPO }}:${SOURCE}" "${{ env.GHCR_REPO }}:${TARGET}" || {
              echo "::error::Failed to promote on GHCR after retries"
              exit 1
            }

            # Sync promoted tag to Docker Hub from GHCR (GHCR is primary)
            echo "Syncing promoted tag to Docker Hub from GHCR..."
            crane_copy_rl "$TARGET: GHCR->DH" "${{ env.GHCR_REPO }}:${TARGET}" "${{ env.DOCKERHUB_REPO }}:${TARGET}" || \
              echo "::warning::Docker Hub sync failed -- sync-registries will retry on next run"
          fi

          echo "Successfully promoted $SOURCE to $TARGET"

          # Output promotion data for state tracking
          if [ "$TARGET" == "stable" ]; then
            PROMOTED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
            echo "promoted_version=$SOURCE" >> $GITHUB_OUTPUT
            echo "promoted_digest=$SOURCE_DIGEST" >> $GITHUB_OUTPUT
            echo "promoted_at=$PROMOTED_AT" >> $GITHUB_OUTPUT
          fi

          echo "## Image Promotion" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "| Source | Target | Status |" >> $GITHUB_STEP_SUMMARY
          echo "|--------|--------|--------|" >> $GITHUB_STEP_SUMMARY
          echo "| $SOURCE | $TARGET | Success |" >> $GITHUB_STEP_SUMMARY

  # ===========================================================================
  # Job 5: Mark stable (5-day rule)
  # ===========================================================================
  mark-stable:
    timeout-minutes: 15
    name: Mark Stable
    needs: [check-releases, merge-manifest, sync-registries, promote-image]
    if: |
      always() &&
      needs.check-releases.result == 'success' &&
      (
        needs.check-releases.outputs.action_type == 'promote-stable' ||
        needs.check-releases.outputs.action_type == 'force-promote-stable' ||
        (needs.check-releases.outputs.action_type == 'auto-check' &&
         needs.check-releases.outputs.should_mark_stable == 'true' &&
         needs.merge-manifest.result != 'failure')
      )
    runs-on: ${{ vars.ACTION_RUNNER_VERSION || 'ubuntu-latest' }}
    concurrency:
      group: docker-build-stable-promotion
      cancel-in-progress: false
    outputs:
      promoted_version: ${{ steps.promote-stable.outputs.promoted_version }}
      promoted_digest: ${{ steps.promote-stable.outputs.promoted_digest }}
      promoted_at: ${{ steps.promote-stable.outputs.promoted_at }}

    steps:
      - name: Checkout
        uses: actions/checkout@v7

      - name: Setup build environment
        uses: ./.github/actions/setup-build-env
        with:
          do_checkout: 'false'
          setup_qemu: 'false'
          setup_buildx: 'false'
          login_dockerhub: 'true'
          login_ghcr: 'true'
          dockerhub_username: ${{ secrets.DOCKERHUB_USERNAME }}
          dockerhub_token: ${{ secrets.DOCKERHUB_TOKEN }}
          ghcr_actor: ${{ github.actor }}
          ghcr_token: ${{ secrets.GITHUB_TOKEN }}

      - name: Check and promote to stable
        id: promote-stable
        run: |
          STABLE_VERSION="${{ needs.check-releases.outputs.stable_version }}"
          ACTION="${{ needs.check-releases.outputs.action_type }}"
          FORCE_PROMOTE_STABLE="false"
          if [[ "$ACTION" == "force-promote-stable" ]]; then
            FORCE_PROMOTE_STABLE="true"
          fi

          echo "promoted_version=" >> $GITHUB_OUTPUT
          echo "promoted_digest=" >> $GITHUB_OUTPUT
          echo "promoted_at=" >> $GITHUB_OUTPUT

          if [ -z "$STABLE_VERSION" ]; then
            echo "No version eligible for stable promotion"
            exit 0
          fi

          # -- Digest cache (crane digest: single HEAD request) --------
          declare -A _DIGEST_CACHE=()
          cached_digest() {
            local ref="$1"
            if [[ -n "${_DIGEST_CACHE[$ref]+x}" ]]; then
              echo "${_DIGEST_CACHE[$ref]}"; return 0
            fi
            local result
            if result=$(crane digest "${ref}" 2>/dev/null); then
              _DIGEST_CACHE[$ref]="$result"; echo "$result"; return 0
            fi
            # Fallback to skopeo
            if result=$(skopeo inspect "docker://${ref}" 2>/dev/null | jq -r '.Digest'); then
              [[ -n "$result" && "$result" != "null" ]] || return 1
              _DIGEST_CACHE[$ref]="$result"; echo "$result"; return 0
            fi
            return 1
          }

          # -- Rate-limit tolerant copy (crane copy: server-side, OCI mount API) --------
          crane_copy_rl() {
            local desc="$1" src="$2" dst="$3"
            local err_file; err_file="$(mktemp)"
            for attempt in 1 2 3 4 5; do
              echo "  [attempt $attempt/5] $desc"
              if crane copy "${src}" "${dst}" 2>"$err_file"; then
                rm -f "$err_file"; return 0
              fi
              if grep -qiE '429|toomanyrequests|rate limit' "$err_file"; then
                local wait=$(( attempt * 60 ))
                echo "  ::warning::Rate limited -- backing off ${wait}s"
              else
                # Fallback to skopeo on crane failure
                echo "  crane failed, trying skopeo..."
                if skopeo copy --all "docker://${src}" "docker://${dst}" 2>"$err_file"; then
                  rm -f "$err_file"; return 0
                fi
                local wait=$(( attempt * 15 ))
                cat "$err_file" >&2
              fi
              sleep "$wait"
            done
            rm -f "$err_file"; return 1
          }

          # -- Stable promotion age check via pipeline-state branch --------
          if [[ "$FORCE_PROMOTE_STABLE" != "true" ]]; then
            # Parse STABLE_PROMOTION_AGE (e.g. "5d", "7d", "30d", "12h"; empty = disabled)
            PROMO_AGE="${{ env.STABLE_PROMOTION_AGE }}"
            if [[ -z "$PROMO_AGE" ]]; then
              echo "::warning::STABLE_PROMOTION_AGE is empty -- stable promotion is disabled. Set it to a value like '5d', '12h', or '30m' to enable."
              exit 0
            fi
            PROMO_VALUE=$(echo "$PROMO_AGE" | grep -oP '^\d+')
            PROMO_UNIT=$(echo "$PROMO_AGE" | grep -oP '[a-z]+$')
            case "$PROMO_UNIT" in
              m) PROMO_SECONDS=$(( PROMO_VALUE * 60 )) ;;
              h) PROMO_SECONDS=$(( PROMO_VALUE * 3600 )) ;;
              d) PROMO_SECONDS=$(( PROMO_VALUE * 86400 )) ;;
              *) PROMO_SECONDS=$(( PROMO_VALUE * 86400 )) ;;
            esac

            LATEST_VERSION_SINCE=""
            if git fetch origin pipeline-state --depth=1 2>/dev/null; then
              LATEST_VERSION_SINCE=$(git show origin/pipeline-state:latest-version-since 2>/dev/null || echo "")
            fi

            if [[ -n "$LATEST_VERSION_SINCE" ]]; then
              SINCE_EPOCH=$(date -d "$LATEST_VERSION_SINCE" +%s 2>/dev/null || echo "0")
              NOW_EPOCH=$(date +%s)
              AGE_SECONDS=$(( NOW_EPOCH - SINCE_EPOCH ))
              HOURS_AS_LATEST=$(( AGE_SECONDS / 3600 ))
              echo "Version $STABLE_VERSION has been latest for ${HOURS_AS_LATEST}h (since $LATEST_VERSION_SINCE, threshold: $PROMO_AGE)"

              if (( AGE_SECONDS < PROMO_SECONDS )); then
                echo "Version not old enough for stable promotion (age: ${HOURS_AS_LATEST}h, threshold: $PROMO_AGE)"
                exit 0
              fi
            else
              echo "::warning::No latest-version-since in pipeline-state -- falling back to release date check"
              # Fall through to the release-date-based check already done in check-releases
            fi
          fi

          echo "Checking if stable tag needs update..."

          # Get digests from primary registry (GHCR) -- single HEAD request per ref
          VERSION_DIGEST=$(cached_digest "${{ env.GHCR_REPO }}:${STABLE_VERSION}") || {
            echo "::error::Could not resolve candidate $STABLE_VERSION in GHCR"
            exit 1
          }

          CURRENT_STABLE="none"
          CURRENT_STABLE=$(cached_digest "${{ env.GHCR_REPO }}:stable" 2>/dev/null) || CURRENT_STABLE="none"

          echo "Current stable digest: $CURRENT_STABLE"
          echo "Version $STABLE_VERSION digest: $VERSION_DIGEST"

          # Idempotency check: if GHCR stable already points to candidate,
          # just ensure Docker Hub is also in sync.
          if [[ "$CURRENT_STABLE" == "$VERSION_DIGEST" && "$CURRENT_STABLE" != "none" ]]; then
            echo "GHCR stable already points to version $STABLE_VERSION"

            DH_STABLE="none"
            DH_STABLE=$(cached_digest "${{ env.DOCKERHUB_REPO }}:stable" 2>/dev/null) || DH_STABLE="none"
            if [[ "$DH_STABLE" != "$CURRENT_STABLE" ]]; then
              echo "Docker Hub stable out of sync -- syncing from GHCR..."
              crane_copy_rl "stable: GHCR->DH" "${{ env.GHCR_REPO }}:stable" "${{ env.DOCKERHUB_REPO }}:stable" || \
                echo "::warning::Could not sync stable to Docker Hub (will retry on next run)"
            fi

            PROMOTED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
            echo "promoted_version=$STABLE_VERSION" >> $GITHUB_OUTPUT
            echo "promoted_digest=$VERSION_DIGEST" >> $GITHUB_OUTPUT
            echo "promoted_at=$PROMOTED_AT" >> $GITHUB_OUTPUT
            exit 0
          fi

          echo "Marking version $STABLE_VERSION as stable..."

          # Promote on GHCR first
          crane_copy_rl "$STABLE_VERSION->stable (GHCR)" \
            "${{ env.GHCR_REPO }}:${STABLE_VERSION}" "${{ env.GHCR_REPO }}:stable" || {
            echo "::error::Failed to promote stable on GHCR"
            exit 1
          }

          # Sync to Docker Hub from GHCR (non-fatal)
          crane_copy_rl "stable: GHCR->DH" \
            "${{ env.GHCR_REPO }}:stable" "${{ env.DOCKERHUB_REPO }}:stable" || \
            echo "::warning::Docker Hub stable sync failed -- sync-registries will retry"

          PROMOTED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
          echo "promoted_version=$STABLE_VERSION" >> $GITHUB_OUTPUT
          echo "promoted_digest=$VERSION_DIGEST" >> $GITHUB_OUTPUT
          echo "promoted_at=$PROMOTED_AT" >> $GITHUB_OUTPUT

          echo "Successfully marked $STABLE_VERSION as stable"

          echo "## Stable Promotion" >> $GITHUB_STEP_SUMMARY
          echo "" >> $GITHUB_STEP_SUMMARY
          echo "Version **$STABLE_VERSION** has been marked as **stable**" >> $GITHUB_STEP_SUMMARY

  # ===========================================================================
  # Job 6: Update repository metadata (main branch)
  # ===========================================================================
  update-metadata:
    timeout-minutes: 15
    name: Update Metadata
    needs: [check-releases, merge-manifest, sync-registries, mark-stable, promote-image]
    if: |
      always() &&
      (needs.merge-manifest.result == 'success' ||
       needs.mark-stable.result == 'success' ||
       needs.promote-image.result == 'success')
    runs-on: ${{ vars.ACTION_RUNNER_VERSION || 'ubuntu-latest' }}
    permissions:
      contents: write

    steps:
      - name: Checkout repository
        uses: actions/checkout@v7
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          fetch-depth: 0

      - name: Update build metadata
        run: |
          LATEST_VERSION="${{ needs.check-releases.outputs.latest_version }}"
          CURRENT_TIME=$(date -u +"%Y-%m-%dT%H:%M:%SZ")

          # Update tag file
          echo "$LATEST_VERSION" > resources/tag

          # Update build timestamp
          echo "Last successful build: $CURRENT_TIME" > resources/build-timestamp

          # Update version file (underscore format)
          VERSION_UNDERSCORE="${LATEST_VERSION//./_}"
          mkdir -p resources/build_data
          echo "$VERSION_UNDERSCORE" > resources/build_data/version

          # Track stable promotion from either mark-stable or promote-image (to stable).
          PROMOTED_VERSION=""
          PROMOTED_DIGEST=""
          PROMOTED_AT=""

          if [ "${{ needs.mark-stable.result }}" == "success" ]; then
            PROMOTED_VERSION="$(echo "${{ needs.mark-stable.outputs.promoted_version }}" | xargs)"
            PROMOTED_DIGEST="$(echo "${{ needs.mark-stable.outputs.promoted_digest }}" | xargs)"
            PROMOTED_AT="$(echo "${{ needs.mark-stable.outputs.promoted_at }}" | xargs)"
          fi

          if [ -z "$PROMOTED_VERSION" ] || [ "$PROMOTED_VERSION" == "null" ]; then
            if [ "${{ needs.promote-image.result }}" == "success" ]; then
              PROMOTED_VERSION="$(echo "${{ needs.promote-image.outputs.promoted_version }}" | xargs)"
              PROMOTED_DIGEST="$(echo "${{ needs.promote-image.outputs.promoted_digest }}" | xargs)"
              PROMOTED_AT="$(echo "${{ needs.promote-image.outputs.promoted_at }}" | xargs)"
            fi
          fi

          if [ -n "$PROMOTED_VERSION" ] && [ "$PROMOTED_VERSION" != "null" ] && \
             [ -n "$PROMOTED_DIGEST" ] && [ "$PROMOTED_DIGEST" != "null" ] && \
             [ -n "$PROMOTED_AT" ] && [ "$PROMOTED_AT" != "null" ]; then
            echo "$PROMOTED_VERSION" > resources/build_data/stable-version
            printf '{"version":"%s","digest":"%s","promoted_at":"%s"}\n' "$PROMOTED_VERSION" "$PROMOTED_DIGEST" "$PROMOTED_AT" > resources/build_data/stable-state.json
          fi

          # Update .agentdvr-version
          echo "$LATEST_VERSION" > .agentdvr-version

          echo "Updated metadata files"

      - name: Commit changes
        run: |
          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"

          git add resources/tag resources/build-timestamp resources/build_data/version resources/build_data/stable-version resources/build_data/stable-state.json .agentdvr-version 2>/dev/null || true

          if git diff --staged --quiet; then
            echo "No changes to commit"
          else
            git commit -m "chore: update build metadata

            Version: ${{ needs.check-releases.outputs.latest_version }}
            Built: $(date -u +"%Y-%m-%dT%H:%M:%SZ")

            [skip ci]"

            for attempt in 1 2 3; do
              echo "Push attempt ${attempt}/3"
              if git pull --rebase origin "${GITHUB_REF_NAME}" && git push origin HEAD:"${GITHUB_REF_NAME}"; then
                echo "Metadata push succeeded"
                break
              fi
              git rebase --abort 2>/dev/null || true
              if [ "$attempt" -eq 3 ]; then
                echo "::warning::Could not push metadata after retries due to concurrent updates."
                exit 0
              fi
              sleep 3
            done
          fi

  # ===========================================================================
  # Job 7: Update pipeline state (orphan branch for quick-check)
  # ===========================================================================
  update-state:
    timeout-minutes: 15
    name: Update Pipeline State
    runs-on: ${{ vars.ACTION_RUNNER_VERSION || 'ubuntu-latest' }}
    needs: [check-releases, merge-manifest, sync-registries]
    if: |
      always() &&
      needs.check-releases.result == 'success' &&
      needs.merge-manifest.result == 'success' &&
      (needs.sync-registries.result == 'success' || needs.sync-registries.result == 'skipped')
    permissions:
      contents: write

    steps:
      - name: Checkout
        uses: actions/checkout@v7
        with:
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Write pipeline state
        run: |
          LATEST_VERSION="${{ needs.check-releases.outputs.latest_version }}"
          NOW="$(date -u +%Y-%m-%dT%H:%M:%SZ)"

          # Read previous state before switching branches
          PREV_VERSION=""
          PREV_LATEST_SINCE=""
          if git fetch origin pipeline-state --depth=1 2>/dev/null; then
            PREV_VERSION=$(git show origin/pipeline-state:last-built-version 2>/dev/null || echo "")
            PREV_LATEST_SINCE=$(git show origin/pipeline-state:latest-version-since 2>/dev/null || echo "")
          fi

          # Early-exit: version unchanged and state already initialized. Avoids
          # the daily-churn commit that would otherwise fire on every cron tick
          # — last-built-time is rewritten to NOW each run, so git diff sees a
          # non-empty staged delta even when nothing meaningful changed.
          if [[ -n "$PREV_VERSION" && "$LATEST_VERSION" == "$PREV_VERSION" && -n "$PREV_LATEST_SINCE" ]]; then
            echo "Version unchanged ($LATEST_VERSION) and state already initialized -- skipping commit"
            exit 0
          fi

          # Determine latest-version-since: reset if version changed, preserve if same
          if [[ "$LATEST_VERSION" != "$PREV_VERSION" || -z "$PREV_LATEST_SINCE" ]]; then
            LATEST_SINCE="$NOW"
            echo "Version changed (${PREV_VERSION:-none} -> $LATEST_VERSION) -- resetting latest-version-since to $NOW"
          else
            LATEST_SINCE="$PREV_LATEST_SINCE"
            echo "Version unchanged ($LATEST_VERSION) -- preserving latest-version-since=$LATEST_SINCE"
          fi

          # Create or update the pipeline-state orphan branch
          git checkout pipeline-state 2>/dev/null || \
            git checkout --orphan pipeline-state

          # Clean working tree for orphan branch
          git rm -rf --ignore-unmatch . > /dev/null 2>&1 || true

          echo "$LATEST_VERSION" > last-built-version
          echo "$NOW"             > last-built-time
          echo "$LATEST_SINCE"    > latest-version-since

          git add last-built-version last-built-time latest-version-since
          if ! git diff --cached --quiet; then
            git -c user.name="github-actions[bot]" \
                -c user.email="github-actions[bot]@users.noreply.github.com" \
                commit -m "pipeline-state: built $LATEST_VERSION (latest_since=$LATEST_SINCE)"
            git push origin pipeline-state
          else
            echo "No changes to pipeline state -- skipping commit"
          fi

          echo "Updated pipeline-state branch with version $LATEST_VERSION, latest-version-since=$LATEST_SINCE"

  # ===========================================================================
  # Job 9: Update README version after promote
  # ===========================================================================
  update-readme-version:
    timeout-minutes: 15
    name: Update README version
    # Also runs after a beta-only publish: promote-image handles the :latest rolling
    # tag and is skipped for betas, so gating solely on it left the beta row in the
    # tag table frozen at whatever version was current when it was last hand-edited.
    needs: [check-releases, merge-manifest, promote-image]
    if: |
      always() &&
      needs.check-releases.outputs.latest_version != '' &&
      (needs.promote-image.result == 'success' || needs.merge-manifest.result == 'success')
    runs-on: ${{ vars.ACTION_RUNNER_VERSION || 'ubuntu-latest' }}
    permissions:
      contents: write
    steps:
      - name: Checkout main
        uses: actions/checkout@v7
        with:
          ref: main
          token: ${{ github.token }}

      - name: Update version in README
        shell: bash
        run: |
          set -euo pipefail
          LATEST="${{ needs.check-releases.outputs.latest_version }}"
          BETA="${{ needs.check-releases.outputs.latest_beta_version }}"
          echo "Updating README -- stable: $LATEST, beta: ${BETA:-<none>}"

          # Stable row. The version cell is matched with a closing </td> right after the
          # digits so it cannot also swallow the "<version>-beta" cell below it.
          sed -i -E "s|<td align=\"center\">[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+</td>|<td align=\"center\">${LATEST}</td>|" README.md
          sed -i -E "s|Pinned release [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+|Pinned release ${LATEST}|" README.md

          # Beta row. Only touched when a beta is actually ahead of stable; when the
          # beta channel has no live version the row is left as the last published beta.
          if [ -n "$BETA" ]; then
            sed -i -E "s|<td align=\"center\">[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+-beta</td>|<td align=\"center\">${BETA}-beta</td>|" README.md
            sed -i -E "s|Beta release [0-9]+\.[0-9]+\.[0-9]+\.[0-9]+|Beta release ${BETA}|" README.md
          fi

          if git diff --quiet README.md; then
            echo "README already up to date"
            exit 0
          fi

          git config user.name "github-actions[bot]"
          git config user.email "github-actions[bot]@users.noreply.github.com"
          git add README.md
          git commit -m "docs: update README versions (stable ${LATEST}${BETA:+, beta ${BETA}})"

          for attempt in 1 2 3; do
            if git pull --rebase origin main && git push origin main; then
              echo "README push succeeded"
              break
            fi
            git rebase --abort 2>/dev/null || true
            if [ "$attempt" -eq 3 ]; then
              echo "::warning::Could not push README update after retries"
              exit 0
            fi
            sleep 3
          done

  # ===========================================================================
  # Job 10: Sync README to Docker Hub description
  # ===========================================================================
  update-dockerhub-description:
    timeout-minutes: 15
    name: Update Docker Hub description
    needs: [promote-image, update-readme-version]
    if: |
      always() &&
      needs.promote-image.result == 'success'
    runs-on: ${{ vars.ACTION_RUNNER_VERSION || 'ubuntu-latest' }}
    steps:
      - name: Checkout
        uses: actions/checkout@v7
        with:
          ref: main

      - name: Sync README to Docker Hub
        uses: peter-evans/dockerhub-description@v5
        continue-on-error: true
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
          repository: ${{ env.DOCKERHUB_REPO }}