diff --git a/.github/actions/base_images/action.yml b/.github/actions/base_images/action.yml index 3a5a1354..f0105281 100644 --- a/.github/actions/base_images/action.yml +++ b/.github/actions/base_images/action.yml @@ -1,15 +1,16 @@ --- name: Build Base Images description: Build the base images (pulp/base & pulp/pulp-ci-centos9) if needed -# Both ARM64 & x86-64 versions of each are built -# Use hashFiles(base_image_files, pulp-ci_image_files) as the key to the cache +# Use hashFiles(base_image_files, pulp-ci_image_files) for rebuild check +inputs: + image_variants: + description: "Whether build is nightly or stable, nightly never rebuilds" + default: "stable" + required: true outputs: - base_cache_key: - value: ${{ steps.hash_key.outputs.base_cache_key }} - description: "The cache key the built images were uploaded to." rebuilt_images: - value: ${{ env.BUILD_IMAGES }} - description: "The images that were rebuilt or empty" + value: ${{ steps.rebuild_needed.outputs.build }} + description: "true/false if the base images were rebuilt" runs: using: "composite" @@ -17,100 +18,52 @@ runs: - uses: actions/checkout@v4 - name: Calculate base images hash - id: hash_key run: | hash=${{ hashFiles('images/Containerfile.core.base', 'images/pulp_ci_centos/Containerfile', 'images/assets/**', 'images/s6_assets/**') }} echo "base image hash is ${hash}" - echo "base_cache_key=${hash}" >> $GITHUB_OUTPUT + echo "HASH=${hash}" >> $GITHUB_ENV shell: bash - - name: Restore previously cached images - id: cache - uses: actions/cache/restore@v3 - with: - key: base-images=${{ steps.hash_key.outputs.base_cache_key }} - path: base-images.tar.gz + - name: Set up oras + uses: oras-project/setup-oras@v1 - - name: Extract images if cached - if: steps.cache.outputs.cache-hit == 'true' + - name: Check for updates on base images + id: rebuild_needed run: | - echo "Base Images were in cache" - podman load -i base-images.tar.gz - shell: bash - - - name: Check for updates on cached images - if: steps.cache.outputs.cache-hit == 'true' - run: | - # Enable running/building ARM64 images: https://github.com/multiarch/qemu-user-static - sudo podman run --rm --privileged multiarch/qemu-user-static --reset -p yes - IMAGES=() - for ARCH in arm64 amd64; do - echo "Checking if rebuild needed for base:${ARCH} & pulp-ci-centos9:${ARCH}" - if ! podman run --pull=never pulp/base:ci-${ARCH} bash -c "dnf check-upgrade"; then - echo "Rebuild needed for base:${ARCH} & pulp-ci-centos9:${ARCH}" - IMAGES+=("base:${ARCH}" "pulp-ci-centos9:${ARCH}") - elif ! podman run --pull=never pulp/pulp-ci-centos9:ci-${ARCH} bash -c "dnf check-upgrade"; then - echo "Rebuild needed for just pulp-ci-centos9:${ARCH}" - IMAGES+=("pulp-ci-centos9:${ARCH}") - fi - done - if [ ${#IMAGES[@]} -eq 0 ]; then - echo "No rebuilds needed :)" + if [[ "${{ inputs.image_variants }}" == "nightly" ]]; then + build=false else - echo "BUILD_IMAGES=[$(echo ${IMAGES[@]@Q} | sed 's/ /, /g')]" >> $GITHUB_ENV + branch=${{ github.base_ref || github.ref_name }} + config=$(oras manifest fetch-config --platform linux/amd64 ghcr.io/pulp/pulp-ci-centos9:${branch} | jq -r '.config.Labels') + latest_hash=$(echo "${config}" | jq -r '."org.pulp.basefiles-hash"') + build=true + if [[ "${latest_hash}" == "${HASH}" ]]; then + if podman run ghcr.io/pulp/pulp-ci-centos9:${branch} bash -c "dnf check-upgrade"; then + echo "No base images rebuild needed :)" + build=false + fi + fi fi + echo "build=${build}" >> "$GITHUB_OUTPUT" + echo "Going to rebuild: ${build}" shell: bash - - name: Set images to build on cache miss - if: steps.cache.outputs.cache-hit != 'true' + - name: Download base images if no rebuild needed + if: steps.rebuild_needed.outputs.build == 'false' run: | - echo "BUILD_IMAGES=['base:arm64', 'base:amd64', 'pulp-ci-centos9:arm64', 'pulp-ci-centos9:amd64']" >> $GITHUB_ENV - sudo podman run --rm --privileged multiarch/qemu-user-static --reset -p yes + echo "Downloading base images" + branch=${{ github.base_ref || github.ref_name }} + podman pull ghcr.io/pulp/base:${branch} + podman pull ghcr.io/pulp/pulp-ci-centos9:${branch} + podman tag ghcr.io/pulp/base:${branch} pulp/base:ci + podman tag ghcr.io/pulp/pulp-ci-centos9:${branch} pulp/pulp-ci-centos9:ci shell: bash - name: Build images - if: env.BUILD_IMAGES + if: steps.rebuild_needed.outputs.build == 'true' run: | - IMAGES=(${{ join(fromJSON(env.BUILD_IMAGES), ' ') }}) - echo "Going to build images: ${IMAGES[@]}" podman version buildah version - for IMAGE in "${IMAGES[@]}"; do - echo "Building image ${IMAGE}" - ARCH=${IMAGE##*:} - case $IMAGE in - base:*) - podman build --platform "linux/${ARCH}" --format docker --file images/Containerfile.core.base --tag "pulp/base:ci-${ARCH}" . - ;; - pulp-ci-centos9:*) - podman build --platform "linux/${ARCH}" --format docker --file images/pulp_ci_centos/Containerfile --tag "pulp/pulp-ci-centos9:ci-${ARCH}" --build-arg FROM_TAG="ci-${ARCH}" . - ;; - esac - done + podman build --format docker --file images/Containerfile.core.base --tag "pulp/base:ci" --label "org.pulp.basefiles-hash=${HASH}" . + podman build --format docker --file images/pulp_ci_centos/Containerfile --tag "pulp/pulp-ci-centos9:ci" --build-arg FROM_TAG="ci" . shell: bash - # we use the docker format (default), even though it may not be the fastest, - # because it supports saving both images at once. - # However, it seems to export the common layers twice. - # We should look into whether its possible to export just pulp-ci-centos, - # and tag the base image manually. - - name: Save podman images to tarball - if: env.BUILD_IMAGES - run: | - rm -f base-images.tar.gz - podman save -m -o base-images.tar pulp/base:ci-arm64 pulp/base:ci-amd64 pulp/pulp-ci-centos9:ci-arm64 pulp/pulp-ci-centos9:ci-amd64 - gzip base-images.tar - shell: bash - - - name: Clear cache for next upload - if: env.BUILD_IMAGES && steps.cache.outputs.cache-hit == 'true' && github.event_name != 'pull_request' - run: | - echo "Deleting existing cache for ${{ steps.hash_key.outputs.base_cache_key }}" - gh cache delete "base-images=${{ steps.hash_key.outputs.base_cache_key }}" -R ${{ github.repository }} - shell: bash - - - name: Cache podman images - if: env.BUILD_IMAGES - uses: actions/cache/save@v3 - with: - key: base-images=${{ steps.hash_key.outputs.base_cache_key }} - path: base-images.tar.gz diff --git a/.github/actions/build_image/action.yml b/.github/actions/build_image/action.yml index c4db9c7e..331ace36 100644 --- a/.github/actions/build_image/action.yml +++ b/.github/actions/build_image/action.yml @@ -9,15 +9,12 @@ inputs: image_name: description: 'Name of the image to be built' required: true - image_cache_key: - description: 'The key value used to store the base images in the cache' - required: true latest_ui: description: 'Use the latest pulp-ui when building the image' default: 'false' required: false - built_base_images: - description: 'A JSON list of the base-images that were freshly rebuilt prior' + rebuilt_base_images: + description: 'true/false if the base images were rebuilt prior' required: true outputs: app_version: @@ -47,17 +44,6 @@ runs: - name: Set up oras uses: oras-project/setup-oras@v1 - - name: Restore podman images from cache - uses: actions/cache/restore@v4 - with: - key: base-images=${{ inputs.image_cache_key }} - path: base-images.tar.gz - - - name: Load podman images from tarball - run: | - podman load -i base-images.tar.gz - shell: bash - - name: Find latest ui version if: inputs.latest_ui != 'false' run: | @@ -79,7 +65,7 @@ runs: # 2. Base images were rebuilt # 3. New pulp versions were released build=true - if [[ "${{ github.event_name }}" != "pull_request" && "${{ inputs.image_variant }}" != "nightly" && -z "${{ inputs.built_base_images }}" ]]; then + if [[ "${{ github.event_name }}" != "pull_request" && "${{ inputs.image_variant }}" != "nightly" && "${{ inputs.rebuilt_base_images }}" == "false" ]]; then # Fetch the latest plugin versions from the last published built image branch=${{ github.base_ref || github.ref_name }} config=$(oras manifest fetch-config --platform linux/amd64 ghcr.io/pulp/${{ inputs.image_name }}:${branch} | jq -r '.config.Labels') @@ -99,17 +85,13 @@ runs: run: | podman version buildah version - sudo podman run --rm --privileged multiarch/qemu-user-static --reset -p yes - for ARCH in arm64 amd64 - do - if [[ "${{ inputs.image_name }}" == "pulp-minimal" ]]; then - base_image=$(echo ${{ inputs.image_name }} | cut -d '-' -f1) - podman build --platform linux/${ARCH} --format docker --pull=false --file images/${{ inputs.image_name }}/${{ inputs.image_variant }}/Containerfile.core --tag pulp/${{ inputs.image_name }}:ci-${ARCH} --build-arg FROM_TAG=ci-${ARCH} . - podman build --platform linux/${ARCH} --format docker --pull=false --file images/${{ inputs.image_name }}/${{ inputs.image_variant }}/Containerfile.webserver --tag pulp/${base_image}-web:ci-${ARCH} --build-arg FROM_TAG=ci-${ARCH} . - else - podman build --platform linux/${ARCH} --format docker --pull=false --file images/${{ inputs.image_name }}/${{ inputs.image_variant }}/Containerfile --tag pulp/${{ inputs.image_name }}:ci-${ARCH} --build-arg FROM_TAG=ci-${ARCH} ${{ env.BUILD_UI_ARG }} . - fi - done + if [[ "${{ inputs.image_name }}" == "pulp-minimal" ]]; then + base_image=$(echo ${{ inputs.image_name }} | cut -d '-' -f1) + podman build --format docker --pull=false --file images/${{ inputs.image_name }}/${{ inputs.image_variant }}/Containerfile.core --tag pulp/${{ inputs.image_name }}:ci --build-arg FROM_TAG=ci . + podman build --format docker --pull=false --file images/${{ inputs.image_name }}/${{ inputs.image_variant }}/Containerfile.webserver --tag pulp/${base_image}-web:ci --build-arg FROM_TAG=ci . + else + podman build --format docker --pull=false --file images/${{ inputs.image_name }}/${{ inputs.image_variant }}/Containerfile --tag pulp/${{ inputs.image_name }}:ci --build-arg FROM_TAG=ci ${{ env.BUILD_UI_ARG }} . + fi podman images -a shell: bash @@ -117,7 +99,7 @@ runs: id: image_version_branch run: | if [[ "${{ steps.rebuild_needed.outputs.build }}" == "true" ]]; then - app_version=$(podman run --pull=never pulp/${{ inputs.image_name }}:ci-amd64 bash -c "pip show pulpcore | sed -n -e 's/Version: //p'") + app_version=$(podman run --pull=never pulp/${{ inputs.image_name }}:ci bash -c "pip show pulpcore | sed -n -e 's/Version: //p'") else app_version=$(grep pulpcore versions.freeze | sed -n -e 's/pulpcore==//p') fi @@ -132,16 +114,13 @@ runs: - name: Label image with metadata if: steps.rebuild_needed.outputs.build == 'true' run: | - packages=$(podman run --pull=never pulp/${{ inputs.image_name }}:ci-amd64 bash -c "pip list --format json") + packages=$(podman run --pull=never pulp/${{ inputs.image_name }}:ci bash -c "pip list --format json") plugin_versions=$(echo $packages | jq -r '.[] | select(.name | contains("pulp")) | .name + "==" + .version') - postgres=$(podman run --pull=never pulp/${{ inputs.image_name }}:ci-amd64 bash -c "postgres --version | sed -n -e 's/postgres (PostgreSQL) //p'") - python_version=$(podman run --pull=never pulp/${{ inputs.image_name }}:ci-amd64 bash -c "python3 --version | sed -n -e 's/Python //p'") + postgres=$(podman run --pull=never pulp/${{ inputs.image_name }}:ci bash -c "postgres --version | sed -n -e 's/postgres (PostgreSQL) //p'") + python_version=$(podman run --pull=never pulp/${{ inputs.image_name }}:ci bash -c "python3 --version | sed -n -e 's/Python //p'") files_hash="${{ hashFiles(format('images/{0}/{1}/**', inputs.image_name, inputs.image_variant), 'images/Containerfile.core.base', 'images/pulp_ci_centos/Containerfile', 'images/assets/**', 'images/s6_assets/**') }}" - for ARCH in arm64 amd64 - do - tag="pulp/${{ inputs.image_name }}:ci-${ARCH}" - echo "FROM ${tag}" | podman build --pull=false --platform linux/${ARCH} --format docker --label "org.pulp.plugins=${plugin_versions}" --label "org.pulp.postgres-version=${postgres}" --label "org.pulp.python-version=${python_version}" --label "org.pulp.containerfiles-hash=${files_hash}" --tag ${tag} - - done - podman image inspect pulp/${{ inputs.image_name }}:ci-amd64 --format='{{ .Config.Labels }}' + tag="pulp/${{ inputs.image_name }}:ci" + echo "FROM ${tag}" | podman build --pull=false --format docker --label "org.pulp.plugins=${plugin_versions}" --label "org.pulp.postgres-version=${postgres}" --label "org.pulp.python-version=${python_version}" --label "org.pulp.containerfiles-hash=${files_hash}" --tag ${tag} - + podman image inspect ${tag} --format='{{ .Config.Labels }}' shell: bash diff --git a/.github/actions/test_image/action.yml b/.github/actions/test_image/action.yml index ccc20063..ab3b0fd4 100644 --- a/.github/actions/test_image/action.yml +++ b/.github/actions/test_image/action.yml @@ -23,7 +23,7 @@ runs: if: inputs.image_name == 'pulp' run: | # 3.73 has postgres 13 rather than 16 - images/s6_assets/test.sh "pulp/${{ inputs.image_name }}:ci-amd64" http "quay.io/pulp/pulp:3.73" + images/s6_assets/test.sh "pulp/${{ inputs.image_name }}:ci" http "quay.io/pulp/pulp:3.73" podman stop pulp podman rm pulp shell: bash @@ -32,5 +32,5 @@ runs: if: inputs.image_name == 'pulp-minimal' run: | base_image=$(echo ${{ inputs.image_name }} | cut -d '-' -f1) - images/compose/test.sh "${{ inputs.image_name }}:ci-amd64" "${base_image}-web:ci-amd64" "compose.yml" + images/compose/test.sh "${{ inputs.image_name }}:ci" "${base_image}-web:ci" "compose.yml" shell: bash diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0e184d0b..f746cdfd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -53,11 +53,15 @@ jobs: done shell: bash - base-images: - runs-on: ubuntu-latest - outputs: - base_cache_key: "${{ steps.build_base_images.outputs.base_cache_key }}" - rebuilt_images: "${{ steps.build_base_images.outputs.rebuilt_images }}" + build-and-test-images: + strategy: + fail-fast: false + matrix: + os: + - ubuntu-24.04 + - ubuntu-24.04-arm + + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -65,36 +69,35 @@ jobs: id: build_base_images uses: "./.github/actions/base_images" - app-images: - needs: base-images - runs-on: ubuntu-latest - outputs: - app_version: ${{ steps.build_image.outputs.app_version }} - app_branch: ${{ steps.build_image.outputs.app_branch }} - strategy: - fail-fast: false - matrix: - image_name: - - pulp-minimal - - pulp - steps: - - uses: actions/checkout@v4 + - name: Build pulp-minimal image + id: build_pulp_minimal_image + uses: "./.github/actions/build_image" + with: + image_name: "pulp-minimal" + image_variant: "stable" + latest_ui: ${{ github.base_ref == 'latest' }} + rebuilt_base_images: ${{ steps.build_base_images.outputs.rebuilt_images }} - - name: Build App Image - id: build_image + - name: Build pulp image + id: build_pulp_image uses: "./.github/actions/build_image" with: - image_name: ${{ matrix.image_name }} + image_name: "pulp" image_variant: "stable" - image_cache_key: ${{ needs.base-images.outputs.base_cache_key }} latest_ui: ${{ github.base_ref == 'latest' }} - built_base_images: ${{ needs.base-images.outputs.rebuilt_images }} + rebuilt_base_images: ${{ steps.build_base_images.outputs.rebuilt_images }} + + - name: Test pulp-minimal image + uses: "./.github/actions/test_image" + with: + image_name: "pulp-minimal" + app_branch: ${{ steps.build_pulp_minimal_image.outputs.app_branch }} - - name: Test App Image + - name: Test pulp image uses: "./.github/actions/test_image" with: - image_name: ${{ matrix.image_name }} - app_branch: ${{ steps.build_image.outputs.app_branch }} + image_name: "pulp" + app_branch: ${{ steps.build_pulp_image.outputs.app_branch }} - name: Logs if: always() diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1874cfdf..4a10b558 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,12 +51,10 @@ jobs: env: GITHUB_TOKEN: ${{ secrets.RELEASE_TOKEN }} - base-images: + base: runs-on: ubuntu-latest outputs: - base_cache_key: ${{ steps.build_base_images.outputs.base_cache_key }} image_variants: ${{ steps.image_variants.outputs.image_variants }} - rebuilt_images: ${{ steps.build_base_images.outputs.rebuilt_images }} steps: - uses: actions/checkout@v4 @@ -87,130 +85,132 @@ jobs: echo "image_variants=[\"stable\"]" >> "$GITHUB_OUTPUT" fi - - name: Build base images - id: build_base_images - uses: "./.github/actions/base_images" - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - # The published base images are tagged with the pulpcore version + python version, however - # the base images don't have Pulp installed. Will need to use our context clues of this run - # to figure out which pulpcore version will be installed. - - name: Find pulpcore versions - run: | - pulpcore_version=$(python .ci/scripts/find_pulpcore_version.py --branch "${{ github.ref_name }}") - pulpcore_branch=$(echo ${pulpcore_version} | grep -oP '\d+\.\d+') - echo "Found pulpcore version $pulpcore_version on branch $pulpcore_branch" - echo "PULPCORE_VERSION=${pulpcore_version}" >> $GITHUB_ENV - echo "PULPCORE_BRANCH=${pulpcore_branch}" >> $GITHUB_ENV - if [ "${{ github.ref_name }}" == "latest" ]; then - # We also tag the latest base images with the nightly(main-branch) version of pulpcore - nightly_version=$(python .ci/scripts/find_pulpcore_version.py --branch main) - nightly_branch=$(echo ${nightly_version} | grep -oP '\d+\.\d+') - echo "Found nightly pulpcore version $nightly_version on branch $nightly_branch" - echo "NIGHTLY_VERSION=${nightly_version}" >> $GITHUB_ENV - echo "NIGHTLY_BRANCH=${nightly_branch}" >> $GITHUB_ENV - fi - - # Base images on latest will also publish under the next pulpcore version on main. - # If python_version==3.9 (our default python) then publish with just the pulpcore version - # to maintain our prior tagging scheme before customizable python versions - - name: Set image tags - run: | - tags="${tags} ${PULPCORE_VERSION} ${PULPCORE_BRANCH}" - if [ "${{ github.ref_name }}" == "latest" ]; then - tags="${tags} ${NIGHTLY_VERSION} ${NIGHTLY_BRANCH} latest" - fi - tags=$(echo "${tags}" | xargs -n1 | sort -u | xargs) - echo "Set tags to: $tags" - echo "TAGS=${tags}" >> $GITHUB_ENV - - - name: Publish base images - uses: "./.github/actions/publish_images" - with: - image_names: "base pulp-ci-centos9" - tags: ${{ env.TAGS }} - github_token: ${{ secrets.GITHUB_TOKEN }} - docker_bot_username: ${{ secrets.DOCKER_BOT_USERNAME }} - docker_bot_password: ${{ secrets.DOCKER_BOT_PASSWORD }} - quay_bot_username: ${{ secrets.QUAY_BOT_USERNAME }} - quay_bot_password: ${{ secrets.QUAY_BOT_PASSWORD }} - - app-images: - needs: base-images - runs-on: ubuntu-latest - outputs: - app_version: ${{ steps.build_image.outputs.app_version }} - app_branch: ${{ steps.build_image.outputs.app_branch }} + build-and-test-images: + needs: base strategy: fail-fast: false matrix: - image_variant: ${{ fromJSON(needs.base-images.outputs.image_variants) }} - image_name: - - pulp-minimal - - pulp + image_variant: ${{ fromJSON(needs.base.outputs.image_variants) }} + os: + - ubuntu-24.04 + - ubuntu-24.04-arm + runs-on: ${{ matrix.os }} + outputs: + # repo_digests format: [ + # rebuilt-base-images bool, rebuilt-minimal-images bool, rebuilt-pulp-image bool + # base sha, pulp-ci-centos9 sha, pulp-minimal sha, pulp-web sha, pulp sha, + # ] + stable_arm: ${{ steps.repo_digests.outputs.stable_arm }} + stable_amd: ${{ steps.repo_digests.outputs.stable_amd }} + nightly_arm: ${{ steps.repo_digests.outputs.nightly_arm }} + nightly_amd: ${{ steps.repo_digests.outputs.nightly_amd }} + stable_app_version: ${{ steps.app_version.outputs.stable_app_version }} + nightly_app_version: ${{ steps.app_version.outputs.nightly_app_version }} + steps: - uses: actions/checkout@v4 - - name: Build App Image - id: build_image + - name: Build base images + id: build_base_images + uses: "./.github/actions/base_images" + with: + image_variants: ${{ matrix.image_variant }} + + - name: Build pulp-minimal image + id: build_pulp_minimal_image uses: "./.github/actions/build_image" with: - image_name: ${{ matrix.image_name }} + image_name: "pulp-minimal" image_variant: ${{ matrix.image_variant }} - image_cache_key: ${{ needs.base-images.outputs.base_cache_key }} - latest_ui: ${{ github.ref_name == 'latest' }} - built_base_images: ${{ needs.base-images.outputs.rebuilt_images }} - env: - GH_TOKEN: ${{ github.token }} + latest_ui: ${{ github.base_ref == 'latest' }} + rebuilt_base_images: ${{ steps.build_base_images.outputs.rebuilt_images }} + + - name: Build pulp image + id: build_pulp_image + uses: "./.github/actions/build_image" + with: + image_name: "pulp" + image_variant: ${{ matrix.image_variant }} + latest_ui: ${{ github.base_ref == 'latest' }} + rebuilt_base_images: ${{ steps.build_base_images.outputs.rebuilt_images }} + + - name: Set app version + id: app_version + run: | + echo "${{ matrix.image_variant }}_app_version=${{ steps.build_pulp_image.outputs.app_version }}" >> $GITHUB_OUTPUT + - name: Test pulp-minimal image + if: ${{ matrix.image_variant != 'nightly' && steps.build_pulp_minimal_image.outputs.rebuilt_images == 'true' }} + uses: "./.github/actions/test_image" + with: + image_name: "pulp-minimal" + app_branch: ${{ steps.build_pulp_minimal_image.outputs.app_branch }} - - name: Test App Image - if: ${{ matrix.image_variant != 'nightly' && steps.build_image.outputs.rebuilt_images == 'true' }} + - name: Test pulp image + if: ${{ matrix.image_variant != 'nightly' && steps.build_pulp_image.outputs.rebuilt_images == 'true' }} uses: "./.github/actions/test_image" with: - image_name: ${{ matrix.image_name }} - app_branch: ${{ steps.build_image.outputs.app_branch }} + image_name: "pulp" + app_branch: ${{ steps.build_pulp_image.outputs.app_branch }} - - name: Set tags - if: ${{ steps.build_image.outputs.rebuilt_images == 'true' }} + # We are doing a temporary upload of the arm or amd images to ghcr.io for the next job to use + - name: Login to ghcr.io + env: + PULP_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PULP_GITHUB_USERNAME: ${{ github.actor }} + run: echo "$PULP_GITHUB_TOKEN" | podman login -u "$PULP_GITHUB_USERNAME" --password-stdin ghcr.io + + - name: Set tags for temp upload run: | - base_image=$(echo ${{ matrix.image_name }} | cut -d '-' -f1) - if [[ "${{ matrix.image_name }}" == "pulp" ]]; then - images="${{ matrix.image_name }}" + if [[ "${{ runner.arch }}" == "ARM64" ]]; then + ARCH="arm" else - images="${{ matrix.image_name }} ${base_image}-web" + ARCH="amd" fi - echo "Going to publish app images: $images" - echo "IMAGES=${images}" >> $GITHUB_ENV - - if [ "${{ matrix.image_variant }}" == "stable" ]; then - app_branch=${{ steps.build_image.outputs.app_branch }} - app_version=${{ steps.build_image.outputs.app_version }} - # latest branch stable variant gets tagged as both "latest" and "stable" - if [ "${GITHUB_REF_NAME%/*}" == "latest" ]; then - tags="${app_branch} ${app_version} stable latest" - else - tags="${app_branch} ${app_version}" - fi + if [[ ${{ matrix.image_variant }} == "stable" ]]; then + TAG="${{ steps.app_version.outputs.stable_app_version }}-$ARCH" else - tags="nightly" + TAG="nightly-$ARCH" fi - tags=$(echo "${tags}" | xargs -n1 | sort -u | xargs) - echo "Going to publish with tags: $tags" - echo "TAGS=${tags}" >> $GITHUB_ENV + echo "TAG=${TAG}" >> $GITHUB_ENV + echo "ARCH=${ARCH}" >> $GITHUB_ENV - - name: Publish App Image - if: ${{ steps.build_image.outputs.rebuilt_images == 'true' }} - uses: "./.github/actions/publish_images" - with: - image_names: ${{ env.IMAGES }} - tags: ${{ env.TAGS }} - github_token: ${{ secrets.GITHUB_TOKEN }} - docker_bot_username: ${{ secrets.DOCKER_BOT_USERNAME }} - docker_bot_password: ${{ secrets.DOCKER_BOT_PASSWORD }} - quay_bot_username: ${{ secrets.QUAY_BOT_USERNAME }} - quay_bot_password: ${{ secrets.QUAY_BOT_PASSWORD }} + - name: Temp upload images to ghcr.io + run: | + trimmed_tag="${TAG::-4}" + if [[ ${{ steps.build_base_images.outputs.rebuilt_images }} == "true" ]]; then + podman push pulp/base:ci ghcr.io/pulp/base:${TAG} + podman push pulp/pulp-ci-centos9:ci ghcr.io/pulp/pulp-ci-centos9:${TAG} + image_names="ghcr.io/pulp/base:${TAG} ghcr.io/pulp/pulp-ci-centos9:${TAG}" + else + image_names="ghcr.io/pulp/base:${trimmed_tag} ghcr.io/pulp/pulp-ci-centos9:${trimmed_tag}" + fi + if [[ ${{ steps.build_pulp_minimal_image.outputs.rebuilt_images }} == "true" ]]; then + podman push pulp/pulp-minimal:ci ghcr.io/pulp/pulp-minimal:${TAG} + podman push pulp/pulp-web:ci ghcr.io/pulp/pulp-web:${TAG} + image_names="${image_names} ghcr.io/pulp/pulp-minimal:${TAG} ghcr.io/pulp/pulp-web:${TAG}" + else + image_names="${image_names} ghcr.io/pulp/pulp-minimal:${trimmed_tag} ghcr.io/pulp/pulp-web:${trimmed_tag}" + fi + if [[ ${{ steps.build_pulp_image.outputs.rebuilt_images }} == "true" ]]; then + podman push pulp/pulp:ci ghcr.io/pulp/pulp:${TAG} + image_names="${image_names} ghcr.io/pulp/pulp:${TAG}" + else + image_names="${image_names} ghcr.io/pulp/pulp:${trimmed_tag}" + fi + echo "image_names=${image_names}" >> $GITHUB_ENV + + - name: Set output of repo digests of uploaded images + id: repo_digests + run: | + repo_digests="${{ steps.build_base_images.outputs.rebuilt_images }}" + repo_digests="${repo_digests} ${{ steps.build_pulp_minimal_image.outputs.rebuilt_images }}" + repo_digests="${repo_digests} ${{ steps.build_pulp_image.outputs.rebuilt_images }}" + for img in ${image_names}; do + # This takes advantage of the fact that skopeo will choose the correct platform for the image + repo_digests="${repo_digests} $(skopeo inspect --format='{{ .Digest }}' registry://${img} )" + done + echo "${{ matrix.image_variant }}_${ARCH}=${repo_digests}" >> $GITHUB_OUTPUT - name: Logs if: always() @@ -230,3 +230,96 @@ jobs: sudo ls -al $VOLUME_PATH sudo tree $VOLUME_PATH http --follow --timeout 30 --check-status --pretty format --print hb http://localhost:8080/pulp/api/v3/status/ || true + + publish-images: + needs: [base,build-and-test-images] + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image_variant: ${{ fromJSON(needs.base.outputs.image_variants) }} + steps: + - uses: actions/checkout@v4 + + # Merging the arm & amd images together and determining if they need to be published + - name: Get previous output + run: | + repo_digests_arm=(${{ needs.build-and-test-images.outputs[format('{0}_arm', matrix.image_variant)] }}) + repo_digests_amd=(${{ needs.build-and-test-images.outputs[format('{0}_amd', matrix.image_variant)] }}) + rebuilt_base_images="${repo_digests_arm[0]} ${repo_digests_amd[0]}" + rebuilt_minimal_images="${repo_digests_arm[1]} ${repo_digests_amd[1]}" + rebuilt_pulp_image="${repo_digests_arm[2]} ${repo_digests_amd[2]}" + base_digests="${repo_digests_arm[3]} ${repo_digests_amd[3]}" + pulp_ci_centos9_digests="${repo_digests_arm[4]} ${repo_digests_amd[4]}" + pulp_minimal_digests="${repo_digests_arm[5]} ${repo_digests_amd[5]}" + pulp_web_digests="${repo_digests_arm[6]} ${repo_digests_amd[6]}" + pulp_digests="${repo_digests_arm[7]} ${repo_digests_amd[7]}" + echo "rebuilt_base_images=${rebuilt_base_images}" >> $GITHUB_ENV + echo "rebuilt_minimal_images=${rebuilt_minimal_images}" >> $GITHUB_ENV + echo "rebuilt_pulp_image=${rebuilt_pulp_image}" >> $GITHUB_ENV + echo "base_digests=${base_digests}" >> $GITHUB_ENV + echo "pulp_ci_centos9_digests=${pulp_ci_centos9_digests}" >> $GITHUB_ENV + echo "pulp_minimal_digests=${pulp_minimal_digests}" >> $GITHUB_ENV + echo "pulp_web_digests=${pulp_web_digests}" >> $GITHUB_ENV + echo "pulp_digests=${pulp_digests}" >> $GITHUB_ENV + + - name: Pull images if needed + run: | + if [[ ${{ contains(env.rebuilt_base_images, 'true') }} == 'true' ]]; then + base_digests=(${{ env.base_digests }}) + pulp_ci_centos9_digests=(${{ env.pulp_ci_centos9_digests }}) + podman pull "ghcr.io/pulp/base@${base_digests[0]}" "ghcr.io/pulp/base@${base_digests[1]}" + podman tag "ghcr.io/pulp/base@${base_digests[0]}" pulp/base:ci-arm64 + podman tag "ghcr.io/pulp/base@${base_digests[1]}" pulp/base:ci-amd64 + podman pull "ghcr.io/pulp/pulp-ci-centos9@${pulp_ci_centos9_digests[0]}" "ghcr.io/pulp/pulp-ci-centos9@${pulp_ci_centos9_digests[1]}" + podman tag "ghcr.io/pulp/pulp-ci-centos9@${pulp_ci_centos9_digests[1]}" pulp/pulp-ci-centos9:ci-arm64 + podman tag "ghcr.io/pulp/pulp-ci-centos9@${pulp_ci_centos9_digests[1]}" pulp/pulp-ci-centos9:ci-amd64 + images="base pulp-ci-centos9" + fi + if [[ ${{ contains(env.rebuilt_minimal_images, 'true') }} == 'true' ]]; then + pulp_minimal_digests=(${{ env.pulp_minimal_digests }}) + pulp_web_digests=(${{ env.pulp_web_digests }}) + podman pull "ghcr.io/pulp/pulp-minimal@${pulp_minimal_digests[0]}" "ghcr.io/pulp/pulp-minimal@${pulp_minimal_digests[1]}" + podman pull "ghcr.io/pulp/pulp-web@${pulp_web_digests[0]}" "ghcr.io/pulp/pulp-web@${pulp_web_digests[1]}" + podman tag "ghcr.io/pulp/pulp-minimal@${pulp_minimal_digests[0]}" pulp/pulp-minimal:ci-arm64 + podman tag "ghcr.io/pulp/pulp-minimal@${pulp_minimal_digests[1]}" pulp/pulp-minimal:ci-amd64 + podman tag "ghcr.io/pulp/pulp-web@${pulp_web_digests[0]}" pulp/pulp-web:ci-arm64 + images="${images} pulp-minimal pulp-web" + fi + if [[ ${{ contains(env.rebuilt_pulp_image, 'true') }} == 'true' ]]; then + pulp_digests=(${{ env.pulp_digests }}) + podman pull "ghcr.io/pulp/pulp@${pulp_digests[0]}" "ghcr.io/pulp/pulp@${pulp_digests[1]}" + podman tag "ghcr.io/pulp/pulp@${pulp_digests[0]}" pulp/pulp:ci-arm64 + podman tag "ghcr.io/pulp/pulp@${pulp_digests[1]}" pulp/pulp:ci-amd64 + images="${images} pulp" + fi + echo "Images to publish: $images" + echo "IMAGES=${images}" >> $GITHUB_ENV + + - name: Set image tags + run: | + version="${{ needs.build-and-test-images.outputs[format('{0}_app_version', matrix.image_variant)] }}" + branch=$(echo ${version} | grep -oP '\d+\.\d+') + tags="${branch} ${version}" + if [[ ${{ matrix.image_variant }} == "stable" ]]; then + if [ "${{ github.ref_name }}" == "latest" ]; then + tags="${tags} latest stable" + fi + else + tags="${tags} nightly" + fi + tags=$(echo "${tags}" | xargs -n1 | sort -u | xargs) + echo "Set tags to: $tags" + echo "TAGS=${tags}" >> $GITHUB_OUTPUT + + - name: Publish images + if: ${{ env.IMAGES }} + uses: "./.github/actions/publish_images" + with: + image_names: ${{ env.IMAGES }} + tags: ${{ env.TAGS }} + github_token: ${{ secrets.GITHUB_TOKEN }} + docker_bot_username: ${{ secrets.DOCKER_BOT_USERNAME }} + docker_bot_password: ${{ secrets.DOCKER_BOT_PASSWORD }} + quay_bot_username: ${{ secrets.QUAY_BOT_USERNAME }} + quay_bot_password: ${{ secrets.QUAY_BOT_PASSWORD }} diff --git a/images/s6_assets/test.sh b/images/s6_assets/test.sh index f676b5ab..1beff93f 100755 --- a/images/s6_assets/test.sh +++ b/images/s6_assets/test.sh @@ -31,6 +31,7 @@ start_container_and_wait() { -e PULP_DOMAIN_ENABLED=${domain_enabled} \ "$1" + sleep 3 # Wait for the container to start podman exec pulp s6-rc -ba list for _ in $(seq 30) do @@ -77,7 +78,7 @@ if [ "$old_image" != "" ]; then start_container_and_wait $old_image podman rm -f pulp fi -if [[ "$image" == "pulp/pulp:ci-amd64" ]]; then +if [[ "$image" == "pulp/pulp:ci" ]]; then domain_enabled=true fi start_container_and_wait $image