add cross-arch aarch64 container image builds

.github/workflows/stackhpc-container-image-build.yml

@@ -74,8 +74,8 @@ jobs:
       - name: Generate build matrix
         id: set-matrix
         run: |
-          comma=""
           echo -n "matrix={\"distro\": [" >> $GITHUB_OUTPUT
+          comma=""
           if [[ ${{ inputs.rocky-linux-9 }} == 'true' ]]; then
             echo -n "$comma\"rocky\"" >> $GITHUB_OUTPUT
             comma=", "
@@ -84,7 +84,7 @@ jobs:
             echo -n "$comma\"ubuntu\"" >> $GITHUB_OUTPUT
             comma=", "
           fi
-          echo "]}" >> $GITHUB_OUTPUT
+          echo "], \"arch\": [\"amd64\", \"aarch64\"]}" >> $GITHUB_OUTPUT
 
       - name: Display container datetime tag
         run: |
@@ -99,6 +99,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix: ${{ fromJson(needs.generate-tag.outputs.matrix) }}
+      # Exclude ubuntu aarch64 builds for now
+      exclude:
+        - distro: ubuntu
+          arch: aarch64
     needs:
       - generate-tag
     steps:
@@ -149,7 +153,12 @@ jobs:
 
       - name: Get Kolla tag
         id: write-kolla-tag
-        run: echo "kolla-tag=${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ matrix.distro == 'rocky' && '9' || 'jammy' }}-${{ needs.generate-tag.outputs.datetime_tag }}"  >> $GITHUB_OUTPUT
+        run: |
+          kolla_tag="${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ needs.generate-tag.outputs.datetime_tag }}"
+          if [[ "${{ matrix.distro }}" == 'rocky' ]]; then
+            kolla_tag="$kolla_tag-${{ matrix.arch }}"
+          fi
+          echo "kolla-tag=$kolla_tag" >> $GITHUB_OUTPUT
 
       - name: Configure localhost as a seed
         run: |
@@ -179,8 +188,11 @@ jobs:
         continue-on-error: true
         run: |
           args="${{ inputs.regexes }}"
+          if [[ "${{ matrix.arch }}" = 'aarch64' ]]; then
+            args="$args -e kolla_base_arch=${{ matrix.arch }}"
+          fi
           args="$args -e kolla_base_distro=${{ matrix.distro }}"
-          args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}"
+          args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}
           args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
@@ -193,26 +205,37 @@ jobs:
         run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-overcloud.log
         if: inputs.overcloud
 
+      - name: Copy build configs to output directory
+        run: sudo cp -rnL /opt/kayobe/etc/kolla/* image-build-logs/
+        if: inputs.overcloud
+
+      - name: Get Kolla tag for seed images
+        id: write-kolla-tag-seed
+        run: |
+          kolla_tag_seed="${{ needs.generate-tag.outputs.openstack_release }}-${{ matrix.distro }}-${{ needs.generate-tag.outputs.datetime_tag }}"
+          echo "kolla-tag-seed=$kolla_tag_seed" >> $GITHUB_OUTPUT
+        if: inputs.seed
+
       - name: Build kolla seed images
         id: build_seed_images
         continue-on-error: true
         run: |
           args="-e kolla_base_distro=${{ matrix.distro }}"
-          args="$args -e kolla_tag=${{ steps.write-kolla-tag.outputs.kolla-tag }}"
+          args="$args -e kolla_tag=${{ steps.write-kolla-tag-seed.outputs.kolla-tag-seed }}"
           args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
           source venvs/kayobe/bin/activate &&
           source src/kayobe-config/kayobe-env --environment ci-builder &&
           kayobe seed container image build $args
         env:
           KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
-        if: inputs.seed
+        if: inputs.seed && matrix.arch == 'amd64'
 
       - name: Copy seed container image build logs to output directory
         run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-seed.log
         if: inputs.seed
 
       - name: Get built container images
-        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}" > ${{ matrix.distro }}-container-images
+        run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/*:${{ steps.write-kolla-tag.outputs.kolla-tag }}*" > ${{ matrix.distro }}-container-images
 
       - name: Fail if no images have been built
         run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi
@@ -273,7 +296,7 @@ jobs:
       - name: Upload output artifact
         uses: actions/upload-artifact@v4
         with:
-          name: ${{ matrix.distro }}-logs
+          name: ${{ matrix.distro }}-${{ matrix.arch }}-logs
           path: image-build-logs
           retention-days: 7
         if: ${{ !cancelled() }}

etc/kayobe/environments/ci-builder/stackhpc-ci.yml

@@ -48,32 +48,34 @@ stackhpc_repo_mirror_password: !vault |
 
 # Build against released Pulp repository versions.
 stackhpc_repo_grafana_version: "{{ stackhpc_pulp_repo_grafana_version }}"
-stackhpc_repo_rhel9_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version }}"
+stackhpc_repo_rhel9_rabbitmq_erlang_version: "{{ stackhpc_pulp_repo_rhel9_rabbitmq_erlang_aarch64_version if kolla_base_arch == 'aarch64' else stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version }}"
 stackhpc_repo_rhel9_rabbitmq_server_version: "{{ stackhpc_pulp_repo_rhel9_rabbitmq_server_version }}"
 stackhpc_repo_ubuntu_jammy_version: "{{ stackhpc_pulp_repo_ubuntu_jammy_version }}"
 stackhpc_repo_ubuntu_jammy_security_version: "{{ stackhpc_pulp_repo_ubuntu_jammy_security_version }}"
 stackhpc_repo_ubuntu_jammy_cve_2024_6387_version: ""
 stackhpc_repo_ubuntu_cloud_archive_version: "{{ stackhpc_pulp_repo_ubuntu_cloud_archive_version }}"
 stackhpc_repo_docker_ce_ubuntu_jammy_version: "{{ stackhpc_pulp_repo_docker_ce_ubuntu_jammy_version }}"
-stackhpc_repo_centos_stream_9_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version }}"
-stackhpc_repo_centos_stream_9_openstack_caracal_version: "{{ stackhpc_pulp_repo_centos_stream_9_openstack_caracal_version }}"
-stackhpc_repo_centos_stream_9_opstools_version: "{{ stackhpc_pulp_repo_centos_stream_9_opstools_version }}"
-stackhpc_repo_centos_stream_9_storage_ceph_reef_version: "{{ stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_version }}"
-stackhpc_repo_centos_stream_9_docker_version: "{{ stackhpc_pulp_repo_centos_stream_9_docker_version }}"
-stackhpc_repo_rhel_9_treasuredata_5_version: "{{ stackhpc_pulp_repo_rhel_9_treasuredata_5_version }}"
-stackhpc_repo_rhel_9_mariadb_10_11_version: "{{ stackhpc_pulp_repo_rhel_9_mariadb_10_11_version }}"
-stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_pulp_repo_rhel_9_influxdb_version }}"
-stackhpc_repo_epel_9_version: "{{ stackhpc_pulp_repo_epel_9_version }}"
+stackhpc_repo_ceph_reef_debian_version: "{{ stackhpc_pulp_repo_ceph_reef_debian_version }}"
+stackhpc_repo_centos_stream_9_nfv_openvswitch_version: "{{ stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_aarch64_version if kolla_base_arch == 'aarch64' else stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version }}"
+stackhpc_repo_centos_stream_9_openstack_caracal_version: "{{ stackhpc_pulp_repo_centos_stream_9_openstack_caracal_aarch64_version if kolla_base_arch == 'aarch64' else stackhpc_pulp_repo_centos_stream_9_openstack_caracal_version }}"
+stackhpc_repo_centos_stream_9_opstools_version: "{{ stackhpc_pulp_repo_centos_stream_9_opstools_aarch64_version if kolla_base_arch == 'aarch64' else stackhpc_pulp_repo_centos_stream_9_opstools_version }}"
+stackhpc_repo_centos_stream_9_storage_ceph_reef_version: "{{ stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_aarch64_version if kolla_base_arch == 'aarch64' else stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_version }}"
+stackhpc_repo_centos_stream_9_docker_version: "{{ stackhpc_pulp_repo_centos_stream_9_docker_aarch64_version if kolla_base_arch == 'aarch64' else stackhpc_pulp_repo_centos_stream_9_docker_version }}"
+stackhpc_repo_rhel_9_treasuredata_5_version: "{{ stackhpc_pulp_repo_rhel_9_treasuredata_5_aarch64_version if kolla_base_arch == 'aarch64' else stackhpc_pulp_repo_rhel_9_treasuredata_5_version }}"
+stackhpc_repo_rhel_9_mariadb_10_11_version: "{{ stackhpc_pulp_repo_rhel_9_mariadb_10_11_aarch64_version if kolla_base_arch == 'aarch64' else stackhpc_pulp_repo_rhel_9_mariadb_10_11_version }}"
+stackhpc_repo_rhel_9_influxdb_version: "{{ stackhpc_pulp_repo_rhel_9_influxdb_aarch64_version if kolla_base_arch == 'aarch64' else stackhpc_pulp_repo_rhel_9_influxdb_version }}"
+stackhpc_repo_epel_9_version: "{{ stackhpc_pulp_repo_epel_9_aarch64_version if kolla_base_arch == 'aarch64' else stackhpc_pulp_repo_epel_9_version }}"
 stackhpc_repo_opensearch_2_x_version: "{{ stackhpc_pulp_repo_opensearch_2_x_version }}"
 stackhpc_repo_opensearch_dashboards_2_x_version: "{{ stackhpc_pulp_repo_opensearch_dashboards_2_x_version }}"
+
 ## Use derived vars from etc/kayobe/pulp.yml to switch between
 ## minor Rocky versions using stackhpc_pulp_repo_rocky_x_minor_version
 stackhpc_repo_rocky_9_baseos_version: "{{ stackhpc_pulp_repo_rocky_9_baseos_version }}"
 stackhpc_repo_rocky_9_appstream_version: "{{ stackhpc_pulp_repo_rocky_9_appstream_version }}"
 stackhpc_repo_rocky_9_extras_version: "{{ stackhpc_pulp_repo_rocky_9_extras_version }}"
 stackhpc_repo_rocky_9_crb_version: "{{ stackhpc_pulp_repo_rocky_9_crb_version }}"
 stackhpc_repo_rocky_9_highavailability_version: "{{ stackhpc_pulp_repo_rocky_9_highavailability_version }}"
-stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_pulp_repo_rocky_9_sig_security_common_version }}"
+stackhpc_repo_rocky_9_sig_security_common_version: "{{ stackhpc_pulp_repo_rocky_9_sig_security_common_aarch64_version if kolla_base_arch == 'aarch64' else stackhpc_pulp_repo_rocky_9_sig_security_common_version }}"
 
 # Rocky-and-CI-specific Pulp urls
 stackhpc_include_os_minor_version_in_repo_url: true

etc/kayobe/kolla.yml

@@ -202,8 +202,8 @@ stackhpc_rocky_9_third_party_repos:
     file: "CentOS-Ceph-Reef.repo"
     tag: "centos-ceph-reef"
   - url: "{{ stackhpc_repo_rhel9_rabbitmq_erlang_url }}"
-    file: "rabbitmq_rabbitmq-erlang.repo"
-    tag: "rabbitmq_rabbitmq-erlang"
+    file: "{{ 'copr-rabbitmq-erlang.repo' if kolla_base_arch == 'aarch64' else 'rabbitmq_rabbitmq-erlang.repo' }}"
+    tag: "{{ 'openstack-kolla-copr-rabbitmq-kolla' if kolla_base_arch == 'aarch64' else 'rabbitmq_rabbitmq-erlang' }}"
   - url: "{{ stackhpc_repo_rhel9_rabbitmq_server_url }}"
     file: "rabbitmq_rabbitmq-server.repo"
     tag: "rabbitmq_rabbitmq-server"
@@ -254,13 +254,18 @@ kolla_build_blocks:
         mkdir -p /etc/yum.repos.d.backup && \
         tar -czf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d . && \
     {% endif %}
+        echo "===== Before sed modifications I =====" && \
+        cat /etc/yum.repos.d/*.repo && \
     {% for repo in stackhpc_yum_repos %}
         sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} && \
     {% endif %}
     {% endfor %}
+        && \
+        echo "===== After sed modifications I =====" && \
+        cat /etc/yum.repos.d/*.repo
     {% else %}
     RUN \
         rm /etc/apt/sources.list && \
@@ -281,13 +286,18 @@ kolla_build_blocks:
         tar -xzf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d && \
         tar -czf /etc/yum.repos.d.backup/repos.tar.gz -C /etc/yum.repos.d . && \
     {% endif %}
+        echo "===== Before sed modifications II =====" && \
+        cat /etc/yum.repos.d/*.repo && \
     {% for repo in base_centos_repo_overrides_post_yum_list %}
         sed -i -e '/\[{{ repo.tag }}\]/,/^\[/ s/^\(mirrorlist *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(baseurl *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[ \t]*\(https.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s/^[# ]*\(metalink *=.*\)/#\1/g' \
                -e '/\[{{ repo.tag }}\]/,/^\[/ s|^\(name.*\)|\1\nbaseurl={{ repo.url }}|' /etc/yum.repos.d/{{ repo.file }}{% if not loop.last %} &&{% endif %} \
     {% endfor %}
+        && \
+        echo "===== After sed modifications II =====" && \
+        cat /etc/yum.repos.d/*.repo \
     {% endif %}
   # With the UCA keyring installed we can now add all repos.
   base_ubuntu_package_sources_list: |

etc/kayobe/pulp-repo-versions.yml

@@ -1,23 +1,34 @@
 ---
 # Do not edit! This file is autogenerated by Ansible.
 stackhpc_pulp_repo_centos_stream_9_docker_version: 20240829T093746
+stackhpc_pulp_repo_centos_stream_9_docker_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_version: 20240829T093746
+stackhpc_pulp_repo_centos_stream_9_nfv_openvswitch_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_centos_stream_9_openstack_caracal_version: 20240902T080424
+stackhpc_pulp_repo_centos_stream_9_openstack_caracal_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_centos_stream_9_opstools_version: 20231213T031318
+stackhpc_pulp_repo_centos_stream_9_opstools_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_version: 20240502T000614
+stackhpc_pulp_repo_centos_stream_9_storage_ceph_reef_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_docker_ce_ubuntu_jammy_version: 20240910T001721
 stackhpc_pulp_repo_elrepo_9_version: 20240902T122220
+stackhpc_pulp_repo_elrepo_9_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_epel_9_version: 20240902T080424
+stackhpc_pulp_repo_epel_9_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_grafana_version: 20240902T080424
 stackhpc_pulp_repo_opensearch_2_x_version: 20240807T235120
 stackhpc_pulp_repo_opensearch_dashboards_2_x_version: 20240807T235120
 stackhpc_pulp_repo_rhel9_rabbitmq_erlang_version: 20240925T093206
+stackhpc_pulp_repo_rhel9_rabbitmq_erlang_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_rhel9_rabbitmq_server_version: 20240925T111913
 stackhpc_pulp_repo_rhel_9_influxdb_version: 20240817T001913
+stackhpc_pulp_repo_rhel_9_influxdb_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_rhel_9_mariadb_10_11_version: 20240810T001640
+stackhpc_pulp_repo_rhel_9_mariadb_10_11_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_rhel_9_rabbitmq_erlang_version: 20240711T091318
 stackhpc_pulp_repo_rhel_9_rabbitmq_server_version: 20240711T091318
 stackhpc_pulp_repo_rhel_9_treasuredata_5_version: 20240711T091318
+stackhpc_pulp_repo_rhel_9_treasuredata_5_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_rocky_9_1_appstream_version: 20231207T013715
 stackhpc_pulp_repo_rocky_9_1_baseos_version: 20231206T014015
 stackhpc_pulp_repo_rocky_9_1_crb_version: 20231211T120328
@@ -34,11 +45,17 @@ stackhpc_pulp_repo_rocky_9_3_crb_version: 20240429T003818
 stackhpc_pulp_repo_rocky_9_3_extras_version: 20240506T000343
 stackhpc_pulp_repo_rocky_9_3_highavailability_version: 20240510T001129
 stackhpc_pulp_repo_rocky_9_4_appstream_version: 20240816T002610
+stackhpc_pulp_repo_rocky_9_4_appstream_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_rocky_9_4_baseos_version: 20240816T002610
+stackhpc_pulp_repo_rocky_9_4_baseos_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_rocky_9_4_crb_version: 20240816T002610
+stackhpc_pulp_repo_rocky_9_4_crb_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_rocky_9_4_extras_version: 20240816T002610
+stackhpc_pulp_repo_rocky_9_4_extras_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_rocky_9_4_highavailability_version: 20240816T002610
+stackhpc_pulp_repo_rocky_9_4_highavailability_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_rocky_9_sig_security_common_version: 20240718T001130
+stackhpc_pulp_repo_rocky_9_sig_security_common_aarch64_version: 20240927T073838
 stackhpc_pulp_repo_ubuntu_cloud_archive_version: 20240911T041957
 stackhpc_pulp_repo_ubuntu_jammy_security_version: 20240911T063424
 stackhpc_pulp_repo_ubuntu_jammy_version: 20240911T063424

etc/kayobe/pulp.yml

@@ -208,11 +208,11 @@ stackhpc_pulp_sync_rocky_9: "{{ os_distribution == 'rocky' }}"
 stackhpc_pulp_repo_rocky_9_minor_version: 4
 # Rocky 9 Snapshot versions. The defaults use the appropriate version from
 # pulp-repo-versions.yml for the selected minor release.
-stackhpc_pulp_repo_rocky_9_appstream_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_%s_appstream_version' % stackhpc_pulp_repo_rocky_9_minor_version) }}"
-stackhpc_pulp_repo_rocky_9_baseos_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_%s_baseos_version' % stackhpc_pulp_repo_rocky_9_minor_version) }}"
-stackhpc_pulp_repo_rocky_9_extras_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_%s_extras_version' % stackhpc_pulp_repo_rocky_9_minor_version) }}"
-stackhpc_pulp_repo_rocky_9_crb_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_%s_crb_version' % stackhpc_pulp_repo_rocky_9_minor_version) }}"
-stackhpc_pulp_repo_rocky_9_highavailability_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_%s_highavailability_version' % stackhpc_pulp_repo_rocky_9_minor_version) }}"
+stackhpc_pulp_repo_rocky_9_baseos_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_baseos' ~ ('_aarch64' if kolla_base_arch == 'aarch64' else '') ~ '_version') }}"
+stackhpc_pulp_repo_rocky_9_appstream_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_appstream' ~ ('_aarch64' if kolla_base_arch == 'aarch64' else '') ~ '_version') }}"
+stackhpc_pulp_repo_rocky_9_extras_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_extras' ~ ('_aarch64' if kolla_base_arch == 'aarch64' else '') ~ '_version') }}"
+stackhpc_pulp_repo_rocky_9_crb_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_crb' ~ ('_aarch64' if kolla_base_arch == 'aarch64' else '') ~ '_version') }}"
+stackhpc_pulp_repo_rocky_9_highavailability_version: "{{ lookup('vars', 'stackhpc_pulp_repo_rocky_9_' ~ stackhpc_pulp_repo_rocky_9_minor_version ~ '_highavailability' ~ ('_aarch64' if kolla_base_arch == 'aarch64' else '') ~ '_version') }}"
 
 # Whether to sync packages common to all RHEL 9 derivatives.
 stackhpc_pulp_sync_el_9: "{{ stackhpc_pulp_sync_rocky_9 | bool }}"