Skip to content

Build Kolla container images #889

Build Kolla container images

Build Kolla container images #889

---
name: Build Kolla container images
on:
workflow_dispatch:
inputs:
regexes:
description: Space-separated list of regular expressions matching overcloud images to build
type: string
required: false
default: ""
overcloud:
description: Build overcloud images?
type: boolean
required: false
default: true
seed:
description: Build seed images?
type: boolean
required: false
default: false
centos-stream-8:
description: Build CentOS Stream 8 images?
type: boolean
required: false
default: true
rocky-linux-9:
description: Build Rocky Linux 9 images?
type: boolean
required: false
default: true
ubuntu-focal:
description: Build Ubuntu Focal 20.04 images?
type: boolean
required: false
default: true
push:
description: Whether to push images
type: boolean
required: false
default: true
push-dirty:
description: Push scanned images that have critical vulnerabilities?
type: boolean
required: false
default: false
env:
ANSIBLE_FORCE_COLOR: True
jobs:
generate-tag:
name: Generate container image tag
if: github.repository == 'stackhpc/stackhpc-kayobe-config'
runs-on: ubuntu-latest
permissions: {}
outputs:
kolla_tag: ${{ steps.kolla_tag.outputs.kolla_tag }}
matrix: ${{ steps.set-matrix.outputs.matrix }}
openstack_release: ${{ steps.openstack_release.outputs.openstack_release }}
steps:
- name: Checkout
uses: actions/checkout@v4
- name: Determine OpenStack release
id: openstack_release
run: |
BRANCH=$(awk -F'=' '/defaultbranch/ {print $2}' .gitreview)
echo "openstack_release=${BRANCH}" | sed -E "s,(stable|unmaintained)/,," >> $GITHUB_OUTPUT
# Generate a tag to apply to all built container images.
# Without this, each kayobe * container image build command would use a different tag.
- name: Generate container image tag
id: kolla_tag
run: |
echo "kolla_tag=$(date +${{ steps.openstack_release.outputs.openstack_release }}-%Y%m%dT%H%M%S)" >> $GITHUB_OUTPUT
# Dynamically define job matrix.
# We need a separate matrix entry for each distribution, when the relevant input is true.
# https://stackoverflow.com/questions/65384420/how-do-i-make-a-github-action-matrix-element-conditional
- name: Generate build matrix
id: set-matrix
run: |
comma=""
echo -n "matrix={\"distro\": [" >> $GITHUB_OUTPUT
if [[ ${{ inputs.centos-stream-8 }} == 'true' ]]; then
echo -n "$comma\"centos\"" >> $GITHUB_OUTPUT
comma=", "
fi
if [[ ${{ inputs.rocky-linux-9 }} == 'true' ]]; then
echo -n "$comma\"rocky\"" >> $GITHUB_OUTPUT
comma=", "
fi
if [[ ${{ inputs.ubuntu-focal }} == 'true' ]]; then
echo -n "$comma\"ubuntu\"" >> $GITHUB_OUTPUT
comma=", "
fi
echo "]}" >> $GITHUB_OUTPUT
- name: Display container image tag
run: |
echo "${{ steps.kolla_tag.outputs.kolla_tag }}"
container-image-build:
name: Build Kolla container images
if: github.repository == 'stackhpc/stackhpc-kayobe-config'
runs-on: arc-skc-container-image-builder-runner
timeout-minutes: 720
permissions: {}
strategy:
fail-fast: false
matrix: ${{ fromJson(needs.generate-tag.outputs.matrix) }}
needs:
- generate-tag
steps:
- name: Install package dependencies
run: |
sudo apt update
sudo apt install -y build-essential git unzip nodejs python3-wheel python3-pip python3-venv curl jq wget
- name: Install gh
run: |
sudo mkdir -p -m 755 /etc/apt/keyrings && wget -qO- https://cli.github.com/packages/githubcli-archive-keyring.gpg | sudo tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null
sudo chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg
echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | sudo tee /etc/apt/sources.list.d/github-cli.list > /dev/null
sudo apt update
sudo apt install gh -y
- name: Checkout
uses: actions/checkout@v4
with:
path: src/kayobe-config
- name: Clone StackHPC Kayobe repository
uses: actions/checkout@v4
with:
repository: stackhpc/kayobe
ref: refs/heads/stackhpc/${{ needs.generate-tag.outputs.openstack_release }}
path: src/kayobe
- name: Make sure dockerd is running and test Docker
run: |
docker ps
- name: Install Trivy
run: |
curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sudo sh -s -- -b /usr/local/bin v0.49.0
- name: Install Kayobe
run: |
mkdir -p venvs &&
pushd venvs &&
python3 -m venv kayobe &&
source kayobe/bin/activate &&
pip install -U pip &&
pip install ../src/kayobe
# Required for Pulp auth proxy deployment and Docker registry login.
# Normally installed during host configure.
- name: Install Docker Python SDK
run: |
sudo pip install docker
- name: Configure localhost as a seed
run: |
cat > src/kayobe-config/etc/kayobe/environments/ci-builder/inventory/hosts << EOF
# A 'seed' host used for building images.
# Use localhost for container image builds.
[seed]
localhost ansible_connection=local ansible_python_interpreter=/usr/bin/python3
EOF
# See etc/kayobe/ansible/roles/pulp_auth_proxy/README.md for details.
# NOTE: We override pulp_auth_proxy_conf_path to a path shared by the
# runner and dind containers.
- name: Deploy an authenticating package repository mirror proxy
run: |
source venvs/kayobe/bin/activate &&
source src/kayobe-config/kayobe-env --environment ci-builder &&
kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/pulp-auth-proxy.yml -e pulp_auth_proxy_conf_path=/home/runner/_work/pulp_proxy
env:
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
- name: Create build logs output directory
run: mkdir image-build-logs
- name: Build kolla overcloud images
id: build_overcloud_images
continue-on-error: true
run: |
args="${{ inputs.regexes }}"
args="$args -e kolla_base_distro=${{ matrix.distro }}"
args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
source venvs/kayobe/bin/activate &&
source src/kayobe-config/kayobe-env --environment ci-builder &&
kayobe overcloud container image build $args
env:
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
if: inputs.overcloud
- name: Copy overcloud container image build logs to output directory
run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-overcloud.log
if: inputs.overcloud
- name: Build kolla seed images
id: build_seed_images
continue-on-error: true
run: |
args="-e kolla_base_distro=${{ matrix.distro }}"
args="$args -e kolla_tag=${{ needs.generate-tag.outputs.kolla_tag }}"
args="$args -e stackhpc_repo_mirror_auth_proxy_enabled=true"
source venvs/kayobe/bin/activate &&
source src/kayobe-config/kayobe-env --environment ci-builder &&
kayobe seed container image build $args
env:
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
if: inputs.seed
- name: Copy seed container image build logs to output directory
run: sudo mv /var/log/kolla-build.log image-build-logs/kolla-build-seed.log
if: inputs.seed
- name: Get built container images
run: docker image ls --filter "reference=ark.stackhpc.com/stackhpc-dev/${{ matrix.distro }}-*:${{ needs.generate-tag.outputs.kolla_tag }}" > ${{ matrix.distro }}-container-images
- name: Fail if no images have been built
run: if [ $(wc -l < ${{ matrix.distro }}-container-images) -le 1 ]; then exit 1; fi
- name: Scan built container images
run: src/kayobe-config/tools/scan-images.sh ${{ matrix.distro }} ${{ needs.generate-tag.outputs.kolla_tag }}
- name: Move image scan logs to output artifact
run: mv image-scan-output image-build-logs/image-scan-output
- name: Fail if no images have passed scanning
run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then exit 1; fi
if: ${{ !inputs.push-dirty }}
- name: Copy clean images to push-attempt-images list
run: cp image-build-logs/image-scan-output/clean-images.txt image-build-logs/push-attempt-images.txt
if: inputs.push
# NOTE(seunghun1ee): This always appends dirty images with CVEs severity lower than critical.
# This should be reverted when it's decided to filter high level CVEs as well.
- name: Append dirty images to push list
run: |
cat image-build-logs/image-scan-output/dirty-images.txt >> image-build-logs/push-attempt-images.txt
if: ${{ inputs.push }}
- name: Append images with critical vulnerabilities to push list
run: |
cat image-build-logs/image-scan-output/critical-images.txt >> image-build-logs/push-attempt-images.txt
if: ${{ inputs.push && inputs.push-dirty }}
- name: Push images
run: |
touch image-build-logs/push-failed-images.txt
source venvs/kayobe/bin/activate &&
source src/kayobe-config/kayobe-env --environment ci-builder &&
kayobe playbook run ${KAYOBE_CONFIG_PATH}/ansible/docker-registry-login.yml &&
while read -r image; do
# Retries!
for i in {1..5}; do
if docker push $image; then
echo "Pushed $image"
break
elif [ $i -eq 5 ] ; then
echo "Failed to push $image"
echo $image >> image-build-logs/push-failed-images.txt
else
echo "Failed on retry $i"
sleep 5
fi;
done
done < image-build-logs/push-attempt-images.txt
shell: bash
env:
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
if: inputs.push
- name: Upload output artifact
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.distro }}-logs
path: image-build-logs
retention-days: 7
if: ${{ !cancelled() }}
- name: Fail when images failed to build
run: echo "An image build failed. Check the workflow artifact for build logs" && exit 1
if: ${{ steps.build_overcloud_images.outcome == 'failure' || steps.build_seed_images.outcome == 'failure' }}
- name: Fail when images failed to push
run: if [ $(wc -l < image-build-logs/push-failed-images.txt) -gt 0 ]; then cat image-build-logs/push-failed-images.txt && exit 1; fi
if: ${{ !cancelled() }}
# NOTE(seunghun1ee): Currently we want to mark the job fail only when critical CVEs are detected.
# This can be used again instead of "Fail when critical vulnerabilities are found" when it's
# decided to fail the job on detecting high CVEs as well.
# - name: Fail when images failed scanning
# run: if [ $(wc -l < image-build-logs/image-scan-output/dirty-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/dirty-images.txt && exit 1; fi
# if: ${{ !inputs.push-dirty && !cancelled() }}
- name: Fail when critical vulnerabilities are found
run: if [ $(wc -l < image-build-logs/image-scan-output/critical-images.txt) -gt 0 ]; then cat image-build-logs/image-scan-output/critical-images.txt && exit 1; fi
if: ${{ !inputs.push-dirty && !cancelled() }}
# NOTE(mgoddard): Trigger another CI workflow in the
# stackhpc-release-train repository.
- name: Trigger container image repository sync
run: |
filter='${{ inputs.regexes }}'
if [[ -n $filter ]] && [[ ${{ inputs.seed }} == 'true' ]]; then
filter="$filter bifrost"
fi
gh workflow run \
container-sync.yml \
--repo stackhpc/stackhpc-release-train \
--ref main \
-f filter="$filter" \
-f sync-new-images=false
env:
GITHUB_TOKEN: ${{ secrets.STACKHPC_RELEASE_TRAIN_TOKEN }}
if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }}
- name: Display link to container image repository sync workflows
run: |
echo "::notice Container image repository sync workflows: https://github.com/stackhpc/stackhpc-release-train/actions/workflows/container-sync.yml"
if: ${{ github.repository == 'stackhpc/stackhpc-kayobe-config' && inputs.push && !cancelled() }}