Skip to content

Use Quorum queues in Rabbit #9

Use Quorum queues in Rabbit

Use Quorum queues in Rabbit #9

---
# This reusable workflow deploys a VM on a cloud using Terraform, then deploys
# OpenStack in the VM via Kayobe. Tempest is then used to test the cloud.
name: All in one
on:
workflow_call:
inputs:
kayobe_image:
description: Kayobe container image
type: string
required: true
os_distribution:
description: Host OS distribution
type: string
default: rocky
os_release:
description: Host OS release
type: string
default: '9'
ssh_username:
description: User for terraform to access the all-in-one VM
type: string
default: cloud-user
neutron_plugin:
description: Neutron ML2 plugin
type: string
required: true
vm_image_override:
description: Full name of an image to use instead of the default
type: string
default: ""
vm_interface:
description: Default network interface name
type: string
default: ens3
vm_flavor:
description: Flavor for the all-in-one VM
type: string
default: en1.medium
vm_network:
description: Network for the all-in-one VM
type: string
default: stackhpc-ci
vm_subnet:
description: Subnet for the all-in-one VM
type: string
default: stackhpc-ci
OS_CLOUD:
description: Name of cloud in clouds.yaml
type: string
required: true
if:
description: Whether to run the workflow (workaround for required status checks issue)
type: boolean
default: true
upgrade:
description: Whether to perform an upgrade
type: boolean
default: false
secrets:
KAYOBE_VAULT_PASSWORD:
required: true
CLOUDS_YAML:
required: true
OS_APPLICATION_CREDENTIAL_ID:
required: true
OS_APPLICATION_CREDENTIAL_SECRET:
required: true
jobs:
# NOTE: Runner needs unzip and nodejs packages.
all-in-one:
name: All in one
if: ${{ inputs.if && !cancelled() }}
runs-on: arc-skc-aio-runner
permissions: {}
env:
KAYOBE_ENVIRONMENT: ci-aio
KAYOBE_VAULT_PASSWORD: ${{ secrets.KAYOBE_VAULT_PASSWORD }}
KAYOBE_IMAGE: ${{ inputs.kayobe_image }}
# NOTE(upgrade): Reference the PREVIOUS release here.
PREVIOUS_KAYOBE_IMAGE: ghcr.io/stackhpc/stackhpc-kayobe-config:stackhpc-2023.1
# NOTE(upgrade): Reference the PREVIOUS release branch here.
PREVIOUS_BRANCH: stackhpc/2023.1
steps:
- name: Install Package
uses: ConorMacBride/install-package@main
with:
apt: git unzip nodejs
# If testing upgrade, checkout previous release, otherwise checkout current branch
- name: Checkout ${{ inputs.upgrade && 'previous release' || 'current' }} config
uses: actions/checkout@v4
with:
ref: ${{ inputs.upgrade && env.PREVIOUS_BRANCH || github.ref }}
submodules: true
- name: Output Kayobe image
id: kayobe_image
run: |
if ${{ inputs.upgrade }}; then
kayobe_image=$PREVIOUS_KAYOBE_IMAGE
else
kayobe_image=$KAYOBE_IMAGE
fi
echo kayobe_image=$kayobe_image >> $GITHUB_OUTPUT
- name: Make sure dockerd is running and test Docker
run: |
docker ps
- name: Output image tag
id: image_tag
run: |
echo image_tag=$(grep stackhpc_${{ inputs.os_distribution }}_$(sed s/-/_/ <(echo "${{ inputs.os_release }}"))_overcloud_host_image_version: etc/kayobe/pulp-host-image-versions.yml | awk '{print $2}') >> $GITHUB_OUTPUT
# Use the image override if set, otherwise use overcloud-os_distribution-os_release-tag
- name: Output image name
id: image_name
run: |
if [ -z "${{ inputs.vm_image_override }}" ]; then
echo image_name=overcloud-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ steps.image_tag.outputs.image_tag }} >> $GITHUB_OUTPUT
else
echo image_name=${{ inputs.vm_image_override }} >> $GITHUB_OUTPUT
fi
- name: Install terraform
uses: hashicorp/setup-terraform@v2
- name: Initialise terraform
run: terraform init
working-directory: ${{ github.workspace }}/terraform/aio
- name: Generate SSH keypair
run: ssh-keygen -f id_rsa -N ''
working-directory: ${{ github.workspace }}/terraform/aio
- name: Generate clouds.yaml
run: |
cat << EOF > clouds.yaml
${{ secrets.CLOUDS_YAML }}
EOF
working-directory: ${{ github.workspace }}/terraform/aio
- name: Generate terraform.tfvars
run: |
cat << EOF > terraform.tfvars
ssh_public_key = "id_rsa.pub"
ssh_username = "${{ env.SSH_USERNAME }}"
aio_vm_interface = "${{ env.VM_INTERFACE }}"
aio_vm_name = "${{ env.VM_NAME }}"
aio_vm_image = "${{ env.VM_IMAGE }}"
aio_vm_flavor = "${{ env.VM_FLAVOR }}"
aio_vm_network = "${{ env.VM_NETWORK }}"
aio_vm_subnet = "${{ env.VM_SUBNET }}"
aio_vm_volume_size = "${{ env.VM_VOLUME_SIZE }}"
aio_vm_tags = ${{ env.VM_TAGS }}
EOF
working-directory: ${{ github.workspace }}/terraform/aio
env:
SSH_USERNAME: "${{ inputs.ssh_username }}"
VM_NAME: "skc-ci-aio-${{ inputs.neutron_plugin }}-${{ github.run_id }}"
VM_IMAGE: ${{ steps.image_name.outputs.image_name }}
VM_FLAVOR: ${{ inputs.vm_flavor }}
VM_NETWORK: ${{ inputs.vm_network }}
VM_SUBNET: ${{ inputs.vm_subnet }}
VM_INTERFACE: ${{ inputs.vm_interface }}
VM_VOLUME_SIZE: ${{ inputs.upgrade && '55' || '40' }}
VM_TAGS: '["skc-ci-aio", "PR=${{ github.event.number }}"]'
- name: Terraform Plan
run: terraform plan
working-directory: ${{ github.workspace }}/terraform/aio
env:
OS_CLOUD: ${{ inputs.OS_CLOUD }}
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
- name: Terraform Apply
id: tf_apply
run: |
for attempt in $(seq 5); do
if terraform apply -auto-approve; then
echo "Created infrastructure on attempt $attempt"
exit 0
fi
echo "Failed to create infrastructure on attempt $attempt"
sleep 10
terraform destroy -auto-approve
sleep 60
done
echo "Failed to create infrastructure after $attempt attempts"
exit 1
working-directory: ${{ github.workspace }}/terraform/aio
env:
OS_CLOUD: ${{ inputs.OS_CLOUD }}
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
- name: Get Terraform outputs
id: tf_outputs
run: |
terraform output -json
working-directory: ${{ github.workspace }}/terraform/aio
- name: Write Terraform outputs
run: |
cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-outputs.yml
${{ steps.tf_outputs.outputs.stdout }}
EOF
- name: Write Terraform network config
run: |
cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/tf-networks.yml
admin_oc_net_name: admin
admin_cidr: "{{ access_cidr.value }}"
admin_allocation_pool_start: 0.0.0.0
admin_allocation_pool_end: 0.0.0.0
admin_gateway: "{{ access_gw.value }}"
admin_bootproto: dhcp
admin_ips:
controller0: "{{ access_ip_v4.value }}"
EOF
- name: Write Terraform network interface config
run: |
cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/inventory/group_vars/controllers/tf-network-interfaces
admin_interface: "{{ access_interface.value }}"
EOF
- name: Write all-in-one scenario config
run: |
cat << EOF > etc/kayobe/environments/$KAYOBE_ENVIRONMENT/zz-aio-scenario.yml
---
os_distribution: ${{ env.OS_DISTRIBUTION }}
os_release: "${{ env.OS_RELEASE }}"
kolla_enable_ovn: ${{ env.ENABLE_OVN }}
EOF
env:
ENABLE_OVN: ${{ inputs.neutron_plugin == 'ovn' }}
OS_DISTRIBUTION: ${{ inputs.os_distribution }}
OS_RELEASE: ${{ inputs.os_release }}
# Use a heredoc to define a multiline string output
# https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings
- name: Set SSH key output
id: ssh_key
run: |
echo "ssh_key<<EOF" >> $GITHUB_OUTPUT
cat terraform/aio/id_rsa >> $GITHUB_OUTPUT
echo "EOF" >> $GITHUB_OUTPUT
# The same tag may be reused (e.g. stackhpc/yoga), so ensure we have the latest image.
- name: Pull previous Kayobe image
run: |
docker image pull ${{ steps.kayobe_image.outputs.kayobe_image }}
if: inputs.upgrade
# The same tag may be reused (e.g. pr-123), so ensure we have the latest image.
- name: Pull current Kayobe image
run: |
docker image pull $KAYOBE_IMAGE
- name: Reboot
run: |
docker run -t --rm \
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
${{ steps.kayobe_image.outputs.kayobe_image }} \
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/reboot.yml -e reboot_with_bootstrap_user=true
env:
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
- name: Run growroot
run: |
docker run -t --rm \
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
${{ steps.kayobe_image.outputs.kayobe_image }} \
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/growroot.yml'
env:
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
- name: Host configure
run: |
docker run -t --rm \
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
${{ steps.kayobe_image.outputs.kayobe_image }} \
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh
env:
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
- name: Service deploy
run: |
docker run -t --rm \
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
${{ steps.kayobe_image.outputs.kayobe_image }} \
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-deploy.sh
env:
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
- name: Configure aio resources
run: |
docker run -t --rm \
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
${{ steps.kayobe_image.outputs.kayobe_image }} \
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml
env:
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
- name: Change rabbit queues from HA to Quorum
run: sed -i -e 's/om_enable_rabbitmq_high_availability: true/om_enable_rabbitmq_high_availability: false' -e 's/om_enable_rabbitmq_quorum_queues: false/om_enable_rabbitmq_quorum_queues: true' etc/kayobe/environments/ci-aio/kolla/globals.yml

Check failure on line 318 in .github/workflows/stackhpc-all-in-one.yml

View workflow run for this annotation

GitHub Actions / .github/workflows/stackhpc-all-in-one.yml

Invalid workflow file

You have an error in your yaml syntax on line 318
if: inputs.upgrade
- name: Migrate RabbitMQ queues
run: |
docker run -t --rm \
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
${{ steps.kayobe_image.outputs.kayobe_image }} \
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/migrate-rabbitmq-queues.yml --tags rabbit-queue-migration
env:
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
if: inputs.upgrade
# If testing upgrade, checkout the current release branch
# Stash changes to tracked files, and set clean=false to avoid removing untracked files.
- name: Stash config changes
run: git stash
if: inputs.upgrade
- name: Checkout current release config
uses: actions/checkout@v4
with:
submodules: true
clean: false
if: inputs.upgrade
- name: Pop stashed config changes
run: git stash pop
if: inputs.upgrade
# Now begin upgrade
- name: Host upgrade
run: |
docker run -t --rm \
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
$KAYOBE_IMAGE \
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-upgrade.sh
env:
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
if: inputs.upgrade
- name: Host configure
run: |
docker run -t --rm \
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
$KAYOBE_IMAGE \
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-host-configure.sh
env:
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
if: inputs.upgrade
- name: Service upgrade
run: |
docker run -t --rm \
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
$KAYOBE_IMAGE \
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/overcloud-service-upgrade.sh
env:
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
if: inputs.upgrade
- name: Ensure we have IP on breth1 to reach the instances
# NOTE(wszumski): Whilst we don't need to create resources again, in some circumstances
# we can lose the IP address that allows us to connect to the instances. This playbook
# also fixes that issue.
run: |
docker run -t --rm \
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
${{ steps.kayobe_image.outputs.kayobe_image }} \
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh etc/kayobe/ansible/configure-aio-resources.yml
env:
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
if: inputs.upgrade
- name: Tempest tests
id: tempest
run: |
mkdir -p tempest-artifacts
docker run -t --rm \
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
-v $(pwd)/tempest-artifacts:/stack/tempest-artifacts \
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
$KAYOBE_IMAGE \
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/tempest.sh -e ansible_user=stack -e rally_no_sensitive_log=false
env:
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
- name: StackHPC OpenStack tests
id: stackhpc-openstack-tests
continue-on-error: true
run: |
mkdir -p sot-results
docker run -t --rm \
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
-v $(pwd)/sot-results:/stack/sot-results \
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
$KAYOBE_IMAGE \
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/stackhpc-openstack-tests.yml'
env:
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
- name: Collect diagnostic information
id: diagnostics
run: |
mkdir -p diagnostics
sudo -E docker run -t --rm \
-v $(pwd):/stack/kayobe-automation-env/src/kayobe-config \
-v $(pwd)/diagnostics:/stack/diagnostics \
-e KAYOBE_ENVIRONMENT -e KAYOBE_VAULT_PASSWORD -e KAYOBE_AUTOMATION_SSH_PRIVATE_KEY \
$KAYOBE_IMAGE \
/stack/kayobe-automation-env/src/kayobe-config/.automation/pipeline/playbook-run.sh '$KAYOBE_CONFIG_PATH/ansible/diagnostics.yml'
env:
KAYOBE_AUTOMATION_SSH_PRIVATE_KEY: ${{ steps.ssh_key.outputs.ssh_key }}
if: ${{ !cancelled() && steps.tf_apply.outcome == 'success' }}
- name: Upload test result artifacts
uses: actions/upload-artifact@v4
with:
name: test-results-${{ inputs.os_distribution }}-${{ inputs.os_release }}-${{ inputs.neutron_plugin }}${{ inputs.upgrade && '-upgrade' || '' }}
path: |
diagnostics/
tempest-artifacts/
sot-results/
if: ${{ !cancelled() && (steps.tempest.outcome == 'success' || steps.stackhpc-openstack-tests.outcome == 'success' || steps.diagnostics.outcome == 'success') }}
- name: Fail if any Tempest tests failed
run: |
test $(wc -l < tempest-artifacts/failed-tests) -lt 1
- name: Fail if any StackHPC OpenStack tests failed
run: |
echo "Some StackHPC OpenStack tests failed."
echo "See HTML results artifact (sot-results) for details."
exit 1
if: steps.stackhpc-openstack-tests.outcome == 'failure'
- name: Destroy
run: terraform destroy -auto-approve
working-directory: ${{ github.workspace }}/terraform/aio
env:
OS_CLOUD: ${{ inputs.OS_CLOUD }}
OS_APPLICATION_CREDENTIAL_ID: ${{ secrets.OS_APPLICATION_CREDENTIAL_ID }}
OS_APPLICATION_CREDENTIAL_SECRET: ${{ secrets.OS_APPLICATION_CREDENTIAL_SECRET }}
if: always()