From 22eded5569d7696d2fb8fb1f11626fc0a0f5a763 Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 15 Jan 2024 16:17:24 +0000 Subject: [PATCH 1/3] vault: Support generating external TLS certificates for testing These should not generally be used in production, since the CA is self-signed. --- doc/source/configuration/vault.rst | 37 ++++++++++++- .../vault-generate-test-external-tls.yml | 55 +++++++++++++++++++ etc/kayobe/inventory/group_vars/all/vault | 3 + 3 files changed, 92 insertions(+), 3 deletions(-) create mode 100644 etc/kayobe/ansible/vault-generate-test-external-tls.yml diff --git a/doc/source/configuration/vault.rst b/doc/source/configuration/vault.rst index d598a63a5..87bdf8151 100644 --- a/doc/source/configuration/vault.rst +++ b/doc/source/configuration/vault.rst @@ -137,6 +137,30 @@ Setup Vault HA on the overcloud hosts Certificates generation ======================= +Create the external TLS certificates (testing only) +--------------------------------------------------- + +Typically external API TLS certificates should be generated by a organisation's trusted internal or third-party CA. +For test and development purposes it is possible to use Vault as a CA for the external API. + +1. Run the playbook + + .. code-block:: + + kayobe playbook run $KAYOBE_CONFIG_PATH/ansible/vault-generate-test-external-tls.yml + +2. Use ansible-vault to encrypt the PEM bundle in $KAYOBE_CONFIG_PATH/kolla/certificates/haproxy.pem. Commit the PEM bundle to the kayobe configuration. + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/kolla/certificates/haproxy.pem + + Or if environments are being used + + .. code-block:: + + ansible-vault encrypt --vault-password-file ~/vault.pass $KAYOBE_CONFIG_PATH/environments/$KAYOBE_ENVIRONMENT/kolla/certificates/haproxy.pem + Create the internal TLS certificates ------------------------------------ @@ -189,14 +213,21 @@ Certificates deployment Enable the required TLS variables in kayobe and kolla ----------------------------------------------------- -1. Set the following in kayobe-config/etc/kayobe/kolla.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla.yml +1. If using Vault as a CA for the external API, set the following in kayobe-config/etc/kayobe/kolla.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla.yml + + .. code-block:: + + # Whether TLS is enabled for the external API endpoints. Default is 'no'. + kolla_enable_tls_external: yes + +2. Set the following in kayobe-config/etc/kayobe/kolla.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla.yml .. code-block:: # Whether TLS is enabled for the internal API endpoints. Default is 'no'. kolla_enable_tls_internal: yes -2. Set the following in etc/kayobe/kolla/globals.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/globals.yml +3. Set the following in etc/kayobe/kolla/globals.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla/globals.yml .. code-block:: @@ -213,7 +244,7 @@ Enable the required TLS variables in kayobe and kolla # If using RabbitMQ TLS: rabbitmq_enable_tls: "yes" -3. Deploy backend and internal TLS +4. Deploy OpenStack .. warning:: diff --git a/etc/kayobe/ansible/vault-generate-test-external-tls.yml b/etc/kayobe/ansible/vault-generate-test-external-tls.yml new file mode 100644 index 000000000..39645e05d --- /dev/null +++ b/etc/kayobe/ansible/vault-generate-test-external-tls.yml @@ -0,0 +1,55 @@ +--- +- name: Generate external API certificate (for testing only) + hosts: controllers + run_once: true + vars: + vault_api_addr: "https://{{ kolla_internal_fqdn }}:8200" + # NOTE: Using the same CA as internal TLS. + vault_intermediate_ca_name: "OS-TLS-INT" + tasks: + - name: Include Vault keys + include_vars: + file: "{{ kayobe_env_config_path }}/vault/overcloud-vault-keys.json" + name: vault_keys + + - name: Issue a certificate for external TLS + hashivault_pki_cert_issue: + url: "{{ vault_api_addr }}" + ca_cert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if ansible_facts.os_family == 'RedHat' else '/usr/local/share/ca-certificates/OS-TLS-ROOT.crt' }}" + token: "{{ vault_keys.root_token }}" + mount_point: "{{ vault_intermediate_ca_name }}" + role: "{{ overcloud_vault_pki_external_tls_role_name }}" + common_name: "{% if kolla_external_fqdn != kolla_external_vip_address %}{{ kolla_external_fqdn }}{% endif %}" + extra_params: + ip_sans: "{{ kolla_external_vip_address }}" + register: external_cert + + - name: Ensure certificates directory exists + file: + path: "{{ kayobe_env_config_path }}/kolla/certificates" + state: directory + delegate_to: localhost + + - name: Ensure CA certificates directory exists + file: + path: "{{ kayobe_env_config_path }}/kolla/certificates/ca" + state: directory + delegate_to: localhost + + - name: Copy external API PEM bundle + no_log: true + copy: + dest: "{{ kayobe_env_config_path }}/kolla/certificates/haproxy.pem" + content: | + {{ external_cert.data.certificate }} + {{ external_cert.data.issuing_ca }} + {{ external_cert.data.private_key }} + mode: 0600 + delegate_to: localhost + + - name: Copy root CA + copy: + src: "{{ kayobe_env_config_path }}/vault/OS-TLS-ROOT.pem" + dest: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt" + mode: 0600 + delegate_to: localhost diff --git a/etc/kayobe/inventory/group_vars/all/vault b/etc/kayobe/inventory/group_vars/all/vault index eda95114f..22e89a455 100644 --- a/etc/kayobe/inventory/group_vars/all/vault +++ b/etc/kayobe/inventory/group_vars/all/vault @@ -64,6 +64,9 @@ overcloud_vault_pki_internal_tls_role_name: "{{ overcloud_vault_pki_default_role # Overcloud Vault PKI Backend TLS Role name overcloud_vault_pki_backend_tls_role_name: "{{ overcloud_vault_pki_default_role_name }}" +# Overcloud Vault PKI External TLS Role name (for testing only) +overcloud_vault_pki_external_tls_role_name: "{{ overcloud_vault_pki_default_role_name }}" + # Overcloud Vault PKI Roles definition overcloud_vault_pki_roles: - name: "{{ overcloud_vault_pki_default_role_name }}" From 88f83b9f5869e67f228821563939b32b4fe54ade Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Wed, 10 Apr 2024 15:27:17 +0100 Subject: [PATCH 2/3] ci-multinode: Update configuration for external TLS using Vault CA --- .../tempest/tempest-ci-multinode.overrides.conf | 2 +- etc/kayobe/environments/ci-multinode/kolla.yml | 8 ++++++-- etc/kayobe/environments/ci-multinode/tempest.yml | 3 +++ 3 files changed, 10 insertions(+), 3 deletions(-) create mode 100644 etc/kayobe/environments/ci-multinode/tempest.yml diff --git a/.automation.conf/tempest/tempest-ci-multinode.overrides.conf b/.automation.conf/tempest/tempest-ci-multinode.overrides.conf index 0ff616f76..663b384df 100644 --- a/.automation.conf/tempest/tempest-ci-multinode.overrides.conf +++ b/.automation.conf/tempest/tempest-ci-multinode.overrides.conf @@ -32,4 +32,4 @@ max_microversion = 3.70 build_timeout = 600 [dashboard] -dashboard_url = http://192.168.39.2 +dashboard_url = https://192.168.39.2 diff --git a/etc/kayobe/environments/ci-multinode/kolla.yml b/etc/kayobe/environments/ci-multinode/kolla.yml index 0fc7b05f5..076529742 100644 --- a/etc/kayobe/environments/ci-multinode/kolla.yml +++ b/etc/kayobe/environments/ci-multinode/kolla.yml @@ -8,12 +8,16 @@ kolla_enable_designate: true kolla_enable_redis: true kolla_enable_barbican: true -# The multinode environment supports Backend and internal TLS , but it must be -# enabled in the correct order. See +# The multinode environment supports backend, external and internal TLS , but +# it must be enabled in the correct order. See # https://stackhpc-kayobe-config.readthedocs.io/en/stackhpc-yoga/configuration/vault.html # for details. +# kolla_enable_tls_external: true # kolla_enable_tls_internal: true +kolla_public_openrc_cacert: "{{ '/etc/pki/tls/certs/ca-bundle.crt' if os_distribution in ['centos', 'rocky'] else '/etc/ssl/certs/ca-certificates.crt' }}" +kolla_admin_openrc_cacert: "{{ kolla_public_openrc_cacert }}" + # The multinode environment supports Manila but it is not enabled by default. # kolla_enable_manila: true # kolla_enable_manila_backend_cephfs_native: true diff --git a/etc/kayobe/environments/ci-multinode/tempest.yml b/etc/kayobe/environments/ci-multinode/tempest.yml new file mode 100644 index 000000000..93a7cdfe2 --- /dev/null +++ b/etc/kayobe/environments/ci-multinode/tempest.yml @@ -0,0 +1,3 @@ +--- +# Add the Vault CA certificate to the rally container when running tempest. +tempest_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt" From cd9ec7be366234e422dff1813561c141398c040f Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Mon, 15 Apr 2024 16:06:15 +0100 Subject: [PATCH 3/3] Update .automation submodule for Tempest CA cert support --- .automation | 2 +- doc/source/configuration/vault.rst | 2 ++ doc/source/operations/tempest.rst | 19 +++++++++++++++++++ .../tempest-cacert-33e2ae1cf0ba88cf.yaml | 4 ++++ 4 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 releasenotes/notes/tempest-cacert-33e2ae1cf0ba88cf.yaml diff --git a/.automation b/.automation index b00f285be..98e92aae8 160000 --- a/.automation +++ b/.automation @@ -1 +1 @@ -Subproject commit b00f285be240e34c643c4bd93a877e56587f71fa +Subproject commit 98e92aae8460db84cd4bf9813e4ef1ba02c5e034 diff --git a/doc/source/configuration/vault.rst b/doc/source/configuration/vault.rst index 87bdf8151..8c177570a 100644 --- a/doc/source/configuration/vault.rst +++ b/doc/source/configuration/vault.rst @@ -220,6 +220,8 @@ Enable the required TLS variables in kayobe and kolla # Whether TLS is enabled for the external API endpoints. Default is 'no'. kolla_enable_tls_external: yes + See :ref:`tempest-cacert` for information on adding CA certificates to the trust store when running Tempest. + 2. Set the following in kayobe-config/etc/kayobe/kolla.yml or if environments are being used etc/kayobe/environments/$KAYOBE_ENVIRONMENT/kolla.yml .. code-block:: diff --git a/doc/source/operations/tempest.rst b/doc/source/operations/tempest.rst index c747b5377..e110e208f 100644 --- a/doc/source/operations/tempest.rst +++ b/doc/source/operations/tempest.rst @@ -251,6 +251,25 @@ ever contain one host. The seed is usually used as the tempest runner however it is also common to use the Ansible control host or an infrastructure VM. The main requirement of the host is that it can reach the OpenStack API. +.. _tempest-cacert: + +Tempest CA certificate +---------------------- + +If your public OpenStack API uses TLS with a Certificate Authority (CA) that is +not trusted by the Python CA trust store, it may be necessary to add a CA +certificate to the trust store in the container that runs Tempest. This can be +done by defining a ``tempest_cacert`` Ansible variable to a path containing the +CA certificate. You may wish to use ``kayobe_config_path`` or +``kayobe_env_config_path`` to be agnostic to the path where kayobe-config is +mounted within the container. For example: + +.. code-block:: yaml + :caption: ``etc/kayobe/tempest.yml`` + + # Add the Vault CA certificate to the rally container when running tempest. + tempest_cacert: "{{ kayobe_env_config_path }}/kolla/certificates/ca/vault.crt" + Running Tempest =============== diff --git a/releasenotes/notes/tempest-cacert-33e2ae1cf0ba88cf.yaml b/releasenotes/notes/tempest-cacert-33e2ae1cf0ba88cf.yaml new file mode 100644 index 000000000..83ddb8102 --- /dev/null +++ b/releasenotes/notes/tempest-cacert-33e2ae1cf0ba88cf.yaml @@ -0,0 +1,4 @@ +--- +features: + - | + Supports adding CA certificates to the Tempest container trust store.