From 520cb10f8f52f73126c38f44bae8421286ac9242 Mon Sep 17 00:00:00 2001 From: Matt Pryor Date: Tue, 5 Dec 2023 11:18:21 +0000 Subject: [PATCH] Make ingress IP optional for HA cluster (#240) * Make ingress IP optional for HA cluster * Support for running the tests against an ephemeral HA cluster --------- Co-authored-by: Matt Anson --- playbooks/generate_tests.yml | 22 ++++++++++ roles/capi_cluster/defaults/main.yml | 29 +++++-------- roles/capi_cluster/tasks/main.yml | 62 ++++++++++++++++++++++------ 3 files changed, 83 insertions(+), 30 deletions(-) diff --git a/playbooks/generate_tests.yml b/playbooks/generate_tests.yml index e859a7ae..6b5b4524 100644 --- a/playbooks/generate_tests.yml +++ b/playbooks/generate_tests.yml @@ -13,6 +13,28 @@ # Gather required facts from the seed - hosts: k3s tasks: + # If running in HA mode with no explicit base domain or explicit ingress load balancer IP, + # we need to discover it + - block: + - name: Get ingress controller service info + command: >- + kubectl get service ingress-nginx-controller + --namespace ingress-nginx + --output json + register: capi_cluster_ingress_controller_svc_cmd + + - name: Set ingress controller load balancer IP fact + set_fact: + capi_cluster_addons_ingress_load_balancer_ip: >- + {{- + capi_cluster_ingress_controller_svc_cmd.stdout | + from_json | + json_query('status.loadBalancer.ingress[0].ip') + }} + when: + - install_mode == 'ha' + - capi_cluster_addons_ingress_load_balancer_ip is not defined + - name: Get installed cluster types command: kubectl get clustertypes -o json register: generate_tests_cluster_types_cmd diff --git a/roles/capi_cluster/defaults/main.yml b/roles/capi_cluster/defaults/main.yml index 1affdb39..ebbea7cb 100644 --- a/roles/capi_cluster/defaults/main.yml +++ b/roles/capi_cluster/defaults/main.yml @@ -159,24 +159,17 @@ capi_cluster_worker_root_volume_availability_zone: "{{ capi_cluster_root_volume_ # Configuration for addons # Determines if the ingress controller should be enabled capi_cluster_addons_ingress_enabled: "{{ ingress_controller_enabled | default(true) }}" -# Require the specification of a pre-allocated IP for the ingress load balancer -#  This IP should have the wildcard domain assigned to it -capi_cluster_addons_ingress_load_balancer_ip: >- - {{- - undef(hint = 'capi_cluster_addons_ingress_load_balancer_ip is required') - if capi_cluster_addons_ingress_enabled - else None - }} +# A pre-allocated IP for the ingress load balancer +#  If given, this IP should have the wildcard domain assigned to it +capi_cluster_addons_ingress_load_balancer_ip: # Options for LoadBalancer services - #  https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/openstack-cloud-controller-manager/using-openstack-cloud-controller-manager.md#load-balancer capi_cluster_addons_openstack_loadbalancer_method: >- {{- 'SOURCE_IP_PORT' if capi_cluster_addons_openstack_loadbalancer_provider == 'ovn' else None - }} - + }} capi_cluster_addons_openstack_loadbalancer_provider: capi_cluster_addons_openstack_loadbalancer_create_monitor: capi_cluster_addons_openstack_loadbalancer_monitor_delay: @@ -342,11 +335,12 @@ capi_cluster_release_defaults: allowSnippetAnnotations: true service: >- {{- - { - "loadBalancerIP": capi_cluster_addons_ingress_load_balancer_ip, - } - if capi_cluster_addons_ingress_enabled - else {} + {} | + combine( + { "loadBalancerIP": capi_cluster_addons_ingress_load_balancer_ip } + if capi_cluster_addons_ingress_load_balancer_ip + else {} + ) }} # Configure monitoring and alerting monitoring: @@ -492,5 +486,4 @@ capi_cluster_release_values: >- }} # The name of the file into which the kubeconfig of the cluster should be output -# If not given, the kubeconfig is not output -capi_cluster_kubeconfig_path: +capi_cluster_kubeconfig_path: "{{ ansible_env.HOME }}/kubeconfig" diff --git a/roles/capi_cluster/tasks/main.yml b/roles/capi_cluster/tasks/main.yml index e685282c..b4f48b1e 100644 --- a/roles/capi_cluster/tasks/main.yml +++ b/roles/capi_cluster/tasks/main.yml @@ -11,6 +11,23 @@ release_state: present release_values: "{{ capi_cluster_release_values }}" create_namespace: yes + register: capi_cluster_helm_release + + # The CAPI and CAPO controllers take some time to react and update the Ready condition + # However some kinds of update, e.g. only addons, do not affect the cluster conditions, + # so we only allow up to two minutes for that to happen + - name: Wait for cluster to become unready + command: >- + kubectl wait clusters.cluster.x-k8s.io/{{ capi_cluster_release_name }} + --for=condition=Ready=false + --namespace {{ capi_cluster_release_namespace }} + --timeout 0s + changed_when: false + register: capi_cluster_not_ready + until: capi_cluster_not_ready is succeeded + retries: 12 + delay: 10 + when: capi_cluster_helm_release is changed - name: Wait for cluster to become ready command: >- @@ -52,21 +69,42 @@ - manifests - helmreleases + - name: Generate kubeconfig for cluster + command: >- + kubectl get secret {{ capi_cluster_release_name }}-kubeconfig + --namespace {{ capi_cluster_release_namespace }} + --output jsonpath='{.data.value}' + changed_when: false + register: capi_cluster_kubeconfig_cmd + + - name: Write kubeconfig file for cluster + copy: + content: "{{ capi_cluster_kubeconfig_cmd.stdout | b64decode }}" + dest: "{{ capi_cluster_kubeconfig_path }}" + mode: u=rw,g=,o= + + # When the ingress controller is enabled, get the IP - block: - - name: Generate kubeconfig for cluster + - name: Get ingress controller service info command: >- - kubectl get secret {{ capi_cluster_release_name }}-kubeconfig - --namespace {{ capi_cluster_release_namespace }} - --output jsonpath='{.data.value}' - changed_when: false - register: capi_cluster_kubeconfig_cmd + kubectl get service ingress-nginx-controller + --namespace ingress-nginx + --output json + environment: + KUBECONFIG: "{{ capi_cluster_kubeconfig_path }}" + register: capi_cluster_ingress_controller_svc_cmd - - name: Write kubeconfig file for cluster - copy: - content: "{{ capi_cluster_kubeconfig_cmd.stdout | b64decode }}" - dest: "{{ capi_cluster_kubeconfig_path }}" - mode: u=rw,g=,o= - when: capi_cluster_kubeconfig_path is defined + - name: Set ingress controller load balancer IP fact + set_fact: + capi_cluster_addons_ingress_load_balancer_ip: >- + {{- + capi_cluster_ingress_controller_svc_cmd.stdout | + from_json | + json_query('status.loadBalancer.ingress[0].ip') + }} + when: + - capi_cluster_addons_ingress_enabled + - not capi_cluster_addons_ingress_load_balancer_ip when: capi_cluster_release_state == 'present' - block: