Make ingress IP optional for HA cluster (#240)

* Make ingress IP optional for HA cluster * Support for running the tests against an ephemeral HA cluster --------- Co-authored-by: Matt Anson <[email protected]>
azimuth-cloud · Dec 5, 2023 · 520cb10 · 520cb10
1 parent bfae3d2
commit 520cb10
Show file tree

Hide file tree

Showing 3 changed files with 83 additions and 30 deletions.
diff --git a/playbooks/generate_tests.yml b/playbooks/generate_tests.yml
@@ -13,6 +13,28 @@
 # Gather required facts from the seed
 - hosts: k3s
   tasks:
+    # If running in HA mode with no explicit base domain or explicit ingress load balancer IP,
+    # we need to discover it
+    - block:
+        - name: Get ingress controller service info
+          command: >-
+            kubectl get service ingress-nginx-controller
+              --namespace ingress-nginx
+              --output json
+          register: capi_cluster_ingress_controller_svc_cmd
+
+        - name: Set ingress controller load balancer IP fact
+          set_fact:
+            capi_cluster_addons_ingress_load_balancer_ip: >-
+              {{-
+                capi_cluster_ingress_controller_svc_cmd.stdout |
+                  from_json |
+                  json_query('status.loadBalancer.ingress[0].ip')
+              }}
+      when:
+        - install_mode == 'ha'
+        - capi_cluster_addons_ingress_load_balancer_ip is not defined
+
     - name: Get installed cluster types
       command: kubectl get clustertypes -o json
       register: generate_tests_cluster_types_cmd

diff --git a/roles/capi_cluster/defaults/main.yml b/roles/capi_cluster/defaults/main.yml
@@ -159,24 +159,17 @@ capi_cluster_worker_root_volume_availability_zone: "{{ capi_cluster_root_volume_
 # Configuration for addons
 #   Determines if the ingress controller should be enabled
 capi_cluster_addons_ingress_enabled: "{{ ingress_controller_enabled | default(true) }}"
-#   Require the specification of a pre-allocated IP for the ingress load balancer
-#   This IP should have the wildcard domain assigned to it
-capi_cluster_addons_ingress_load_balancer_ip: >-
-  {{-
-    undef(hint = 'capi_cluster_addons_ingress_load_balancer_ip is required')
-    if capi_cluster_addons_ingress_enabled
-    else None
-  }}
+#   A pre-allocated IP for the ingress load balancer
+#   If given, this IP should have the wildcard domain assigned to it
+capi_cluster_addons_ingress_load_balancer_ip:
 #   Options for LoadBalancer services
-
 #   https://github.com/kubernetes/cloud-provider-openstack/blob/master/docs/openstack-cloud-controller-manager/using-openstack-cloud-controller-manager.md#load-balancer
 capi_cluster_addons_openstack_loadbalancer_method: >-
   {{-
     'SOURCE_IP_PORT'
     if capi_cluster_addons_openstack_loadbalancer_provider == 'ovn'
     else None
-    }}
-
+  }}
 capi_cluster_addons_openstack_loadbalancer_provider:
 capi_cluster_addons_openstack_loadbalancer_create_monitor:
 capi_cluster_addons_openstack_loadbalancer_monitor_delay:
@@ -342,11 +335,12 @@ capi_cluster_release_defaults:
               allowSnippetAnnotations: true
               service: >-
                 {{-
-                  {
-                    "loadBalancerIP": capi_cluster_addons_ingress_load_balancer_ip,
-                  }
-                  if capi_cluster_addons_ingress_enabled
-                  else {}
+                  {} |
+                    combine(
+                      { "loadBalancerIP": capi_cluster_addons_ingress_load_balancer_ip }
+                      if capi_cluster_addons_ingress_load_balancer_ip
+                      else {}
+                    )
                 }}
     # Configure monitoring and alerting
     monitoring:
@@ -492,5 +486,4 @@ capi_cluster_release_values: >-
   }}
 
 # The name of the file into which the kubeconfig of the cluster should be output
-# If not given, the kubeconfig is not output
-capi_cluster_kubeconfig_path:
+capi_cluster_kubeconfig_path: "{{ ansible_env.HOME }}/kubeconfig"
diff --git a/roles/capi_cluster/tasks/main.yml b/roles/capi_cluster/tasks/main.yml
@@ -11,6 +11,23 @@
         release_state: present
         release_values: "{{ capi_cluster_release_values }}"
         create_namespace: yes
+      register: capi_cluster_helm_release
+
+    # The CAPI and CAPO controllers take some time to react and update the Ready condition
+    # However some kinds of update, e.g. only addons, do not affect the cluster conditions,
+    # so we only allow up to two minutes for that to happen
+    - name: Wait for cluster to become unready
+      command: >-
+        kubectl wait clusters.cluster.x-k8s.io/{{ capi_cluster_release_name }}
+          --for=condition=Ready=false
+          --namespace {{ capi_cluster_release_namespace }}
+          --timeout 0s
+      changed_when: false
+      register: capi_cluster_not_ready
+      until: capi_cluster_not_ready is succeeded
+      retries: 12
+      delay: 10
+      when: capi_cluster_helm_release is changed
 
     - name: Wait for cluster to become ready
       command: >-
@@ -52,21 +69,42 @@
         - manifests
         - helmreleases
 
+    - name: Generate kubeconfig for cluster
+      command: >-
+        kubectl get secret {{ capi_cluster_release_name }}-kubeconfig
+          --namespace {{ capi_cluster_release_namespace }}
+          --output jsonpath='{.data.value}'
+      changed_when: false
+      register: capi_cluster_kubeconfig_cmd
+
+    - name: Write kubeconfig file for cluster
+      copy:
+        content: "{{ capi_cluster_kubeconfig_cmd.stdout | b64decode }}"
+        dest: "{{ capi_cluster_kubeconfig_path }}"
+        mode: u=rw,g=,o=
+
+    # When the ingress controller is enabled, get the IP
     - block:
-        - name: Generate kubeconfig for cluster
+        - name: Get ingress controller service info
           command: >-
-            kubectl get secret {{ capi_cluster_release_name }}-kubeconfig
-              --namespace {{ capi_cluster_release_namespace }}
-              --output jsonpath='{.data.value}'
-          changed_when: false
-          register: capi_cluster_kubeconfig_cmd
+            kubectl get service ingress-nginx-controller
+              --namespace ingress-nginx
+              --output json
+          environment:
+            KUBECONFIG: "{{ capi_cluster_kubeconfig_path }}"
+          register: capi_cluster_ingress_controller_svc_cmd
 
-        - name: Write kubeconfig file for cluster
-          copy:
-            content: "{{ capi_cluster_kubeconfig_cmd.stdout | b64decode }}"
-            dest: "{{ capi_cluster_kubeconfig_path }}"
-            mode: u=rw,g=,o=
-      when: capi_cluster_kubeconfig_path is defined
+        - name: Set ingress controller load balancer IP fact
+          set_fact:
+            capi_cluster_addons_ingress_load_balancer_ip: >-
+              {{-
+                capi_cluster_ingress_controller_svc_cmd.stdout |
+                  from_json |
+                  json_query('status.loadBalancer.ingress[0].ip')
+              }}
+      when:
+        - capi_cluster_addons_ingress_enabled
+        - not capi_cluster_addons_ingress_load_balancer_ip
   when: capi_cluster_release_state == 'present'
 
 - block: