feat: remove hetzner load balancer in favour of metallb

This gives great flexibility for using UDP ingress, required to run
the unifi controller in the cluster

modules/hetzner/networks.tf

@@ -58,6 +58,22 @@ resource "hcloud_firewall" "firewall" {
         ]
         protocol = "udp"
       },
+      {
+        description = "Allow TCP access to port 80"
+        source_ips = [
+          local.global_ipv4_cidr,
+          local.global_ipv6_cidr,
+        ]
+        port = 80
+      },
+      {
+        description = "Allow TCP access to port 443"
+        source_ips = [
+          local.global_ipv4_cidr,
+          local.global_ipv6_cidr,
+        ]
+        port = 443
+      },
       # Direct public access only allowed if single manager node
       {
         description = "Allow access to Kubernetes API"

modules/hetzner/variables.tf

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 variable "firewall_allow_api_access" {
   type        = list(string)
   description = "CIDR range to allow access to the Kubernetes API"

modules/kubernetes/README.md

@@ -8,15 +8,13 @@
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0.0 |
 | <a name="requirement_helm"></a> [helm](#requirement\_helm) | >= 2.14.0, < 3.0.0 |
 | <a name="requirement_kubernetes"></a> [kubernetes](#requirement\_kubernetes) | >= 2.31.0, < 3.0.0 |
-| <a name="requirement_random"></a> [random](#requirement\_random) | >= 3.6.2, < 4.0.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
 | <a name="provider_helm"></a> [helm](#provider\_helm) | 2.14.1 |
 | <a name="provider_kubernetes"></a> [kubernetes](#provider\_kubernetes) | 2.31.0 |
-| <a name="provider_random"></a> [random](#provider\_random) | 3.6.2 |
 
 ## Modules
 
@@ -29,13 +27,14 @@ No modules.
 | [helm_release.argocd](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [helm_release.hcloud_ccm](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [helm_release.hcloud_csi](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
-| [helm_release.ingress_nginx](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [kubernetes_config_map_v1.metallb](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/config_map_v1) | resource |
 | [kubernetes_namespace_v1.argocd](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace_v1) | resource |
 | [kubernetes_namespace_v1.external_secrets](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace_v1) | resource |
+| [kubernetes_namespace_v1.metallb](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace_v1) | resource |
 | [kubernetes_secret_v1.github_secret](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource |
 | [kubernetes_secret_v1.hcloud](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource |
 | [kubernetes_secret_v1.infisical](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource |
-| [random_integer.ingress_load_balancer_id](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/integer) | resource |
+| [kubernetes_nodes.cluster](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/data-sources/nodes) | data source |
 
 ## Inputs
 
@@ -54,12 +53,9 @@ No modules.
 | <a name="input_hetzner_csi_driver_version"></a> [hetzner\_csi\_driver\_version](#input\_hetzner\_csi\_driver\_version) | Tag of the CSI driver to use - defaults to latest | `string` | `null` | no |
 | <a name="input_infisical_client_id"></a> [infisical\_client\_id](#input\_infisical\_client\_id) | Infisical client ID | `string` | n/a | yes |
 | <a name="input_infisical_client_secret"></a> [infisical\_client\_secret](#input\_infisical\_client\_secret) | Infisical client secret | `string` | n/a | yes |
-| <a name="input_ingress_nginx_version"></a> [ingress\_nginx\_version](#input\_ingress\_nginx\_version) | Version of Ingress Nginx to install - defaults to latest | `string` | `null` | no |
 | <a name="input_k3s_cluster_cidr"></a> [k3s\_cluster\_cidr](#input\_k3s\_cluster\_cidr) | CIDR used for the k3s cluster | `string` | `"10.244.0.0/16"` | no |
 | <a name="input_kube_context"></a> [kube\_context](#input\_kube\_context) | Kubernetes context to use | `string` | `"default"` | no |
 | <a name="input_kubeconfig"></a> [kubeconfig](#input\_kubeconfig) | Kubeconfig for the cluster | `string` | n/a | yes |
-| <a name="input_load_balancer_location"></a> [load\_balancer\_location](#input\_load\_balancer\_location) | Location to use for the load balancer | `string` | n/a | yes |
-| <a name="input_load_balancer_type"></a> [load\_balancer\_type](#input\_load\_balancer\_type) | Type of load balancer to use | `string` | `"lb11"` | no |
 
 ## Outputs
 

modules/kubernetes/metallb.tf

@@ -0,0 +1,51 @@
+# Copyright 2024 Simon Emms <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+data "kubernetes_nodes" "cluster" {
+  depends_on = [
+    helm_release.hcloud_ccm,
+    helm_release.hcloud_csi,
+  ]
+}
+
+resource "kubernetes_namespace_v1" "metallb" {
+  metadata {
+    name = "metallb-system"
+  }
+}
+
+resource "kubernetes_config_map_v1" "metallb" {
+  metadata {
+    name      = "nodes"
+    namespace = kubernetes_namespace_v1.metallb.metadata[0].name
+  }
+
+  data = {
+    resource = yamlencode({
+      apiVersion = "metallb.io/v1beta1"
+      kind       = "IPAddressPool"
+      metadata = {
+        name      = "nodes"
+        namespace = kubernetes_namespace_v1.metallb.metadata[0].name
+      }
+      spec = {
+        addresses = [
+          for n in flatten(data.kubernetes_nodes.cluster.nodes[*].status[*].addresses) : "${n.address}/32" if n.type == "ExternalIP"
+        ]
+      }
+    })
+  }
+
+  immutable = false
+}

modules/kubernetes/terraform.tf

@@ -23,10 +23,6 @@ terraform {
       source  = "hashicorp/kubernetes"
       version = ">= 2.31.0, < 3.0.0"
     }
-    random = {
-      source  = "hashicorp/random"
-      version = ">= 3.6.2, < 4.0.0"
-    }
   }
 }
 

modules/kubernetes/variables.tf

@@ -94,12 +94,6 @@ variable "kube_context" {
   default     = "default"
 }
 
-variable "ingress_nginx_version" {
-  type        = string
-  description = "Version of Ingress Nginx to install - defaults to latest"
-  default     = null
-}
-
 variable "infisical_client_id" {
   type        = string
   description = "Infisical client ID"
@@ -111,14 +105,3 @@ variable "infisical_client_secret" {
   description = "Infisical client secret"
   sensitive   = true
 }
-
-variable "load_balancer_location" {
-  type        = string
-  description = "Location to use for the load balancer"
-}
-
-variable "load_balancer_type" {
-  type        = string
-  description = "Type of load balancer to use"
-  default     = "lb11"
-}

registry/clusters/dev/components/ingress-nginx.yaml

@@ -0,0 +1,24 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: ingress-nginx-components
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "20"
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/mrsimonemms/infrastructure
+    path: registry/components/ingress-nginx
+    targetRevision: HEAD
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: ingress-nginx
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

registry/clusters/dev/components/metallb.yaml

@@ -0,0 +1,24 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: metallb-components
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "10"
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/mrsimonemms/infrastructure
+    path: registry/components/metallb
+    targetRevision: HEAD
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: metallb-system
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

registry/clusters/prod/components/ingress-nginx.yaml

@@ -0,0 +1,24 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: ingress-nginx-components
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "20"
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/mrsimonemms/infrastructure
+    path: registry/components/ingress-nginx
+    targetRevision: HEAD
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: ingress-nginx
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

registry/clusters/prod/components/metallb.yaml

@@ -0,0 +1,24 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: metallb-components
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "10"
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    repoURL: https://github.com/mrsimonemms/infrastructure
+    path: registry/components/metallb
+    targetRevision: HEAD
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: metallb-system
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

registry/components/ingress-nginx/application.yaml

@@ -0,0 +1,33 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Application
+metadata:
+  name: ingress-nginx
+  namespace: argocd
+  annotations:
+    argocd.argoproj.io/sync-wave: "10"
+  finalizers:
+    - resources-finalizer.argocd.argoproj.io
+spec:
+  project: default
+  source:
+    chart: ingress-nginx
+    repoURL: https://kubernetes.github.io/ingress-nginx
+    targetRevision: 4.11.3
+    helm:
+      valuesObject:
+        controller:
+          # Not strictly necessary, but feels safer running on all nodes
+          kind: DaemonSet
+          config:
+            use-proxy-protocol: false
+          extraArgs:
+            enable-ssl-passthrough: true
+  destination:
+    server: https://kubernetes.default.svc
+    namespace: ingress-nginx
+  syncPolicy:
+    automated:
+      prune: true
+      selfHeal: true
+    syncOptions:
+      - CreateNamespace=true

registry/components/ingress-nginx/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - application.yaml
+  - namespace.yaml