feat(kubernetes): configure autoscaling nodes

modules/hetzner/README.md

@@ -74,4 +74,5 @@ No modules.
 | <a name="output_kubeconfig"></a> [kubeconfig](#output\_kubeconfig) | Kubeconfig file |
 | <a name="output_location"></a> [location](#output\_location) | Location to use. This is a single datacentre. |
 | <a name="output_region"></a> [region](#output\_region) | Region to use. This covers multiple datacentres. |
+| <a name="output_worker_pools"></a> [worker\_pools](#output\_worker\_pools) | Worker pool configuration for Cluster Autoscaler |
 <!-- END_TF_DOCS -->

modules/hetzner/output.tf

@@ -43,3 +43,60 @@ output "region" {
   description = "Region to use. This covers multiple datacentres."
   value       = var.region
 }
+
+output "worker_pools" {
+  sensitive   = true
+  description = "Worker pool configuration for Cluster Autoscaler"
+  value = [
+    for w in var.k3s_worker_pools : {
+      firewall_id = hcloud_firewall.name.id
+      network_id  = hcloud_network.network.id
+      ssh_key_id  = hcloud_ssh_key.server.id
+      pool = {
+        instanceType = w.server_type
+        minSize      = w.autoscaling.min
+        maxSize      = w.autoscaling.max
+        name         = w.name
+        region       = w.location != null ? w.location : var.location
+      }
+      config = {
+        imagesForArch = {
+          arm64 = "ubuntu-24.04"
+          amd64 = "ubuntu-24.04"
+        }
+        nodeConfigs = {
+          (w.name) = {
+            cloudInit = templatefile("${path.module}/files/k3s-worker.yaml", {
+              k3s_config = {
+                # node-label = [for l in w.labels : "${l.key}=${l.value}"]
+                # node-taint = [for t in local.k3s_worker_nodes[count.index].taints : "${t.key}=${t.value}:${t.effect}"]
+                server = local.k3s_server_url
+                token  = local.k3s_join_token
+              }
+              k3s_download_url = var.k3s_download_url
+              sshPort          = var.ssh_port
+              publicKey        = hcloud_ssh_key.server.public_key
+              user             = local.machine_user
+            })
+            labels = merge(
+              {
+                "node.kubernetes.io/role"             = "autoscaler-node",
+                format(local.label_namespace, "pool") = w.name
+              },
+              { for l in w.labels : l.key => l.value }
+            )
+            taints = concat([
+              {
+                key    = "node.kubernetes.io/role",
+                value  = "autoscaler-node",
+                effect = "NoExecute"
+              }
+              ],
+              w.taints,
+            )
+          }
+        }
+      }
+    } if lookup(w.autoscaling, "enabled", false) == true
+  ]
+}

modules/kubernetes/README.md

@@ -25,23 +25,28 @@ No modules.
 | Name | Type |
 |------|------|
 | [helm_release.cilium](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
+| [helm_release.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [helm_release.hcloud_ccm](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [helm_release.hcloud_csi](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource |
 | [kubernetes_annotations.hcloud_ccm](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/annotations) | resource |
+| [kubernetes_namespace.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource |
+| [kubernetes_secret_v1.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource |
 | [kubernetes_secret_v1.hcloud](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource |
 
 ## Inputs
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_cilium_version"></a> [cilium\_version](#input\_cilium\_version) | Version of Cilium to use - defaults to latest | `string` | `null` | no |
+| <a name="input_cluster_autoscaler_version"></a> [cluster\_autoscaler\_version](#input\_cluster\_autoscaler\_version) | Version of Cluster Autoscaler to use - defaults to latest | `string` | `null` | no |
 | <a name="input_hcloud_network_name"></a> [hcloud\_network\_name](#input\_hcloud\_network\_name) | Hetzner network name | `string` | n/a | yes |
 | <a name="input_hcloud_token"></a> [hcloud\_token](#input\_hcloud\_token) | Hetzner API token | `string` | n/a | yes |
 | <a name="input_hetzner_cloud_config_manager_version"></a> [hetzner\_cloud\_config\_manager\_version](#input\_hetzner\_cloud\_config\_manager\_version) | Version of the HCloud CCM to use - defaults to latest | `string` | `null` | no |
 | <a name="input_hetzner_csi_driver_version"></a> [hetzner\_csi\_driver\_version](#input\_hetzner\_csi\_driver\_version) | Tag of the CSI driver to use - defaults to latest | `string` | `null` | no |
 | <a name="input_k3s_cluster_cidr"></a> [k3s\_cluster\_cidr](#input\_k3s\_cluster\_cidr) | CIDR used for the k3s cluster | `string` | `"10.244.0.0/16"` | no |
 | <a name="input_kube_context"></a> [kube\_context](#input\_kube\_context) | Kubernetes context to use | `string` | `"default"` | no |
 | <a name="input_kubeconfig"></a> [kubeconfig](#input\_kubeconfig) | Kubeconfig for the cluster | `string` | n/a | yes |
+| <a name="input_worker_pools"></a> [worker\_pools](#input\_worker\_pools) | Cluster autoscaler configuration | <pre>list(object({<br>    firewall_id = string<br>    network_id  = string<br>    ssh_key_id  = string<br>    pool = object({<br>      instanceType = string<br>      minSize      = number<br>      maxSize      = number<br>      name         = string<br>      region       = string<br>    })<br>    config = object({<br>      imagesForArch = object({<br>        arm64 = string<br>        amd64 = string<br>      })<br>      nodeConfigs = map(object({<br>        cloudInit = string<br>        labels    = map(string)<br>        taints = list(object({<br>          key    = string<br>          value  = string<br>          effect = string<br>        }))<br>      }))<br>    })<br>  }))</pre> | `[]` | no |
 
 ## Outputs
 

modules/kubernetes/autoscaler.tf

@@ -0,0 +1,82 @@
+# Copyright 2024 Simon Emms <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+resource "kubernetes_namespace" "cluster_autoscaler" {
+  count = length(var.worker_pools) > 0 ? 1 : 0
+
+  metadata {
+    name = "cluster-autoscaler"
+  }
+
+  depends_on = [helm_release.cilium]
+}
+
+resource "kubernetes_secret_v1" "cluster_autoscaler" {
+  count = length(var.worker_pools)
+
+  metadata {
+    name      = "hetzner-${var.worker_pools[count.index].pool.name}"
+    namespace = kubernetes_namespace.cluster_autoscaler[0].metadata[0].name
+  }
+
+  data = {
+    HCLOUD_TOKEN          = var.hcloud_token
+    HCLOUD_NETWORK        = var.hcloud_network_name
+    HCLOUD_FIREWALL       = var.worker_pools[count.index].firewall_id
+    HCLOUD_SSH_KEY        = var.worker_pools[count.index].ssh_key_id
+    HCLOUD_CLUSTER_CONFIG = base64encode(jsonencode(var.worker_pools[count.index].config))
+  }
+}
+
+resource "helm_release" "cluster_autoscaler" {
+  count = length(var.worker_pools)
+
+  chart           = "cluster-autoscaler"
+  name            = "cluster-autoscaler-${var.worker_pools[count.index].pool.name}"
+  atomic          = true
+  cleanup_on_fail = true
+  namespace       = kubernetes_namespace.cluster_autoscaler[0].metadata[0].name
+  repository      = "https://kubernetes.github.io/autoscaler"
+  reset_values    = true
+  version         = var.cluster_autoscaler_version
+  wait            = true
+
+  set {
+    name  = "cloudProvider"
+    value = "hetzner"
+  }
+
+  set {
+    name  = "envFromSecret"
+    value = kubernetes_secret_v1.cluster_autoscaler[count.index].metadata[0].name
+  }
+
+  dynamic "set" {
+    for_each = [for k, v in var.worker_pools[count.index].pool : {
+      name  = "autoscalingGroups[0].${k}"
+      value = v
+    }]
+    iterator = each
+
+    content {
+      name  = each.value.name
+      value = each.value.value
+    }
+  }
+
+  set {
+    name  = "podAnnotations.secret"
+    value = sha512(yamlencode(kubernetes_secret_v1.cluster_autoscaler[count.index].data))
+  }
+}

modules/kubernetes/hetzner.tf

@@ -22,8 +22,6 @@ resource "kubernetes_secret_v1" "hcloud" {
     network = var.hcloud_network_name # Required by the CCM
     token   = var.hcloud_token        # Required by the CSI
   }
-
-  depends_on = [helm_release.cilium]
 }
 
 resource "helm_release" "hcloud_ccm" {
@@ -77,11 +75,4 @@ resource "helm_release" "hcloud_csi" {
     name  = "controller.podAnnotations.secret"
     value = sha512(yamlencode(kubernetes_secret_v1.hcloud.data))
   }
-
-  # set {
-  #   name = "controller.nodeSelector.node-role\.kubernetes\.io/control-plane"
-  #   value = ""
-  # }
-
-  depends_on = [kubernetes_secret_v1.hcloud]
 }

modules/kubernetes/variables.tf

@@ -12,13 +12,18 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-
 variable "cilium_version" {
   type        = string
   description = "Version of Cilium to use - defaults to latest"
   default     = null
 }
 
+variable "cluster_autoscaler_version" {
+  type        = string
+  description = "Version of Cluster Autoscaler to use - defaults to latest"
+  default     = null
+}
+
 variable "hcloud_network_name" {
   type        = string
   description = "Hetzner network name"
@@ -59,3 +64,36 @@ variable "kube_context" {
   description = "Kubernetes context to use"
   default     = "default"
 }
+
+variable "worker_pools" {
+  type = list(object({
+    firewall_id = string
+    network_id  = string
+    ssh_key_id  = string
+    pool = object({
+      instanceType = string
+      minSize      = number
+      maxSize      = number
+      name         = string
+      region       = string
+    })
+    config = object({
+      imagesForArch = object({
+        arm64 = string
+        amd64 = string
+      })
+      nodeConfigs = map(object({
+        cloudInit = string
+        labels    = map(string)
+        taints = list(object({
+          key    = string
+          value  = string
+          effect = string
+        }))
+      }))
+    })
+  }))
+  description = "Cluster autoscaler configuration"
+  # sensitive   = true
+  default = []
+}

stacks/dev/kubernetes/terragrunt.hcl

@@ -27,11 +27,13 @@ dependency "hetzner" {
     hcloud_network_name = "some-network-name"
     k3s_cluster_cidr    = "some-cluster-cidr"
     kubeconfig          = "some-kubeconfig"
+    worker_pools        = []
   }
 }
 
 inputs = {
   hcloud_network_name = dependency.hetzner.outputs.hcloud_network_name
   k3s_cluster_cidr    = dependency.hetzner.outputs.k3s_cluster_cidr
   kubeconfig          = dependency.hetzner.outputs.kubeconfig
+  worker_pools        = dependency.hetzner.outputs.worker_pools
 }