feat(hetzner): add static worker nodes to the cluster

modules/hetzner/README.md

@@ -32,7 +32,9 @@ No modules.
 | [hcloud_network.network](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/network) | resource |
 | [hcloud_network_subnet.subnet](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/network_subnet) | resource |
 | [hcloud_placement_group.managers](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/placement_group) | resource |
+| [hcloud_placement_group.workers](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/placement_group) | resource |
 | [hcloud_server.manager](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/server) | resource |
+| [hcloud_server.workers](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/server) | resource |
 | [hcloud_ssh_key.server](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/ssh_key) | resource |
 | [ssh_resource.initial_manager](https://registry.terraform.io/providers/loafoe/ssh/latest/docs/resources/resource) | resource |
 | [ssh_resource.server_ready](https://registry.terraform.io/providers/loafoe/ssh/latest/docs/resources/resource) | resource |
@@ -51,6 +53,7 @@ No modules.
 | <a name="input_k3s_manager_load_balancer_algorithm"></a> [k3s\_manager\_load\_balancer\_algorithm](#input\_k3s\_manager\_load\_balancer\_algorithm) | Algorithm to use for the k3s manager load balancer | `string` | `"round_robin"` | no |
 | <a name="input_k3s_manager_load_balancer_type"></a> [k3s\_manager\_load\_balancer\_type](#input\_k3s\_manager\_load\_balancer\_type) | Load balancer type for the k3s manager nodes | `string` | `"lb11"` | no |
 | <a name="input_k3s_manager_pool"></a> [k3s\_manager\_pool](#input\_k3s\_manager\_pool) | Manager pool configuration | <pre>object({<br>    name        = optional(string, "manager")<br>    server_type = optional(string, "cx22")<br>    count       = optional(number, 1)<br>    image       = optional(string, "ubuntu-24.04")<br>    labels = optional(<br>      list(object({<br>        key   = string<br>        value = string<br>      })),<br>      [],<br>    )<br>    taints = optional(<br>      list(object({<br>        key    = string<br>        value  = string<br>        effect = string<br>      })),<br>      []<br>    )<br>  })</pre> | `{}` | no |
+| <a name="input_k3s_worker_pools"></a> [k3s\_worker\_pools](#input\_k3s\_worker\_pools) | Worker pools configuration | <pre>list(object({<br>    name        = string<br>    server_type = optional(string, "cx22")<br>    count       = optional(number, 1)<br>    image       = optional(string, "ubuntu-24.04")<br>    location    = optional(string) # Defaults to var.location if not set<br>    labels = optional(<br>      list(object({<br>        key   = string<br>        value = string<br>      })),<br>      [],<br>    )<br>    taints = optional(<br>      list(object({<br>        key    = string<br>        value  = string<br>        effect = string<br>      })),<br>      []<br>    )<br>    autoscaling = optional(<br>      object({<br>        enabled = bool<br>        min     = number<br>        max     = number<br>      }),<br>      {<br>        enabled = false<br>        min     = null<br>        max     = null<br>      },<br>    )<br>  }))</pre> | `[]` | no |
 | <a name="input_location"></a> [location](#input\_location) | Location to use. This is a single datacentre. | `string` | `"nbg1"` | no |
 | <a name="input_name"></a> [name](#input\_name) | Name of project | `string` | `"infrastructure"` | no |
 | <a name="input_network_subnet"></a> [network\_subnet](#input\_network\_subnet) | Subnet of the main network | `string` | `"10.0.0.0/16"` | no |

modules/hetzner/files/k3s-worker.yaml

@@ -0,0 +1,35 @@
+#cloud-config
+
+package_reboot_if_required: true
+package_update: true
+package_upgrade: true
+packages:
+  - curl
+runcmd:
+  - [service, sshd, restart]
+  - [rm, -f, /root/.ssh/authorized_keys]
+  - chown ${user}:${user} "/home/${user}"
+  - curl -sfL ${k3s_download_url} | INSTALL_K3S_EXEC="agent" sh -
+timezone: UTC
+users:
+  - default
+  - name: "${user}"
+    gecos: "${user}"
+    sudo: ALL=(ALL) NOPASSWD:ALL
+    lock_passwd: true
+    shell: /bin/bash
+    ssh_authorized_keys:
+      - "${publicKey}"
+write_files:
+  - path: /etc/ssh/sshd_config.d/ssh.conf
+    content: |
+      PasswordAuthentication no
+      PermitRootLogin no
+      Port ${sshPort}
+  - path: /etc/environment
+    content: |
+      KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
+    append: true
+  - path: /etc/rancher/k3s/config.yaml
+    content: |
+      ${indent(6, yamlencode(k3s_config))}

modules/hetzner/k3s.tf

@@ -51,7 +51,7 @@ locals {
   k3s_initial_manager_private_ip = tolist(local.k3s_initial_manager.network)[0].ip
   k3s_join_token                 = chomp(ssh_sensitive_resource.join_token.result)
   k3s_kubeconfig                 = chomp(ssh_sensitive_resource.kubeconfig.result)
-  k3s_server_url                 = "https://${var.k3s_manager_pool.count > 1 ? hcloud_load_balancer.k3s_manager[0].ipv4 : local.k3s_initial_manager_private_ip}:6443"
+  k3s_server_url                 = "https://${var.k3s_manager_pool.count > 1 ? hcloud_load_balancer.k3s_manager[0].ipv4 : local.k3s_initial_manager_private_ip}:${local.kubernetes_api_port}"
 }
 
 resource "ssh_resource" "initial_manager" {

modules/hetzner/locals.tf

@@ -22,13 +22,43 @@ locals {
   k3s_manager_labels = merge(local.labels, {
     format(local.label_namespace, "type") = "manager"
   })
+  k3s_worker_labels = merge(local.labels, {
+    format(local.label_namespace, "type") = "worker"
+  })
+  k3s_worker_nodes = flatten([for w in var.k3s_worker_pools : [
+    # If autoscaling, don't create any nodes
+    for n in range(w.autoscaling.enabled ? 0 : w.count) :
+    merge(
+      w,
+      {
+        pool     = w.name
+        name     = "${w.name}-${n}"
+        location = w.location != null ? w.location : var.location
+        labels = concat(
+          // Add the pool name as a label
+          [
+            {
+              key   = format(local.label_namespace, "pool"),
+              value = w.name
+            },
+          ],
+          w.labels,
+        )
+    })
+  ]])
   kubernetes_api_port = 6443
   label_namespace     = "simonemms.com/%s"
   machine_user        = "k3s"
   name_format = join("-", [
     "hetzner",
     "%s", # resource name
     local.workspace_name
-  ])                                                     # use `format(local.name_format, "<name>")` to use this
+  ]) # use `format(local.name_format, "<name>")` to use this
+  worker_placement_groups = {
+    for i, w in hcloud_placement_group.workers : var.k3s_worker_pools[i].name => {
+      id   = w.id
+      name = w.name
+    }
+  }
   workspace_name = replace(var.workspace, "/[\\W]/", "") # alphanumeric workspace name
 }

modules/hetzner/server.tf

@@ -12,18 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+##########
+# Common #
+##########
 resource "hcloud_ssh_key" "server" {
   name       = format(local.name_format, "ssh-key")
   public_key = file(var.ssh_key_public)
 
   labels = merge(local.labels, {})
 }
 
+############
+# Managers #
+############
 resource "hcloud_placement_group" "managers" {
   name = format(local.name_format, "k3s-managers")
   type = "spread"
 
-  labels = merge(local.labels, {})
+  labels = merge(local.k3s_manager_labels, {})
 }
 
 resource "hcloud_server" "manager" {
@@ -88,3 +94,66 @@ resource "ssh_resource" "server_ready" {
     hcloud_server.manager
   ]
 }
+
+##################
+# Static workers #
+##################
+resource "hcloud_placement_group" "workers" {
+  count = length(var.k3s_worker_pools)
+
+  name = format(local.name_format, "k3s-worker-${var.k3s_worker_pools[count.index].name}")
+  type = "spread"
+
+  labels = merge(local.k3s_worker_labels, {
+    format(local.label_namespace, "pool") = var.k3s_worker_pools[count.index].name
+  })
+}
+
+resource "hcloud_server" "workers" {
+  count = length(local.k3s_worker_nodes)
+
+  name        = format(local.name_format, local.k3s_worker_nodes[count.index].name)
+  image       = local.k3s_worker_nodes[count.index].image
+  server_type = local.k3s_worker_nodes[count.index].server_type
+  location    = local.k3s_worker_nodes[count.index].location
+  ssh_keys = [
+    hcloud_ssh_key.server.id
+  ]
+  placement_group_id = local.worker_placement_groups[local.k3s_worker_nodes[count.index].pool].id
+
+  user_data = templatefile("${path.module}/files/k3s-worker.yaml", {
+    k3s_config = {
+      node-label = [for l in local.k3s_worker_nodes[count.index].labels : "${l.key}=${l.value}"]
+      node-name  = format(local.name_format, local.k3s_worker_nodes[count.index].name)
+      node-taint = [for t in local.k3s_worker_nodes[count.index].taints : "${t.key}=${t.value}:${t.effect}"]
+      server     = local.k3s_server_url
+      token      = local.k3s_join_token
+    }
+    k3s_download_url = var.k3s_download_url
+    sshPort          = var.ssh_port
+    publicKey        = hcloud_ssh_key.server.public_key
+    user             = local.machine_user
+  })
+
+  network {
+    network_id = hcloud_network.network.id
+    # Set the alias_ips to avoid this triggering an update each run
+    # @link https://github.com/hetznercloud/terraform-provider-hcloud/issues/650#issuecomment-1497160625
+    alias_ips = []
+  }
+
+  public_net {
+    ipv4_enabled = true
+    ipv6_enabled = true
+  }
+
+  labels = merge(local.k3s_worker_labels, {
+    format(local.label_namespace, "pool") = local.k3s_worker_nodes[count.index].pool
+  })
+
+  lifecycle {
+    ignore_changes = [
+      ssh_keys
+    ]
+  }
+}

modules/hetzner/variables.tf

@@ -92,6 +92,45 @@ variable "k3s_manager_load_balancer_type" {
   default     = "lb11"
 }
 
+variable "k3s_worker_pools" {
+  type = list(object({
+    name        = string
+    server_type = optional(string, "cx22")
+    count       = optional(number, 1)
+    image       = optional(string, "ubuntu-24.04")
+    location    = optional(string) # Defaults to var.location if not set
+    labels = optional(
+      list(object({
+        key   = string
+        value = string
+      })),
+      [],
+    )
+    taints = optional(
+      list(object({
+        key    = string
+        value  = string
+        effect = string
+      })),
+      []
+    )
+    autoscaling = optional(
+      object({
+        enabled = bool
+        min     = number
+        max     = number
+      }),
+      {
+        enabled = false
+        min     = null
+        max     = null
+      },
+    )
+  }))
+  description = "Worker pools configuration"
+  default     = []
+}
+
 variable "name" {
   type        = string
   description = "Name of project"

stacks/dev/hetzner/terragrunt.hcl

@@ -22,7 +22,21 @@ include {
 
 inputs = {
   k3s_manager_pool = {
-    count = 3
+    count       = 3
+    server_type = "cpx11"
   }
+  k3s_worker_pools = [
+    {
+      name  = "pool1"
+      count = 0
+      // location = "fsn1"
+      server_type = "cpx11"
+      // autoscaling = {
+      //   enabled = true
+      //   min = 1
+      //   max = 3
+      // }
+    }
+  ]
   network_subnet = "10.2.0.0/16"
 }