From 8f2eb2f6c7459c41c3f5dcb5227ad6967b753419 Mon Sep 17 00:00:00 2001 From: Simon Emms Date: Sun, 30 Jun 2024 20:14:16 +0000 Subject: [PATCH] feat(hetzner): add static worker nodes to the cluster --- modules/hetzner/README.md | 4 ++ modules/hetzner/files/k3s-worker.yaml | 35 +++++++++++++++ modules/hetzner/k3s.tf | 2 +- modules/hetzner/locals.tf | 23 +++++++++- modules/hetzner/output.tf | 5 +++ modules/hetzner/server.tf | 62 ++++++++++++++++++++++++++- modules/hetzner/variables.tf | 38 ++++++++++++++++ 7 files changed, 166 insertions(+), 3 deletions(-) create mode 100644 modules/hetzner/files/k3s-worker.yaml diff --git a/modules/hetzner/README.md b/modules/hetzner/README.md index 5ea26f7..ac8979f 100644 --- a/modules/hetzner/README.md +++ b/modules/hetzner/README.md @@ -32,7 +32,9 @@ No modules. | [hcloud_network.network](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/network) | resource | | [hcloud_network_subnet.subnet](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/network_subnet) | resource | | [hcloud_placement_group.managers](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/placement_group) | resource | +| [hcloud_placement_group.workers](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/placement_group) | resource | | [hcloud_server.manager](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/server) | resource | +| [hcloud_server.workers](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/server) | resource | | [hcloud_ssh_key.server](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/ssh_key) | resource | | [ssh_resource.initial_manager](https://registry.terraform.io/providers/loafoe/ssh/latest/docs/resources/resource) | resource | | [ssh_resource.server_ready](https://registry.terraform.io/providers/loafoe/ssh/latest/docs/resources/resource) | resource | @@ -51,6 +53,7 @@ No modules. | [k3s\_manager\_load\_balancer\_algorithm](#input\_k3s\_manager\_load\_balancer\_algorithm) | Algorithm to use for the k3s manager load balancer | `string` | `"round_robin"` | no | | [k3s\_manager\_load\_balancer\_type](#input\_k3s\_manager\_load\_balancer\_type) | Load balancer type for the k3s manager nodes | `string` | `"lb11"` | no | | [k3s\_manager\_pool](#input\_k3s\_manager\_pool) | Manager pool configuration | 
object({
    name        = optional(string, "manager")
    server_type = optional(string, "cx22")
    count       = optional(number, 1)
    image       = optional(string, "ubuntu-24.04")
    labels = optional(
      list(object({
        key   = string
        value = string
      })),
      [],
    )
    taints = optional(
      list(object({
        key    = string
        value  = string
        effect = string
      })),
      []
    )
  })
| `{}` | no | +| [k3s\_worker\_pools](#input\_k3s\_worker\_pools) | Worker pools configuration | 
list(object({
    name        = string
    server_type = optional(string, "cx22")
    count       = optional(number, 1)
    image       = optional(string, "ubuntu-24.04")
    labels = optional(
      list(object({
        key   = string
        value = string
      })),
      [],
    )
    taints = optional(
      list(object({
        key    = string
        value  = string
        effect = string
      })),
      []
    )
    autoscaling = optional(
      object({
        enabled = bool
        min     = number
        max     = number
      }),
      {
        enabled = false
        min     = null
        max     = null
      },
    )
  }))
| `[]` | no | | [location](#input\_location) | Location to use. This is a single datacentre. | `string` | `"nbg1"` | no | | [name](#input\_name) | Name of project | `string` | `"infrastructure"` | no | | [network\_subnet](#input\_network\_subnet) | Subnet of the main network | `string` | `"10.0.0.0/16"` | no | @@ -71,4 +74,5 @@ No modules. | [kubeconfig](#output\_kubeconfig) | Kubeconfig file | | [location](#output\_location) | Location to use. This is a single datacentre. | | [region](#output\_region) | Region to use. This covers multiple datacentres. | +| [worker\_placement\_groups](#output\_worker\_placement\_groups) | Placement groups of worker node pools | diff --git a/modules/hetzner/files/k3s-worker.yaml b/modules/hetzner/files/k3s-worker.yaml new file mode 100644 index 0000000..f316a61 --- /dev/null +++ b/modules/hetzner/files/k3s-worker.yaml @@ -0,0 +1,35 @@ +#cloud-config + +package_reboot_if_required: true +package_update: true +package_upgrade: true +packages: + - curl +runcmd: + - [service, sshd, restart] + - [rm, -f, /root/.ssh/authorized_keys] + - chown ${user}:${user} "/home/${user}" + - curl -sfL ${k3s_download_url} | INSTALL_K3S_EXEC="agent" sh - +timezone: UTC +users: + - default + - name: "${user}" + gecos: "${user}" + sudo: ALL=(ALL) NOPASSWD:ALL + lock_passwd: true + shell: /bin/bash + ssh_authorized_keys: + - "${publicKey}" +write_files: + - path: /etc/ssh/sshd_config.d/ssh.conf + content: | + PasswordAuthentication no + PermitRootLogin no + Port ${sshPort} + - path: /etc/environment + content: | + KUBECONFIG="/etc/rancher/k3s/k3s.yaml" + append: true + - path: /etc/rancher/k3s/config.yaml + content: | + ${indent(6, yamlencode(k3s_config))} diff --git a/modules/hetzner/k3s.tf b/modules/hetzner/k3s.tf index 6d5bc45..cb01dea 100644 --- a/modules/hetzner/k3s.tf +++ b/modules/hetzner/k3s.tf @@ -51,7 +51,7 @@ locals { k3s_initial_manager_private_ip = tolist(local.k3s_initial_manager.network)[0].ip k3s_join_token = chomp(ssh_sensitive_resource.join_token.result) k3s_kubeconfig = chomp(ssh_sensitive_resource.kubeconfig.result) - k3s_server_url = "https://${var.k3s_manager_pool.count > 1 ? hcloud_load_balancer.k3s_manager[0].ipv4 : local.k3s_initial_manager_private_ip}:6443" + k3s_server_url = "https://${var.k3s_manager_pool.count > 1 ? hcloud_load_balancer.k3s_manager[0].ipv4 : local.k3s_initial_manager_private_ip}:${local.kubernetes_api_port}" } resource "ssh_resource" "initial_manager" { diff --git a/modules/hetzner/locals.tf b/modules/hetzner/locals.tf index 668f2cb..5bd6b40 100644 --- a/modules/hetzner/locals.tf +++ b/modules/hetzner/locals.tf @@ -22,6 +22,21 @@ locals { k3s_manager_labels = merge(local.labels, { format(local.label_namespace, "type") = "manager" }) + k3s_worker_labels = merge(local.labels, { + format(local.label_namespace, "type") = "worker" + }) + k3s_worker_nodes = flatten([for w in var.k3s_worker_pools : [ + # If autoscaling, don't create any nodes + for n in range(w.autoscaling.enabled ? 0 : w.count) : + { + pool = w.name + name = "${w.name}-${n}" + image = w.image + server_type = w.server_type + labels = w.labels + taints = w.taints + } + ]]) kubernetes_api_port = 6443 label_namespace = "simonemms.com/%s" machine_user = "k3s" @@ -29,6 +44,12 @@ locals { "hetzner", "%s", # resource name local.workspace_name - ]) # use `format(local.name_format, "")` to use this + ]) # use `format(local.name_format, "")` to use this + worker_placement_groups = { + for i, w in hcloud_placement_group.workers : var.k3s_worker_pools[i].name => { + id = w.id + name = w.name + } + } workspace_name = replace(var.workspace, "/[\\W]/", "") # alphanumeric workspace name } diff --git a/modules/hetzner/output.tf b/modules/hetzner/output.tf index 3a72f8f..69abaca 100644 --- a/modules/hetzner/output.tf +++ b/modules/hetzner/output.tf @@ -43,3 +43,8 @@ output "region" { description = "Region to use. This covers multiple datacentres." value = var.region } + +output "worker_placement_groups" { + description = "Placement groups of worker node pools" + value = local.worker_placement_groups +} diff --git a/modules/hetzner/server.tf b/modules/hetzner/server.tf index e09fb76..576630b 100644 --- a/modules/hetzner/server.tf +++ b/modules/hetzner/server.tf @@ -23,7 +23,7 @@ resource "hcloud_placement_group" "managers" { name = format(local.name_format, "k3s-managers") type = "spread" - labels = merge(local.labels, {}) + labels = merge(local.k3s_manager_labels, {}) } resource "hcloud_server" "manager" { @@ -88,3 +88,63 @@ resource "ssh_resource" "server_ready" { hcloud_server.manager ] } + +resource "hcloud_placement_group" "workers" { + count = length(var.k3s_worker_pools) + + name = format(local.name_format, "k3s-worker-${var.k3s_worker_pools[count.index].name}") + type = "spread" + + labels = merge(local.k3s_worker_labels, { + format(local.label_namespace, "pool") = var.k3s_worker_pools[count.index].name + }) +} + +resource "hcloud_server" "workers" { + count = length(local.k3s_worker_nodes) + + name = format(local.name_format, local.k3s_worker_nodes[count.index].name) + image = local.k3s_worker_nodes[count.index].image + server_type = local.k3s_worker_nodes[count.index].server_type + location = var.location + ssh_keys = [ + hcloud_ssh_key.server.id + ] + placement_group_id = local.worker_placement_groups[local.k3s_worker_nodes[count.index].pool].id + + user_data = templatefile("${path.module}/files/k3s-worker.yaml", { + k3s_config = { + node-label = [for l in local.k3s_worker_nodes[count.index].labels : "${l.key}=${l.value}"] + node-name = format(local.name_format, local.k3s_worker_nodes[count.index].name) + node-taint = [for t in local.k3s_worker_nodes[count.index].taints : "${t.key}=${t.value}:${t.effect}"] + server = local.k3s_server_url + token = local.k3s_join_token + } + k3s_download_url = var.k3s_download_url + sshPort = var.ssh_port + publicKey = hcloud_ssh_key.server.public_key + user = local.machine_user + }) + + network { + network_id = hcloud_network.network.id + # Set the alias_ips to avoid this triggering an update each run + # @link https://github.com/hetznercloud/terraform-provider-hcloud/issues/650#issuecomment-1497160625 + alias_ips = [] + } + + public_net { + ipv4_enabled = true + ipv6_enabled = true + } + + labels = merge(local.k3s_worker_labels, { + format(local.label_namespace, "pool") = local.k3s_worker_nodes[count.index].pool + }) + + lifecycle { + ignore_changes = [ + ssh_keys + ] + } +} diff --git a/modules/hetzner/variables.tf b/modules/hetzner/variables.tf index 2b2ad5d..41da535 100644 --- a/modules/hetzner/variables.tf +++ b/modules/hetzner/variables.tf @@ -92,6 +92,44 @@ variable "k3s_manager_load_balancer_type" { default = "lb11" } +variable "k3s_worker_pools" { + type = list(object({ + name = string + server_type = optional(string, "cx22") + count = optional(number, 1) + image = optional(string, "ubuntu-24.04") + labels = optional( + list(object({ + key = string + value = string + })), + [], + ) + taints = optional( + list(object({ + key = string + value = string + effect = string + })), + [] + ) + autoscaling = optional( + object({ + enabled = bool + min = number + max = number + }), + { + enabled = false + min = null + max = null + }, + ) + })) + description = "Worker pools configuration" + default = [] +} + variable "name" { type = string description = "Name of project"