From 5832cffd442ba36ed5332dec75e72022b3a68671 Mon Sep 17 00:00:00 2001 From: Simon Emms Date: Thu, 4 Jul 2024 20:54:46 +0100 Subject: [PATCH] autoscaling --- modules/hetzner/files/k3s-manager.yaml | 2 +- modules/hetzner/files/k3s-worker.yaml | 2 +- modules/hetzner/output.tf | 58 ++++++++++++-------------- modules/kubernetes/autoscaler.tf | 11 ++--- modules/kubernetes/variables.tf | 17 +------- stacks/dev/hetzner/terragrunt.hcl | 19 +++++---- 6 files changed, 48 insertions(+), 61 deletions(-) diff --git a/modules/hetzner/files/k3s-manager.yaml b/modules/hetzner/files/k3s-manager.yaml index 33c4b8b..830e8a0 100644 --- a/modules/hetzner/files/k3s-manager.yaml +++ b/modules/hetzner/files/k3s-manager.yaml @@ -18,7 +18,7 @@ users: lock_passwd: true shell: /bin/bash ssh_authorized_keys: - - "${publicKey}" + - "${chomp(publicKey)}" write_files: - path: /etc/ssh/sshd_config.d/ssh.conf content: | diff --git a/modules/hetzner/files/k3s-worker.yaml b/modules/hetzner/files/k3s-worker.yaml index f316a61..3fd7486 100644 --- a/modules/hetzner/files/k3s-worker.yaml +++ b/modules/hetzner/files/k3s-worker.yaml @@ -19,7 +19,7 @@ users: lock_passwd: true shell: /bin/bash ssh_authorized_keys: - - "${publicKey}" + - "${chomp(publicKey)}" write_files: - path: /etc/ssh/sshd_config.d/ssh.conf content: | diff --git a/modules/hetzner/output.tf b/modules/hetzner/output.tf index 7d45a96..16d6465 100644 --- a/modules/hetzner/output.tf +++ b/modules/hetzner/output.tf @@ -59,44 +59,40 @@ output "worker_pools" { name = w.name region = w.location != null ? w.location : var.location } - config = { - imagesForArch = { - arm64 = "ubuntu-24.04" - amd64 = "ubuntu-24.04" - } - nodeConfigs = { - (w.name) = { - cloudInit = templatefile("${path.module}/files/k3s-worker.yaml", { - k3s_config = { - # node-label = [for l in w.labels : "${l.key}=${l.value}"] - # node-taint = [for t in local.k3s_worker_nodes[count.index].taints : "${t.key}=${t.value}:${t.effect}"] - server = local.k3s_server_url - token = local.k3s_join_token - } - k3s_download_url = var.k3s_download_url - sshPort = var.ssh_port - publicKey = hcloud_ssh_key.server.public_key - user = local.machine_user - }) - labels = merge( + image = w.image, + cloudInit = base64encode(templatefile("${path.module}/files/k3s-worker.yaml", { + k3s_config = { + node-label = [for l in concat( + [ + { + key = "node.kubernetes.io/role" + value = "autoscaler-node" + }, { - "node.kubernetes.io/role" = "autoscaler-node", - format(local.label_namespace, "pool") = w.name + key = format(local.label_namespace, "pool") + value = w.name }, - { for l in w.labels : l.key => l.value } - ) - taints = concat([ + ], + w.labels, + ) : "${l.key}=${l.value}"] + node-taint = [for t in concat( + [ { key = "node.kubernetes.io/role", value = "autoscaler-node", effect = "NoExecute" - } - ], - w.taints, - ) - } + }, + ], + w.taints, + ) : "${t.key}=${t.value}:${t.effect}"] + server = local.k3s_server_url + token = local.k3s_join_token } - } + k3s_download_url = var.k3s_download_url + sshPort = var.ssh_port + publicKey = hcloud_ssh_key.server.public_key + user = local.machine_user + })) } if lookup(w.autoscaling, "enabled", false) == true ] } diff --git a/modules/kubernetes/autoscaler.tf b/modules/kubernetes/autoscaler.tf index 1fe24e3..7b48eb8 100644 --- a/modules/kubernetes/autoscaler.tf +++ b/modules/kubernetes/autoscaler.tf @@ -31,11 +31,12 @@ resource "kubernetes_secret_v1" "cluster_autoscaler" { } data = { - HCLOUD_TOKEN = var.hcloud_token - HCLOUD_NETWORK = var.hcloud_network_name - HCLOUD_FIREWALL = var.worker_pools[count.index].firewall_id - HCLOUD_SSH_KEY = var.worker_pools[count.index].ssh_key_id - HCLOUD_CLUSTER_CONFIG = base64encode(jsonencode(var.worker_pools[count.index].config)) + HCLOUD_TOKEN = var.hcloud_token + HCLOUD_NETWORK = var.hcloud_network_name + HCLOUD_FIREWALL = var.worker_pools[count.index].firewall_id + HCLOUD_SSH_KEY = var.worker_pools[count.index].ssh_key_id + HCLOUD_CLOUD_INIT = base64encode(jsonencode(var.worker_pools[count.index].cloudInit)) + HCLOUD_IMAGE = var.worker_pools[count.index].image } } diff --git a/modules/kubernetes/variables.tf b/modules/kubernetes/variables.tf index fb1df31..d1296ae 100644 --- a/modules/kubernetes/variables.tf +++ b/modules/kubernetes/variables.tf @@ -77,21 +77,8 @@ variable "worker_pools" { name = string region = string }) - config = object({ - imagesForArch = object({ - arm64 = string - amd64 = string - }) - nodeConfigs = map(object({ - cloudInit = string - labels = map(string) - taints = list(object({ - key = string - value = string - effect = string - })) - })) - }) + image = string + cloudInit = string })) description = "Cluster autoscaler configuration" # sensitive = true diff --git a/stacks/dev/hetzner/terragrunt.hcl b/stacks/dev/hetzner/terragrunt.hcl index 32a0d28..7c7a23a 100644 --- a/stacks/dev/hetzner/terragrunt.hcl +++ b/stacks/dev/hetzner/terragrunt.hcl @@ -27,15 +27,18 @@ inputs = { } k3s_worker_pools = [ { - name = "pool1" - count = 0 - // location = "fsn1" + name = "static" + count = 3 server_type = "cpx11" - // autoscaling = { - // enabled = true - // min = 1 - // max = 3 - // } + }, + { + name = "scaling1" + server_type = "cpx11" + autoscaling = { + enabled = true + min = 1 + max = 3 + } } ] network_subnet = "10.2.0.0/16"