From 46f356016d3bd191559a061faf1a35f5ec35b2fe Mon Sep 17 00:00:00 2001 From: Simon Emms Date: Wed, 3 Jul 2024 21:18:20 +0000 Subject: [PATCH] feat(kubernetes): configure autoscaling nodes --- modules/hetzner/README.md | 1 + modules/hetzner/output.tf | 57 +++++++++++++++++++ modules/kubernetes/README.md | 5 ++ modules/kubernetes/autoscaler.tf | 82 ++++++++++++++++++++++++++++ modules/kubernetes/hetzner.tf | 9 --- modules/kubernetes/variables.tf | 40 +++++++++++++- stacks/dev/kubernetes/terragrunt.hcl | 2 + 7 files changed, 186 insertions(+), 10 deletions(-) create mode 100644 modules/kubernetes/autoscaler.tf diff --git a/modules/hetzner/README.md b/modules/hetzner/README.md index 01e4eff..9497c7c 100644 --- a/modules/hetzner/README.md +++ b/modules/hetzner/README.md @@ -74,4 +74,5 @@ No modules. | [kubeconfig](#output\_kubeconfig) | Kubeconfig file | | [location](#output\_location) | Location to use. This is a single datacentre. | | [region](#output\_region) | Region to use. This covers multiple datacentres. | +| [worker\_pools](#output\_worker\_pools) | Worker pool configuration for Cluster Autoscaler | diff --git a/modules/hetzner/output.tf b/modules/hetzner/output.tf index 3a72f8f..7d45a96 100644 --- a/modules/hetzner/output.tf +++ b/modules/hetzner/output.tf @@ -43,3 +43,60 @@ output "region" { description = "Region to use. This covers multiple datacentres." value = var.region } + +output "worker_pools" { + sensitive = true + description = "Worker pool configuration for Cluster Autoscaler" + value = [ + for w in var.k3s_worker_pools : { + firewall_id = hcloud_firewall.name.id + network_id = hcloud_network.network.id + ssh_key_id = hcloud_ssh_key.server.id + pool = { + instanceType = w.server_type + minSize = w.autoscaling.min + maxSize = w.autoscaling.max + name = w.name + region = w.location != null ? w.location : var.location + } + config = { + imagesForArch = { + arm64 = "ubuntu-24.04" + amd64 = "ubuntu-24.04" + } + nodeConfigs = { + (w.name) = { + cloudInit = templatefile("${path.module}/files/k3s-worker.yaml", { + k3s_config = { + # node-label = [for l in w.labels : "${l.key}=${l.value}"] + # node-taint = [for t in local.k3s_worker_nodes[count.index].taints : "${t.key}=${t.value}:${t.effect}"] + server = local.k3s_server_url + token = local.k3s_join_token + } + k3s_download_url = var.k3s_download_url + sshPort = var.ssh_port + publicKey = hcloud_ssh_key.server.public_key + user = local.machine_user + }) + labels = merge( + { + "node.kubernetes.io/role" = "autoscaler-node", + format(local.label_namespace, "pool") = w.name + }, + { for l in w.labels : l.key => l.value } + ) + taints = concat([ + { + key = "node.kubernetes.io/role", + value = "autoscaler-node", + effect = "NoExecute" + } + ], + w.taints, + ) + } + } + } + } if lookup(w.autoscaling, "enabled", false) == true + ] +} diff --git a/modules/kubernetes/README.md b/modules/kubernetes/README.md index 1a3aaa5..ab393f8 100644 --- a/modules/kubernetes/README.md +++ b/modules/kubernetes/README.md @@ -25,9 +25,12 @@ No modules. | Name | Type | |------|------| | [helm_release.cilium](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.hcloud_ccm](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.hcloud_csi](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [kubernetes_annotations.hcloud_ccm](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/annotations) | resource | +| [kubernetes_namespace.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace) | resource | +| [kubernetes_secret_v1.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource | | [kubernetes_secret_v1.hcloud](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource | ## Inputs @@ -35,6 +38,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [cilium\_version](#input\_cilium\_version) | Version of Cilium to use - defaults to latest | `string` | `null` | no | +| [cluster\_autoscaler\_version](#input\_cluster\_autoscaler\_version) | Version of Cluster Autoscaler to use - defaults to latest | `string` | `null` | no | | [hcloud\_network\_name](#input\_hcloud\_network\_name) | Hetzner network name | `string` | n/a | yes | | [hcloud\_token](#input\_hcloud\_token) | Hetzner API token | `string` | n/a | yes | | [hetzner\_cloud\_config\_manager\_version](#input\_hetzner\_cloud\_config\_manager\_version) | Version of the HCloud CCM to use - defaults to latest | `string` | `null` | no | @@ -42,6 +46,7 @@ No modules. | [k3s\_cluster\_cidr](#input\_k3s\_cluster\_cidr) | CIDR used for the k3s cluster | `string` | `"10.244.0.0/16"` | no | | [kube\_context](#input\_kube\_context) | Kubernetes context to use | `string` | `"default"` | no | | [kubeconfig](#input\_kubeconfig) | Kubeconfig for the cluster | `string` | n/a | yes | +| [worker\_pools](#input\_worker\_pools) | Cluster autoscaler configuration | 
list(object({
    firewall_id = string
    network_id  = string
    ssh_key_id  = string
    pool = object({
      instanceType = string
      minSize      = number
      maxSize      = number
      name         = string
      region       = string
    })
    config = object({
      imagesForArch = object({
        arm64 = string
        amd64 = string
      })
      nodeConfigs = map(object({
        cloudInit = string
        labels    = map(string)
        taints = list(object({
          key    = string
          value  = string
          effect = string
        }))
      }))
    })
  }))
| `[]` | no | ## Outputs diff --git a/modules/kubernetes/autoscaler.tf b/modules/kubernetes/autoscaler.tf new file mode 100644 index 0000000..1fe24e3 --- /dev/null +++ b/modules/kubernetes/autoscaler.tf @@ -0,0 +1,82 @@ +# Copyright 2024 Simon Emms +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "kubernetes_namespace" "cluster_autoscaler" { + count = length(var.worker_pools) > 0 ? 1 : 0 + + metadata { + name = "cluster-autoscaler" + } + + depends_on = [helm_release.cilium] +} + +resource "kubernetes_secret_v1" "cluster_autoscaler" { + count = length(var.worker_pools) + + metadata { + name = "hetzner-${var.worker_pools[count.index].pool.name}" + namespace = kubernetes_namespace.cluster_autoscaler[0].metadata[0].name + } + + data = { + HCLOUD_TOKEN = var.hcloud_token + HCLOUD_NETWORK = var.hcloud_network_name + HCLOUD_FIREWALL = var.worker_pools[count.index].firewall_id + HCLOUD_SSH_KEY = var.worker_pools[count.index].ssh_key_id + HCLOUD_CLUSTER_CONFIG = base64encode(jsonencode(var.worker_pools[count.index].config)) + } +} + +resource "helm_release" "cluster_autoscaler" { + count = length(var.worker_pools) + + chart = "cluster-autoscaler" + name = "cluster-autoscaler-${var.worker_pools[count.index].pool.name}" + atomic = true + cleanup_on_fail = true + namespace = kubernetes_namespace.cluster_autoscaler[0].metadata[0].name + repository = "https://kubernetes.github.io/autoscaler" + reset_values = true + version = var.cluster_autoscaler_version + wait = true + + set { + name = "cloudProvider" + value = "hetzner" + } + + set { + name = "envFromSecret" + value = kubernetes_secret_v1.cluster_autoscaler[count.index].metadata[0].name + } + + dynamic "set" { + for_each = [for k, v in var.worker_pools[count.index].pool : { + name = "autoscalingGroups[0].${k}" + value = v + }] + iterator = each + + content { + name = each.value.name + value = each.value.value + } + } + + set { + name = "podAnnotations.secret" + value = sha512(yamlencode(kubernetes_secret_v1.cluster_autoscaler[count.index].data)) + } +} diff --git a/modules/kubernetes/hetzner.tf b/modules/kubernetes/hetzner.tf index 16b9f74..c6c3be8 100644 --- a/modules/kubernetes/hetzner.tf +++ b/modules/kubernetes/hetzner.tf @@ -22,8 +22,6 @@ resource "kubernetes_secret_v1" "hcloud" { network = var.hcloud_network_name # Required by the CCM token = var.hcloud_token # Required by the CSI } - - depends_on = [helm_release.cilium] } resource "helm_release" "hcloud_ccm" { @@ -77,11 +75,4 @@ resource "helm_release" "hcloud_csi" { name = "controller.podAnnotations.secret" value = sha512(yamlencode(kubernetes_secret_v1.hcloud.data)) } - - # set { - # name = "controller.nodeSelector.node-role\.kubernetes\.io/control-plane" - # value = "" - # } - - depends_on = [kubernetes_secret_v1.hcloud] } diff --git a/modules/kubernetes/variables.tf b/modules/kubernetes/variables.tf index e881873..fb1df31 100644 --- a/modules/kubernetes/variables.tf +++ b/modules/kubernetes/variables.tf @@ -12,13 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. - variable "cilium_version" { type = string description = "Version of Cilium to use - defaults to latest" default = null } +variable "cluster_autoscaler_version" { + type = string + description = "Version of Cluster Autoscaler to use - defaults to latest" + default = null +} + variable "hcloud_network_name" { type = string description = "Hetzner network name" @@ -59,3 +64,36 @@ variable "kube_context" { description = "Kubernetes context to use" default = "default" } + +variable "worker_pools" { + type = list(object({ + firewall_id = string + network_id = string + ssh_key_id = string + pool = object({ + instanceType = string + minSize = number + maxSize = number + name = string + region = string + }) + config = object({ + imagesForArch = object({ + arm64 = string + amd64 = string + }) + nodeConfigs = map(object({ + cloudInit = string + labels = map(string) + taints = list(object({ + key = string + value = string + effect = string + })) + })) + }) + })) + description = "Cluster autoscaler configuration" + # sensitive = true + default = [] +} diff --git a/stacks/dev/kubernetes/terragrunt.hcl b/stacks/dev/kubernetes/terragrunt.hcl index 840ade4..ea3989e 100644 --- a/stacks/dev/kubernetes/terragrunt.hcl +++ b/stacks/dev/kubernetes/terragrunt.hcl @@ -27,6 +27,7 @@ dependency "hetzner" { hcloud_network_name = "some-network-name" k3s_cluster_cidr = "some-cluster-cidr" kubeconfig = "some-kubeconfig" + worker_pools = [] } } @@ -34,4 +35,5 @@ inputs = { hcloud_network_name = dependency.hetzner.outputs.hcloud_network_name k3s_cluster_cidr = dependency.hetzner.outputs.k3s_cluster_cidr kubeconfig = dependency.hetzner.outputs.kubeconfig + worker_pools = dependency.hetzner.outputs.worker_pools }