feat(hetzner): optionally prevent workloads running on manager nodes

Author: mrsimonemms

modules/hetzner/README.md

@@ -61,6 +61,7 @@ No modules.
 | <a name="input_network_subnet"></a> [network\_subnet](#input\_network\_subnet) | Subnet of the main network | `string` | `"10.0.0.0/16"` | no |
 | <a name="input_network_type"></a> [network\_type](#input\_network\_type) | Type of network to use | `string` | `"cloud"` | no |
 | <a name="input_region"></a> [region](#input\_region) | Region to use. This covers multiple datacentres. | `string` | `"eu-central"` | no |
+| <a name="input_schedule_workloads_on_manager_nodes"></a> [schedule\_workloads\_on\_manager\_nodes](#input\_schedule\_workloads\_on\_manager\_nodes) | Allow scheduling of workloads of manager nodes. | `bool` | `true` | no |
 | <a name="input_ssh_key"></a> [ssh\_key](#input\_ssh\_key) | Path to the private SSH key | `string` | `"~/.ssh/id_ed25519"` | no |
 | <a name="input_ssh_key_public"></a> [ssh\_key\_public](#input\_ssh\_key\_public) | Path to the public SSH key | `string` | `"~/.ssh/id_ed25519.pub"` | no |
 | <a name="input_ssh_port"></a> [ssh\_port](#input\_ssh\_port) | Port to use for SSH access | `number` | `2244` | no |

modules/hetzner/k3s.tf

@@ -34,8 +34,17 @@ locals {
     kube-proxy-arg              = "metrics-bind-address=0.0.0.0"
     kube-scheduler-arg          = "bind-address=0.0.0.0"
     node-label                  = [for l in var.k3s_manager_pool.labels : "${l.key}=${l.value}"]
-    node-taint                  = [for t in var.k3s_manager_pool.taints : "${t.key}=${t.value}:${t.effect}"]
-    service-cidr                = var.k3s_service_cidr
+    node-taint = [for t in concat(
+      var.schedule_workloads_on_manager_nodes ? [] : [
+        {
+          key    = "CriticalAddonsOnly"
+          value  = "true"
+          effect = "NoExecute"
+        }
+      ],
+      var.k3s_manager_pool.taints
+    ) : "${t.key}=${t.value}:${t.effect}"]
+    service-cidr = var.k3s_service_cidr
     tls-san = concat(
       [local.k3s_access_address],
       [for o in hcloud_server.manager : tolist(o.network)[0].ip]

modules/hetzner/variables.tf

@@ -172,6 +172,12 @@ variable "region" {
   default     = "eu-central"
 }
 
+variable "schedule_workloads_on_manager_nodes" {
+  type        = bool
+  description = "Allow scheduling of workloads of manager nodes."
+  default     = true
+}
+
 variable "ssh_key" {
   type        = string
   description = "Path to the private SSH key"

modules/kubernetes/autoscaler.tf

@@ -90,4 +90,26 @@ resource "helm_release" "cluster_autoscaler" {
     name  = "podAnnotations.secret"
     value = sha512(yamlencode(kubernetes_secret_v1.cluster_autoscaler[count.index].data))
   }
+
+  # Allow running on control plane nodes
+  dynamic "set" {
+    for_each = flatten([
+      for i, taint in local.control_plane_taints :
+      [
+        for k, v in taint :
+        [
+          {
+            name  = "tolerations[${i}].${k}"
+            value = v
+          },
+        ]
+      ]
+    ])
+    iterator = each
+
+    content {
+      name  = each.value.name
+      value = each.value.value
+    }
+  }
 }

modules/kubernetes/hetzner.tf

@@ -75,4 +75,30 @@ resource "helm_release" "hcloud_csi" {
     name  = "controller.podAnnotations.secret"
     value = sha512(yamlencode(kubernetes_secret_v1.hcloud.data))
   }
+
+  # Allow running on control plane nodes
+  dynamic "set" {
+    for_each = flatten([
+      for i, taint in local.control_plane_taints :
+      [
+        for k, v in taint :
+        [
+          {
+            name  = "controller.tolerations[${i}].${k}"
+            value = v
+          },
+          {
+            name  = "node.tolerations[${i}].${k}"
+            value = v
+          },
+        ]
+      ]
+    ])
+    iterator = each
+
+    content {
+      name  = each.value.name
+      value = each.value.value
+    }
+  }
 }

modules/kubernetes/locals.tf

@@ -13,6 +13,12 @@
 # limitations under the License.
 
 locals {
+  control_plane_taints = [
+    {
+      key      = "CriticalAddonsOnly"
+      operator = "Exists"
+    },
+  ]
   kubeconfig            = yamldecode(var.kubeconfig)
   kubeconfig_clusters   = { for context in local.kubeconfig.clusters : context.name => context.cluster }
   kubeconfig_users      = { for context in local.kubeconfig.users : context.name => context.user }