feat(infra): install k3s to additional managers

Author: mrsimonemms

infrastructure/README.md

@@ -34,6 +34,7 @@ No modules.
 | [hcloud_placement_group.kubernetes](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/placement_group) | resource |
 | [hcloud_server.manager](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/server) | resource |
 | [hcloud_ssh_key.server](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/ssh_key) | resource |
+| [ssh_resource.additional_managers](https://registry.terraform.io/providers/loafoe/ssh/2.7.0/docs/resources/resource) | resource |
 | [ssh_resource.initial_manager](https://registry.terraform.io/providers/loafoe/ssh/2.7.0/docs/resources/resource) | resource |
 | [ssh_resource.server_ready](https://registry.terraform.io/providers/loafoe/ssh/2.7.0/docs/resources/resource) | resource |
 | [ssh_sensitive_resource.join_token](https://registry.terraform.io/providers/loafoe/ssh/2.7.0/docs/resources/sensitive_resource) | resource |

infrastructure/k3s.tf

@@ -15,9 +15,30 @@
 locals {
   k3s_initial_manager            = hcloud_server.manager[0]
   k3s_initial_manager_private_ip = tolist(local.k3s_initial_manager.network)[0].ip
-  k3s_tls_san                    = var.k3s_manager_count > 1 ? hcloud_load_balancer.k3s_manager[0].ipv4 : local.k3s_initial_manager.ipv4_address
+  k3s_access_address             = var.k3s_manager_count > 1 ? hcloud_load_balancer.k3s_manager[0].ipv4 : local.k3s_initial_manager.ipv4_address
   k3s_join_token                 = chomp(ssh_sensitive_resource.join_token.result)
   k3s_kubeconfig                 = chomp(ssh_sensitive_resource.kubeconfig.result)
+  k3s_manager_install_command = join(" ", concat(
+    [
+      "server",
+      "--write-kubeconfig-mode=0644",
+      "--disable servicelb",
+      "--disable traefik",
+      "--node-name=$(hostname -f)",
+      "--node-external-ip=$(hostname -I | awk '{print $1}')",  # Public IP
+      "--node-ip=$(hostname -I | awk '{print $2}')",           # Private IP
+      "--advertise-address=$(hostname -I | awk '{print $2}')", # Private IP
+    ],
+    # Set TLS SANs - first, add load balancer or managers's public address
+    var.k3s_manager_count > 1 ? [
+      "--cluster-init",
+      "--tls-san=${hcloud_load_balancer.k3s_manager[0].ipv4}"
+      ] : [
+      "--tls-san=${local.k3s_initial_manager_private_ip}"
+    ],
+    # Now, add all the servers
+    [for o in hcloud_server.manager : "--tls-san=${tolist(o.network)[0].ip}"]
+  ))
 }
 
 resource "ssh_resource" "server_ready" {
@@ -26,6 +47,7 @@ resource "ssh_resource" "server_ready" {
   host        = hcloud_server.manager[count.index].ipv4_address
   user        = local.machine_user
   private_key = file(var.ssh_key)
+  port        = var.ssh_port
 
   timeout     = "5m"
   retry_delay = "5s"
@@ -47,26 +69,19 @@ resource "ssh_resource" "initial_manager" {
   host        = local.k3s_initial_manager.ipv4_address
   user        = local.machine_user
   private_key = file(var.ssh_key)
+  port        = var.ssh_port
 
   commands = [
     format(
       "curl -sfL %s | INSTALL_K3S_EXEC=\"%s\" %s sh -",
       var.k3s_download_url,
       // Install configuration
-      join(" ", [
-        "server",
-        "--write-kubeconfig-mode=0644",
-        "--disable servicelb",
-        "--disable traefik",
-        "--tls-san=${local.k3s_tls_san}",
-        "--node-name=$(hostname -f)",
-        "--node-external-ip=$(hostname -I | awk '{print $1}')",  # Public IP
-        "--node-ip=$(hostname -I | awk '{print $2}')",           # Private IP
-        "--advertise-address=$(hostname -I | awk '{print $2}')", # Private IP
-      ]),
+      local.k3s_manager_install_command,
       // Other k3s configuration
       ""
-    )
+    ),
+    # Ensure k3s is running
+    "sudo systemctl start k3s"
   ]
 
   timeout     = "5m"
@@ -86,6 +101,7 @@ resource "ssh_sensitive_resource" "join_token" {
   host        = local.k3s_initial_manager.ipv4_address
   user        = local.machine_user
   private_key = file(var.ssh_key)
+  port        = var.ssh_port
 
   commands = [
     "sudo cat /var/lib/rancher/k3s/server/token"
@@ -103,9 +119,10 @@ resource "ssh_sensitive_resource" "kubeconfig" {
   host        = local.k3s_initial_manager.ipv4_address
   user        = local.machine_user
   private_key = file(var.ssh_key)
+  port        = var.ssh_port
 
   commands = [
-    format("sudo cat /etc/rancher/k3s/k3s.yaml | sed 's/%s/%s/'", "127.0.0.1", local.k3s_tls_san)
+    format("sudo cat /etc/rancher/k3s/k3s.yaml | sed 's/%s/%s/'", "127.0.0.1", local.k3s_access_address)
   ]
 
   timeout     = "5m"
@@ -115,3 +132,40 @@ resource "ssh_sensitive_resource" "kubeconfig" {
     ssh_resource.initial_manager
   ]
 }
+
+resource "ssh_resource" "additional_managers" {
+  count = var.k3s_manager_count - 1
+
+  host        = hcloud_server.manager[count.index + 1].ipv4_address
+  user        = local.machine_user
+  private_key = file(var.ssh_key)
+  port        = var.ssh_port
+
+  commands = [
+    format(
+      "curl -sfL %s | INSTALL_K3S_EXEC=\"%s\" %s sh -",
+      var.k3s_download_url,
+      // Install configuration
+      local.k3s_manager_install_command,
+      // Other k3s configuration
+      join(" ", [
+        "K3S_URL=https://${local.k3s_initial_manager_private_ip}:6443",
+        "K3S_TOKEN=${local.k3s_join_token}"
+      ])
+    ),
+    # Ensure k3s is running
+    "sudo systemctl start k3s"
+  ]
+
+  timeout     = "5m"
+  retry_delay = "5s"
+
+
+  triggers = {
+    always_run = timestamp()
+  }
+
+  depends_on = [
+    ssh_resource.initial_manager
+  ]
+}

infrastructure/network.tf

@@ -56,6 +56,7 @@ resource "hcloud_firewall" "name" {
         source_ips = [
           hcloud_network.network.ip_range
         ]
+        protocol = "udp"
       },
       {
         description = "Allow access to Kubernetes API"

infrastructure/server.tf

@@ -46,6 +46,9 @@ resource "hcloud_server" "manager" {
 
   network {
     network_id = hcloud_network.network.id
+    # Set the alias_ips to avoid this triggering an update each run
+    # @link https://github.com/hetznercloud/terraform-provider-hcloud/issues/650#issuecomment-1497160625
+    alias_ips = []
   }
 
   public_net {