From c34397c7ed3480b00e2db1212927d6cbb7326446 Mon Sep 17 00:00:00 2001
From: Simon Emms <simon@simonemms.com>
Date: Sun, 23 Jun 2024 21:33:43 +0000
Subject: [PATCH] install k3s properly with cilium

---
 modules/hetzner/README.md              |  1 +
 modules/hetzner/files/k3s-manager.yaml |  4 ++
 modules/hetzner/k3s.tf                 | 52 +++++++++++++++++++++++++-
 stacks/dev/hetzner/terragrunt.hcl      |  3 +-
 4 files changed, 58 insertions(+), 2 deletions(-)

diff --git a/modules/hetzner/README.md b/modules/hetzner/README.md
index 1682c13..132266d 100644
--- a/modules/hetzner/README.md
+++ b/modules/hetzner/README.md
@@ -34,6 +34,7 @@ No modules.
 | [hcloud_placement_group.managers](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/placement_group) | resource |
 | [hcloud_server.manager](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/server) | resource |
 | [hcloud_ssh_key.server](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/ssh_key) | resource |
+| [ssh_resource.cilium_install](https://registry.terraform.io/providers/loafoe/ssh/latest/docs/resources/resource) | resource |
 | [ssh_resource.initial_manager](https://registry.terraform.io/providers/loafoe/ssh/latest/docs/resources/resource) | resource |
 | [ssh_resource.server_ready](https://registry.terraform.io/providers/loafoe/ssh/latest/docs/resources/resource) | resource |
 | [ssh_sensitive_resource.additional_managers](https://registry.terraform.io/providers/loafoe/ssh/latest/docs/resources/sensitive_resource) | resource |
diff --git a/modules/hetzner/files/k3s-manager.yaml b/modules/hetzner/files/k3s-manager.yaml
index 88df068..33c4b8b 100644
--- a/modules/hetzner/files/k3s-manager.yaml
+++ b/modules/hetzner/files/k3s-manager.yaml
@@ -25,3 +25,7 @@ write_files:
       PasswordAuthentication no
       PermitRootLogin no
       Port ${sshPort}
+  - path: /etc/environment
+    content: |
+      KUBECONFIG="/etc/rancher/k3s/k3s.yaml"
+    append: true
diff --git a/modules/hetzner/k3s.tf b/modules/hetzner/k3s.tf
index c546a72..0652597 100644
--- a/modules/hetzner/k3s.tf
+++ b/modules/hetzner/k3s.tf
@@ -21,6 +21,12 @@ locals {
       "servicelb",
       "traefik"
     ]
+    disable-cloud-controller = true
+    disable-network-policy   = true
+    flannel-backend          = "none"
+    kubelet-arg = [
+      "cloud-provider=external"
+    ]
     tls-san = concat(
       [local.k3s_access_address],
       [for o in hcloud_server.manager : tolist(o.network)[0].ip]
@@ -70,6 +76,50 @@ resource "ssh_resource" "initial_manager" {
     ssh_resource.server_ready
   ]
 }
+resource "ssh_resource" "cilium_install" {
+  host        = local.k3s_initial_manager.ipv4_address
+  user        = local.machine_user
+  private_key = file(var.ssh_key)
+  port        = var.ssh_port
+
+  # https://docs.cilium.io/en/stable/gettingstarted/k8s-install-default
+  commands = [
+    "echo $(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt) > /tmp/cli_version",
+    "echo amd64 > /tmp/cli_arch",
+    "if [ \"$(uname -m)\" = \"aarch64\" ]; then echo arm64 > /tmp/cli_arch; fi",
+    "curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/$(cat /tmp/cli_version)/cilium-linux-$(cat /tmp/cli_arch).tar.gz{,.sha256sum}",
+    "sha256sum --check cilium-linux-$(cat /tmp/cli_arch).tar.gz.sha256sum",
+    "sudo tar xzvfC cilium-linux-$(cat /tmp/cli_arch).tar.gz /usr/local/bin",
+    "rm cilium-linux-$(cat /tmp/cli_arch).tar.gz{,.sha256sum}",
+    "cilium install"
+  ]
+
+  timeout     = "1m"
+  retry_delay = "5s"
+
+  depends_on = [
+    ssh_resource.initial_manager
+  ]
+}
+
+# resource "ssh_resource" "cilium_ready" {
+#   host        = local.k3s_initial_manager.ipv4_address
+#   user        = local.machine_user
+#   private_key = file(var.ssh_key)
+#   port        = var.ssh_port
+
+#   commands = [
+#     "cilium status --wait",
+#     "cilium connectivity test"
+#   ]
+
+#   timeout     = "5m"
+#   retry_delay = "5s"
+
+#   depends_on = [
+#     ssh_resource.cilium_install
+#   ]
+# }
 
 resource "ssh_sensitive_resource" "join_token" {
   host        = local.k3s_initial_manager.ipv4_address
@@ -117,7 +167,7 @@ resource "ssh_sensitive_resource" "additional_managers" {
 
   commands = local.k3s_install_command
 
-  file { # systemctl status k3s.service" and "journalctl -xeu k3s.service
+  file {
     content = yamlencode(merge(local.k3s_common_manager_config, {
       advertise-address = tolist(hcloud_server.manager[count.index + 1].network)[0].ip # Private IP
       node-name         = hcloud_server.manager[count.index + 1].name
diff --git a/stacks/dev/hetzner/terragrunt.hcl b/stacks/dev/hetzner/terragrunt.hcl
index 5f2e7e4..94befd1 100644
--- a/stacks/dev/hetzner/terragrunt.hcl
+++ b/stacks/dev/hetzner/terragrunt.hcl
@@ -21,5 +21,6 @@ include {
 }
 
 inputs = {
-  network_subnet = "10.2.0.0/16"
+  k3s_manager_count = 3
+  network_subnet    = "10.2.0.0/16"
 }