From 94d354b6995138a63c34a8e6905710846499a815 Mon Sep 17 00:00:00 2001 From: Simon Emms Date: Sat, 3 Aug 2024 10:44:35 +0000 Subject: [PATCH] feat(hetzner): build servers --- modules/hetzner/README.md | 24 ++++- modules/hetzner/files/cloud-config.yaml | 32 ++++++ modules/hetzner/load_balancer.tf | 60 ++++++++++++ modules/hetzner/locals.tf | 32 +++++- modules/hetzner/networks.tf | 16 ++- modules/hetzner/output.tf | 50 ++++++++++ modules/hetzner/server.tf | 125 ++++++++++++++++++++++++ modules/hetzner/variables.tf | 58 +++++++++-- stacks/dev/hetzner/terragrunt.hcl | 13 +++ 9 files changed, 397 insertions(+), 13 deletions(-) create mode 100644 modules/hetzner/files/cloud-config.yaml create mode 100644 modules/hetzner/load_balancer.tf create mode 100644 modules/hetzner/output.tf create mode 100644 modules/hetzner/server.tf diff --git a/modules/hetzner/README.md b/modules/hetzner/README.md index d51184c..22e8a35 100644 --- a/modules/hetzner/README.md +++ b/modules/hetzner/README.md @@ -23,8 +23,17 @@ No modules. | Name | Type | |------|------| | [hcloud_firewall.firewall](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/firewall) | resource | +| [hcloud_load_balancer.k3s_manager](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/load_balancer) | resource | +| [hcloud_load_balancer_network.k3s_manager](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/load_balancer_network) | resource | +| [hcloud_load_balancer_service.k3s_manager](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/load_balancer_service) | resource | +| [hcloud_load_balancer_target.k3s_manager](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/load_balancer_target) | resource | | [hcloud_network.network](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/network) | resource | | [hcloud_network_subnet.subnet](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/network_subnet) | resource | +| [hcloud_placement_group.managers](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/placement_group) | resource | +| [hcloud_placement_group.workers](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/placement_group) | resource | +| [hcloud_server.manager](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/server) | resource | +| [hcloud_server.workers](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/server) | resource | +| [hcloud_ssh_key.server](https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/ssh_key) | resource | ## Inputs @@ -32,14 +41,25 @@ No modules. |------|-------------|------|---------|:--------:| | [firewall\_allow\_api\_access](#input\_firewall\_allow\_api\_access) | CIDR range to allow access to the Kubernetes API | `list(string)` | 
[
  "0.0.0.0/0",
  "::/0"
]
| no | | [firewall\_allow\_ssh\_access](#input\_firewall\_allow\_ssh\_access) | CIDR range to allow access to the servers via SSH | `list(string)` | 
[
  "0.0.0.0/0",
  "::/0"
]
| no | -| [name](#input\_name) | Name of project | `string` | `"infrastructure"` | no | +| [k3s\_manager\_load\_balancer\_algorithm](#input\_k3s\_manager\_load\_balancer\_algorithm) | Algorithm to use for the k3s manager load balancer | `string` | `"round_robin"` | no | +| [k3s\_manager\_load\_balancer\_type](#input\_k3s\_manager\_load\_balancer\_type) | Load balancer type for the k3s manager nodes | `string` | `"lb11"` | no | +| [k3s\_manager\_pool](#input\_k3s\_manager\_pool) | Manager pool configuration | 
object({
    name        = optional(string, "manager")
    server_type = optional(string, "cx22")
    count       = optional(number, 1)
    image       = optional(string, "ubuntu-24.04")
  })
| `{}` | no | +| [k3s\_worker\_pools](#input\_k3s\_worker\_pools) | Worker pools configuration | 
list(object({
    name        = string
    server_type = optional(string, "cx22")
    count       = optional(number, 1)
    image       = optional(string, "ubuntu-24.04")
    location    = optional(string) # Defaults to var.location if not set
  }))
| `[]` | no | +| [location](#input\_location) | Location to use. This is a single datacentre. | `string` | `"nbg1"` | no | +| [name](#input\_name) | Name of project | `string` | `"k3s"` | no | | [network\_subnet](#input\_network\_subnet) | Subnet of the main network | `string` | `"10.0.0.0/16"` | no | | [network\_type](#input\_network\_type) | Type of network to use | `string` | `"cloud"` | no | | [region](#input\_region) | Region to use. This covers multiple datacentres. | `string` | `"eu-central"` | no | +| [ssh\_key\_public](#input\_ssh\_key\_public) | Path to the public SSH key | `string` | `"~/.ssh/id_ed25519.pub"` | no | | [ssh\_port](#input\_ssh\_port) | Port to use for SSH access | `number` | `22` | no | | [workspace](#input\_workspace) | Terraform workspace name | `string` | `"default"` | no | ## Outputs -No outputs. +| Name | Description | +|------|-------------| +| [kube\_api\_server](#output\_kube\_api\_server) | Kubernetes API server address | +| [pools](#output\_pools) | Servers created | +| [ssh\_port](#output\_ssh\_port) | SSH port for server | +| [ssh\_user](#output\_ssh\_user) | SSH user for server | diff --git a/modules/hetzner/files/cloud-config.yaml b/modules/hetzner/files/cloud-config.yaml new file mode 100644 index 0000000..62afb1b --- /dev/null +++ b/modules/hetzner/files/cloud-config.yaml @@ -0,0 +1,32 @@ +#cloud-config + +package_reboot_if_required: true +package_update: true +package_upgrade: true +packages: + - curl + - yq +runcmd: + - [service, sshd, restart] + - [rm, -f, /root/.ssh/authorized_keys] + - chown ${user}:${user} "/home/${user}" +timezone: UTC +users: + - default + - name: "${user}" + gecos: "${user}" + sudo: ALL=(ALL) NOPASSWD:ALL + lock_passwd: true + shell: /bin/bash + ssh_authorized_keys: + - "${chomp(publicKey)}" +write_files: + - path: /etc/ssh/sshd_config.d/ssh.conf + content: | + PasswordAuthentication no + PermitRootLogin no + Port ${sshPort} + - path: /etc/environment + content: | + KUBECONFIG="/etc/rancher/k3s/k3s.yaml" + append: true diff --git a/modules/hetzner/load_balancer.tf b/modules/hetzner/load_balancer.tf new file mode 100644 index 0000000..3cd4a1b --- /dev/null +++ b/modules/hetzner/load_balancer.tf @@ -0,0 +1,60 @@ +# Copyright 2024 Simon Emms +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "hcloud_load_balancer" "k3s_manager" { + count = var.k3s_manager_pool.count > 1 ? 1 : 0 + + name = format(local.name_format, "load_balancer") + load_balancer_type = var.k3s_manager_load_balancer_type + network_zone = var.region + + algorithm { + type = var.k3s_manager_load_balancer_algorithm + } + + labels = merge(local.labels, {}) +} + +resource "hcloud_load_balancer_network" "k3s_manager" { + count = var.k3s_manager_pool.count > 1 ? 1 : 0 + + load_balancer_id = hcloud_load_balancer.k3s_manager[count.index].id + network_id = hcloud_network.network.id + + depends_on = [ + hcloud_network_subnet.subnet + ] +} + +resource "hcloud_load_balancer_service" "k3s_manager" { + count = var.k3s_manager_pool.count > 1 ? 1 : 0 + + load_balancer_id = hcloud_load_balancer.k3s_manager[count.index].id + protocol = "tcp" + listen_port = local.kubernetes_api_port + destination_port = local.kubernetes_api_port +} + +resource "hcloud_load_balancer_target" "k3s_manager" { + count = var.k3s_manager_pool.count > 1 ? 1 : 0 + + load_balancer_id = hcloud_load_balancer.k3s_manager[count.index].id + type = "label_selector" + label_selector = join(",", [for key, value in local.k3s_manager_labels : "${key}=${value}"]) + use_private_ip = true + + depends_on = [ + hcloud_load_balancer_network.k3s_manager + ] +} diff --git a/modules/hetzner/locals.tf b/modules/hetzner/locals.tf index cfe229d..f77519c 100644 --- a/modules/hetzner/locals.tf +++ b/modules/hetzner/locals.tf @@ -15,17 +15,43 @@ locals { global_ipv4_cidr = "0.0.0.0/0" global_ipv6_cidr = "::/0" + k3s_manager_labels = merge(local.labels, { + format(local.label_namespace, "type") = "manager" + }) + k3s_worker_labels = merge(local.labels, { + format(local.label_namespace, "type") = "worker" + }) + # Convert pools into individual servers + k3s_worker_pools = flatten([ + for w in var.k3s_worker_pools : [ + for n in range(w.count) : + merge( + w, + { + location = w.location != null ? w.location : var.location + name = "${w.name}-${n}" + pool = w.name + } + ) + ] + ]) + kubernetes_api_port = 6443 labels = { format(local.label_namespace, "project") = var.name format(local.label_namespace, "workspace") = local.workspace_name } - kubernetes_api_port = 6443 - label_namespace = "simonemms.com/%s" + label_namespace = "simonemms.com/%s" name_format = join("-", [ "hetzner", var.name, "%s", # resource name local.workspace_name - ]) # use `format(local.name_format, "")` to use this + ]) # use `format(local.name_format, "")` to use this + ssh_user = "k3smanager" + user_data = templatefile("${path.module}/files/cloud-config.yaml", { + sshPort = var.ssh_port + publicKey = hcloud_ssh_key.server.public_key + user = local.ssh_user + }) workspace_name = replace(var.workspace, "/[\\W]/", "") # alphanumeric workspace name } diff --git a/modules/hetzner/networks.tf b/modules/hetzner/networks.tf index 8190bcd..9aa4c42 100644 --- a/modules/hetzner/networks.tf +++ b/modules/hetzner/networks.tf @@ -30,12 +30,21 @@ resource "hcloud_firewall" "firewall" { name = format(local.name_format, "firewall") dynamic "rule" { - for_each = [ + for_each = [for each in [ { description = "SSH port" port = var.ssh_port source_ips = var.firewall_allow_ssh_access }, + { + description = "Allow ICMP (ping)" + source_ips = [ + local.global_ipv4_cidr, + local.global_ipv6_cidr, + ] + protocol = "icmp" + port = null + }, { description = "Allow all TCP traffic on private network" source_ips = [ @@ -49,12 +58,15 @@ resource "hcloud_firewall" "firewall" { ] protocol = "udp" }, + # Direct public access only allowed if single manager node { description = "Allow access to Kubernetes API" port = local.kubernetes_api_port source_ips = var.firewall_allow_api_access + disabled = var.k3s_manager_pool.count > 1 } - ] + ] : each if lookup(each, "disabled", false) != true] + content { description = lookup(rule.value, "description", "") destination_ips = lookup(rule.value, "destination_ips", []) diff --git a/modules/hetzner/output.tf b/modules/hetzner/output.tf new file mode 100644 index 0000000..f43ad77 --- /dev/null +++ b/modules/hetzner/output.tf @@ -0,0 +1,50 @@ +# Copyright 2024 Simon Emms +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +output "kube_api_server" { + description = "Kubernetes API server address" + value = var.k3s_manager_pool.count > 1 ? hcloud_load_balancer.k3s_manager[0].ipv4 : hcloud_server.manager[0].ipv4_address +} + +output "pools" { + description = "Servers created" + value = merge( + { + managers : [ + for m in hcloud_server.manager : { + name = m.name + ipv4_address = m.ipv4_address + ipv6_address = m.ipv6_address + } + ] + }, + { + for w in local.k3s_worker_pools : w.pool => { + name = hcloud_server.workers[w.name].name + ipv4_address = hcloud_server.workers[w.name].ipv4_address + ipv6_address = hcloud_server.workers[w.name].ipv6_address + }... + } + ) +} + +output "ssh_port" { + description = "SSH port for server" + value = var.ssh_port +} + +output "ssh_user" { + description = "SSH user for server" + value = local.ssh_user +} diff --git a/modules/hetzner/server.tf b/modules/hetzner/server.tf new file mode 100644 index 0000000..dd8ded2 --- /dev/null +++ b/modules/hetzner/server.tf @@ -0,0 +1,125 @@ +# Copyright 2024 Simon Emms +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +########## +# Common # +########## +resource "hcloud_ssh_key" "server" { + name = format(local.name_format, "ssh_key") + public_key = file(var.ssh_key_public) + + labels = merge(local.k3s_manager_labels, {}) +} + +############ +# Managers # +############ +resource "hcloud_placement_group" "managers" { + count = var.k3s_manager_pool.count > 1 ? 1 : 0 + + name = format(local.name_format, "manager") + type = "spread" + + labels = merge(local.k3s_manager_labels, {}) +} + +resource "hcloud_server" "manager" { + count = var.k3s_manager_pool.count + + name = format(local.name_format, "manager-${count.index}") + image = var.k3s_manager_pool.image + server_type = var.k3s_manager_pool.server_type + location = var.location + ssh_keys = [ + hcloud_ssh_key.server.id + ] + + # No placement group if single node manager + placement_group_id = try(hcloud_placement_group.managers[0].id, null) + + user_data = local.user_data + + network { + network_id = hcloud_network.network.id + # Set the alias_ips to avoid this triggering an update each run + # @link https://github.com/hetznercloud/terraform-provider-hcloud/issues/650#issuecomment-1497160625 + alias_ips = [] + } + + public_net { + ipv4_enabled = true + ipv6_enabled = true + } + + labels = merge(local.k3s_manager_labels, {}) + + depends_on = [ + hcloud_load_balancer_network.k3s_manager + ] + + lifecycle { + ignore_changes = [ + ssh_keys + ] + } +} + +################## +# Static workers # +################## +resource "hcloud_placement_group" "workers" { + for_each = toset([for i in var.k3s_worker_pools : i.name]) + + name = format(local.name_format, each.value) + type = "spread" + + labels = merge(local.k3s_worker_labels, {}) +} + +resource "hcloud_server" "workers" { + for_each = { for i in local.k3s_worker_pools : i.name => i } + + name = format(local.name_format, each.value.name) + image = each.value.image + server_type = each.value.server_type + location = each.value.location + ssh_keys = [ + hcloud_ssh_key.server.id + ] + placement_group_id = hcloud_placement_group.workers[each.value.pool].id + + user_data = local.user_data + + network { + network_id = hcloud_network.network.id + # Set the alias_ips to avoid this triggering an update each run + # @link https://github.com/hetznercloud/terraform-provider-hcloud/issues/650#issuecomment-1497160625 + alias_ips = [] + } + + public_net { + ipv4_enabled = true + ipv6_enabled = true + } + + labels = merge(local.k3s_worker_labels, { + format(local.label_namespace, "pool") = each.value.pool + }) + + lifecycle { + ignore_changes = [ + ssh_keys + ] + } +} diff --git a/modules/hetzner/variables.tf b/modules/hetzner/variables.tf index 8f548c9..b83ca27 100644 --- a/modules/hetzner/variables.tf +++ b/modules/hetzner/variables.tf @@ -31,16 +31,56 @@ variable "firewall_allow_ssh_access" { ] } -# variable "location" { -# type = string -# description = "Location to use. This is a single datacentre." -# default = "nbg1" -# } +variable "k3s_manager_load_balancer_algorithm" { + type = string + description = "Algorithm to use for the k3s manager load balancer" + default = "round_robin" +} + +variable "k3s_manager_load_balancer_type" { + type = string + description = "Load balancer type for the k3s manager nodes" + default = "lb11" +} + +variable "k3s_manager_pool" { + type = object({ + name = optional(string, "manager") + server_type = optional(string, "cx22") + count = optional(number, 1) + image = optional(string, "ubuntu-24.04") + }) + description = "Manager pool configuration" + default = {} + + validation { + condition = var.k3s_manager_pool.count >= 1 && var.k3s_manager_pool.count % 2 == 1 + error_message = "Invalid k3s_manager_pool.count given." + } +} + +variable "k3s_worker_pools" { + type = list(object({ + name = string + server_type = optional(string, "cx22") + count = optional(number, 1) + image = optional(string, "ubuntu-24.04") + location = optional(string) # Defaults to var.location if not set + })) + description = "Worker pools configuration" + default = [] +} + +variable "location" { + type = string + description = "Location to use. This is a single datacentre." + default = "nbg1" +} variable "name" { type = string description = "Name of project" - default = "infrastructure" + default = "k3s" } variable "network_type" { @@ -66,6 +106,12 @@ variable "region" { default = "eu-central" } +variable "ssh_key_public" { + type = string + description = "Path to the public SSH key" + default = "~/.ssh/id_ed25519.pub" +} + variable "ssh_port" { type = number description = "Port to use for SSH access" diff --git a/stacks/dev/hetzner/terragrunt.hcl b/stacks/dev/hetzner/terragrunt.hcl index 5f2e7e4..b155666 100644 --- a/stacks/dev/hetzner/terragrunt.hcl +++ b/stacks/dev/hetzner/terragrunt.hcl @@ -21,5 +21,18 @@ include { } inputs = { + k3s_manager_pool = { + count = 3 + } + k3s_worker_pools = [ + { + count = 2 + name = "pool1" + }, + { + count = 1 + name = "pool2" + } + ] network_subnet = "10.2.0.0/16" }