From e3e8d233f599b84d03ccd556c47e19bd6673072d Mon Sep 17 00:00:00 2001 From: Simon Emms Date: Mon, 24 Jun 2024 21:15:50 +0000 Subject: [PATCH] feat(kubernetes): add hetzner ccm and csi --- modules/kubernetes/.terraform.lock.hcl | 23 ++++++ modules/kubernetes/README.md | 15 ++++ modules/kubernetes/hetzner.tf | 96 +++++++++++++++++++++++ modules/kubernetes/terraform.tf | 5 ++ modules/kubernetes/variables.tf | 35 +++++++++ stacks/dev/kubernetes/.terraform.lock.hcl | 23 ++++++ stacks/dev/kubernetes/terragrunt.hcl | 5 +- 7 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 modules/kubernetes/hetzner.tf diff --git a/modules/kubernetes/.terraform.lock.hcl b/modules/kubernetes/.terraform.lock.hcl index 7de0879..32a5de9 100644 --- a/modules/kubernetes/.terraform.lock.hcl +++ b/modules/kubernetes/.terraform.lock.hcl @@ -40,3 +40,26 @@ provider "registry.terraform.io/hashicorp/kubernetes" { "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } + +provider "registry.terraform.io/integrations/github" { + version = "6.2.2" + constraints = ">= 6.2.2, < 7.0.0" + hashes = [ + "h1:3gbrNGsK0dQ5zpN0qeHm3uNdWJl+f760+VtV2GJZ8Vg=", + "zh:43d7e5f1e11d67e38ca717016d209d6d9a6fa03321b489f91984351bfb143b69", + "zh:46e788395034b410bf59dfa43eb748a3d81ecfd23fc442349990fd7d92bd856a", + "zh:5234b7d5c5817ff7ebec29756050708372a071a701e2c8236e714a0bd29ef160", + "zh:74c485a241cc8e8cb99f988d38116fb14e51de896761fc9ca35a34ca5c999a7e", + "zh:7606789521c50937913ea13f851150828b5f9b8804ba80c5b2538c0b019339d8", + "zh:760fb0e74590459689c7159456b6e76f165634f7d0f89f5572d56b57d387f645", + "zh:7979d9085d809bb7d0db2c67e6c3443d1c18d12e51b72220dcb4cc5e883cd64a", + "zh:8bed25d8199bf8b2e7ccf67edc1a4a2fc041bd490b2c11565c669b80be43896c", + "zh:9ff82a6279fb7ae0cd9e44f1e73b64dd2aeca43d4d3096f3f2866b1ebbcb9431", + "zh:a886055ecd63ccb9b880e3c3301c0eca9acb108580d12519617554ae2be9a393", + "zh:c1f20386704919c7964a95daffcb29f494efb061abc28469840df4532833cecf", + "zh:cb6e9c4e33d6a57770073867e174c09c0eed401ee70473a688d20cb1cf0394f7", + "zh:f89ca130cc90b87dc25d036fe8f8cadb6fb53dc33368a032c5cee6275f3bcddc", + "zh:f94a2d1174091f04ed361192cdda9503baa3d161849d4f218c55a96bfb1ea33d", + "zh:fbd1fee2c9df3aa19cf8851ce134dea6e45ea01cb85695c1726670c285797e25", + ] +} diff --git a/modules/kubernetes/README.md b/modules/kubernetes/README.md index a8fb8b7..9e8211c 100644 --- a/modules/kubernetes/README.md +++ b/modules/kubernetes/README.md @@ -6,6 +6,7 @@ | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.0.0 | +| [github](#requirement\_github) | >= 6.2.2, < 7.0.0 | | [helm](#requirement\_helm) | >= 2.14.0, < 3.0.0 | | [kubernetes](#requirement\_kubernetes) | >= 2.31.0, < 3.0.0 | @@ -13,7 +14,9 @@ | Name | Version | |------|---------| +| [github](#provider\_github) | 6.2.2 | | [helm](#provider\_helm) | 2.14.0 | +| [kubernetes](#provider\_kubernetes) | 2.31.0 | ## Modules @@ -24,12 +27,24 @@ No modules. | Name | Type | |------|------| | [helm_release.cilium](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.hcloud_ccm](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [kubernetes_annotations.hcloud_ccm](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/annotations) | resource | +| [kubernetes_manifest.csi_driver](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/manifest) | resource | +| [kubernetes_secret_v1.hcloud](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/secret_v1) | resource | +| [github_release.csi_driver](https://registry.terraform.io/providers/integrations/github/latest/docs/data-sources/release) | data source | +| [github_repository_file.csi_driver](https://registry.terraform.io/providers/integrations/github/latest/docs/data-sources/repository_file) | data source | ## Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| | [cilium\_version](#input\_cilium\_version) | Version of Cilium to use - defaults to latest | `string` | `null` | no | +| [hcloud\_network\_name](#input\_hcloud\_network\_name) | Hetzner network name | `string` | n/a | yes | +| [hcloud\_token](#input\_hcloud\_token) | Hetzner API token | `string` | n/a | yes | +| [hetzner\_cloud\_config\_manager\_version](#input\_hetzner\_cloud\_config\_manager\_version) | Version of the HCloud CCM to use - defaults to latest | `string` | `null` | no | +| [hetzner\_csi\_driver\_owner](#input\_hetzner\_csi\_driver\_owner) | GitHub owner to get the CSI driver from | `string` | `"hetznercloud"` | no | +| [hetzner\_csi\_driver\_repo](#input\_hetzner\_csi\_driver\_repo) | GitHub repo to get the CSI driver from | `string` | `"csi-driver"` | no | +| [hetzner\_csi\_driver\_version](#input\_hetzner\_csi\_driver\_version) | Tag of the CSI driver to use - provide the tag name or latest | `string` | `"latest"` | no | | [k3s\_cluster\_cidr](#input\_k3s\_cluster\_cidr) | CIDR used for the k3s cluster | `string` | `"10.244.0.0/16"` | no | | [kube\_context](#input\_kube\_context) | Kubernetes context to use | `string` | `"default"` | no | | [kubeconfig](#input\_kubeconfig) | Kubeconfig for the cluster | `string` | n/a | yes | diff --git a/modules/kubernetes/hetzner.tf b/modules/kubernetes/hetzner.tf new file mode 100644 index 0000000..254c776 --- /dev/null +++ b/modules/kubernetes/hetzner.tf @@ -0,0 +1,96 @@ +# Copyright 2024 Simon Emms +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +resource "kubernetes_secret_v1" "hcloud" { + metadata { + name = "hcloud" + namespace = "kube-system" + } + + data = { + network = var.hcloud_network_name # Required by the CCM + token = var.hcloud_token # Required by the CSI + } + + depends_on = [helm_release.cilium] +} + +resource "helm_release" "hcloud_ccm" { + chart = "hcloud-cloud-controller-manager" + name = "hccm" + atomic = true + cleanup_on_fail = true + namespace = "kube-system" + repository = "https://charts.hetzner.cloud" + reset_values = true + version = var.hetzner_cloud_config_manager_version + wait = true + + set { + name = "networking.enabled" + value = "true" + } + + set { + name = "networking.clusterCIDR" + value = var.k3s_cluster_cidr + } + + depends_on = [kubernetes_secret_v1.hcloud] +} + +resource "kubernetes_annotations" "hcloud_ccm" { + api_version = "apps/v1" + kind = "Deployment" + metadata { + name = helm_release.hcloud_ccm.chart + namespace = helm_release.hcloud_ccm.namespace + } + template_annotations = { + "secret" = sha512(yamlencode(kubernetes_secret_v1.hcloud.data)) + } +} + +data "github_release" "csi_driver" { + repository = var.hetzner_csi_driver_repo + owner = var.hetzner_csi_driver_owner + + retrieve_by = var.hetzner_csi_driver_version == "latest" ? var.hetzner_csi_driver_version : "tag" + release_tag = var.hetzner_csi_driver_version != "latest" ? var.hetzner_csi_driver_version : null +} + +data "github_repository_file" "csi_driver" { + repository = "${var.hetzner_csi_driver_owner}/${var.hetzner_csi_driver_repo}" + branch = data.github_release.csi_driver.release_tag + file = "deploy/kubernetes/hcloud-csi.yml" +} + +resource "kubernetes_manifest" "csi_driver" { + for_each = { + for m in provider::kubernetes::manifest_decode_multi(data.github_repository_file.csi_driver.content) : + "${m.apiVersion}.${m.kind}.${m.metadata.name}" => m + } + manifest = each.value + + // The hcloud-csi-controller deployment has 5 containers + computed_fields = [ + "spec.template.spec.containers[0].resources", + "spec.template.spec.containers[1].resources", + "spec.template.spec.containers[2].resources", + "spec.template.spec.containers[3].resources", + "spec.template.spec.containers[4].resources", + ] + + depends_on = [kubernetes_secret_v1.hcloud] +} diff --git a/modules/kubernetes/terraform.tf b/modules/kubernetes/terraform.tf index 9909e16..6e869d4 100644 --- a/modules/kubernetes/terraform.tf +++ b/modules/kubernetes/terraform.tf @@ -15,6 +15,11 @@ terraform { required_version = ">= 1.0.0" required_providers { + # Should use a GITHUB_TOKEN envvar to avoid rate-limiting + github = { + source = "integrations/github" + version = ">= 6.2.2, < 7.0.0" + } helm = { source = "hashicorp/helm" version = ">= 2.14.0, < 3.0.0" diff --git a/modules/kubernetes/variables.tf b/modules/kubernetes/variables.tf index 171688b..10ababb 100644 --- a/modules/kubernetes/variables.tf +++ b/modules/kubernetes/variables.tf @@ -19,6 +19,41 @@ variable "cilium_version" { default = null } +variable "hcloud_network_name" { + type = string + description = "Hetzner network name" +} + +variable "hcloud_token" { + type = string + description = "Hetzner API token" + sensitive = true +} + +variable "hetzner_cloud_config_manager_version" { + type = string + description = "Version of the HCloud CCM to use - defaults to latest" + default = null +} + +variable "hetzner_csi_driver_owner" { + type = string + description = "GitHub owner to get the CSI driver from" + default = "hetznercloud" +} + +variable "hetzner_csi_driver_repo" { + type = string + description = "GitHub repo to get the CSI driver from" + default = "csi-driver" +} + +variable "hetzner_csi_driver_version" { + type = string + description = "Tag of the CSI driver to use - provide the tag name or latest" + default = "latest" +} + variable "k3s_cluster_cidr" { type = string description = "CIDR used for the k3s cluster" diff --git a/stacks/dev/kubernetes/.terraform.lock.hcl b/stacks/dev/kubernetes/.terraform.lock.hcl index 7de0879..32a5de9 100644 --- a/stacks/dev/kubernetes/.terraform.lock.hcl +++ b/stacks/dev/kubernetes/.terraform.lock.hcl @@ -40,3 +40,26 @@ provider "registry.terraform.io/hashicorp/kubernetes" { "zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c", ] } + +provider "registry.terraform.io/integrations/github" { + version = "6.2.2" + constraints = ">= 6.2.2, < 7.0.0" + hashes = [ + "h1:3gbrNGsK0dQ5zpN0qeHm3uNdWJl+f760+VtV2GJZ8Vg=", + "zh:43d7e5f1e11d67e38ca717016d209d6d9a6fa03321b489f91984351bfb143b69", + "zh:46e788395034b410bf59dfa43eb748a3d81ecfd23fc442349990fd7d92bd856a", + "zh:5234b7d5c5817ff7ebec29756050708372a071a701e2c8236e714a0bd29ef160", + "zh:74c485a241cc8e8cb99f988d38116fb14e51de896761fc9ca35a34ca5c999a7e", + "zh:7606789521c50937913ea13f851150828b5f9b8804ba80c5b2538c0b019339d8", + "zh:760fb0e74590459689c7159456b6e76f165634f7d0f89f5572d56b57d387f645", + "zh:7979d9085d809bb7d0db2c67e6c3443d1c18d12e51b72220dcb4cc5e883cd64a", + "zh:8bed25d8199bf8b2e7ccf67edc1a4a2fc041bd490b2c11565c669b80be43896c", + "zh:9ff82a6279fb7ae0cd9e44f1e73b64dd2aeca43d4d3096f3f2866b1ebbcb9431", + "zh:a886055ecd63ccb9b880e3c3301c0eca9acb108580d12519617554ae2be9a393", + "zh:c1f20386704919c7964a95daffcb29f494efb061abc28469840df4532833cecf", + "zh:cb6e9c4e33d6a57770073867e174c09c0eed401ee70473a688d20cb1cf0394f7", + "zh:f89ca130cc90b87dc25d036fe8f8cadb6fb53dc33368a032c5cee6275f3bcddc", + "zh:f94a2d1174091f04ed361192cdda9503baa3d161849d4f218c55a96bfb1ea33d", + "zh:fbd1fee2c9df3aa19cf8851ce134dea6e45ea01cb85695c1726670c285797e25", + ] +} diff --git a/stacks/dev/kubernetes/terragrunt.hcl b/stacks/dev/kubernetes/terragrunt.hcl index c06f9f9..bd0f2c5 100644 --- a/stacks/dev/kubernetes/terragrunt.hcl +++ b/stacks/dev/kubernetes/terragrunt.hcl @@ -25,6 +25,7 @@ dependency "hetzner" { } inputs = { - k3s_cluster_cidr = dependency.hetzner.outputs.k3s_cluster_cidr - kubeconfig = dependency.hetzner.outputs.kubeconfig + hcloud_network_name = dependency.hetzner.outputs.hcloud_network_name + k3s_cluster_cidr = dependency.hetzner.outputs.k3s_cluster_cidr + kubeconfig = dependency.hetzner.outputs.kubeconfig }