Merge pull request #28 from rossf7/cluster-infra

Provision cluster and bootstrap flux

.github/workflows/tofu.yaml

@@ -0,0 +1,47 @@
+name: OpenTofu
+
+on:
+  push:
+    branches:
+    - main
+    paths:
+    - 'infrastructure/**'
+
+defaults:
+  run:
+    working-directory: infrastructure/equinix-metal
+
+jobs:
+  tofu:
+    name: OpenTofu
+    runs-on: ubuntu-latest
+    env:
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      TF_VAR_equinix_auth_token: ${{ secrets.EQUINIX_AUTH_TOKEN }}
+      TF_VAR_equinix_project_id: ${{ secrets.EQUINIX_PROJECT_ID }}
+      TF_VAR_flux_github_token: ${{ secrets.FLUX_GITHUB_TOKEN }}
+      TF_VAR_k3s_token: ${{ secrets.K3S_TOKEN }}
+      TF_VAR_ssh_public_key: ${{ secrets.SSH_PUBLIC_KEY }}
+    steps:
+    - uses: actions/checkout@v3
+    - uses: opentofu/setup-opentofu@v1
+
+    - name: Add SSH key
+      run: |
+        mkdir -p ~/.ssh
+        echo "${{ secrets.SSH_PRIVATE_KEY }}" > ~/.ssh/id_rsa
+        chmod 600 ~/.ssh/id_rsa
+        ssh-agent -a $SSH_AUTH_SOCK > /dev/null
+        ssh-add ~/.ssh/id_rsa
+      env:
+        SSH_AUTH_SOCK: /tmp/ssh_agent.sock
+
+    - name: tofu init
+      run: tofu init
+
+    - name: tofu plan
+      run: tofu plan
+
+    - name: tofu apply
+      run: tofu apply -auto-approve

.gitignore

@@ -0,0 +1,2 @@
+infrastructure/equinix-metal/.terraform/
+infrastructure/equinix-metal/terraform.tfvars

infrastructure/equinix-metal/main.tf

@@ -0,0 +1,155 @@
+terraform {
+  required_providers {
+    equinix = {
+      source  = "equinix/equinix"
+      version = "1.22.0"
+    }
+    null = {
+      source = "hashicorp/null"
+      version = "3.2.2"
+    }
+  }
+
+  backend "s3" {
+    bucket  = "green-reviews-state-bucket"
+    key     = "opentofu/terraform.tfstate"
+    region  = "eu-central-1"
+    encrypt = true
+  }
+}
+
+provider "equinix" {
+  auth_token = var.equinix_auth_token
+}
+
+resource "equinix_metal_project_ssh_key" "ssh_key" {
+  name       = var.cluster_name
+  project_id = var.equinix_project_id
+  public_key = var.ssh_public_key
+}
+
+resource "equinix_metal_device" "control_plane" {
+  hostname            = "${var.cluster_name}-control-plane"
+  plan                = var.device_plan
+  metro               = var.device_metro
+  operating_system    = var.device_os
+  billing_cycle       = var.billing_cycle
+  project_id          = var.equinix_project_id
+  depends_on          = [equinix_metal_project_ssh_key.ssh_key]
+  project_ssh_key_ids = [equinix_metal_project_ssh_key.ssh_key.id]
+
+  behavior {
+    allow_changes = [
+      "custom_data",
+      "user_data"
+    ]
+  }
+
+  connection {
+    user = "root"
+    private_key = file(var.ssh_private_key_path)
+    host = self.access_public_ipv4
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "curl -sfL https://get.k3s.io | INSTALL_K3S_CHANNEL=${var.k3s_version} K3S_TOKEN=${var.k3s_token} sh -s - server --node-taint node-role.kubernetes.io/control-plane=:NoSchedule --flannel-backend=none --disable-network-policy --disable=traefik",
+      "systemctl is-active --quiet k3s.service",
+    ]
+  }
+}
+
+resource "equinix_metal_device" "worker" {
+  for_each            = toset(var.worker_nodes)
+  hostname            = "${var.cluster_name}-${each.value}"
+  plan                = var.device_plan
+  metro               = var.device_metro
+  operating_system    = var.device_os
+  billing_cycle       = var.billing_cycle
+  project_id          = var.equinix_project_id
+  project_ssh_key_ids = [equinix_metal_project_ssh_key.ssh_key.id]
+  depends_on          = [equinix_metal_device.control_plane]
+  user_data           = <<EOF
+#!/bin/bash
+curl -sfL https://get.k3s.io | INSTALL_K3S_CHANNEL="${var.k3s_version}" sh -s - agent --token "${var.k3s_token}" --server "https://${equinix_metal_device.control_plane.access_private_ipv4}:6443"
+EOF
+
+  behavior {
+    allow_changes = [
+      "custom_data",
+      "user_data"
+    ]
+  }
+}
+
+resource "null_resource" "install_cilium_cni" {
+  depends_on          = [equinix_metal_device.control_plane]
+  triggers = {
+    always_run = "${timestamp()}"
+  }
+
+  connection {
+    user = "root"
+    private_key = file(var.ssh_private_key_path)
+    host = equinix_metal_device.control_plane.access_public_ipv4
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "echo '@@@@@@ Installing Cilium @@@@@@'",
+      "CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt)",
+      "CLI_ARCH=amd64",
+      "curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/$${CILIUM_CLI_VERSION}/cilium-linux-$${CLI_ARCH}.tar.gz{,.sha256sum}",
+      "sha256sum --check cilium-linux-$${CLI_ARCH}.tar.gz.sha256sum",
+      "sudo tar xzvfC cilium-linux-$${CLI_ARCH}.tar.gz /usr/local/bin",
+      "rm cilium-linux-$${CLI_ARCH}.tar.gz{,.sha256sum}",
+      "echo '@@@@@@ Installed Cilium @@@@@@'",
+      "export KUBECONFIG=/etc/rancher/k3s/k3s.yaml",
+      "echo '@@@@@@ Adding Cilium CNI to cluster @@@@@@'",
+      "cilium install --version ${var.cilium_version}",
+      "cilium status --wait"
+    ]
+  }
+}
+
+resource "null_resource" "flux_env_vars" {
+  depends_on          = [null_resource.install_cilium_cni]
+  triggers = {
+    always_run = "${timestamp()}"
+  }
+
+  connection {
+    user = "root"
+    private_key = file(var.ssh_private_key_path)
+    host = equinix_metal_device.control_plane.access_public_ipv4
+  }
+
+  provisioner "file" {
+    content = <<EOF
+GITHUB_TOKEN=${var.flux_github_token}
+KUBECONFIG=/etc/rancher/k3s/k3s.yaml
+EOF
+    destination = "/tmp/flux_env_vars"
+  }
+}
+
+resource "null_resource" "bootstrap_flux" {
+  depends_on          = [null_resource.flux_env_vars]
+  triggers = {
+    always_run = "${timestamp()}"
+  }
+
+  connection {
+    user = "root"
+    private_key = file(var.ssh_private_key_path)
+    host = equinix_metal_device.control_plane.access_public_ipv4
+  }
+
+  provisioner "remote-exec" {
+    inline = [
+      "export $(cat /tmp/flux_env_vars | xargs) && env && rm /tmp/flux_env_vars",
+      "curl -s https://fluxcd.io/install.sh | sudo FLUX_VERSION=${var.flux_version} bash",
+      "flux bootstrap github --owner=${var.flux_github_user} --repository=${var.flux_github_repo} --path=clusters --branch=${var.flux_branch}"
+    ]
+  }
+}

infrastructure/equinix-metal/variables.tf

@@ -0,0 +1,107 @@
+variable "billing_cycle" {
+  description = "Billing cycle for the Equinix Metal device"
+  type        = string
+  default     = "hourly"
+}
+
+variable "cilium_version" {
+  description = "cilium version for the cluster"
+  type        = string
+  default     = "1.14.4"
+}
+
+variable "cluster_name" {
+  description = "Name of the cluster"
+  type        = string
+  default     = "green-reviews"
+}
+
+variable "device_metro" {
+  description = "Metro location for the Equinix Metal device"
+  type        = string
+  default     = "pa"
+}
+
+variable "device_os" {
+  description = "Operating system for the Equinix Metal device"
+  type        = string
+  default     = "ubuntu_22_04"
+}
+
+variable "device_plan" {
+  description = "Plan type for the Equinix Metal device"
+  type        = string
+  default     = "m3.small.x86"
+}
+
+variable "equinix_auth_token" {
+  description = "Authentication token for Equinix Metal"
+  type        = string
+  sensitive   = true
+}
+
+variable "equinix_project_id" {
+  description = "Project ID for the Equinix Metal resources"
+  type        = string
+  sensitive   = true
+}
+
+variable "flux_branch" {
+  description = "Git branch for Flux"
+  type        = string
+  default     = "main"
+}
+
+variable "flux_github_token" {
+  description = "GitHub token for Flux"
+  type        = string
+  sensitive = true
+}
+
+variable "flux_github_repo" {
+  description = "GitHub repository for Flux"
+  type        = string
+  default     = "green-reviews-tooling"
+}
+
+variable "flux_github_user" {
+  description = "GitHub user for Flux"
+  type        = string
+  default     = "cncf-tags"
+}
+
+variable "flux_version" {
+  description = "Flux CLI version"
+  type        = string
+  default     = "2.1.2"
+}
+
+variable "k3s_token" {
+  description = "k3s token for joining nodes to the cluster"
+  type = string
+  sensitive = true
+}
+
+variable "k3s_version" {
+  description = "k3s version for the cluster"
+  type        = string
+  default     = "v1.29.0+k3s1"
+}
+
+variable "ssh_public_key" {
+  description = "SSH public key for the Equinix Metal device"
+  type        = string
+  sensitive   = true
+}
+
+variable "ssh_private_key_path" {
+  description = "SSH private key path for the Equinix Metal device"
+  type        = string
+  default = "~/.ssh/id_rsa"
+}
+
+variable "worker_nodes" {
+  description = "List of worker node names"
+  type        = list(string)
+  default     = ["worker1"]
+}