feat: kube-proxy mode (#62)

* feat:configure kube-proxy setup

* feat:node label

* feat:kube proxy resources

* feat:add node labels

* feat:add node labels

* feat:add node labels

* feat:node label variable

* feat: node label per node

* feat:node annotations

---------

Co-authored-by: Kirubel-Fikru <[email protected]>

Author: zifeo

main.tf

@@ -62,6 +62,9 @@ module "servers" {
 
   network_id    = openstack_networking_network_v2.net.id
   subnet_id     = openstack_networking_subnet_v2.servers.id
+  cluster_cidr  = var.cluster_cidr
+  service_cidr  = var.service_cidr
+  cni           = var.cni
   secgroup_id   = openstack_networking_secgroup_v2.server.id
   internal_vip  = local.internal_vip
   vip_interface = var.vip_interface
@@ -103,9 +106,10 @@ module "servers" {
         cluster_name        = var.name
       }),
       "patches/rke2-cilium.yaml" : templatefile("${path.module}/patches/rke2-cilium.yaml.tpl", {
-        operator_replica = local.operator_replica
-        cluster_name     = var.name
-        cluster_id       = var.cluster_id
+        operator_replica  = local.operator_replica
+        cluster_name      = var.name
+        cluster_id        = var.cluster_id
+        ff_with_kubeproxy = var.ff_with_kubeproxy
       }),
       "patches/rke2-coredns.yaml" : templatefile("${path.module}/patches/rke2-coredns.yaml.tpl", {
         operator_replica = local.operator_replica
@@ -160,9 +164,14 @@ module "servers" {
   kube_scheduler_resources          = var.kube_scheduler_resources
   kube_controller_manager_resources = var.kube_controller_manager_resources
   etcd_resources                    = var.etcd_resources
+  kube_proxy_resources              = var.kube_proxy_resources
 
   ff_autoremove_agent = null
   ff_wait_ready       = var.ff_wait_ready
+  ff_with_kubeproxy   = var.ff_with_kubeproxy
+  
+  node_taints = each.value.node_taints
+  node_labels = each.value.node_labels
 }
 
 module "agents" {
@@ -202,12 +211,18 @@ module "agents" {
 
   network_id    = openstack_networking_network_v2.net.id
   subnet_id     = openstack_networking_subnet_v2.agents.id
+  cluster_cidr  = var.cluster_cidr
+  service_cidr  = var.service_cidr
+  cni           = var.cni
   secgroup_id   = openstack_networking_secgroup_v2.agent.id
   internal_vip  = local.internal_vip
   vip_interface = var.vip_interface
   bastion_host  = local.external_ip
 
   ff_autoremove_agent = var.ff_autoremove_agent
   ff_wait_ready       = var.ff_wait_ready
-
+  ff_with_kubeproxy   = var.ff_with_kubeproxy
+ 
+  node_taints = each.value.node_taints
+  node_labels = each.value.node_labels
 }

node/cloud-init.yaml.tpl

@@ -223,6 +223,8 @@ write_files:
     server: "https://${internal_vip}:9345"
     %{~ endif ~}
     node-ip: "${node_ip}"
+    cluster-cidr: "${cluster_cidr}"
+    service-cidr: "${service_cidr}"
     cloud-provider-name: external
     advertise-address: "${node_ip}"
     write-kubeconfig-mode: "0640"
@@ -250,14 +252,19 @@ write_files:
     control-plane-resource-limits: "${control_plane_limits}"
     %{~ endif ~}
     disable-cloud-controller: true
-    disable-kube-proxy: true
+    disable-kube-proxy: ${ff_with_kubeproxy ? "false" : "true"}
     disable: rke2-ingress-nginx
-    cni: cilium
-    node-label:
-      - node.kubernetes.io/exclude-from-external-load-balancers=true
+    cni: "${cni}"
     node-taint:
-      - CriticalAddonsOnly=true:NoExecute
-    ${indent(4,rke2_conf)}
+      - CriticalAddonsOnly=true:NoExecute  
+  %{ for k, v in node_taints ~}
+    - "${k}=${v}"
+  %{ endfor ~}  
+  node-label:
+      - node.kubernetes.io/exclude-from-external-load-balancers=true
+  %{ for k, v in node_labels ~}
+    - ${k}=${v}
+  %{ endfor ~}
 %{~ else ~}
 - path: /etc/rancher/rke2/config.yaml
   permissions: "0600"
@@ -267,7 +274,14 @@ write_files:
     server: https://${internal_vip}:9345
     node-ip: ${node_ip}
     cloud-provider-name: external
-    ${indent(4,rke2_conf)}
+    node-taint:
+  %{ for k, v in node_taints ~}
+    - "${k}=${v}"
+  %{ endfor ~}
+  node-label:
+  %{ for k, v in node_labels ~}
+    - "${k}=${v}"
+  %{ endfor ~}
 %{~ endif ~}
 
 runcmd:

node/main.tf

@@ -88,6 +88,9 @@ resource "openstack_compute_instance_v2" "instance" {
     internal_vip  = var.internal_vip
     vip_interface = var.vip_interface
     node_ip       = openstack_networking_port_v2.port[count.index].all_fixed_ips[0]
+    cluster_cidr  = var.cluster_cidr
+    service_cidr  = var.service_cidr
+    cni           = var.cni
     san           = var.is_server ? var.san : []
     manifests_files = var.is_server ? merge(
       var.manifests_folder != "" ? {
@@ -107,6 +110,8 @@ resource "openstack_compute_instance_v2" "instance" {
       try("kube-controller-manager-memory=${var.kube_controller_manager_resources.requests.memory}", ""),
       try("etcd-cpu=${var.etcd_resources.requests.cpu}", ""),
       try("etcd-memory=${var.etcd_resources.requests.memory}", ""),
+      try("kube-proxy-cpu=${var.kube_proxy_resources.requests.cpu}", ""),
+      try("kube-proxy-memory=${var.kube_proxy_resources.requests.memory}", ""),
     ] : limit if limit != ""])
     control_plane_limits = join(",", [for limit in [
       try("kube-apiserver-cpu=${var.kube_apiserver_resources.limits.cpu}", ""),
@@ -117,10 +122,15 @@ resource "openstack_compute_instance_v2" "instance" {
       try("kube-controller-manager-memory=${var.kube_controller_manager_resources.limits.memory}", ""),
       try("etcd-cpu=${var.etcd_resources.limits.cpu}", ""),
       try("etcd-memory=${var.etcd_resources.limits.memory}", ""),
+      try("kube-proxy-cpu=${var.kube_proxy_resources.limits.cpu}", ""),
+      try("kube-proxy-memory=${var.kube_proxy_resources.limits.memory}", ""),
     ] : limit if limit != ""])
-    system_user       = var.system_user
-    authorized_keys   = var.ssh_authorized_keys
-    ff_wait_apiserver = false
+    system_user        = var.system_user
+    authorized_keys    = var.ssh_authorized_keys
+    ff_wait_apiserver  = false
+    ff_with_kubeproxy  = var.ff_with_kubeproxy
+    node_taints  = var.node_taints
+    node_labels  = var.node_labels
   }))
 }
 

node/variables.tf

@@ -59,6 +59,18 @@ variable "subnet_id" {
   type = string
 }
 
+variable "cluster_cidr" {
+  type    = string
+}
+
+variable "service_cidr" {
+  type    = string
+}
+
+variable "cni" {
+  type    = string
+}
+
 variable "san" {
   type    = list(string)
   default = []
@@ -196,6 +208,20 @@ variable "etcd_resources" {
   default = null
 }
 
+variable "kube_proxy_resources" {
+  type = object({
+    requests = optional(object({
+      cpu    = optional(string)
+      memory = optional(string)
+    }))
+    limits = optional(object({
+      cpu    = optional(string)
+      memory = optional(string)
+    }))
+  })
+  default = null
+}
+
 variable "manifests_folder" {
   type    = string
   default = ""
@@ -218,3 +244,15 @@ variable "ff_wait_ready" {
   type    = bool
   default = false
 }
+
+variable "ff_with_kubeproxy" {
+  type = bool
+}
+
+variable "node_taints" {
+  type = map(string)
+}
+
+variable "node_labels" {
+  type    = map(string)
+} 

patches/rke2-cilium.yaml.tpl

@@ -3,7 +3,7 @@ cluster:
   id: ${cluster_id}
 eni:
   enabled: true
-kubeProxyReplacement: "true"
+kubeProxyReplacement: "${ff_with_kubeproxy ? false : true}"
 k8sServiceHost: 127.0.0.1
 k8sServicePort: 6443
 operator:

variables.tf

@@ -84,6 +84,21 @@ variable "subnet_lb_cidr" {
   default = "192.168.44.0/24"
 }
 
+variable "cluster_cidr" {
+  type    = string
+  default = "10.42.0.0/16"
+}
+
+variable "service_cidr" {
+  type    = string
+  default = "10.43.0.0/16"
+}
+
+variable "cni" {
+  type    = string
+  default = "cilium"
+}
+
 variable "vip_interface" {
   type    = string
   default = "ens3"
@@ -127,6 +142,8 @@ variable "servers" {
     rke2_volume_size   = number
     rke2_volume_type   = optional(string)
     rke2_volume_device = optional(string)
+    node_taints = optional(map(string), {})
+    node_labels  = optional(map(string), {})  
   }))
   validation {
     condition = (
@@ -159,6 +176,8 @@ variable "agents" {
     rke2_volume_size   = number
     rke2_volume_type   = optional(string)
     rke2_volume_device = optional(string)
+    node_taints = optional(map(string), {})
+    node_labels  = optional(map(string), {})
   }))
   validation {
     condition = (
@@ -249,6 +268,20 @@ variable "etcd_resources" {
   default = null
 }
 
+variable "kube_proxy_resources" {
+  type = object({
+    requests = optional(object({
+      cpu    = optional(string)
+      memory = optional(string)
+    }))
+    limits = optional(object({
+      cpu    = optional(string)
+      memory = optional(string)
+    }))
+  })
+  default = null
+}
+
 variable "manifests_folder" {
   type    = string
   default = ""
@@ -297,3 +330,8 @@ variable "ff_infomaniak_sc" {
   type    = bool
   default = false
 }
+
+variable "ff_with_kubeproxy" {
+  type    = bool
+  default = false
+}