diff --git a/README.md b/README.md index b72a61d9..8bfc627b 100644 --- a/README.md +++ b/README.md @@ -272,7 +272,7 @@ module "karpenter" { | [cluster\_security\_group\_name](#input\_cluster\_security\_group\_name) | Cluster security group name | `string` | `null` | no | | [cluster\_service\_ipv4\_cidr](#input\_cluster\_service\_ipv4\_cidr) | The CIDR block to assign Kubernetes service IP addresses from. If you don't specify a block, Kubernetes assigns addresses from either the 10.100.0.0/16 or 172.20.0.0/16 CIDR blocks | `string` | `null` | no | | [cluster\_service\_ipv6\_cidr](#input\_cluster\_service\_ipv6\_cidr) | The CIDR block to assign Kubernetes pod and service IP addresses from if `ipv6` was specified when the cluster was created. Kubernetes assigns service addresses from the unique local address range (fc00::/7) because you can't specify a custom IPv6 CIDR block when you create the cluster | `string` | `null` | no | -| [cluster\_version](#input\_cluster\_version) | EKS Cluster Version | `string` | `"1.31"` | no | +| [cluster\_version](#input\_cluster\_version) | EKS Cluster Version | `string` | `"1.32"` | no | | [create\_aws\_auth\_configmap](#input\_create\_aws\_auth\_configmap) | Determines whether to create the aws-auth configmap. NOTE - this is only intended for scenarios where the configmap does not exist (i.e. - when using only self-managed node groups). Most users should use `manage_aws_auth_configmap` | `bool` | `false` | no | | [create\_aws\_observability\_ns](#input\_create\_aws\_observability\_ns) | Whether to create AWS Observability Namespace. | `bool` | `true` | no | | [create\_aws\_observability\_ns\_for\_karpenter](#input\_create\_aws\_observability\_ns\_for\_karpenter) | Create aws-observability namespace flag | `bool` | `false` | no | @@ -307,11 +307,12 @@ module "karpenter" { | [force\_imdsv2](#input\_force\_imdsv2) | Force IMDSv2 metadata server. | `bool` | `true` | no | | [force\_irsa](#input\_force\_irsa) | Force usage of IAM Roles for Service Account | `bool` | `true` | no | | [iam\_role\_additional\_policies](#input\_iam\_role\_additional\_policies) | Additional policies to be added to the IAM role | `set(string)` | `[]` | no | -| [karpenter\_chart\_version](#input\_karpenter\_chart\_version) | Chart version for Karpenter | `string` | `"1.2.1"` | no | -| [karpenter\_crd\_chart\_version](#input\_karpenter\_crd\_chart\_version) | Chart version for Karpenter CRDs same version as `karpenter_chart_version` | `string` | `"1.2.1"` | no | +| [karpenter\_chart\_version](#input\_karpenter\_chart\_version) | Chart version for Karpenter | `string` | `"1.3.3"` | no | +| [karpenter\_crd\_chart\_version](#input\_karpenter\_crd\_chart\_version) | Chart version for Karpenter CRDs same version as `karpenter_chart_version` | `string` | `"1.3.3"` | no | | [karpenter\_default\_subnet\_selector\_tags](#input\_karpenter\_default\_subnet\_selector\_tags) | Subnet selector tags for Karpenter default node class | `map(string)` | 
{
  "kubernetes.io/role/internal-elb": "1"
}
| no | | [karpenter\_ephemeral\_volume\_size](#input\_karpenter\_ephemeral\_volume\_size) | Ephemeral volume size for Karpenter node groups | `string` | `"50Gi"` | no | -| [karpenter\_nodeclasses](#input\_karpenter\_nodeclasses) | List of nodetemplate maps | 
list(object({
    nodeclass_name                         = string
    karpenter_subnet_selector_maps         = list(map(any))
    karpenter_security_group_selector_maps = list(map(any))
    karpenter_ami_selector_maps            = list(map(any))
    karpenter_node_role                    = string
    karpenter_node_tags_map                = map(string)
    karpenter_node_user_data               = string
    karpenter_node_metadata_options        = map(any)
    karpenter_block_device_mapping = list(object({
      deviceName = string
      ebs = object({
        encrypted           = bool
        volumeSize          = string
        volumeType          = string
        kmsKeyID            = optional(string)
        deleteOnTermination = bool
      })
    }))
  }))
| `[]` | no | +| [karpenter\_nodeclass\_kubelet\_clusterdns\_ips](#input\_karpenter\_nodeclass\_kubelet\_clusterdns\_ips) | Cluster DNS IPs for Karpenter node classes | `list(string)` | `[]` | no | +| [karpenter\_nodeclasses](#input\_karpenter\_nodeclasses) | List of nodetemplate maps | 
list(object({
    nodeclass_name                         = string
    karpenter_subnet_selector_maps         = list(map(any))
    karpenter_security_group_selector_maps = list(map(any))
    karpenter_ami_selector_maps            = list(map(any))
    karpenter_node_role                    = string
    karpenter_node_tags_map                = map(string)
    karpenter_node_user_data               = string
    karpenter_node_metadata_options        = map(any)
    karpenter_node_kubelet_yaml            = map(any)
    karpenter_block_device_mapping = list(object({
      deviceName = string
      ebs = object({
        encrypted           = bool
        volumeSize          = string
        volumeType          = string
        kmsKeyID            = optional(string)
        deleteOnTermination = bool
      })
    }))
  }))
| `[]` | no | | [karpenter\_nodepools](#input\_karpenter\_nodepools) | List of Provisioner maps | 
list(object({
    nodepool_name                     = string
    nodeclass_name                    = string
    karpenter_nodepool_node_labels    = map(string)
    karpenter_nodepool_annotations    = map(string)
    karpenter_nodepool_node_taints    = list(map(string))
    karpenter_nodepool_startup_taints = list(map(string))
    karpenter_requirements = list(object({
      key      = string
      operator = string
      values   = list(string)
      })
    )
    karpenter_nodepool_disruption = object({
      consolidation_policy = string
      consolidate_after    = string
      expire_after         = string
    })
    karpenter_nodepool_disruption_budgets = list(map(any))
    karpenter_nodepool_weight             = number
  }))
| 
[
  {
    "karpenter_nodepool_annotations": {},
    "karpenter_nodepool_disruption": {
      "consolidate_after": "10m",
      "consolidation_policy": "WhenEmptyOrUnderutilized",
      "expire_after": "168h"
    },
    "karpenter_nodepool_disruption_budgets": [
      {
        "nodes": "10%"
      }
    ],
    "karpenter_nodepool_node_labels": {
      "bottlerocket.aws/updater-interface-version": "2.0.0"
    },
    "karpenter_nodepool_node_taints": [],
    "karpenter_nodepool_startup_taints": [],
    "karpenter_nodepool_weight": 10,
    "karpenter_requirements": [
      {
        "key": "karpenter.k8s.aws/instance-category",
        "operator": "In",
        "values": [
          "t",
          "m"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "In",
        "values": [
          "2",
          "4"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-memory",
        "operator": "Gt",
        "values": [
          "2048"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-generation",
        "operator": "Gt",
        "values": [
          "2"
        ]
      },
      {
        "key": "karpenter.sh/capacity-type",
        "operator": "In",
        "values": [
          "on-demand"
        ]
      },
      {
        "key": "kubernetes.io/arch",
        "operator": "In",
        "values": [
          "amd64"
        ]
      },
      {
        "key": "kubernetes.io/os",
        "operator": "In",
        "values": [
          "linux"
        ]
      }
    ],
    "nodeclass_name": "default",
    "nodepool_name": "default"
  }
]
| no | | [karpenter\_pod\_resources](#input\_karpenter\_pod\_resources) | Karpenter Pod Resource | 
object({
    requests = object({
      cpu    = string
      memory = string
    })
    limits = object({
      cpu    = string
      memory = string
    })
  })
| 
{
  "limits": {
    "cpu": "1",
    "memory": "2Gi"
  },
  "requests": {
    "cpu": "1",
    "memory": "2Gi"
  }
}
| no | | [karpenter\_security\_group\_selector\_terms\_type](#input\_karpenter\_security\_group\_selector\_terms\_type) | Type of terms to use in the security group selector | `string` | `"tags"` | no | diff --git a/karpenter.tf b/karpenter.tf index 5af84977..560d8e24 100644 --- a/karpenter.tf +++ b/karpenter.tf @@ -48,6 +48,9 @@ locals { } } ] + karpenter_node_kubelet = { + clusterDNS = var.karpenter_nodeclass_kubelet_clusterdns_ips + } }, ]) diff --git a/main.tf b/main.tf index 103029d4..5f66256b 100644 --- a/main.tf +++ b/main.tf @@ -170,10 +170,18 @@ module "eks" { memory = "256M" } } + autoScaling = { + enabled = true + } }) } : { most_recent = true resolve_conflicts_on_update = "OVERWRITE" + configuration_values = jsonencode({ + autoScaling = { + enabled = true + } + }) } eks-pod-identity-agent = var.cluster_ip_family == "ipv4" ? { most_recent = true diff --git a/modules/essentials/README.md b/modules/essentials/README.md index cafad33f..14584648 100644 --- a/modules/essentials/README.md +++ b/modules/essentials/README.md @@ -66,6 +66,7 @@ module "eks_essentials" { | [aws](#requirement\_aws) | >= 5.70 | | [helm](#requirement\_helm) | >= 2.16 | | [kubernetes](#requirement\_kubernetes) | >= 2.33 | +| [random](#requirement\_random) | >= 3.5 | ## Providers @@ -74,12 +75,14 @@ module "eks_essentials" { | [aws](#provider\_aws) | >= 5.70 | | [helm](#provider\_helm) | >= 2.16 | | [kubernetes](#provider\_kubernetes) | >= 2.33 | +| [random](#provider\_random) | >= 3.5 | ## Modules | Name | Source | Version | |------|--------|---------| | [cluster\_autoscaler\_irsa\_role](#module\_cluster\_autoscaler\_irsa\_role) | terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks | ~> 5.47 | +| [fluentbit\_s3\_bucket](#module\_fluentbit\_s3\_bucket) | terraform-aws-modules/s3-bucket/aws | ~> 4.6.1 | | [helm\_fluent\_bit](#module\_helm\_fluent\_bit) | SPHTech-Platform/release/helm | ~> 0.1.4 | | [helm\_kube\_state\_metrics](#module\_helm\_kube\_state\_metrics) | SPHTech-Platform/release/helm | ~> 0.1.4 | | [helm\_metrics\_server](#module\_helm\_metrics\_server) | SPHTech-Platform/release/helm | ~> 0.1.4 | @@ -104,10 +107,12 @@ module "eks_essentials" { | [helm_release.cert_manager](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.cluster_autoscaler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [helm_release.node_termination_handler](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | +| [helm_release.nodelocaldns](https://registry.terraform.io/providers/hashicorp/helm/latest/docs/resources/release) | resource | | [kubernetes_annotations.gp2_storage_class](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/annotations) | resource | | [kubernetes_namespace_v1.namespaces](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/namespace_v1) | resource | | [kubernetes_pod_disruption_budget_v1.coredns](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/pod_disruption_budget_v1) | resource | | [kubernetes_storage_class_v1.default](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/resources/storage_class_v1) | resource | +| [random_string.s3_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | | [aws_arn.node_termination_handler_sqs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/arn) | data source | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_eks_addon_version.latest_adot](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source | @@ -117,6 +122,7 @@ module "eks_essentials" { | [aws_iam_policy_document.node_termination_handler_sqs](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | | [aws_sqs_queue.node_termination_handler](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/sqs_queue) | data source | +| [kubernetes_service.kube_dns](https://registry.terraform.io/providers/hashicorp/kubernetes/latest/docs/data-sources/service) | data source | ## Inputs @@ -211,14 +217,21 @@ module "eks_essentials" { | [fargate\_cluster](#input\_fargate\_cluster) | Deploying workloads on Fargate, set this to TRUE | `bool` | `false` | no | | [fargate\_mix\_node\_groups](#input\_fargate\_mix\_node\_groups) | Deploying mix workloads as in EKS Manage Node Groups and Fragate Node Groups, set this to TRUE | `bool` | `false` | no | | [feature\_gates](#input\_feature\_gates) | Feature gates to enable on the pod | `list(any)` | `[]` | no | +| [fluent\_bit\_enable\_cw\_output](#input\_fluent\_bit\_enable\_cw\_output) | Enable cloudwatch logging | `bool` | `true` | no | +| [fluent\_bit\_enable\_s3\_output](#input\_fluent\_bit\_enable\_s3\_output) | Enable S3 output logging | `bool` | `false` | no | | [fluent\_bit\_enabled](#input\_fluent\_bit\_enabled) | Enable fluent-bit helm charts installation. | `bool` | `true` | no | +| [fluent\_bit\_excluded\_namespaces](#input\_fluent\_bit\_excluded\_namespaces) | Namespaces to exclude from fluent-bit | `list(string)` | `[]` | no | | [fluent\_bit\_helm\_config](#input\_fluent\_bit\_helm\_config) | Helm provider config for AWS for Fluent Bit. | `any` | `{}` | no | | [fluent\_bit\_helm\_config\_defaults](#input\_fluent\_bit\_helm\_config\_defaults) | Helm provider default config for Fluent Bit. | `any` | 
{
  "chart": "fluent-bit",
  "description": "Fluent Bit helm Chart deployment configuration",
  "name": "fluent-bit",
  "namespace": "logging",
  "repository": "https://fluent.github.io/helm-charts",
  "version": "0.47.9"
}
| no | | [fluent\_bit\_image\_repository](#input\_fluent\_bit\_image\_repository) | Fluent Bit Image repo | `string` | `"public.ecr.aws/aws-observability/aws-for-fluent-bit"` | no | | [fluent\_bit\_image\_tag](#input\_fluent\_bit\_image\_tag) | Fluent Bit Image tag | `string` | `"2.32.0"` | no | +| [fluent\_bit\_liveness\_probe](#input\_fluent\_bit\_liveness\_probe) | Liveness probe for fluent-bit | `map(any)` | 
{
  "httpGet": {
    "path": "/",
    "port": 2020
  }
}
| no | | [fluent\_bit\_log\_group\_retention](#input\_fluent\_bit\_log\_group\_retention) | Number of days to retain the cloudwatch logs | `number` | `30` | no | | [fluent\_bit\_overwrite\_helm\_values](#input\_fluent\_bit\_overwrite\_helm\_values) | helm values for overwrite configuration | `string` | `""` | no | +| [fluent\_bit\_readiness\_probe](#input\_fluent\_bit\_readiness\_probe) | Readiness probe for fluent-bit | `map(any)` | 
{
  "httpGet": {
    "path": "/api/v1/health",
    "port": 2020
  }
}
| no | +| [fluent\_bit\_resources](#input\_fluent\_bit\_resources) | Resources for fluent-bit | `map(any)` | 
{
  "limits": {
    "cpu": "100m",
    "memory": "128Mi"
  },
  "requests": {
    "cpu": "100m",
    "memory": "128Mi"
  }
}
| no | | [fluent\_bit\_role\_policy\_arns](#input\_fluent\_bit\_role\_policy\_arns) | ARNs of any policies to attach to the IAM role | `map(string)` | `{}` | no | +| [fluent\_bit\_tolerations](#input\_fluent\_bit\_tolerations) | Tolerations for fluent-bit | `list(any)` | 
[
  {
    "effect": "NoSchedule",
    "operator": "Exists"
  }
]
| no | | [helm\_release\_max\_history](#input\_helm\_release\_max\_history) | The maximum number of history releases to keep track in each Helm release | `number` | `20` | no | | [image\_pull\_secrets](#input\_image\_pull\_secrets) | Secrets for image pulling | `list(any)` | `[]` | no | | [image\_repository](#input\_image\_repository) | Image repository | `string` | `"quay.io/jetstack/cert-manager-controller"` | no | @@ -270,6 +283,15 @@ module "eks_essentials" { | [node\_termination\_handler\_taint\_node](#input\_node\_termination\_handler\_taint\_node) | Taint node upon spot interruption termination notice | `bool` | `true` | no | | [node\_termination\_namespace](#input\_node\_termination\_namespace) | Namespace to deploy Node Termination Handler | `string` | `"kube-system"` | no | | [node\_termination\_service\_account](#input\_node\_termination\_service\_account) | Service account for Node Termination Handler pods | `string` | `"node-termination-handler"` | no | +| [nodelocaldns\_affinity](#input\_nodelocaldns\_affinity) | Node Local DNS Cache affinity | `map(string)` | 
{
  "nodeAffinity": {
    "requiredDuringSchedulingIgnoredDuringExecution": {
      "nodeSelectorTerms": [
        {
          "matchExpressions": [
            {
              "key": "kubernetes.io/os",
              "operator": "In",
              "values": [
                "linux"
              ]
            },
            {
              "key": "kubernetes.io/arch",
              "operator": "In",
              "values": [
                "amd64",
                "arm64"
              ]
            },
            {
              "key": "eks.amazonaws.com/compute-type",
              "operator": "NotIn",
              "values": [
                "fargate",
                "auto"
              ]
            }
          ]
        }
      ]
    }
  }
}
| no | +| [nodelocaldns\_chart\_name](#input\_nodelocaldns\_chart\_name) | Chart name for Node Local DNS Cache | `string` | `"node-local-dns"` | no | +| [nodelocaldns\_chart\_repository](#input\_nodelocaldns\_chart\_repository) | Chart Repository URL for Node Local DNS Cache | `string` | `"https://lablabs.github.io/k8s-nodelocaldns-helm"` | no | +| [nodelocaldns\_chart\_version](#input\_nodelocaldns\_chart\_version) | Chart version for Node Local DNS Cache | `string` | `"2.1.0"` | no | +| [nodelocaldns\_enabled](#input\_nodelocaldns\_enabled) | Enable Node Local DNS Cache | `bool` | `false` | no | +| [nodelocaldns\_image\_tag](#input\_nodelocaldns\_image\_tag) | Node Local DNS Cache image tag, Refer https://github.com/kubernetes/dns/releases to get tag | `string` | `"1.25.0"` | no | +| [nodelocaldns\_localdns\_ip](#input\_nodelocaldns\_localdns\_ip) | Node Local DNS Cache IP | `string` | `"169.254.20.10"` | no | +| [nodelocaldns\_namespace](#input\_nodelocaldns\_namespace) | Namespace to deploy Node Local DNS Cache | `string` | `"kube-system"` | no | +| [nodelocaldns\_release\_name](#input\_nodelocaldns\_release\_name) | Release name for Node Local DNS Cache | `string` | `"node-local-dns"` | no | | [oidc\_provider\_arn](#input\_oidc\_provider\_arn) | ARN of the OIDC Provider for IRSA | `string` | n/a | yes | | [pod\_annotations](#input\_pod\_annotations) | Extra annotations for pods | `map(string)` | `{}` | no | | [pod\_labels](#input\_pod\_labels) | Extra labels for pods | `map(string)` | `{}` | no | diff --git a/modules/essentials/nodelocaldns.tf b/modules/essentials/nodelocaldns.tf new file mode 100644 index 00000000..de5c2ef4 --- /dev/null +++ b/modules/essentials/nodelocaldns.tf @@ -0,0 +1,25 @@ +locals { + nodelocaldns_values = { + tag = var.nodelocaldns_image_tag + local_dns_ip = var.nodelocaldns_localdns_ip + affinity = jsonencode(var.nodelocaldns_affinity) + } +} + +resource "helm_release" "nodelocaldns" { + count = var.nodelocaldns_enabled ? 1 : 0 + + name = var.nodelocaldns_release_name + chart = var.nodelocaldns_chart_name + repository = var.nodelocaldns_chart_repository + version = var.nodelocaldns_chart_version + + create_namespace = true + namespace = var.nodelocaldns_namespace + + max_history = 10 + + values = [ + templatefile("${path.module}/templates/nodelocaldns.yaml", local.nodelocaldns_values), + ] +} diff --git a/modules/essentials/templates/nodelocaldns.yaml b/modules/essentials/templates/nodelocaldns.yaml new file mode 100644 index 00000000..15b4d535 --- /dev/null +++ b/modules/essentials/templates/nodelocaldns.yaml @@ -0,0 +1,26 @@ +image: + tag: ${tag} +config: + localDnsIp: ${local_dns_ip} + zones: + cluster.local:53: + plugins: + errors: true + reload: true + log: + format: combined + classes: all + cache: + success: + size: 9984 + ttl: 30 + denial: + size: 9984 + ttl: 5 + forward: + parameters: __PILLAR__CLUSTER__DNS__ + force_tcp: true + prometheus: true + health: + port: 8080 +affinity: ${affinity} diff --git a/modules/essentials/variables.tf b/modules/essentials/variables.tf index fbc0ad6e..5c9aac17 100644 --- a/modules/essentials/variables.tf +++ b/modules/essentials/variables.tf @@ -1576,3 +1576,92 @@ variable "fluent_bit_enable_cw_output" { type = bool default = true } + +######################## +# Node Local DNS Cache # +######################## +variable "nodelocaldns_enabled" { + description = "Enable Node Local DNS Cache" + type = bool + default = false +} + +variable "nodelocaldns_release_name" { + description = "Release name for Node Local DNS Cache" + type = string + default = "node-local-dns" +} + +variable "nodelocaldns_chart_name" { + description = "Chart name for Node Local DNS Cache" + type = string + default = "node-local-dns" +} + +variable "nodelocaldns_chart_repository" { + description = "Chart Repository URL for Node Local DNS Cache" + type = string + default = "https://lablabs.github.io/k8s-nodelocaldns-helm" +} + +variable "nodelocaldns_chart_version" { + description = "Chart version for Node Local DNS Cache" + type = string + default = "2.1.0" +} + +variable "nodelocaldns_namespace" { + description = "Namespace to deploy Node Local DNS Cache" + type = string + default = "kube-system" +} + +variable "nodelocaldns_image_tag" { + description = "Node Local DNS Cache image tag, Refer https://github.com/kubernetes/dns/releases to get tag " + type = string + default = "1.25.0" +} + +variable "nodelocaldns_localdns_ip" { + description = "Node Local DNS Cache IP" + type = string + default = "169.254.20.10" +} + +variable "nodelocaldns_affinity" { + description = "Node Local DNS Cache affinity" + type = map(any) + default = { + "nodeAffinity" = { + "requiredDuringSchedulingIgnoredDuringExecution" = { + "nodeSelectorTerms" = [ + { + "matchExpressions" = [ + { + "key" = "kubernetes.io/os" + "operator" = "In" + "values" = ["linux"] + }, + { + "key" = "kubernetes.io/arch" + "operator" = "In" + "values" = [ + "amd64", + "arm64" + ] + }, + { + "key" = "eks.amazonaws.com/compute-type" + "operator" = "NotIn" + "values" = [ + "fargate", + "auto" + ] + } + ] + } + ] + } + } + } +} diff --git a/modules/karpenter/README.md b/modules/karpenter/README.md index 053ca247..217a7ae8 100644 --- a/modules/karpenter/README.md +++ b/modules/karpenter/README.md @@ -66,14 +66,14 @@ | [enable\_v1\_permissions](#input\_enable\_v1\_permissions) | Determines whether to enable permissions suitable for v1+ (`true`) or for v0.33.x-v0.37.x (`false`) | `bool` | `true` | no | | [karpenter\_chart\_name](#input\_karpenter\_chart\_name) | Chart name for Karpenter | `string` | `"karpenter"` | no | | [karpenter\_chart\_repository](#input\_karpenter\_chart\_repository) | Chart repository for Karpenter | `string` | `"oci://public.ecr.aws/karpenter"` | no | -| [karpenter\_chart\_version](#input\_karpenter\_chart\_version) | Chart version for Karpenter | `string` | `"1.0.8"` | no | +| [karpenter\_chart\_version](#input\_karpenter\_chart\_version) | Chart version for Karpenter | `string` | `"1.3.3"` | no | | [karpenter\_crd\_chart\_name](#input\_karpenter\_crd\_chart\_name) | Chart name for Karpenter | `string` | `"karpenter-crd"` | no | | [karpenter\_crd\_chart\_repository](#input\_karpenter\_crd\_chart\_repository) | Chart repository for Karpenter | `string` | `"oci://public.ecr.aws/karpenter"` | no | -| [karpenter\_crd\_chart\_version](#input\_karpenter\_crd\_chart\_version) | Chart version for Karpenter | `string` | `"1.0.8"` | no | +| [karpenter\_crd\_chart\_version](#input\_karpenter\_crd\_chart\_version) | Chart version for Karpenter | `string` | `"1.3.3"` | no | | [karpenter\_crd\_namespace](#input\_karpenter\_crd\_namespace) | Namespace to deploy karpenter | `string` | `"kube-system"` | no | | [karpenter\_crd\_release\_name](#input\_karpenter\_crd\_release\_name) | Release name for Karpenter | `string` | `"karpenter-crd"` | no | | [karpenter\_namespace](#input\_karpenter\_namespace) | Namespace to deploy karpenter | `string` | `"kube-system"` | no | -| [karpenter\_nodeclasses](#input\_karpenter\_nodeclasses) | List of nodetemplate maps | 
list(object({
    nodeclass_name                         = string
    karpenter_subnet_selector_maps         = list(map(any))
    karpenter_security_group_selector_maps = list(map(any))
    karpenter_ami_selector_maps            = list(map(any))
    karpenter_node_role                    = string
    karpenter_node_tags_map                = map(string)
    karpenter_node_user_data               = string
    karpenter_node_metadata_options        = map(any)
    karpenter_block_device_mapping = list(object({
      deviceName = string
      ebs = object({
        encrypted           = bool
        volumeSize          = string
        volumeType          = string
        kmsKeyID            = optional(string)
        deleteOnTermination = bool
      })
    }))
  }))
| 
[
  {
    "karpenter_ami_selector_maps": [],
    "karpenter_block_device_mapping": [],
    "karpenter_node_metadata_options": {
      "httpEndpoint": "enabled",
      "httpProtocolIPv6": "disabled",
      "httpPutResponseHopLimit": 1,
      "httpTokens": "required"
    },
    "karpenter_node_role": "module.eks.worker_iam_role_name",
    "karpenter_node_tags_map": {},
    "karpenter_node_user_data": "",
    "karpenter_security_group_selector_maps": [],
    "karpenter_subnet_selector_maps": [],
    "nodeclass_name": "default"
  }
]
| no | +| [karpenter\_nodeclasses](#input\_karpenter\_nodeclasses) | List of nodetemplate maps | 
list(object({
    nodeclass_name                         = string
    karpenter_subnet_selector_maps         = list(map(any))
    karpenter_security_group_selector_maps = list(map(any))
    karpenter_ami_selector_maps            = list(map(any))
    karpenter_node_role                    = string
    karpenter_node_tags_map                = map(string)
    karpenter_node_user_data               = string
    karpenter_node_metadata_options        = map(any)
    karpenter_node_kubelet_yaml            = map(any)
    karpenter_block_device_mapping = list(object({
      deviceName = string
      ebs = object({
        encrypted           = bool
        volumeSize          = string
        volumeType          = string
        kmsKeyID            = optional(string)
        deleteOnTermination = bool
      })
    }))
  }))
| 
[
  {
    "karpenter_ami_selector_maps": [],
    "karpenter_block_device_mapping": [],
    "karpenter_node_kubelet_yaml": {},
    "karpenter_node_metadata_options": {
      "httpEndpoint": "enabled",
      "httpProtocolIPv6": "disabled",
      "httpPutResponseHopLimit": 1,
      "httpTokens": "required"
    },
    "karpenter_node_role": "module.eks.worker_iam_role_name",
    "karpenter_node_tags_map": {},
    "karpenter_node_user_data": "",
    "karpenter_security_group_selector_maps": [],
    "karpenter_subnet_selector_maps": [],
    "nodeclass_name": "default"
  }
]
| no | | [karpenter\_nodepools](#input\_karpenter\_nodepools) | List of Provisioner maps | 
list(object({
    nodepool_name                     = string
    nodeclass_name                    = string
    karpenter_nodepool_node_labels    = map(string)
    karpenter_nodepool_annotations    = map(string)
    karpenter_nodepool_node_taints    = list(map(string))
    karpenter_nodepool_startup_taints = list(map(string))
    karpenter_requirements = list(object({
      key      = string
      operator = string
      values   = list(string)
      })
    )
    karpenter_nodepool_disruption = object({
      consolidation_policy = string
      consolidate_after    = string
      expire_after         = string
    })
    karpenter_nodepool_disruption_budgets = list(map(any))
    karpenter_nodepool_weight             = number
  }))
| 
[
  {
    "karpenter_nodepool_annotations": {},
    "karpenter_nodepool_disruption": {
      "consolidate_after": "5m",
      "consolidation_policy": "WhenEmptyOrUnderutilized",
      "expire_after": "168h"
    },
    "karpenter_nodepool_disruption_budgets": [
      {
        "nodes": "10%"
      }
    ],
    "karpenter_nodepool_node_labels": {},
    "karpenter_nodepool_node_taints": [],
    "karpenter_nodepool_startup_taints": [],
    "karpenter_nodepool_weight": 10,
    "karpenter_requirements": [
      {
        "key": "karpenter.k8s.aws/instance-category",
        "operator": "In",
        "values": [
          "m"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-cpu",
        "operator": "In",
        "values": [
          "4,8,16"
        ]
      },
      {
        "key": "karpenter.k8s.aws/instance-generation",
        "operator": "Gt",
        "values": [
          "5"
        ]
      },
      {
        "key": "karpenter.sh/capacity-type",
        "operator": "In",
        "values": [
          "on-demand"
        ]
      },
      {
        "key": "kubernetes.io/arch",
        "operator": "In",
        "values": [
          "amd64"
        ]
      },
      {
        "key": "kubernetes.io/os",
        "operator": "In",
        "values": [
          "linux"
        ]
      }
    ],
    "nodeclass_name": "default",
    "nodepool_name": "default"
  }
]
| no | | [karpenter\_pod\_resources](#input\_karpenter\_pod\_resources) | Karpenter Pod Resource | 
object({
    requests = object({
      cpu    = string
      memory = string
    })
    limits = object({
      cpu    = string
      memory = string
    })
  })
| 
{
  "limits": {
    "cpu": "1",
    "memory": "2Gi"
  },
  "requests": {
    "cpu": "1",
    "memory": "2Gi"
  }
}
| no | | [karpenter\_release\_name](#input\_karpenter\_release\_name) | Release name for Karpenter | `string` | `"karpenter"` | no | diff --git a/modules/karpenter/karpenter.tf b/modules/karpenter/karpenter.tf index a7e8e5d5..0e09ba5b 100644 --- a/modules/karpenter/karpenter.tf +++ b/modules/karpenter/karpenter.tf @@ -92,7 +92,7 @@ resource "kubectl_manifest" "karpenter_nodeclass" { karpenter_node_tags_map_yaml = length(keys(each.value.karpenter_node_tags_map)) == 0 ? "" : yamlencode(each.value.karpenter_node_tags_map) karpenter_node_metadata_options_yaml = length(keys(each.value.karpenter_node_metadata_options)) == 0 ? "" : replace(yamlencode(each.value.karpenter_node_metadata_options), "/\"([0-9]+)\"/", "$1") karpenter_block_device_mapping_yaml = length(each.value.karpenter_block_device_mapping) == 0 ? "" : yamlencode(each.value.karpenter_block_device_mapping) - + karpenter_node_kubelet_yaml = length(each.value.karpenter_node_kubelet.clusterDNS) == 0 ? "" : yamlencode(each.value.karpenter_node_kubelet) }) depends_on = [ diff --git a/modules/karpenter/templates/nodeclass.tftpl b/modules/karpenter/templates/nodeclass.tftpl index 34645f27..76b6d752 100644 --- a/modules/karpenter/templates/nodeclass.tftpl +++ b/modules/karpenter/templates/nodeclass.tftpl @@ -38,6 +38,11 @@ spec: ${indent(4,karpenter_node_metadata_options_yaml)} %{ endif } + %{ if karpenter_node_kubelet_yaml != ""} + kubelet: + ${indent(4,karpenter_node_kubelet_yaml)} + %{ endif } + # optional, configures storage devices for the instance %{ if karpenter_block_device_mapping_yaml != ""} blockDeviceMappings: diff --git a/modules/karpenter/variables.tf b/modules/karpenter/variables.tf index c4730bbf..4dc24bbb 100644 --- a/modules/karpenter/variables.tf +++ b/modules/karpenter/variables.tf @@ -143,6 +143,7 @@ variable "karpenter_nodeclasses" { karpenter_node_tags_map = map(string) karpenter_node_user_data = string karpenter_node_metadata_options = map(any) + karpenter_node_kubelet = map(any) karpenter_block_device_mapping = list(object({ deviceName = string ebs = object({ @@ -163,6 +164,7 @@ variable "karpenter_nodeclasses" { karpenter_subnet_selector_maps = [] karpenter_security_group_selector_maps = [] karpenter_node_tags_map = {} + karpenter_node_kubelet = {} karpenter_node_metadata_options = { httpEndpoint = "enabled" httpProtocolIPv6 = "disabled" diff --git a/variables.tf b/variables.tf index 5d1acd5d..a921f0b1 100644 --- a/variables.tf +++ b/variables.tf @@ -517,6 +517,7 @@ variable "karpenter_nodeclasses" { karpenter_node_tags_map = map(string) karpenter_node_user_data = string karpenter_node_metadata_options = map(any) + karpenter_node_kubelet = map(any) karpenter_block_device_mapping = list(object({ deviceName = string ebs = object({ @@ -610,6 +611,12 @@ variable "karpenter_ephemeral_volume_size" { default = "50Gi" } +variable "karpenter_nodeclass_kubelet_clusterdns_ips" { + description = "Cluster DNS IPs for Karpenter node classes" + type = list(string) + default = [] +} + variable "karpenter_pod_resources" { description = "Karpenter Pod Resource" type = object({