From d8441bbc6f856a6ff359474a25fdbdfa679049ff Mon Sep 17 00:00:00 2001 From: Christopher Markieta Date: Mon, 4 Mar 2024 17:30:46 -0500 Subject: [PATCH 1/7] fix: :bug: enable/disable observability inputs & `logmon_config_is_set` --- README.md | 5 +++-- autogen/main/cluster.tf.tmpl | 8 ++++++-- autogen/main/main.tf.tmpl | 4 ++-- autogen/main/variables.tf.tmpl | 10 ++++++++-- cluster.tf | 8 ++++++-- main.tf | 2 +- modules/beta-private-cluster-update-variant/README.md | 5 +++-- modules/beta-private-cluster-update-variant/cluster.tf | 8 ++++++-- modules/beta-private-cluster-update-variant/main.tf | 2 +- .../beta-private-cluster-update-variant/variables.tf | 10 ++++++++-- modules/beta-private-cluster/README.md | 5 +++-- modules/beta-private-cluster/cluster.tf | 8 ++++++-- modules/beta-private-cluster/main.tf | 2 +- modules/beta-private-cluster/variables.tf | 10 ++++++++-- modules/beta-public-cluster-update-variant/README.md | 5 +++-- modules/beta-public-cluster-update-variant/cluster.tf | 8 ++++++-- modules/beta-public-cluster-update-variant/main.tf | 2 +- .../beta-public-cluster-update-variant/variables.tf | 10 ++++++++-- modules/beta-public-cluster/README.md | 5 +++-- modules/beta-public-cluster/cluster.tf | 8 ++++++-- modules/beta-public-cluster/main.tf | 2 +- modules/beta-public-cluster/variables.tf | 10 ++++++++-- modules/private-cluster-update-variant/README.md | 5 +++-- modules/private-cluster-update-variant/cluster.tf | 8 ++++++-- modules/private-cluster-update-variant/main.tf | 2 +- modules/private-cluster-update-variant/variables.tf | 10 ++++++++-- modules/private-cluster/README.md | 5 +++-- modules/private-cluster/cluster.tf | 8 ++++++-- modules/private-cluster/main.tf | 2 +- modules/private-cluster/variables.tf | 10 ++++++++-- variables.tf | 10 ++++++++-- 31 files changed, 142 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index 23a04c47f5..522939ba79 100644 --- a/README.md +++ b/README.md @@ -190,8 +190,9 @@ Then perform the following commands on the root folder: | maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `null` | no | | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 1228e29a95..991550c45e 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -121,11 +121,15 @@ resource "google_container_cluster" "primary" { {% endif %} content{ enable_components = var.monitoring_enabled_components - managed_prometheus { - enabled = var.monitoring_enable_managed_prometheus + dynamic managed_prometheus { + for_each = var.monitoring_enable_managed_prometheus != null ? [1] : [] + content{ + enabled = var.monitoring_enable_managed_prometheus + } } advanced_datapath_observability_config { enable_metrics = var.monitoring_enable_observability_metrics + enable_relay = var.monitoring_enable_observability_relay relay_mode = var.monitoring_observability_metrics_relay_mode } } diff --git a/autogen/main/main.tf.tmpl b/autogen/main/main.tf.tmpl index 49b847fcba..ddece38d66 100644 --- a/autogen/main/main.tf.tmpl +++ b/autogen/main/main.tf.tmpl @@ -105,8 +105,8 @@ locals { enabled = false provider = null }] - cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] - logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] + logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] {% endif %} diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index e625e3d4ce..bc82d32fdb 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -737,13 +737,19 @@ variable "timeouts" { variable "monitoring_enable_managed_prometheus" { type = bool description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." - default = false + default = null } variable "monitoring_enable_observability_metrics" { type = bool description = "Whether or not the advanced datapath metrics are enabled." - default = false + default = null +} + +variable "monitoring_enable_observability_relay" { + type = bool + description = "Whether or not the advanced datapath relay is enabled." + default = null } variable "monitoring_observability_metrics_relay_mode" { diff --git a/cluster.tf b/cluster.tf index 2236446061..e78456aa7f 100644 --- a/cluster.tf +++ b/cluster.tf @@ -94,11 +94,15 @@ resource "google_container_cluster" "primary" { for_each = local.logmon_config_is_set || local.logmon_config_is_set ? [1] : [] content { enable_components = var.monitoring_enabled_components - managed_prometheus { - enabled = var.monitoring_enable_managed_prometheus + dynamic "managed_prometheus" { + for_each = var.monitoring_enable_managed_prometheus != null ? [1] : [] + content { + enabled = var.monitoring_enable_managed_prometheus + } } advanced_datapath_observability_config { enable_metrics = var.monitoring_enable_observability_metrics + enable_relay = var.monitoring_enable_observability_relay relay_mode = var.monitoring_observability_metrics_relay_mode } } diff --git a/main.tf b/main.tf index 4025b6063c..af82fe3acb 100644 --- a/main.tf +++ b/main.tf @@ -91,7 +91,7 @@ locals { provider = null }] cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] - logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 3e31d9d529..59479ea2bd 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -239,8 +239,9 @@ Then perform the following commands on the root folder: | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `null` | no | | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 4057938a0a..286a889dc7 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -100,11 +100,15 @@ resource "google_container_cluster" "primary" { for_each = local.cluster_telemetry_type_is_set || local.logmon_config_is_set ? [1] : [] content { enable_components = var.monitoring_enabled_components - managed_prometheus { - enabled = var.monitoring_enable_managed_prometheus + dynamic "managed_prometheus" { + for_each = var.monitoring_enable_managed_prometheus != null ? [1] : [] + content { + enabled = var.monitoring_enable_managed_prometheus + } } advanced_datapath_observability_config { enable_metrics = var.monitoring_enable_observability_metrics + enable_relay = var.monitoring_enable_observability_relay relay_mode = var.monitoring_observability_metrics_relay_mode } } diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index f7ef1e33d2..c59479b579 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -91,7 +91,7 @@ locals { provider = null }] cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] - logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index eeefad4291..3d13d63ac5 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -703,13 +703,19 @@ variable "timeouts" { variable "monitoring_enable_managed_prometheus" { type = bool description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." - default = false + default = null } variable "monitoring_enable_observability_metrics" { type = bool description = "Whether or not the advanced datapath metrics are enabled." - default = false + default = null +} + +variable "monitoring_enable_observability_relay" { + type = bool + description = "Whether or not the advanced datapath relay is enabled." + default = null } variable "monitoring_observability_metrics_relay_mode" { diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index b9379d4554..dce2535e3e 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -217,8 +217,9 @@ Then perform the following commands on the root folder: | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `null` | no | | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index c702c838e7..aed3ab4152 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -100,11 +100,15 @@ resource "google_container_cluster" "primary" { for_each = local.cluster_telemetry_type_is_set || local.logmon_config_is_set ? [1] : [] content { enable_components = var.monitoring_enabled_components - managed_prometheus { - enabled = var.monitoring_enable_managed_prometheus + dynamic "managed_prometheus" { + for_each = var.monitoring_enable_managed_prometheus != null ? [1] : [] + content { + enabled = var.monitoring_enable_managed_prometheus + } } advanced_datapath_observability_config { enable_metrics = var.monitoring_enable_observability_metrics + enable_relay = var.monitoring_enable_observability_relay relay_mode = var.monitoring_observability_metrics_relay_mode } } diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index f7ef1e33d2..c59479b579 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -91,7 +91,7 @@ locals { provider = null }] cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] - logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index eeefad4291..3d13d63ac5 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -703,13 +703,19 @@ variable "timeouts" { variable "monitoring_enable_managed_prometheus" { type = bool description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." - default = false + default = null } variable "monitoring_enable_observability_metrics" { type = bool description = "Whether or not the advanced datapath metrics are enabled." - default = false + default = null +} + +variable "monitoring_enable_observability_relay" { + type = bool + description = "Whether or not the advanced datapath relay is enabled." + default = null } variable "monitoring_observability_metrics_relay_mode" { diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 4b30d2352a..4bf4563bbc 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -228,8 +228,9 @@ Then perform the following commands on the root folder: | maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `null` | no | | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index 12d39c2ed7..eb4a2f7070 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -100,11 +100,15 @@ resource "google_container_cluster" "primary" { for_each = local.cluster_telemetry_type_is_set || local.logmon_config_is_set ? [1] : [] content { enable_components = var.monitoring_enabled_components - managed_prometheus { - enabled = var.monitoring_enable_managed_prometheus + dynamic "managed_prometheus" { + for_each = var.monitoring_enable_managed_prometheus != null ? [1] : [] + content { + enabled = var.monitoring_enable_managed_prometheus + } } advanced_datapath_observability_config { enable_metrics = var.monitoring_enable_observability_metrics + enable_relay = var.monitoring_enable_observability_relay relay_mode = var.monitoring_observability_metrics_relay_mode } } diff --git a/modules/beta-public-cluster-update-variant/main.tf b/modules/beta-public-cluster-update-variant/main.tf index d92b11fadd..33dd316b35 100644 --- a/modules/beta-public-cluster-update-variant/main.tf +++ b/modules/beta-public-cluster-update-variant/main.tf @@ -91,7 +91,7 @@ locals { provider = null }] cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] - logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index ff2f84c094..4a5d55dd6a 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -673,13 +673,19 @@ variable "timeouts" { variable "monitoring_enable_managed_prometheus" { type = bool description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." - default = false + default = null } variable "monitoring_enable_observability_metrics" { type = bool description = "Whether or not the advanced datapath metrics are enabled." - default = false + default = null +} + +variable "monitoring_enable_observability_relay" { + type = bool + description = "Whether or not the advanced datapath relay is enabled." + default = null } variable "monitoring_observability_metrics_relay_mode" { diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 8c4288e7f0..1a54af89e2 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -206,8 +206,9 @@ Then perform the following commands on the root folder: | maintenance\_recurrence | Frequency of the recurring maintenance window in RFC5545 format. | `string` | `""` | no | | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `null` | no | | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 779c0e19c8..e2f88f7cb2 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -100,11 +100,15 @@ resource "google_container_cluster" "primary" { for_each = local.cluster_telemetry_type_is_set || local.logmon_config_is_set ? [1] : [] content { enable_components = var.monitoring_enabled_components - managed_prometheus { - enabled = var.monitoring_enable_managed_prometheus + dynamic "managed_prometheus" { + for_each = var.monitoring_enable_managed_prometheus != null ? [1] : [] + content { + enabled = var.monitoring_enable_managed_prometheus + } } advanced_datapath_observability_config { enable_metrics = var.monitoring_enable_observability_metrics + enable_relay = var.monitoring_enable_observability_relay relay_mode = var.monitoring_observability_metrics_relay_mode } } diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index d92b11fadd..33dd316b35 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -91,7 +91,7 @@ locals { provider = null }] cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] - logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index ff2f84c094..4a5d55dd6a 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -673,13 +673,19 @@ variable "timeouts" { variable "monitoring_enable_managed_prometheus" { type = bool description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." - default = false + default = null } variable "monitoring_enable_observability_metrics" { type = bool description = "Whether or not the advanced datapath metrics are enabled." - default = false + default = null +} + +variable "monitoring_enable_observability_relay" { + type = bool + description = "Whether or not the advanced datapath relay is enabled." + default = null } variable "monitoring_observability_metrics_relay_mode" { diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index c8c38ed6ec..5c94834af4 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -223,8 +223,9 @@ Then perform the following commands on the root folder: | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `null` | no | | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index c468560690..236d61117f 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -94,11 +94,15 @@ resource "google_container_cluster" "primary" { for_each = local.logmon_config_is_set || local.logmon_config_is_set ? [1] : [] content { enable_components = var.monitoring_enabled_components - managed_prometheus { - enabled = var.monitoring_enable_managed_prometheus + dynamic "managed_prometheus" { + for_each = var.monitoring_enable_managed_prometheus != null ? [1] : [] + content { + enabled = var.monitoring_enable_managed_prometheus + } } advanced_datapath_observability_config { enable_metrics = var.monitoring_enable_observability_metrics + enable_relay = var.monitoring_enable_observability_relay relay_mode = var.monitoring_observability_metrics_relay_mode } } diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index f2a6e28ea8..4c5d8bd613 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -91,7 +91,7 @@ locals { provider = null }] cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] - logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 6c7e643132..2d7f8cc8c6 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -679,13 +679,19 @@ variable "timeouts" { variable "monitoring_enable_managed_prometheus" { type = bool description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." - default = false + default = null } variable "monitoring_enable_observability_metrics" { type = bool description = "Whether or not the advanced datapath metrics are enabled." - default = false + default = null +} + +variable "monitoring_enable_observability_relay" { + type = bool + description = "Whether or not the advanced datapath relay is enabled." + default = null } variable "monitoring_observability_metrics_relay_mode" { diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 8a8a961ad2..3d17b7e890 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -201,8 +201,9 @@ Then perform the following commands on the root folder: | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | | master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | -| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | -| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `false` | no | +| monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `null` | no | +| monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `null` | no | | monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 7bb8f67db2..9c989b2365 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -94,11 +94,15 @@ resource "google_container_cluster" "primary" { for_each = local.logmon_config_is_set || local.logmon_config_is_set ? [1] : [] content { enable_components = var.monitoring_enabled_components - managed_prometheus { - enabled = var.monitoring_enable_managed_prometheus + dynamic "managed_prometheus" { + for_each = var.monitoring_enable_managed_prometheus != null ? [1] : [] + content { + enabled = var.monitoring_enable_managed_prometheus + } } advanced_datapath_observability_config { enable_metrics = var.monitoring_enable_observability_metrics + enable_relay = var.monitoring_enable_observability_relay relay_mode = var.monitoring_observability_metrics_relay_mode } } diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index f2a6e28ea8..4c5d8bd613 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -91,7 +91,7 @@ locals { provider = null }] cluster_gce_pd_csi_config = var.gce_pd_csi_driver ? [{ enabled = true }] : [{ enabled = false }] - logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus + logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 6c7e643132..2d7f8cc8c6 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -679,13 +679,19 @@ variable "timeouts" { variable "monitoring_enable_managed_prometheus" { type = bool description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." - default = false + default = null } variable "monitoring_enable_observability_metrics" { type = bool description = "Whether or not the advanced datapath metrics are enabled." - default = false + default = null +} + +variable "monitoring_enable_observability_relay" { + type = bool + description = "Whether or not the advanced datapath relay is enabled." + default = null } variable "monitoring_observability_metrics_relay_mode" { diff --git a/variables.tf b/variables.tf index ddfbb6d741..2a02ef1556 100644 --- a/variables.tf +++ b/variables.tf @@ -649,13 +649,19 @@ variable "timeouts" { variable "monitoring_enable_managed_prometheus" { type = bool description = "Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled." - default = false + default = null } variable "monitoring_enable_observability_metrics" { type = bool description = "Whether or not the advanced datapath metrics are enabled." - default = false + default = null +} + +variable "monitoring_enable_observability_relay" { + type = bool + description = "Whether or not the advanced datapath relay is enabled." + default = null } variable "monitoring_observability_metrics_relay_mode" { From 0ccc8d0ed09b77e98c9de12fbcd73030707524a7 Mon Sep 17 00:00:00 2001 From: Christopher Markieta Date: Thu, 30 May 2024 12:15:57 -0400 Subject: [PATCH 2/7] Revert "Merge branch 'master' into fix-observability-config" This reverts commit 53e34135eef5c92f591f3cf75ec1f60de07c7e26, reversing changes made to fce6c185c0ffde1e99c8a5b16d12c5de721cc4b3. --- CHANGELOG.md | 55 ----- Makefile | 2 +- README.md | 56 ++--- autogen/main/README.md | 51 ++-- autogen/main/cluster.tf.tmpl | 100 +------- autogen/main/main.tf.tmpl | 3 +- autogen/main/outputs.tf.tmpl | 10 +- autogen/main/sa.tf.tmpl | 30 --- autogen/main/variables.tf.tmpl | 78 ++---- autogen/main/versions.tf.tmpl | 10 +- autogen/safer-cluster/main.tf.tmpl | 1 - autogen/safer-cluster/variables.tf.tmpl | 8 +- autogen/safer-cluster/versions.tf.tmpl | 2 +- build/int.cloudbuild.yaml | 2 +- build/lint.cloudbuild.yaml | 2 +- cluster.tf | 140 ----------- docs/upgrading_to_v31.0.md | 5 - .../acm-terraform-blog-part1/terraform/gke.tf | 2 +- .../acm-terraform-blog-part2/terraform/gke.tf | 2 +- .../acm-terraform-blog-part3/terraform/gke.tf | 2 +- examples/deploy_service/main.tf | 2 +- .../island_cluster_with_vm_router/README.md | 38 --- .../island_cluster_with_vm_router/main.tf | 226 ------------------ .../island_cluster_with_vm_router/outputs.tf | 19 -- .../island_cluster_with_vm_router/router.tf | 51 ---- .../terraform.tfvars | 29 --- .../island_cluster_with_vm_router/vars.tf | 64 ----- .../island_cluster_with_vm_router/versions.tf | 28 --- examples/node_pool/main.tf | 31 +-- examples/safer_cluster_iap_bastion/apis.tf | 2 +- examples/simple_autopilot_private/main.tf | 1 + .../main.tf | 1 + examples/simple_regional/main.tf | 1 - main.tf | 2 - modules/acm/versions.tf | 4 +- modules/asm/versions.tf | 4 +- modules/auth/versions.tf | 2 +- .../beta-autopilot-private-cluster/README.md | 14 +- .../beta-autopilot-private-cluster/cluster.tf | 3 - .../beta-autopilot-private-cluster/main.tf | 2 +- .../beta-autopilot-private-cluster/outputs.tf | 10 +- modules/beta-autopilot-private-cluster/sa.tf | 28 --- .../variables.tf | 21 +- .../versions.tf | 6 +- .../beta-autopilot-public-cluster/README.md | 6 +- .../beta-autopilot-public-cluster/cluster.tf | 3 - modules/beta-autopilot-public-cluster/main.tf | 2 +- .../beta-autopilot-public-cluster/outputs.tf | 10 +- modules/beta-autopilot-public-cluster/sa.tf | 28 --- .../variables.tf | 11 +- .../beta-autopilot-public-cluster/versions.tf | 6 +- .../README.md | 62 ++--- .../cluster.tf | 144 ----------- .../main.tf | 3 +- .../outputs.tf | 10 +- .../beta-private-cluster-update-variant/sa.tf | 28 --- .../variables.tf | 73 ++---- .../versions.tf | 6 +- modules/beta-private-cluster/README.md | 62 ++--- modules/beta-private-cluster/cluster.tf | 140 ----------- modules/beta-private-cluster/main.tf | 3 +- modules/beta-private-cluster/outputs.tf | 10 +- modules/beta-private-cluster/sa.tf | 28 --- modules/beta-private-cluster/variables.tf | 73 ++---- modules/beta-private-cluster/versions.tf | 6 +- .../README.md | 56 ++--- .../cluster.tf | 144 ----------- .../main.tf | 3 +- .../outputs.tf | 10 +- .../beta-public-cluster-update-variant/sa.tf | 28 --- .../variables.tf | 65 ++--- .../versions.tf | 6 +- modules/beta-public-cluster/README.md | 56 ++--- modules/beta-public-cluster/cluster.tf | 140 ----------- modules/beta-public-cluster/main.tf | 3 +- modules/beta-public-cluster/outputs.tf | 10 +- modules/beta-public-cluster/sa.tf | 28 --- modules/beta-public-cluster/variables.tf | 65 ++--- modules/beta-public-cluster/versions.tf | 6 +- modules/binary-authorization/main.tf | 2 +- modules/binary-authorization/versions.tf | 2 +- modules/fleet-membership/versions.tf | 2 +- modules/hub-legacy/versions.tf | 2 +- .../private-cluster-update-variant/README.md | 62 ++--- .../private-cluster-update-variant/cluster.tf | 144 ----------- .../private-cluster-update-variant/main.tf | 2 - .../private-cluster-update-variant/outputs.tf | 5 - modules/private-cluster-update-variant/sa.tf | 14 -- .../variables.tf | 61 ++--- .../versions.tf | 4 +- modules/private-cluster/README.md | 62 ++--- modules/private-cluster/cluster.tf | 140 ----------- modules/private-cluster/main.tf | 2 - modules/private-cluster/outputs.tf | 5 - modules/private-cluster/sa.tf | 14 -- modules/private-cluster/variables.tf | 61 ++--- modules/private-cluster/versions.tf | 4 +- .../safer-cluster-update-variant/README.md | 3 +- modules/safer-cluster-update-variant/main.tf | 1 - .../safer-cluster-update-variant/variables.tf | 8 +- .../safer-cluster-update-variant/versions.tf | 2 +- modules/safer-cluster/README.md | 3 +- modules/safer-cluster/main.tf | 1 - modules/safer-cluster/variables.tf | 8 +- modules/safer-cluster/versions.tf | 2 +- modules/services/main.tf | 2 +- modules/services/versions.tf | 2 +- modules/workload-identity/README.md | 45 ---- modules/workload-identity/versions.tf | 2 +- outputs.tf | 5 - sa.tf | 14 -- test/integration/go.mod | 43 ++-- test/integration/go.sum | 86 ++++--- test/integration/node_pool/controls/gcloud.rb | 25 +- .../testdata/TestSimpleRegional.json | 3 - test/setup/main.tf | 6 +- variables.tf | 53 +--- versions.tf | 4 +- 118 files changed, 532 insertions(+), 2858 deletions(-) delete mode 100644 docs/upgrading_to_v31.0.md delete mode 100644 examples/island_cluster_with_vm_router/README.md delete mode 100644 examples/island_cluster_with_vm_router/main.tf delete mode 100644 examples/island_cluster_with_vm_router/outputs.tf delete mode 100644 examples/island_cluster_with_vm_router/router.tf delete mode 100644 examples/island_cluster_with_vm_router/terraform.tfvars delete mode 100644 examples/island_cluster_with_vm_router/vars.tf delete mode 100644 examples/island_cluster_with_vm_router/versions.tf diff --git a/CHANGELOG.md b/CHANGELOG.md index 906450b4e7..c2679604a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,61 +6,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Extending the adopted spec, each change should have a link to its corresponding pull request appended. -## [31.0.0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v30.3.0...v31.0.0) (2024-05-28) - - -### ⚠ BREAKING CHANGES - -* **TPG>=5.25.0:** bump for #1948 ([#1955](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1955)) -* **TPG>=5.21:** add queued_provisioning ([#1950](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1950)) -* **TPG>=5.25.0:** Add support for StatefulHA Addon ([#1948](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1948)) -* **deps:** Update Terraform terraform-google-modules/project-factory/google to v15 ([#1936](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1936)) - -### Features - -* Add local_ssd_ephemeral_count to default configuration ([#1944](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1944)) ([04ebd0c](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/04ebd0c7d3ad236abcfcd9a7b1c54d4c75b1ba28)) -* add missing dns_cache output in non beta ([#1864](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1864)) ([3d5cc9f](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/3d5cc9fc4fd817c35e089222a043e02358d64c1d)) -* Add option image_type in cluster_autoscaling block ([#1905](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1905)) ([2272164](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/22721644102a8b6b875439c80a6571266e6bf723)) -* Add secondary_boot_disks to node_pool configuration. ([#1946](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1946)) ([11bae67](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/11bae6791b07efcce29a0227e0b8b4d6c7070971)) -* add security posture VULNERABILITY_ENTERPRISE ([#1947](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1947)) ([c48c8ab](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/c48c8abe22b78e40b6f56d2940403d8b6e586367)) -* add support for gpu_sharing_config on nodepool ([#1874](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1874)) ([b57387c](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/b57387cc771f90425e5b001fce2b9963dba1b791)) -* Add threads_per_core setting for node config in node pools ([#1942](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1942)) ([e573ced](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/e573cedee0a82517410f9d607ce69be7189124ca)) -* Add upgrade_settings for NAP created node pools ([#1908](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1908)) ([c87bb41](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/c87bb41a4d13a06e8e4b7687e15bed63595ad50b)) -* **TPG>=5.21:** add queued_provisioning ([#1950](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1950)) ([c87333b](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/c87333bcaaf4e91f550816f48862e18657f5596e)) -* **TPG>=5.25.0:** Add support for StatefulHA Addon ([#1948](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1948)) ([acbb453](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/acbb453cc7128ab9784486b1cc1238b0e7616f34)) - - -### Bug Fixes - -* add gpu_driver_version to recreate list + docs ([#1913](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1913)) ([12cf40f](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/12cf40fcfeb341220f639138483897a47c76c6f9)) -* **deps:** Update Terraform terraform-google-modules/project-factory/google to v15 ([#1936](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1936)) ([d01e5bc](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/d01e5bce35c73dba751fcb76f26b1a8f268f0506)) -* **docs:** remove beta from docs from non beta arguments ([#1957](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1957)) ([3de63b3](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/3de63b3a5ce80c4bdcfa106b7254a64dc956d7bb)) -* dynamic block ephemeral_storage_local_ssd_config ([#1951](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1951)) ([26eb7c0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/26eb7c09f6ebb7873719c68caa2fafb92be061ce)) -* local nvme ssd count ([#1937](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1937)) ([dbd90e3](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/dbd90e3e556d74881fb3132457a9159d7fb81e6b)) -* **TPG>=5.25.0:** bump for [#1948](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1948) ([#1955](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1955)) ([b12c2e7](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/b12c2e73ad8f09092bd1566f26f10b988beb5286)) - -## [30.3.0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v30.2.0...v30.3.0) (2024-05-09) - - -### Features - -* add enable_l4_ilb_subsetting for GA TPG ([#1924](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1924)) ([da0476a](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/da0476af8c2911bbe6bbdea65af59421a4fcd61e)) -* adds local_nvme_ssd_block_config to beta-public-cluster ([#1912](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1912)) ([f7c2ed9](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/f7c2ed9102eca18b0d2233784616c60f81180d72)) -* network tags for autoprovisioned node pools ([#1920](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1920)) ([f864e8a](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/f864e8add729b799529f5ede187347ed5cebd5f9)) -* **safer-clusters:** add components selection for monitoring ([#1851](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1851)) ([15b472f](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/15b472fafc6b63afb0b52acec56cea8fd5c25cdf)) - - -### Bug Fixes - -* make master_ipv4_cidr_block optional for autopilot ([#1902](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1902)) ([af43bdd](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/af43bddeaa5902681c0012b7ea724937b161fad6)) - -## [30.2.0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v30.1.0...v30.2.0) (2024-03-08) - - -### Features - -* add cross project fleet service agent ([#1896](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1896)) ([59d36b9](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/59d36b9c2ce27e2830f174afe8dd42416b664833)) - ## [30.1.0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v30.0.0...v30.1.0) (2024-02-26) diff --git a/Makefile b/Makefile index 942ff8cb78..f92db301e4 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ # Make will use bash instead of sh SHELL := /usr/bin/env bash -DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.21 +DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.19 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools REGISTRY_URL := gcr.io/cloud-foundation-cicd DOCKER_BIN ?= docker diff --git a/README.md b/README.md index d613063d65..522939ba79 100644 --- a/README.md +++ b/README.md @@ -50,33 +50,27 @@ module "gke" { network_policy = false horizontal_pod_autoscaling = true filestore_csi_driver = false - dns_cache = false node_pools = [ { - name = "default-node-pool" - machine_type = "e2-medium" - node_locations = "us-central1-b,us-central1-c" - min_count = 1 - max_count = 100 - local_ssd_count = 0 - spot = false - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS_CONTAINERD" - enable_gcfs = false - enable_gvnic = false - logging_variant = "DEFAULT" - auto_repair = true - auto_upgrade = true - service_account = "project-service-account@.iam.gserviceaccount.com" - preemptible = false - initial_node_count = 80 - accelerator_count = 1 - accelerator_type = "nvidia-l4" - gpu_driver_version = "LATEST" - gpu_sharing_strategy = "TIME_SHARING" - max_shared_clients_per_gpu = 2 + name = "default-node-pool" + machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = false + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + enable_gcfs = false + enable_gvnic = false + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 }, ] @@ -143,7 +137,7 @@ Then perform the following commands on the root folder: | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                 = bool
    autoscaling_profile     = string
    min_cpu_cores           = number
    max_cpu_cores           = number
    min_memory_gb           = number
    max_memory_gb           = number
    gpu_resources           = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair             = bool
    auto_upgrade            = bool
    disk_size               = optional(number)
    disk_type               = optional(string)
    image_type              = optional(string)
    strategy                = optional(string)
    max_surge               = optional(number)
    max_unavailable         = optional(number)
    node_pool_soak_duration = optional(string)
    batch_soak_duration     = optional(string)
    batch_percentage        = optional(number)
    batch_node_count        = optional(number)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
    disk_size           = optional(number)
    disk_type           = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -164,7 +158,6 @@ Then perform the following commands on the root folder: | enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | @@ -208,7 +201,6 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | @@ -228,14 +220,13 @@ Then perform the following commands on the root folder: | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | | shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | @@ -249,7 +240,6 @@ Then perform the following commands on the root folder: |------|-------------| | ca\_certificate | Cluster ca certificate (base64 encoded) | | cluster\_id | Cluster ID | -| dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | | fleet\_membership | Fleet membership (if registered) | | gateway\_api\_channel | The gateway api channel of this cluster. | @@ -299,22 +289,18 @@ The node_pools variable takes the following parameters: | enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | -| gpu_driver_version | Mode for how the GPU driver is installed | null | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | -| local_ssd_ephemeral_storage_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | -| local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | | strategy | The upgrade stragey to be used for upgrading the nodes. Valid values of state are: `SURGE` or `BLUE_GREEN` | "SURGE" | Optional | -| threads_per_core | Optional The number of threads per physical core. To disable simultaneous multithreading (SMT) set this to 1. If unset, the maximum number of threads supported per core by the underlying processor is assumed | null | Optional | | max_surge | The number of additional nodes that can be added to the node pool during an upgrade. Increasing max_surge raises the number of nodes that can be upgraded simultaneously. Can be set to 0 or greater. Only works with `SURGE` strategy. | 1 | Optional | | max_unavailable | The number of nodes that can be simultaneously unavailable during an upgrade. Increasing max_unavailable raises the number of nodes that can be upgraded in parallel. Can be set to 0 or greater. Only works with `SURGE` strategy. | 0 | Optional | | node_pool_soak_duration | Time needed after draining the entire blue pool. After this period, the blue pool will be cleaned up. By default, it is set to one hour (3600 seconds). The maximum length of the soak time is 7 days (604,800 seconds). Only works with `BLUE_GREEN` strategy. | "3600s" | Optional | @@ -334,8 +320,6 @@ The node_pools variable takes the following parameters: | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | -| secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | -| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/autogen/main/README.md b/autogen/main/README.md index f47c6171d7..83227dc2f7 100644 --- a/autogen/main/README.md +++ b/autogen/main/README.md @@ -90,38 +90,33 @@ module "gke" { {% if beta_cluster and autopilot_cluster != true %} istio = true cloudrun = true - {% endif %} dns_cache = false + {% endif %} {% if autopilot_cluster != true %} node_pools = [ { - name = "default-node-pool" - machine_type = "e2-medium" - node_locations = "us-central1-b,us-central1-c" - min_count = 1 - max_count = 100 - local_ssd_count = 0 - spot = false + name = "default-node-pool" + machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = false {% if beta_cluster %} - local_ssd_ephemeral_count = 0 + local_ssd_ephemeral_count = 0 {% endif %} - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS_CONTAINERD" - enable_gcfs = false - enable_gvnic = false - logging_variant = "DEFAULT" - auto_repair = true - auto_upgrade = true - service_account = "project-service-account@.iam.gserviceaccount.com" - preemptible = false - initial_node_count = 80 - accelerator_count = 1 - accelerator_type = "nvidia-l4" - gpu_driver_version = "LATEST" - gpu_sharing_strategy = "TIME_SHARING" - max_shared_clients_per_gpu = 2 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + enable_gcfs = false + enable_gvnic = false + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 }, ] @@ -210,25 +205,21 @@ The node_pools variable takes the following parameters: | enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | -| gpu_driver_version | Mode for how the GPU driver is installed | null | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | -| local_ssd_ephemeral_storage_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | {% if beta_cluster %} | local_ssd_ephemeral_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | {% endif %} -| local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | | strategy | The upgrade stragey to be used for upgrading the nodes. Valid values of state are: `SURGE` or `BLUE_GREEN` | "SURGE" | Optional | -| threads_per_core | Optional The number of threads per physical core. To disable simultaneous multithreading (SMT) set this to 1. If unset, the maximum number of threads supported per core by the underlying processor is assumed | null | Optional | | max_surge | The number of additional nodes that can be added to the node pool during an upgrade. Increasing max_surge raises the number of nodes that can be upgraded simultaneously. Can be set to 0 or greater. Only works with `SURGE` strategy. | 1 | Optional | | max_unavailable | The number of nodes that can be simultaneously unavailable during an upgrade. Increasing max_unavailable raises the number of nodes that can be upgraded in parallel. Can be set to 0 or greater. Only works with `SURGE` strategy. | 0 | Optional | | node_pool_soak_duration | Time needed after draining the entire blue pool. After this period, the blue pool will be cleaned up. By default, it is set to one hour (3600 seconds). The maximum length of the soak time is 7 days (604,800 seconds). Only works with `BLUE_GREEN` strategy. | "3600s" | Optional | @@ -258,8 +249,6 @@ The node_pools variable takes the following parameters: | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | -| secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | -| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index b3c21041dc..991550c45e 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -151,30 +151,9 @@ resource "google_container_cluster" "primary" { disk_size = lookup(var.cluster_autoscaling, "disk_size", 100) disk_type = lookup(var.cluster_autoscaling, "disk_type", "pd-standard") - upgrade_settings { - strategy = lookup(var.cluster_autoscaling, "strategy", "SURGE") - max_surge = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_surge", 0) : null - max_unavailable = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_unavailable", 0) : null - - dynamic "blue_green_settings" { - for_each = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "BLUE_GREEN" ? [1] : [] - content { - node_pool_soak_duration = lookup(var.cluster_autoscaling, "node_pool_soak_duration", null) - - standard_rollout_policy { - batch_soak_duration = lookup(var.cluster_autoscaling, "batch_soak_duration", null) - batch_percentage = lookup(var.cluster_autoscaling, "batch_percentage", null) - batch_node_count = lookup(var.cluster_autoscaling, "batch_node_count", null) - } - } - } - } - {% if beta_cluster %} min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") {% endif %} - - image_type = lookup(var.cluster_autoscaling, "image_type", "COS_CONTAINERD") } } autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" @@ -231,10 +210,10 @@ resource "google_container_cluster" "primary" { enabled = identity_service_config.value } } - {% endif %} enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting {% endif %} + {% endif %} {% if beta_cluster %} enable_fqdn_network_policy = var.enable_fqdn_network_policy {% endif %} @@ -264,17 +243,6 @@ resource "google_container_cluster" "primary" { } {% endif %} -{% if autopilot_cluster != true %} - dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] - content { - network_tags { - tags = var.network_tags - } - } - } -{% endif %} - master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -334,14 +302,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "stateful_ha_config" { - for_each = local.stateful_ha_config - - content { - enabled = stateful_ha_config.value.enabled - } - } - config_connector_config { enabled = var.config_connector } @@ -628,10 +588,6 @@ resource "google_container_cluster" "primary" { } } {% endif %} - {% if beta_cluster %} - - depends_on = [google_project_iam_member.service_agent] - {% endif %} } {% if autopilot_cluster != true %} /****************************************** @@ -646,9 +602,6 @@ locals { "accelerator_count", "accelerator_type", "gpu_partition_size", - "gpu_driver_version", - "gpu_sharing_strategy", - "max_shared_clients_per_gpu", "enable_secure_boot", "enable_integrity_monitoring", "local_ssd_count", @@ -664,7 +617,6 @@ locals { "enable_gvnic", "enable_secure_boot", "boot_disk_kms_key", - "queued_provisioning", ] } @@ -818,13 +770,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "queued_provisioning" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - enabled = lookup(each.value, "queued_provisioning", null) - } - } - node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -841,12 +786,6 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } - dynamic "reservation_affinity" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - consume_reservation_type = "NO_RESERVATION" - } - } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -890,12 +829,6 @@ resource "google_container_node_pool" "windows_pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") - dynamic "ephemeral_storage_local_ssd_config" { - for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] - content { - local_ssd_count = ephemeral_storage_local_ssd_config.value - } - } {% if beta_cluster %} dynamic "ephemeral_storage_config" { for_each = lookup(each.value, "local_ssd_ephemeral_count",0) > 0 ? [each.value.local_ssd_ephemeral_count] : [] @@ -905,22 +838,6 @@ resource "google_container_node_pool" "windows_pools" { } {% endif %} - dynamic "local_nvme_ssd_block_config" { - for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] - content { - local_ssd_count = local_nvme_ssd_block_config.value - } - } - - # Supports a single secondary boot disk because `map(any)` must have the same values type. - dynamic "secondary_boot_disks" { - for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] - content { - disk_image = secondary_boot_disks.value - mode = "CONTAINER_IMAGE_CACHE" - } - } - service_account = lookup( each.value, "service_account", @@ -947,21 +864,6 @@ resource "google_container_node_pool" "windows_pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } - - dynamic "gpu_sharing_config" { - for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] - content { - gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") - max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) - } - } - } - } - - dynamic "advanced_machine_features" { - for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] - content { - threads_per_core = lookup(each.value, "threads_per_core", 0) } } diff --git a/autogen/main/main.tf.tmpl b/autogen/main/main.tf.tmpl index 779206e579..ddece38d66 100644 --- a/autogen/main/main.tf.tmpl +++ b/autogen/main/main.tf.tmpl @@ -109,7 +109,6 @@ locals { logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] - stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] {% endif %} {% if beta_cluster and autopilot_cluster != true %} cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { @@ -232,6 +231,7 @@ locals { {% if beta_cluster %} # BETA features cluster_istio_enabled = ! local.cluster_output_istio_disabled + cluster_dns_cache_enabled = var.dns_cache {% if autopilot_cluster != true %} cluster_telemetry_type_is_set = var.cluster_telemetry_type != null {% endif %} @@ -241,7 +241,6 @@ locals { # /BETA features {% endif %} - cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/autogen/main/outputs.tf.tmpl b/autogen/main/outputs.tf.tmpl index 3d5be22ea9..afedce7f55 100644 --- a/autogen/main/outputs.tf.tmpl +++ b/autogen/main/outputs.tf.tmpl @@ -198,11 +198,6 @@ output "peering_name" { value = local.cluster_peering_name } {% endif %} - -output "dns_cache_enabled" { - description = "Whether DNS Cache enabled" - value = local.cluster_dns_cache_enabled -} {% if beta_cluster %} output "cloudrun_enabled" { @@ -219,6 +214,11 @@ output "istio_enabled" { value = local.cluster_istio_enabled } +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} + output "pod_security_policy_enabled" { description = "Whether pod security policy is enabled" value = local.cluster_pod_security_policy_enabled diff --git a/autogen/main/sa.tf.tmpl b/autogen/main/sa.tf.tmpl index f38e3a8687..1ab198a8b5 100644 --- a/autogen/main/sa.tf.tmpl +++ b/autogen/main/sa.tf.tmpl @@ -52,20 +52,6 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/monitoring.metricWriter" - member = google_service_account.cluster_service_account[0].member -} - -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/stackdriver.resourceMetadata.writer" - member = google_service_account.cluster_service_account[0].member -} - resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key @@ -79,19 +65,3 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } -{% if beta_cluster %} - -resource "google_project_service_identity" "fleet_project" { - count = var.fleet_project_grant_service_agent ? 1 : 0 - provider = google-beta - project = var.fleet_project - service = "gkehub.googleapis.com" -} - -resource "google_project_iam_member" "service_agent" { - for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] - project = var.project_id - role = each.value - member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" -} -{% endif %} diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index 1737faa53b..bc82d32fdb 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -244,25 +244,17 @@ variable "enable_resource_consumption_export" { {% if autopilot_cluster != true %} variable "cluster_autoscaling" { type = object({ - enabled = bool - autoscaling_profile = string - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) - image_type = optional(string) - strategy = optional(string) - max_surge = optional(number) - max_unavailable = optional(number) - node_pool_soak_duration = optional(string) - batch_soak_duration = optional(string) - batch_percentage = optional(number) - batch_node_count = optional(number) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) }) default = { enabled = false @@ -276,7 +268,6 @@ variable "cluster_autoscaling" { auto_upgrade = true disk_size = 100 disk_type = "pd-standard" - image_type = "COS_CONTAINERD" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -315,12 +306,13 @@ variable "node_pools_oauth_scopes" { } {% endif %} +{% if autopilot_cluster %} variable "network_tags" { - description = "(Optional) - List of network tags applied to auto-provisioned node pools." + description = "(Optional, Beta) - List of network tags applied to auto-provisioned node pools." type = list(string) default = [] } - +{% endif %} variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -431,30 +423,26 @@ variable "cluster_resource_labels" { variable "deploy_using_private_endpoint" { type = bool - description = "A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." + description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." default = false } variable "enable_private_endpoint" { type = bool - description = "Whether the master's internal IP address is used as the cluster endpoint" + description = "(Beta) Whether the master's internal IP address is used as the cluster endpoint" default = false } variable "enable_private_nodes" { type = bool - description = "Whether nodes have internal IP addresses only" + description = "(Beta) Whether nodes have internal IP addresses only" default = false } variable "master_ipv4_cidr_block" { type = string - description = "The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters." - {% if autopilot_cluster == true%} - default = null - {% else %} + description = "(Beta) The IP range in CIDR notation to use for the hosted master network" default = "10.0.0.0/28" - {% endif %} } variable "master_global_access_enabled" { @@ -589,7 +577,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -734,12 +722,6 @@ variable "gcs_fuse_csi_driver" { default = false } -variable "stateful_ha" { - type = bool - description = "Whether the Stateful HA Addon is enabled for this cluster." - default = false -} - {% endif %} variable "timeouts" { type = map(string) @@ -782,7 +764,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." default = [] } @@ -803,12 +785,6 @@ variable "config_connector" { description = "Whether ConfigConnector is enabled for this cluster." default = false } - -variable "enable_l4_ilb_subsetting" { - type = bool - description = "Enable L4 ILB Subsetting on the cluster" - default = false -} {% endif %} {% if beta_cluster %} {% if autopilot_cluster != true %} @@ -849,6 +825,12 @@ variable "enable_pod_security_policy" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." @@ -887,11 +869,3 @@ variable "fleet_project" { type = string default = null } -{% if beta_cluster %} - -variable "fleet_project_grant_service_agent" { - description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." - type = bool - default = false -} -{% endif %} diff --git a/autogen/main/versions.tf.tmpl b/autogen/main/versions.tf.tmpl index 3e266f9fc4..707a3b2747 100644 --- a/autogen/main/versions.tf.tmpl +++ b/autogen/main/versions.tf.tmpl @@ -24,11 +24,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.25.0, < 6" + version = ">= 5.9.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.25.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -40,13 +40,13 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v30.1.0" } {% else %} required_providers { google = { source = "hashicorp/google" - version = ">= 5.25.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -58,7 +58,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v30.1.0" } {% endif %} } diff --git a/autogen/safer-cluster/main.tf.tmpl b/autogen/safer-cluster/main.tf.tmpl index 5a156c7ccc..3146875d47 100644 --- a/autogen/safer-cluster/main.tf.tmpl +++ b/autogen/safer-cluster/main.tf.tmpl @@ -108,7 +108,6 @@ module "gke" { monitoring_service = var.monitoring_service monitoring_enable_managed_prometheus = var.monitoring_enable_managed_prometheus - monitoring_enabled_components = var.monitoring_enabled_components // We never use the default service account for the cluster. The default // project/editor permissions can create problems if nodes were to be ever diff --git a/autogen/safer-cluster/variables.tf.tmpl b/autogen/safer-cluster/variables.tf.tmpl index 88da0f4adc..0f21e99a4e 100644 --- a/autogen/safer-cluster/variables.tf.tmpl +++ b/autogen/safer-cluster/variables.tf.tmpl @@ -280,12 +280,6 @@ variable "monitoring_enable_managed_prometheus" { default = false } -variable "monitoring_enabled_components" { - type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." - default = [] -} - variable "grant_registry_access" { type = bool description = "Grants created cluster-specific service account storage.objectViewer role." @@ -476,7 +470,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`" + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`." type = string default = null } diff --git a/autogen/safer-cluster/versions.tf.tmpl b/autogen/safer-cluster/versions.tf.tmpl index 3ad61d487a..542b84b2b6 100644 --- a/autogen/safer-cluster/versions.tf.tmpl +++ b/autogen/safer-cluster/versions.tf.tmpl @@ -23,6 +23,6 @@ terraform { required_version = ">=1.3" provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v30.1.0" } } diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 8dc74732aa..dc1ebab01a 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -519,6 +519,6 @@ tags: - 'integration' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.21' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.19' options: machineType: 'N1_HIGHCPU_8' diff --git a/build/lint.cloudbuild.yaml b/build/lint.cloudbuild.yaml index 8a067a2f7f..d2d4c0c066 100644 --- a/build/lint.cloudbuild.yaml +++ b/build/lint.cloudbuild.yaml @@ -22,7 +22,7 @@ tags: - 'lint' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.21' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.19' options: machineType: 'N1_HIGHCPU_8' env: diff --git a/cluster.tf b/cluster.tf index b3b4b7655f..e78456aa7f 100644 --- a/cluster.tf +++ b/cluster.tf @@ -124,27 +124,6 @@ resource "google_container_cluster" "primary" { disk_size = lookup(var.cluster_autoscaling, "disk_size", 100) disk_type = lookup(var.cluster_autoscaling, "disk_type", "pd-standard") - upgrade_settings { - strategy = lookup(var.cluster_autoscaling, "strategy", "SURGE") - max_surge = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_surge", 0) : null - max_unavailable = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_unavailable", 0) : null - - dynamic "blue_green_settings" { - for_each = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "BLUE_GREEN" ? [1] : [] - content { - node_pool_soak_duration = lookup(var.cluster_autoscaling, "node_pool_soak_duration", null) - - standard_rollout_policy { - batch_soak_duration = lookup(var.cluster_autoscaling, "batch_soak_duration", null) - batch_percentage = lookup(var.cluster_autoscaling, "batch_percentage", null) - batch_node_count = lookup(var.cluster_autoscaling, "batch_node_count", null) - } - } - } - } - - - image_type = lookup(var.cluster_autoscaling, "image_type", "COS_CONTAINERD") } } autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" @@ -172,8 +151,6 @@ resource "google_container_cluster" "primary" { enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu - - enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -187,15 +164,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] - content { - network_tags { - tags = var.network_tags - } - } - } - master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -254,14 +222,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "stateful_ha_config" { - for_each = local.stateful_ha_config - - content { - enabled = stateful_ha_config.value.enabled - } - } - config_connector_config { enabled = var.config_connector } @@ -540,13 +500,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "queued_provisioning" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - enabled = lookup(each.value, "queued_provisioning", null) - } - } - node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -563,12 +516,6 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } - dynamic "reservation_affinity" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - consume_reservation_type = "NO_RESERVATION" - } - } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -612,28 +559,6 @@ resource "google_container_node_pool" "pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") - dynamic "ephemeral_storage_local_ssd_config" { - for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] - content { - local_ssd_count = ephemeral_storage_local_ssd_config.value - } - } - - dynamic "local_nvme_ssd_block_config" { - for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] - content { - local_ssd_count = local_nvme_ssd_block_config.value - } - } - - # Supports a single secondary boot disk because `map(any)` must have the same values type. - dynamic "secondary_boot_disks" { - for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] - content { - disk_image = secondary_boot_disks.value - mode = "CONTAINER_IMAGE_CACHE" - } - } service_account = lookup( each.value, @@ -661,21 +586,6 @@ resource "google_container_node_pool" "pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } - - dynamic "gpu_sharing_config" { - for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] - content { - gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") - max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) - } - } - } - } - - dynamic "advanced_machine_features" { - for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] - content { - threads_per_core = lookup(each.value, "threads_per_core", 0) } } @@ -799,13 +709,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "queued_provisioning" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - enabled = lookup(each.value, "queued_provisioning", null) - } - } - node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -822,12 +725,6 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } - dynamic "reservation_affinity" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - consume_reservation_type = "NO_RESERVATION" - } - } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -871,28 +768,6 @@ resource "google_container_node_pool" "windows_pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") - dynamic "ephemeral_storage_local_ssd_config" { - for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] - content { - local_ssd_count = ephemeral_storage_local_ssd_config.value - } - } - - dynamic "local_nvme_ssd_block_config" { - for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] - content { - local_ssd_count = local_nvme_ssd_block_config.value - } - } - - # Supports a single secondary boot disk because `map(any)` must have the same values type. - dynamic "secondary_boot_disks" { - for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] - content { - disk_image = secondary_boot_disks.value - mode = "CONTAINER_IMAGE_CACHE" - } - } service_account = lookup( each.value, @@ -920,21 +795,6 @@ resource "google_container_node_pool" "windows_pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } - - dynamic "gpu_sharing_config" { - for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] - content { - gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") - max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) - } - } - } - } - - dynamic "advanced_machine_features" { - for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] - content { - threads_per_core = lookup(each.value, "threads_per_core", 0) } } diff --git a/docs/upgrading_to_v31.0.md b/docs/upgrading_to_v31.0.md deleted file mode 100644 index a3f5adc170..0000000000 --- a/docs/upgrading_to_v31.0.md +++ /dev/null @@ -1,5 +0,0 @@ -# Upgrading to v31.0 -The v31.0 release of *kubernetes-engine* is a backwards incompatible release. - -### Google Cloud Platform Provider upgrade -The Terraform Kubernetes Engine Module now requires version 5.25 or higher of the Google Cloud Platform Providers. diff --git a/examples/acm-terraform-blog-part1/terraform/gke.tf b/examples/acm-terraform-blog-part1/terraform/gke.tf index 45a627c467..9c997717dc 100644 --- a/examples/acm-terraform-blog-part1/terraform/gke.tf +++ b/examples/acm-terraform-blog-part1/terraform/gke.tf @@ -16,7 +16,7 @@ module "enabled_google_apis" { source = "terraform-google-modules/project-factory/google//modules/project_services" - version = "~> 15.0" + version = "~> 14.0" project_id = var.project disable_services_on_destroy = false diff --git a/examples/acm-terraform-blog-part2/terraform/gke.tf b/examples/acm-terraform-blog-part2/terraform/gke.tf index 816e2bde95..2c0d637463 100644 --- a/examples/acm-terraform-blog-part2/terraform/gke.tf +++ b/examples/acm-terraform-blog-part2/terraform/gke.tf @@ -16,7 +16,7 @@ module "enabled_google_apis" { source = "terraform-google-modules/project-factory/google//modules/project_services" - version = "~> 15.0" + version = "~> 14.0" project_id = var.project disable_services_on_destroy = false diff --git a/examples/acm-terraform-blog-part3/terraform/gke.tf b/examples/acm-terraform-blog-part3/terraform/gke.tf index 6694582d56..9f43da79bc 100644 --- a/examples/acm-terraform-blog-part3/terraform/gke.tf +++ b/examples/acm-terraform-blog-part3/terraform/gke.tf @@ -16,7 +16,7 @@ module "enabled_google_apis" { source = "terraform-google-modules/project-factory/google//modules/project_services" - version = "~> 15.0" + version = "~> 14.0" project_id = var.project disable_services_on_destroy = false diff --git a/examples/deploy_service/main.tf b/examples/deploy_service/main.tf index e825609a1f..a36e40b5fa 100644 --- a/examples/deploy_service/main.tf +++ b/examples/deploy_service/main.tf @@ -55,7 +55,7 @@ resource "kubernetes_pod" "nginx-example" { spec { container { - image = "nginx:1.26.0" + image = "nginx:1.25.3" name = "nginx-example" } } diff --git a/examples/island_cluster_with_vm_router/README.md b/examples/island_cluster_with_vm_router/README.md deleted file mode 100644 index e36d8ed115..0000000000 --- a/examples/island_cluster_with_vm_router/README.md +++ /dev/null @@ -1,38 +0,0 @@ -# GKE island cluster using VM as router - -This example provisions a cluster in an island VPC allowing reuse of the IP address space for multiple clusters in the same project. - -1. An appliance(VM as router) with multiple NICs is used to establish connectivity between the island VPC and the existing network. -1. Outbound connections will go through the router. -1. For inbound connections, use Private Service Connect. - -## Deploy - -1. Update `project_id`, `cluster_name` and `primary_subnet` values in `terraform.tfvars`, and update other variables as needed. -1. Run `terraform apply`. - - -## Inputs - -| Name | Description | Type | Default | Required | -|------|-------------|------|---------|:--------:| -| cluster\_name | n/a | `string` | n/a | yes | -| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | n/a | yes | -| node\_locations | n/a | `list(string)` | n/a | yes | -| primary\_net\_cidrs | n/a | `list(string)` | n/a | yes | -| primary\_subnet | n/a | `string` | n/a | yes | -| project\_id | n/a | `string` | n/a | yes | -| proxy\_subnet\_cidr | n/a | `string` | n/a | yes | -| psc\_subnet\_cidr | n/a | `string` | n/a | yes | -| region | n/a | `string` | n/a | yes | -| router\_machine\_type | n/a | `string` | n/a | yes | -| secondary\_ranges | n/a | `map(string)` | n/a | yes | -| subnet\_cidr | n/a | `string` | n/a | yes | - -## Outputs - -| Name | Description | -|------|-------------| -| cluster\_id | n/a | - - diff --git a/examples/island_cluster_with_vm_router/main.tf b/examples/island_cluster_with_vm_router/main.tf deleted file mode 100644 index 86ce138888..0000000000 --- a/examples/island_cluster_with_vm_router/main.tf +++ /dev/null @@ -1,226 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "random_id" "rand" { - byte_length = 4 -} - -resource "google_service_account" "gke-sa" { - account_id = "gke-sa-${random_id.rand.hex}" - project = var.project_id -} - -module "net" { - source = "terraform-google-modules/network/google" - version = "~> 9.0" - - network_name = "gke-net-${random_id.rand.hex}" - routing_mode = "GLOBAL" - project_id = var.project_id - delete_default_internet_gateway_routes = true - - subnets = [ - { - subnet_name = "${var.cluster_name}-${var.region}-snet" - subnet_ip = var.subnet_cidr - subnet_region = var.region - subnet_private_access = "true" - }, - { - subnet_name = "${var.cluster_name}-${var.region}-proxy-snet" - subnet_ip = var.proxy_subnet_cidr - subnet_region = var.region - purpose = "REGIONAL_MANAGED_PROXY" - role = "ACTIVE" - }, - { - subnet_name = "${var.cluster_name}-${var.region}-psc-snet" - subnet_ip = var.psc_subnet_cidr - subnet_region = var.region - subnet_private_access = "true" - purpose = "PRIVATE_SERVICE_CONNECT" - } - ] - - secondary_ranges = { - "${var.cluster_name}-${var.region}-snet" = [ - { - range_name = "${var.cluster_name}-${var.region}-snet-pods" - ip_cidr_range = var.secondary_ranges["pods"] - }, - { - range_name = "${var.cluster_name}-${var.region}-snet-services" - ip_cidr_range = var.secondary_ranges["services"] - }, - ] - } - - routes = flatten([ - [for k, v in var.primary_net_cidrs : - { - name = "${var.cluster_name}-egress-gke-${k}" - description = "egress through the router for range ${v}" - destination_range = v - tags = "gke-${random_id.rand.hex}" - next_hop_instance = google_compute_instance.vm.self_link - priority = 100 - } - ], - [ - { - name = "${var.cluster_name}-default-igw" - description = "internet through the router" - destination_range = "0.0.0.0/0" - tags = "gke-${random_id.rand.hex}" - next_hop_instance = google_compute_instance.vm.self_link - priority = 100 - } - ] - ]) - - firewall_rules = [ - { - name = "${var.cluster_name}-iap" - direction = "INGRESS" - allow = [ - { - protocol = "TCP" - ports = ["22"] - } - ] - ranges = ["35.235.240.0/20"] - }, - { - name = "${var.cluster_name}-tcp-primary" - direction = "INGRESS" - allow = [ - { - protocol = "TCP" - } - ] - ranges = [ - var.subnet_cidr, - var.secondary_ranges["pods"] - ] - }, - { - name = "${var.cluster_name}-allow-psc" - direction = "INGRESS" - allow = [ - { - protocol = "TCP" - } - ] - ranges = [var.psc_subnet_cidr] - target_service_accounts = [google_service_account.gke-sa.email] - }, - { - name = "${var.cluster_name}-allow-proxy" - direction = "INGRESS" - allow = [ - { - protocol = "TCP" - } - ] - ranges = [var.proxy_subnet_cidr] - target_service_accounts = [google_service_account.gke-sa.email] - }, - ] -} - -module "gke" { - source = "terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster" - version = "~> 30.0" - - depends_on = [google_compute_instance.vm] - - name = var.cluster_name - project_id = var.project_id - region = var.region - release_channel = "RAPID" - zones = var.node_locations - network = module.net.network_name - subnetwork = "${var.cluster_name}-${var.region}-snet" - ip_range_pods = "${var.cluster_name}-${var.region}-snet-pods" - ip_range_services = "${var.cluster_name}-${var.region}-snet-services" - enable_private_endpoint = true - enable_private_nodes = true - datapath_provider = "ADVANCED_DATAPATH" - monitoring_enable_managed_prometheus = false - enable_shielded_nodes = true - master_global_access_enabled = false - master_ipv4_cidr_block = var.secondary_ranges["master_cidr"] - master_authorized_networks = var.master_authorized_networks - deletion_protection = false - remove_default_node_pool = true - disable_default_snat = true - gateway_api_channel = "CHANNEL_STANDARD" - - node_pools = [ - { - name = "default" - machine_type = "e2-highcpu-2" - min_count = 1 - max_count = 100 - local_ssd_count = 0 - spot = true - local_ssd_ephemeral_count = 0 - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS_CONTAINERD" - logging_variant = "DEFAULT" - auto_repair = true - auto_upgrade = true - service_account = google_service_account.gke-sa.email - initial_node_count = 1 - enable_secure_boot = true - }, - ] - - node_pools_tags = { - all = ["gke-${random_id.rand.hex}"] - } - - node_pools_oauth_scopes = { - all = [ - "https://www.googleapis.com/auth/logging.write", - "https://www.googleapis.com/auth/monitoring", - ] - } - - timeouts = { - create = "15m" - update = "15m" - delete = "15m" - } -} - -resource "google_gke_hub_membership" "primary" { - provider = google-beta - - project = var.project_id - membership_id = "${var.project_id}-${module.gke.name}" - location = var.region - - endpoint { - gke_cluster { - resource_link = "//container.googleapis.com/${module.gke.cluster_id}" - } - } - authority { - issuer = "https://container.googleapis.com/v1/${module.gke.cluster_id}" - } -} diff --git a/examples/island_cluster_with_vm_router/outputs.tf b/examples/island_cluster_with_vm_router/outputs.tf deleted file mode 100644 index 640692ed75..0000000000 --- a/examples/island_cluster_with_vm_router/outputs.tf +++ /dev/null @@ -1,19 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -output "cluster_id" { - value = module.gke.cluster_id -} diff --git a/examples/island_cluster_with_vm_router/router.tf b/examples/island_cluster_with_vm_router/router.tf deleted file mode 100644 index aee86f74cf..0000000000 --- a/examples/island_cluster_with_vm_router/router.tf +++ /dev/null @@ -1,51 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -resource "google_compute_instance" "vm" { - project = var.project_id - zone = var.node_locations[0] - name = "${var.cluster_name}-router-${random_id.rand.hex}" - machine_type = var.router_machine_type - allow_stopping_for_update = true - boot_disk { - initialize_params { - image = "debian-cloud/debian-12" - } - } - can_ip_forward = true - shielded_instance_config { - enable_secure_boot = true - } - network_interface { - subnetwork = var.primary_subnet - } - network_interface { - subnetwork = module.net.subnets["${var.region}/${var.cluster_name}-${var.region}-snet"]["self_link"] - } - metadata_startup_script = <<-EOT - #!/bin/bash - set -ex - sudo apt-get update - echo "net.ipv4.ip_forward = 1" | sudo tee -a /etc/sysctl.conf - sudo sysctl -p - sudo iptables -A FORWARD -i ens5 -o ens4 -j ACCEPT - sudo iptables -A FORWARD -i ens4 -o ens5 -m state --state ESTABLISHED,RELATED -j ACCEPT - GWY_URL="http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/1/gateway" - GWY_IP=$(curl $${GWY_URL} -H "Metadata-Flavor: Google") - sudo ip route add ${var.secondary_ranges["pods"]} via $${GWY_IP} dev ens5 - sudo iptables -t nat -A POSTROUTING -o ens4 -s 0.0.0.0/0 -j MASQUERADE - EOT -} diff --git a/examples/island_cluster_with_vm_router/terraform.tfvars b/examples/island_cluster_with_vm_router/terraform.tfvars deleted file mode 100644 index 3e9b74c54b..0000000000 --- a/examples/island_cluster_with_vm_router/terraform.tfvars +++ /dev/null @@ -1,29 +0,0 @@ -project_id = "" -region = "us-central1" -cluster_name = "gke-island-cluster-test" -node_locations = [ - "us-central1-a", - "us-central1-b", - "us-central1-f" -] -subnet_cidr = "100.64.0.0/20" -router_machine_type = "n2-highcpu-4" -primary_subnet = "projects//regions//subnetworks/" -secondary_ranges = { - pods = "100.64.64.0/18" - services = "100.64.128.0/20" - master_cidr = "100.64.144.0/28" -} -proxy_subnet_cidr = "100.64.168.0/24" -psc_subnet_cidr = "100.64.192.0/24" -master_authorized_networks = [ - { - cidr_block = "100.64.0.0/10" - display_name = "cluster net" - } -] -primary_net_cidrs = [ - "10.0.0.0/8", - "192.168.0.0/16", - "172.16.0.0/12" -] diff --git a/examples/island_cluster_with_vm_router/vars.tf b/examples/island_cluster_with_vm_router/vars.tf deleted file mode 100644 index 37cf62d182..0000000000 --- a/examples/island_cluster_with_vm_router/vars.tf +++ /dev/null @@ -1,64 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -variable "project_id" { - type = string -} - -variable "region" { - type = string -} - -variable "cluster_name" { - type = string -} - -variable "node_locations" { - type = list(string) -} - -variable "subnet_cidr" { - type = string -} - -variable "psc_subnet_cidr" { - type = string -} - -variable "proxy_subnet_cidr" { - type = string -} - -variable "secondary_ranges" { - type = map(string) -} - -variable "master_authorized_networks" { - type = list(object({ cidr_block = string, display_name = string })) - description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." -} - -variable "primary_subnet" { - type = string -} - -variable "primary_net_cidrs" { - type = list(string) -} - -variable "router_machine_type" { - type = string -} diff --git a/examples/island_cluster_with_vm_router/versions.tf b/examples/island_cluster_with_vm_router/versions.tf deleted file mode 100644 index 7e297ed82e..0000000000 --- a/examples/island_cluster_with_vm_router/versions.tf +++ /dev/null @@ -1,28 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -terraform { - required_version = ">= 1.6" - - required_providers { - google = { - source = "hashicorp/google" - } - google-beta = { - source = "hashicorp/google-beta" - } - } -} diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index 7626216fb7..cea54d19b6 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -66,25 +66,18 @@ module "gke" { service_account = var.compute_engine_service_account }, { - name = "pool-03" - machine_type = "n1-standard-2" - node_locations = "${var.region}-b,${var.region}-c" - autoscaling = false - node_count = 2 - disk_type = "pd-standard" - auto_upgrade = true - service_account = var.compute_engine_service_account - pod_range = "test" - sandbox_enabled = true - cpu_manager_policy = "static" - cpu_cfs_quota = true - local_ssd_ephemeral_count = 2 - }, - { - name = "pool-04" - min_count = 0 - service_account = var.compute_engine_service_account - queued_provisioning = true + name = "pool-03" + machine_type = "n1-standard-2" + node_locations = "${var.region}-b,${var.region}-c" + autoscaling = false + node_count = 2 + disk_type = "pd-standard" + auto_upgrade = true + service_account = var.compute_engine_service_account + pod_range = "test" + sandbox_enabled = true + cpu_manager_policy = "static" + cpu_cfs_quota = true }, ] diff --git a/examples/safer_cluster_iap_bastion/apis.tf b/examples/safer_cluster_iap_bastion/apis.tf index f72733494f..8665a9b3d1 100644 --- a/examples/safer_cluster_iap_bastion/apis.tf +++ b/examples/safer_cluster_iap_bastion/apis.tf @@ -16,7 +16,7 @@ module "enabled_google_apis" { source = "terraform-google-modules/project-factory/google//modules/project_services" - version = "~> 15.0" + version = "~> 14.0" project_id = var.project_id disable_services_on_destroy = false diff --git a/examples/simple_autopilot_private/main.tf b/examples/simple_autopilot_private/main.tf index 125ba24706..ee628d8459 100644 --- a/examples/simple_autopilot_private/main.tf +++ b/examples/simple_autopilot_private/main.tf @@ -49,6 +49,7 @@ module "gke" { enable_vertical_pod_autoscaling = true enable_private_endpoint = true enable_private_nodes = true + master_ipv4_cidr_block = "172.16.0.0/28" network_tags = [local.cluster_type] deletion_protection = false diff --git a/examples/simple_autopilot_private_non_default_sa/main.tf b/examples/simple_autopilot_private_non_default_sa/main.tf index b9ff6e92a5..1da730e987 100644 --- a/examples/simple_autopilot_private_non_default_sa/main.tf +++ b/examples/simple_autopilot_private_non_default_sa/main.tf @@ -49,6 +49,7 @@ module "gke" { enable_vertical_pod_autoscaling = true enable_private_endpoint = true enable_private_nodes = true + master_ipv4_cidr_block = "172.16.0.0/28" deletion_protection = false master_authorized_networks = [ diff --git a/examples/simple_regional/main.tf b/examples/simple_regional/main.tf index 5fc839611d..11a34fa0c6 100644 --- a/examples/simple_regional/main.tf +++ b/examples/simple_regional/main.tf @@ -45,5 +45,4 @@ module "gke" { gcs_fuse_csi_driver = true fleet_project = var.project_id deletion_protection = false - stateful_ha = true } diff --git a/main.tf b/main.tf index ee6350cf77..af82fe3acb 100644 --- a/main.tf +++ b/main.tf @@ -94,7 +94,6 @@ locals { logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] - stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group @@ -171,7 +170,6 @@ locals { }] : [] - cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/modules/acm/versions.tf b/modules/acm/versions.tf index a9babfe8f8..557f677e42 100644 --- a/modules/acm/versions.tf +++ b/modules/acm/versions.tf @@ -19,11 +19,11 @@ terraform { required_version = ">= 0.13.0" provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v30.1.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v30.1.0" } required_providers { diff --git a/modules/asm/versions.tf b/modules/asm/versions.tf index d45b12a96f..161403d4f9 100644 --- a/modules/asm/versions.tf +++ b/modules/asm/versions.tf @@ -36,10 +36,10 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v30.1.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v30.1.0" } } diff --git a/modules/auth/versions.tf b/modules/auth/versions.tf index bc3a4c4d0c..87862337c5 100644 --- a/modules/auth/versions.tf +++ b/modules/auth/versions.tf @@ -26,6 +26,6 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:auth/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:auth/v30.1.0" } } diff --git a/modules/beta-autopilot-private-cluster/README.md b/modules/beta-autopilot-private-cluster/README.md index 5628cde23f..ff40b31c79 100644 --- a/modules/beta-autopilot-private-cluster/README.md +++ b/modules/beta-autopilot-private-cluster/README.md @@ -54,7 +54,6 @@ module "gke" { enable_private_endpoint = true enable_private_nodes = true master_ipv4_cidr_block = "10.0.0.0/28" - dns_cache = false } ``` @@ -84,7 +83,7 @@ Then perform the following commands on the root folder: | create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | | database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | | deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | | description | The description of the cluster | `string` | `""` | no | | disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | | dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `true` | no | @@ -92,15 +91,14 @@ Then perform the following commands on the root folder: | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | | enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `false` | no | +| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `true` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | @@ -118,11 +116,11 @@ Then perform the following commands on the root folder: | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `null` | no | +| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | | network | The VPC network to host the cluster in (required) | `string` | n/a | yes | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| network\_tags | (Optional, Beta) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | | notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | | project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | @@ -132,7 +130,7 @@ Then perform the following commands on the root folder: | release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | diff --git a/modules/beta-autopilot-private-cluster/cluster.tf b/modules/beta-autopilot-private-cluster/cluster.tf index 397c7755b9..ae074625a1 100644 --- a/modules/beta-autopilot-private-cluster/cluster.tf +++ b/modules/beta-autopilot-private-cluster/cluster.tf @@ -107,7 +107,6 @@ resource "google_container_cluster" "primary" { } } - master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -269,6 +268,4 @@ resource "google_container_cluster" "primary" { topic = var.notification_config_topic } } - - depends_on = [google_project_iam_member.service_agent] } diff --git a/modules/beta-autopilot-private-cluster/main.tf b/modules/beta-autopilot-private-cluster/main.tf index 3354fc71a1..f0d22b5840 100644 --- a/modules/beta-autopilot-private-cluster/main.tf +++ b/modules/beta-autopilot-private-cluster/main.tf @@ -127,12 +127,12 @@ locals { # BETA features cluster_istio_enabled = !local.cluster_output_istio_disabled + cluster_dns_cache_enabled = var.dns_cache cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled # /BETA features - cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/modules/beta-autopilot-private-cluster/outputs.tf b/modules/beta-autopilot-private-cluster/outputs.tf index beb7a8c030..f2888c0aa1 100644 --- a/modules/beta-autopilot-private-cluster/outputs.tf +++ b/modules/beta-autopilot-private-cluster/outputs.tf @@ -159,11 +159,6 @@ output "peering_name" { value = local.cluster_peering_name } -output "dns_cache_enabled" { - description = "Whether DNS Cache enabled" - value = local.cluster_dns_cache_enabled -} - output "cloudrun_enabled" { description = "Whether CloudRun enabled" value = false @@ -174,6 +169,11 @@ output "istio_enabled" { value = local.cluster_istio_enabled } +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} + output "pod_security_policy_enabled" { description = "Whether pod security policy is enabled" value = local.cluster_pod_security_policy_enabled diff --git a/modules/beta-autopilot-private-cluster/sa.tf b/modules/beta-autopilot-private-cluster/sa.tf index fe36dadd32..6f89899bee 100644 --- a/modules/beta-autopilot-private-cluster/sa.tf +++ b/modules/beta-autopilot-private-cluster/sa.tf @@ -52,20 +52,6 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/monitoring.metricWriter" - member = google_service_account.cluster_service_account[0].member -} - -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/stackdriver.resourceMetadata.writer" - member = google_service_account.cluster_service_account[0].member -} - resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key @@ -79,17 +65,3 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } - -resource "google_project_service_identity" "fleet_project" { - count = var.fleet_project_grant_service_agent ? 1 : 0 - provider = google-beta - project = var.fleet_project - service = "gkehub.googleapis.com" -} - -resource "google_project_iam_member" "service_agent" { - for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] - project = var.project_id - role = each.value - member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" -} diff --git a/modules/beta-autopilot-private-cluster/variables.tf b/modules/beta-autopilot-private-cluster/variables.tf index 9e5c223a81..35325d6774 100644 --- a/modules/beta-autopilot-private-cluster/variables.tf +++ b/modules/beta-autopilot-private-cluster/variables.tf @@ -174,11 +174,10 @@ variable "enable_resource_consumption_export" { variable "network_tags" { - description = "(Optional) - List of network tags applied to auto-provisioned node pools." + description = "(Optional, Beta) - List of network tags applied to auto-provisioned node pools." type = list(string) default = [] } - variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -266,26 +265,26 @@ variable "cluster_resource_labels" { variable "deploy_using_private_endpoint" { type = bool - description = "A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." + description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." default = false } variable "enable_private_endpoint" { type = bool - description = "Whether the master's internal IP address is used as the cluster endpoint" + description = "(Beta) Whether the master's internal IP address is used as the cluster endpoint" default = false } variable "enable_private_nodes" { type = bool - description = "Whether nodes have internal IP addresses only" + description = "(Beta) Whether nodes have internal IP addresses only" default = false } variable "master_ipv4_cidr_block" { type = string - description = "The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters." - default = null + description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + default = "10.0.0.0/28" } variable "master_global_access_enabled" { @@ -406,7 +405,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -467,9 +466,3 @@ variable "fleet_project" { type = string default = null } - -variable "fleet_project_grant_service_agent" { - description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." - type = bool - default = false -} diff --git a/modules/beta-autopilot-private-cluster/versions.tf b/modules/beta-autopilot-private-cluster/versions.tf index ef38f35edc..d80b7f0fce 100644 --- a/modules/beta-autopilot-private-cluster/versions.tf +++ b/modules/beta-autopilot-private-cluster/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.25.0, < 6" + version = ">= 5.9.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.25.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-private-cluster/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-private-cluster/v30.1.0" } } diff --git a/modules/beta-autopilot-public-cluster/README.md b/modules/beta-autopilot-public-cluster/README.md index 0e3c169dff..198fde422e 100644 --- a/modules/beta-autopilot-public-cluster/README.md +++ b/modules/beta-autopilot-public-cluster/README.md @@ -48,7 +48,6 @@ module "gke" { ip_range_services = "us-central1-01-gke-01-services" horizontal_pod_autoscaling = true filestore_csi_driver = false - dns_cache = false } ``` @@ -91,7 +90,6 @@ Then perform the following commands on the root folder: | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | @@ -111,7 +109,7 @@ Then perform the following commands on the root folder: | name | The name of the cluster (required) | `string` | n/a | yes | | network | The VPC network to host the cluster in (required) | `string` | n/a | yes | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| network\_tags | (Optional, Beta) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | | notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | | project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | @@ -121,7 +119,7 @@ Then perform the following commands on the root folder: | release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | diff --git a/modules/beta-autopilot-public-cluster/cluster.tf b/modules/beta-autopilot-public-cluster/cluster.tf index 01ea54baf4..39b4c57bbe 100644 --- a/modules/beta-autopilot-public-cluster/cluster.tf +++ b/modules/beta-autopilot-public-cluster/cluster.tf @@ -107,7 +107,6 @@ resource "google_container_cluster" "primary" { } } - master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -250,6 +249,4 @@ resource "google_container_cluster" "primary" { topic = var.notification_config_topic } } - - depends_on = [google_project_iam_member.service_agent] } diff --git a/modules/beta-autopilot-public-cluster/main.tf b/modules/beta-autopilot-public-cluster/main.tf index 9df7f76c14..24bcb36d3d 100644 --- a/modules/beta-autopilot-public-cluster/main.tf +++ b/modules/beta-autopilot-public-cluster/main.tf @@ -126,12 +126,12 @@ locals { # BETA features cluster_istio_enabled = !local.cluster_output_istio_disabled + cluster_dns_cache_enabled = var.dns_cache cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled # /BETA features - cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/modules/beta-autopilot-public-cluster/outputs.tf b/modules/beta-autopilot-public-cluster/outputs.tf index 17e634faf2..9b26f835f8 100644 --- a/modules/beta-autopilot-public-cluster/outputs.tf +++ b/modules/beta-autopilot-public-cluster/outputs.tf @@ -149,11 +149,6 @@ output "tpu_ipv4_cidr_block" { -output "dns_cache_enabled" { - description = "Whether DNS Cache enabled" - value = local.cluster_dns_cache_enabled -} - output "cloudrun_enabled" { description = "Whether CloudRun enabled" value = false @@ -164,6 +159,11 @@ output "istio_enabled" { value = local.cluster_istio_enabled } +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} + output "pod_security_policy_enabled" { description = "Whether pod security policy is enabled" value = local.cluster_pod_security_policy_enabled diff --git a/modules/beta-autopilot-public-cluster/sa.tf b/modules/beta-autopilot-public-cluster/sa.tf index fe36dadd32..6f89899bee 100644 --- a/modules/beta-autopilot-public-cluster/sa.tf +++ b/modules/beta-autopilot-public-cluster/sa.tf @@ -52,20 +52,6 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/monitoring.metricWriter" - member = google_service_account.cluster_service_account[0].member -} - -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/stackdriver.resourceMetadata.writer" - member = google_service_account.cluster_service_account[0].member -} - resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key @@ -79,17 +65,3 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } - -resource "google_project_service_identity" "fleet_project" { - count = var.fleet_project_grant_service_agent ? 1 : 0 - provider = google-beta - project = var.fleet_project - service = "gkehub.googleapis.com" -} - -resource "google_project_iam_member" "service_agent" { - for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] - project = var.project_id - role = each.value - member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" -} diff --git a/modules/beta-autopilot-public-cluster/variables.tf b/modules/beta-autopilot-public-cluster/variables.tf index 0ecca7a1a3..0d3c5cdd5c 100644 --- a/modules/beta-autopilot-public-cluster/variables.tf +++ b/modules/beta-autopilot-public-cluster/variables.tf @@ -174,11 +174,10 @@ variable "enable_resource_consumption_export" { variable "network_tags" { - description = "(Optional) - List of network tags applied to auto-provisioned node pools." + description = "(Optional, Beta) - List of network tags applied to auto-provisioned node pools." type = list(string) default = [] } - variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -376,7 +375,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -437,9 +436,3 @@ variable "fleet_project" { type = string default = null } - -variable "fleet_project_grant_service_agent" { - description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." - type = bool - default = false -} diff --git a/modules/beta-autopilot-public-cluster/versions.tf b/modules/beta-autopilot-public-cluster/versions.tf index bc43de8446..ad536dbc26 100644 --- a/modules/beta-autopilot-public-cluster/versions.tf +++ b/modules/beta-autopilot-public-cluster/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.25.0, < 6" + version = ">= 5.9.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.25.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-public-cluster/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-public-cluster/v30.1.0" } } diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index fffa95b587..59479ea2bd 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -84,30 +84,25 @@ module "gke" { node_pools = [ { - name = "default-node-pool" - machine_type = "e2-medium" - node_locations = "us-central1-b,us-central1-c" - min_count = 1 - max_count = 100 - local_ssd_count = 0 - spot = false - local_ssd_ephemeral_count = 0 - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS_CONTAINERD" - enable_gcfs = false - enable_gvnic = false - logging_variant = "DEFAULT" - auto_repair = true - auto_upgrade = true - service_account = "project-service-account@.iam.gserviceaccount.com" - preemptible = false - initial_node_count = 80 - accelerator_count = 1 - accelerator_type = "nvidia-l4" - gpu_driver_version = "LATEST" - gpu_sharing_strategy = "TIME_SHARING" - max_shared_clients_per_gpu = 2 + name = "default-node-pool" + machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = false + local_ssd_ephemeral_count = 0 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + enable_gcfs = false + enable_gvnic = false + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 }, ] @@ -176,7 +171,7 @@ Then perform the following commands on the root folder: | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                 = bool
    autoscaling_profile     = string
    min_cpu_cores           = number
    max_cpu_cores           = number
    min_memory_gb           = number
    max_memory_gb           = number
    gpu_resources           = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair             = bool
    auto_upgrade            = bool
    disk_size               = optional(number)
    disk_type               = optional(string)
    image_type              = optional(string)
    strategy                = optional(string)
    max_surge               = optional(number)
    max_unavailable         = optional(number)
    node_pool_soak_duration = optional(string)
    batch_soak_duration     = optional(string)
    batch_percentage        = optional(number)
    batch_node_count        = optional(number)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
    disk_size           = optional(number)
    disk_type           = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -190,7 +185,7 @@ Then perform the following commands on the root folder: | datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | | deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | | description | The description of the cluster | `string` | `""` | no | | disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | @@ -207,8 +202,8 @@ Then perform the following commands on the root folder: | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | | enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | -| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `false` | no | +| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | | enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | @@ -217,7 +212,6 @@ Then perform the following commands on the root folder: | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | @@ -256,7 +250,6 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | @@ -277,14 +270,13 @@ Then perform the following commands on the root folder: | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | | shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | @@ -361,23 +353,19 @@ The node_pools variable takes the following parameters: | enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | -| gpu_driver_version | Mode for how the GPU driver is installed | null | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | -| local_ssd_ephemeral_storage_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | | local_ssd_ephemeral_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | -| local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | | strategy | The upgrade stragey to be used for upgrading the nodes. Valid values of state are: `SURGE` or `BLUE_GREEN` | "SURGE" | Optional | -| threads_per_core | Optional The number of threads per physical core. To disable simultaneous multithreading (SMT) set this to 1. If unset, the maximum number of threads supported per core by the underlying processor is assumed | null | Optional | | max_surge | The number of additional nodes that can be added to the node pool during an upgrade. Increasing max_surge raises the number of nodes that can be upgraded simultaneously. Can be set to 0 or greater. Only works with `SURGE` strategy. | 1 | Optional | | max_unavailable | The number of nodes that can be simultaneously unavailable during an upgrade. Increasing max_unavailable raises the number of nodes that can be upgraded in parallel. Can be set to 0 or greater. Only works with `SURGE` strategy. | 0 | Optional | | node_pool_soak_duration | Time needed after draining the entire blue pool. After this period, the blue pool will be cleaned up. By default, it is set to one hour (3600 seconds). The maximum length of the soak time is 7 days (604,800 seconds). Only works with `BLUE_GREEN` strategy. | "3600s" | Optional | @@ -400,8 +388,6 @@ The node_pools variable takes the following parameters: | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | -| secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | -| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 99f22565cc..286a889dc7 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -130,28 +130,7 @@ resource "google_container_cluster" "primary" { disk_size = lookup(var.cluster_autoscaling, "disk_size", 100) disk_type = lookup(var.cluster_autoscaling, "disk_type", "pd-standard") - upgrade_settings { - strategy = lookup(var.cluster_autoscaling, "strategy", "SURGE") - max_surge = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_surge", 0) : null - max_unavailable = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_unavailable", 0) : null - - dynamic "blue_green_settings" { - for_each = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "BLUE_GREEN" ? [1] : [] - content { - node_pool_soak_duration = lookup(var.cluster_autoscaling, "node_pool_soak_duration", null) - - standard_rollout_policy { - batch_soak_duration = lookup(var.cluster_autoscaling, "batch_soak_duration", null) - batch_percentage = lookup(var.cluster_autoscaling, "batch_percentage", null) - batch_node_count = lookup(var.cluster_autoscaling, "batch_node_count", null) - } - } - } - } - min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") - - image_type = lookup(var.cluster_autoscaling, "image_type", "COS_CONTAINERD") } } autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" @@ -210,15 +189,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] - content { - network_tags { - tags = var.network_tags - } - } - } - master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -277,14 +247,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "stateful_ha_config" { - for_each = local.stateful_ha_config - - content { - enabled = stateful_ha_config.value.enabled - } - } - config_connector_config { enabled = var.config_connector } @@ -542,8 +504,6 @@ resource "google_container_cluster" "primary" { } } } - - depends_on = [google_project_iam_member.service_agent] } /****************************************** Create Container Cluster node pools @@ -555,9 +515,6 @@ locals { "accelerator_count", "accelerator_type", "gpu_partition_size", - "gpu_driver_version", - "gpu_sharing_strategy", - "max_shared_clients_per_gpu", "enable_secure_boot", "enable_integrity_monitoring", "local_ssd_count", @@ -573,7 +530,6 @@ locals { "enable_gvnic", "enable_secure_boot", "boot_disk_kms_key", - "queued_provisioning", ] } @@ -704,13 +660,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "queued_provisioning" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - enabled = lookup(each.value, "queued_provisioning", null) - } - } - node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -727,12 +676,6 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } - dynamic "reservation_affinity" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - consume_reservation_type = "NO_RESERVATION" - } - } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -776,12 +719,6 @@ resource "google_container_node_pool" "pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") - dynamic "ephemeral_storage_local_ssd_config" { - for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] - content { - local_ssd_count = ephemeral_storage_local_ssd_config.value - } - } dynamic "ephemeral_storage_config" { for_each = lookup(each.value, "local_ssd_ephemeral_count", 0) > 0 ? [each.value.local_ssd_ephemeral_count] : [] content { @@ -789,22 +726,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "local_nvme_ssd_block_config" { - for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] - content { - local_ssd_count = local_nvme_ssd_block_config.value - } - } - - # Supports a single secondary boot disk because `map(any)` must have the same values type. - dynamic "secondary_boot_disks" { - for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] - content { - disk_image = secondary_boot_disks.value - mode = "CONTAINER_IMAGE_CACHE" - } - } - service_account = lookup( each.value, "service_account", @@ -831,21 +752,6 @@ resource "google_container_node_pool" "pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } - - dynamic "gpu_sharing_config" { - for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] - content { - gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") - max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) - } - } - } - } - - dynamic "advanced_machine_features" { - for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] - content { - threads_per_core = lookup(each.value, "threads_per_core", 0) } } @@ -989,13 +895,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "queued_provisioning" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - enabled = lookup(each.value, "queued_provisioning", null) - } - } - node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -1012,12 +911,6 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } - dynamic "reservation_affinity" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - consume_reservation_type = "NO_RESERVATION" - } - } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -1061,12 +954,6 @@ resource "google_container_node_pool" "windows_pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") - dynamic "ephemeral_storage_local_ssd_config" { - for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] - content { - local_ssd_count = ephemeral_storage_local_ssd_config.value - } - } dynamic "ephemeral_storage_config" { for_each = lookup(each.value, "local_ssd_ephemeral_count", 0) > 0 ? [each.value.local_ssd_ephemeral_count] : [] content { @@ -1074,22 +961,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "local_nvme_ssd_block_config" { - for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] - content { - local_ssd_count = local_nvme_ssd_block_config.value - } - } - - # Supports a single secondary boot disk because `map(any)` must have the same values type. - dynamic "secondary_boot_disks" { - for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] - content { - disk_image = secondary_boot_disks.value - mode = "CONTAINER_IMAGE_CACHE" - } - } - service_account = lookup( each.value, "service_account", @@ -1116,21 +987,6 @@ resource "google_container_node_pool" "windows_pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } - - dynamic "gpu_sharing_config" { - for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] - content { - gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") - max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) - } - } - } - } - - dynamic "advanced_machine_features" { - for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] - content { - threads_per_core = lookup(each.value, "threads_per_core", 0) } } diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index 2e05bedeeb..c59479b579 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -94,7 +94,6 @@ locals { logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] - stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} @@ -191,13 +190,13 @@ locals { # BETA features cluster_istio_enabled = !local.cluster_output_istio_disabled + cluster_dns_cache_enabled = var.dns_cache cluster_telemetry_type_is_set = var.cluster_telemetry_type != null cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled # /BETA features - cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/modules/beta-private-cluster-update-variant/outputs.tf b/modules/beta-private-cluster-update-variant/outputs.tf index 3209aca8ae..2d8e768f7c 100644 --- a/modules/beta-private-cluster-update-variant/outputs.tf +++ b/modules/beta-private-cluster-update-variant/outputs.tf @@ -185,11 +185,6 @@ output "peering_name" { value = local.cluster_peering_name } -output "dns_cache_enabled" { - description = "Whether DNS Cache enabled" - value = local.cluster_dns_cache_enabled -} - output "cloudrun_enabled" { description = "Whether CloudRun enabled" value = local.cluster_cloudrun_enabled @@ -200,6 +195,11 @@ output "istio_enabled" { value = local.cluster_istio_enabled } +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} + output "pod_security_policy_enabled" { description = "Whether pod security policy is enabled" value = local.cluster_pod_security_policy_enabled diff --git a/modules/beta-private-cluster-update-variant/sa.tf b/modules/beta-private-cluster-update-variant/sa.tf index fe36dadd32..6f89899bee 100644 --- a/modules/beta-private-cluster-update-variant/sa.tf +++ b/modules/beta-private-cluster-update-variant/sa.tf @@ -52,20 +52,6 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/monitoring.metricWriter" - member = google_service_account.cluster_service_account[0].member -} - -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/stackdriver.resourceMetadata.writer" - member = google_service_account.cluster_service_account[0].member -} - resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key @@ -79,17 +65,3 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } - -resource "google_project_service_identity" "fleet_project" { - count = var.fleet_project_grant_service_agent ? 1 : 0 - provider = google-beta - project = var.fleet_project - service = "gkehub.googleapis.com" -} - -resource "google_project_iam_member" "service_agent" { - for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] - project = var.project_id - role = each.value - member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" -} diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index 64f685a34d..3d13d63ac5 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -239,25 +239,17 @@ variable "enable_resource_consumption_export" { variable "cluster_autoscaling" { type = object({ - enabled = bool - autoscaling_profile = string - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) - image_type = optional(string) - strategy = optional(string) - max_surge = optional(number) - max_unavailable = optional(number) - node_pool_soak_duration = optional(string) - batch_soak_duration = optional(string) - batch_percentage = optional(number) - batch_node_count = optional(number) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) }) default = { enabled = false @@ -271,7 +263,6 @@ variable "cluster_autoscaling" { auto_upgrade = true disk_size = 100 disk_type = "pd-standard" - image_type = "COS_CONTAINERD" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -309,12 +300,6 @@ variable "node_pools_oauth_scopes" { } } -variable "network_tags" { - description = "(Optional) - List of network tags applied to auto-provisioned node pools." - type = list(string) - default = [] -} - variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -420,25 +405,25 @@ variable "cluster_resource_labels" { variable "deploy_using_private_endpoint" { type = bool - description = "A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." + description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." default = false } variable "enable_private_endpoint" { type = bool - description = "Whether the master's internal IP address is used as the cluster endpoint" + description = "(Beta) Whether the master's internal IP address is used as the cluster endpoint" default = false } variable "enable_private_nodes" { type = bool - description = "Whether nodes have internal IP addresses only" + description = "(Beta) Whether nodes have internal IP addresses only" default = false } variable "master_ipv4_cidr_block" { type = string - description = "The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters." + description = "(Beta) The IP range in CIDR notation to use for the hosted master network" default = "10.0.0.0/28" } @@ -565,7 +550,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -705,12 +690,6 @@ variable "gcs_fuse_csi_driver" { default = false } -variable "stateful_ha" { - type = bool - description = "Whether the Stateful HA Addon is enabled for this cluster." - default = false -} - variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -751,7 +730,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." default = [] } @@ -773,12 +752,6 @@ variable "config_connector" { default = false } -variable "enable_l4_ilb_subsetting" { - type = bool - description = "Enable L4 ILB Subsetting on the cluster" - default = false -} - variable "istio" { description = "(Beta) Enable Istio addon" type = bool @@ -815,6 +788,12 @@ variable "enable_pod_security_policy" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." @@ -844,9 +823,3 @@ variable "fleet_project" { type = string default = null } - -variable "fleet_project_grant_service_agent" { - description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." - type = bool - default = false -} diff --git a/modules/beta-private-cluster-update-variant/versions.tf b/modules/beta-private-cluster-update-variant/versions.tf index d9385ca73d..7fc2de48e4 100644 --- a/modules/beta-private-cluster-update-variant/versions.tf +++ b/modules/beta-private-cluster-update-variant/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.25.0, < 6" + version = ">= 5.9.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.25.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster-update-variant/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster-update-variant/v30.1.0" } } diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 6690b071d2..dce2535e3e 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -62,30 +62,25 @@ module "gke" { node_pools = [ { - name = "default-node-pool" - machine_type = "e2-medium" - node_locations = "us-central1-b,us-central1-c" - min_count = 1 - max_count = 100 - local_ssd_count = 0 - spot = false - local_ssd_ephemeral_count = 0 - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS_CONTAINERD" - enable_gcfs = false - enable_gvnic = false - logging_variant = "DEFAULT" - auto_repair = true - auto_upgrade = true - service_account = "project-service-account@.iam.gserviceaccount.com" - preemptible = false - initial_node_count = 80 - accelerator_count = 1 - accelerator_type = "nvidia-l4" - gpu_driver_version = "LATEST" - gpu_sharing_strategy = "TIME_SHARING" - max_shared_clients_per_gpu = 2 + name = "default-node-pool" + machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = false + local_ssd_ephemeral_count = 0 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + enable_gcfs = false + enable_gvnic = false + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 }, ] @@ -154,7 +149,7 @@ Then perform the following commands on the root folder: | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                 = bool
    autoscaling_profile     = string
    min_cpu_cores           = number
    max_cpu_cores           = number
    min_memory_gb           = number
    max_memory_gb           = number
    gpu_resources           = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair             = bool
    auto_upgrade            = bool
    disk_size               = optional(number)
    disk_type               = optional(string)
    image_type              = optional(string)
    strategy                = optional(string)
    max_surge               = optional(number)
    max_unavailable         = optional(number)
    node_pool_soak_duration = optional(string)
    batch_soak_duration     = optional(string)
    batch_percentage        = optional(number)
    batch_node_count        = optional(number)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
    disk_size           = optional(number)
    disk_type           = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -168,7 +163,7 @@ Then perform the following commands on the root folder: | datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | | deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | | description | The description of the cluster | `string` | `""` | no | | disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | @@ -185,8 +180,8 @@ Then perform the following commands on the root folder: | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | | enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | -| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `false` | no | +| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | | enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | @@ -195,7 +190,6 @@ Then perform the following commands on the root folder: | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | @@ -234,7 +228,6 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | @@ -255,14 +248,13 @@ Then perform the following commands on the root folder: | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | | shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | @@ -339,23 +331,19 @@ The node_pools variable takes the following parameters: | enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | -| gpu_driver_version | Mode for how the GPU driver is installed | null | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | -| local_ssd_ephemeral_storage_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | | local_ssd_ephemeral_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | -| local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | | strategy | The upgrade stragey to be used for upgrading the nodes. Valid values of state are: `SURGE` or `BLUE_GREEN` | "SURGE" | Optional | -| threads_per_core | Optional The number of threads per physical core. To disable simultaneous multithreading (SMT) set this to 1. If unset, the maximum number of threads supported per core by the underlying processor is assumed | null | Optional | | max_surge | The number of additional nodes that can be added to the node pool during an upgrade. Increasing max_surge raises the number of nodes that can be upgraded simultaneously. Can be set to 0 or greater. Only works with `SURGE` strategy. | 1 | Optional | | max_unavailable | The number of nodes that can be simultaneously unavailable during an upgrade. Increasing max_unavailable raises the number of nodes that can be upgraded in parallel. Can be set to 0 or greater. Only works with `SURGE` strategy. | 0 | Optional | | node_pool_soak_duration | Time needed after draining the entire blue pool. After this period, the blue pool will be cleaned up. By default, it is set to one hour (3600 seconds). The maximum length of the soak time is 7 days (604,800 seconds). Only works with `BLUE_GREEN` strategy. | "3600s" | Optional | @@ -378,8 +366,6 @@ The node_pools variable takes the following parameters: | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | -| secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | -| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index de7b0c9caa..aed3ab4152 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -130,28 +130,7 @@ resource "google_container_cluster" "primary" { disk_size = lookup(var.cluster_autoscaling, "disk_size", 100) disk_type = lookup(var.cluster_autoscaling, "disk_type", "pd-standard") - upgrade_settings { - strategy = lookup(var.cluster_autoscaling, "strategy", "SURGE") - max_surge = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_surge", 0) : null - max_unavailable = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_unavailable", 0) : null - - dynamic "blue_green_settings" { - for_each = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "BLUE_GREEN" ? [1] : [] - content { - node_pool_soak_duration = lookup(var.cluster_autoscaling, "node_pool_soak_duration", null) - - standard_rollout_policy { - batch_soak_duration = lookup(var.cluster_autoscaling, "batch_soak_duration", null) - batch_percentage = lookup(var.cluster_autoscaling, "batch_percentage", null) - batch_node_count = lookup(var.cluster_autoscaling, "batch_node_count", null) - } - } - } - } - min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") - - image_type = lookup(var.cluster_autoscaling, "image_type", "COS_CONTAINERD") } } autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" @@ -210,15 +189,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] - content { - network_tags { - tags = var.network_tags - } - } - } - master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -277,14 +247,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "stateful_ha_config" { - for_each = local.stateful_ha_config - - content { - enabled = stateful_ha_config.value.enabled - } - } - config_connector_config { enabled = var.config_connector } @@ -542,8 +504,6 @@ resource "google_container_cluster" "primary" { } } } - - depends_on = [google_project_iam_member.service_agent] } /****************************************** Create Container Cluster node pools @@ -625,13 +585,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "queued_provisioning" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - enabled = lookup(each.value, "queued_provisioning", null) - } - } - node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -648,12 +601,6 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } - dynamic "reservation_affinity" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - consume_reservation_type = "NO_RESERVATION" - } - } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -697,12 +644,6 @@ resource "google_container_node_pool" "pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") - dynamic "ephemeral_storage_local_ssd_config" { - for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] - content { - local_ssd_count = ephemeral_storage_local_ssd_config.value - } - } dynamic "ephemeral_storage_config" { for_each = lookup(each.value, "local_ssd_ephemeral_count", 0) > 0 ? [each.value.local_ssd_ephemeral_count] : [] content { @@ -710,22 +651,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "local_nvme_ssd_block_config" { - for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] - content { - local_ssd_count = local_nvme_ssd_block_config.value - } - } - - # Supports a single secondary boot disk because `map(any)` must have the same values type. - dynamic "secondary_boot_disks" { - for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] - content { - disk_image = secondary_boot_disks.value - mode = "CONTAINER_IMAGE_CACHE" - } - } - service_account = lookup( each.value, "service_account", @@ -752,21 +677,6 @@ resource "google_container_node_pool" "pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } - - dynamic "gpu_sharing_config" { - for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] - content { - gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") - max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) - } - } - } - } - - dynamic "advanced_machine_features" { - for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] - content { - threads_per_core = lookup(each.value, "threads_per_core", 0) } } @@ -909,13 +819,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "queued_provisioning" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - enabled = lookup(each.value, "queued_provisioning", null) - } - } - node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -932,12 +835,6 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } - dynamic "reservation_affinity" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - consume_reservation_type = "NO_RESERVATION" - } - } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -981,12 +878,6 @@ resource "google_container_node_pool" "windows_pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") - dynamic "ephemeral_storage_local_ssd_config" { - for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] - content { - local_ssd_count = ephemeral_storage_local_ssd_config.value - } - } dynamic "ephemeral_storage_config" { for_each = lookup(each.value, "local_ssd_ephemeral_count", 0) > 0 ? [each.value.local_ssd_ephemeral_count] : [] content { @@ -994,22 +885,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "local_nvme_ssd_block_config" { - for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] - content { - local_ssd_count = local_nvme_ssd_block_config.value - } - } - - # Supports a single secondary boot disk because `map(any)` must have the same values type. - dynamic "secondary_boot_disks" { - for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] - content { - disk_image = secondary_boot_disks.value - mode = "CONTAINER_IMAGE_CACHE" - } - } - service_account = lookup( each.value, "service_account", @@ -1036,21 +911,6 @@ resource "google_container_node_pool" "windows_pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } - - dynamic "gpu_sharing_config" { - for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] - content { - gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") - max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) - } - } - } - } - - dynamic "advanced_machine_features" { - for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] - content { - threads_per_core = lookup(each.value, "threads_per_core", 0) } } diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index 2e05bedeeb..c59479b579 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -94,7 +94,6 @@ locals { logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] - stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} @@ -191,13 +190,13 @@ locals { # BETA features cluster_istio_enabled = !local.cluster_output_istio_disabled + cluster_dns_cache_enabled = var.dns_cache cluster_telemetry_type_is_set = var.cluster_telemetry_type != null cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled # /BETA features - cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf index 3209aca8ae..2d8e768f7c 100644 --- a/modules/beta-private-cluster/outputs.tf +++ b/modules/beta-private-cluster/outputs.tf @@ -185,11 +185,6 @@ output "peering_name" { value = local.cluster_peering_name } -output "dns_cache_enabled" { - description = "Whether DNS Cache enabled" - value = local.cluster_dns_cache_enabled -} - output "cloudrun_enabled" { description = "Whether CloudRun enabled" value = local.cluster_cloudrun_enabled @@ -200,6 +195,11 @@ output "istio_enabled" { value = local.cluster_istio_enabled } +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} + output "pod_security_policy_enabled" { description = "Whether pod security policy is enabled" value = local.cluster_pod_security_policy_enabled diff --git a/modules/beta-private-cluster/sa.tf b/modules/beta-private-cluster/sa.tf index fe36dadd32..6f89899bee 100644 --- a/modules/beta-private-cluster/sa.tf +++ b/modules/beta-private-cluster/sa.tf @@ -52,20 +52,6 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/monitoring.metricWriter" - member = google_service_account.cluster_service_account[0].member -} - -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/stackdriver.resourceMetadata.writer" - member = google_service_account.cluster_service_account[0].member -} - resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key @@ -79,17 +65,3 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } - -resource "google_project_service_identity" "fleet_project" { - count = var.fleet_project_grant_service_agent ? 1 : 0 - provider = google-beta - project = var.fleet_project - service = "gkehub.googleapis.com" -} - -resource "google_project_iam_member" "service_agent" { - for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] - project = var.project_id - role = each.value - member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" -} diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 64f685a34d..3d13d63ac5 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -239,25 +239,17 @@ variable "enable_resource_consumption_export" { variable "cluster_autoscaling" { type = object({ - enabled = bool - autoscaling_profile = string - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) - image_type = optional(string) - strategy = optional(string) - max_surge = optional(number) - max_unavailable = optional(number) - node_pool_soak_duration = optional(string) - batch_soak_duration = optional(string) - batch_percentage = optional(number) - batch_node_count = optional(number) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) }) default = { enabled = false @@ -271,7 +263,6 @@ variable "cluster_autoscaling" { auto_upgrade = true disk_size = 100 disk_type = "pd-standard" - image_type = "COS_CONTAINERD" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -309,12 +300,6 @@ variable "node_pools_oauth_scopes" { } } -variable "network_tags" { - description = "(Optional) - List of network tags applied to auto-provisioned node pools." - type = list(string) - default = [] -} - variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -420,25 +405,25 @@ variable "cluster_resource_labels" { variable "deploy_using_private_endpoint" { type = bool - description = "A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." + description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." default = false } variable "enable_private_endpoint" { type = bool - description = "Whether the master's internal IP address is used as the cluster endpoint" + description = "(Beta) Whether the master's internal IP address is used as the cluster endpoint" default = false } variable "enable_private_nodes" { type = bool - description = "Whether nodes have internal IP addresses only" + description = "(Beta) Whether nodes have internal IP addresses only" default = false } variable "master_ipv4_cidr_block" { type = string - description = "The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters." + description = "(Beta) The IP range in CIDR notation to use for the hosted master network" default = "10.0.0.0/28" } @@ -565,7 +550,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -705,12 +690,6 @@ variable "gcs_fuse_csi_driver" { default = false } -variable "stateful_ha" { - type = bool - description = "Whether the Stateful HA Addon is enabled for this cluster." - default = false -} - variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -751,7 +730,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." default = [] } @@ -773,12 +752,6 @@ variable "config_connector" { default = false } -variable "enable_l4_ilb_subsetting" { - type = bool - description = "Enable L4 ILB Subsetting on the cluster" - default = false -} - variable "istio" { description = "(Beta) Enable Istio addon" type = bool @@ -815,6 +788,12 @@ variable "enable_pod_security_policy" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." @@ -844,9 +823,3 @@ variable "fleet_project" { type = string default = null } - -variable "fleet_project_grant_service_agent" { - description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." - type = bool - default = false -} diff --git a/modules/beta-private-cluster/versions.tf b/modules/beta-private-cluster/versions.tf index 7184582867..ded29b9a0e 100644 --- a/modules/beta-private-cluster/versions.tf +++ b/modules/beta-private-cluster/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.25.0, < 6" + version = ">= 5.9.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.25.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster/v30.1.0" } } diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 8717517d0d..4bf4563bbc 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -78,30 +78,25 @@ module "gke" { node_pools = [ { - name = "default-node-pool" - machine_type = "e2-medium" - node_locations = "us-central1-b,us-central1-c" - min_count = 1 - max_count = 100 - local_ssd_count = 0 - spot = false - local_ssd_ephemeral_count = 0 - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS_CONTAINERD" - enable_gcfs = false - enable_gvnic = false - logging_variant = "DEFAULT" - auto_repair = true - auto_upgrade = true - service_account = "project-service-account@.iam.gserviceaccount.com" - preemptible = false - initial_node_count = 80 - accelerator_count = 1 - accelerator_type = "nvidia-l4" - gpu_driver_version = "LATEST" - gpu_sharing_strategy = "TIME_SHARING" - max_shared_clients_per_gpu = 2 + name = "default-node-pool" + machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = false + local_ssd_ephemeral_count = 0 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + enable_gcfs = false + enable_gvnic = false + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 }, ] @@ -170,7 +165,7 @@ Then perform the following commands on the root folder: | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                 = bool
    autoscaling_profile     = string
    min_cpu_cores           = number
    max_cpu_cores           = number
    min_memory_gb           = number
    max_memory_gb           = number
    gpu_resources           = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair             = bool
    auto_upgrade            = bool
    disk_size               = optional(number)
    disk_type               = optional(string)
    image_type              = optional(string)
    strategy                = optional(string)
    max_surge               = optional(number)
    max_unavailable         = optional(number)
    node_pool_soak_duration = optional(string)
    batch_soak_duration     = optional(string)
    batch_percentage        = optional(number)
    batch_node_count        = optional(number)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
    disk_size           = optional(number)
    disk_type           = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -208,7 +203,6 @@ Then perform the following commands on the root folder: | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | @@ -245,7 +239,6 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | @@ -266,14 +259,13 @@ Then perform the following commands on the root folder: | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | | shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | @@ -348,23 +340,19 @@ The node_pools variable takes the following parameters: | enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | -| gpu_driver_version | Mode for how the GPU driver is installed | null | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | -| local_ssd_ephemeral_storage_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | | local_ssd_ephemeral_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | -| local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | | strategy | The upgrade stragey to be used for upgrading the nodes. Valid values of state are: `SURGE` or `BLUE_GREEN` | "SURGE" | Optional | -| threads_per_core | Optional The number of threads per physical core. To disable simultaneous multithreading (SMT) set this to 1. If unset, the maximum number of threads supported per core by the underlying processor is assumed | null | Optional | | max_surge | The number of additional nodes that can be added to the node pool during an upgrade. Increasing max_surge raises the number of nodes that can be upgraded simultaneously. Can be set to 0 or greater. Only works with `SURGE` strategy. | 1 | Optional | | max_unavailable | The number of nodes that can be simultaneously unavailable during an upgrade. Increasing max_unavailable raises the number of nodes that can be upgraded in parallel. Can be set to 0 or greater. Only works with `SURGE` strategy. | 0 | Optional | | node_pool_soak_duration | Time needed after draining the entire blue pool. After this period, the blue pool will be cleaned up. By default, it is set to one hour (3600 seconds). The maximum length of the soak time is 7 days (604,800 seconds). Only works with `BLUE_GREEN` strategy. | "3600s" | Optional | @@ -388,8 +376,6 @@ The node_pools variable takes the following parameters: | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | -| secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | -| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index 742c0b5ced..eb4a2f7070 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -130,28 +130,7 @@ resource "google_container_cluster" "primary" { disk_size = lookup(var.cluster_autoscaling, "disk_size", 100) disk_type = lookup(var.cluster_autoscaling, "disk_type", "pd-standard") - upgrade_settings { - strategy = lookup(var.cluster_autoscaling, "strategy", "SURGE") - max_surge = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_surge", 0) : null - max_unavailable = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_unavailable", 0) : null - - dynamic "blue_green_settings" { - for_each = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "BLUE_GREEN" ? [1] : [] - content { - node_pool_soak_duration = lookup(var.cluster_autoscaling, "node_pool_soak_duration", null) - - standard_rollout_policy { - batch_soak_duration = lookup(var.cluster_autoscaling, "batch_soak_duration", null) - batch_percentage = lookup(var.cluster_autoscaling, "batch_percentage", null) - batch_node_count = lookup(var.cluster_autoscaling, "batch_node_count", null) - } - } - } - } - min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") - - image_type = lookup(var.cluster_autoscaling, "image_type", "COS_CONTAINERD") } } autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" @@ -210,15 +189,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] - content { - network_tags { - tags = var.network_tags - } - } - } - master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -277,14 +247,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "stateful_ha_config" { - for_each = local.stateful_ha_config - - content { - enabled = stateful_ha_config.value.enabled - } - } - config_connector_config { enabled = var.config_connector } @@ -523,8 +485,6 @@ resource "google_container_cluster" "primary" { } } } - - depends_on = [google_project_iam_member.service_agent] } /****************************************** Create Container Cluster node pools @@ -536,9 +496,6 @@ locals { "accelerator_count", "accelerator_type", "gpu_partition_size", - "gpu_driver_version", - "gpu_sharing_strategy", - "max_shared_clients_per_gpu", "enable_secure_boot", "enable_integrity_monitoring", "local_ssd_count", @@ -554,7 +511,6 @@ locals { "enable_gvnic", "enable_secure_boot", "boot_disk_kms_key", - "queued_provisioning", ] } @@ -685,13 +641,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "queued_provisioning" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - enabled = lookup(each.value, "queued_provisioning", null) - } - } - node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -708,12 +657,6 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } - dynamic "reservation_affinity" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - consume_reservation_type = "NO_RESERVATION" - } - } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -757,12 +700,6 @@ resource "google_container_node_pool" "pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") - dynamic "ephemeral_storage_local_ssd_config" { - for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] - content { - local_ssd_count = ephemeral_storage_local_ssd_config.value - } - } dynamic "ephemeral_storage_config" { for_each = lookup(each.value, "local_ssd_ephemeral_count", 0) > 0 ? [each.value.local_ssd_ephemeral_count] : [] content { @@ -770,22 +707,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "local_nvme_ssd_block_config" { - for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] - content { - local_ssd_count = local_nvme_ssd_block_config.value - } - } - - # Supports a single secondary boot disk because `map(any)` must have the same values type. - dynamic "secondary_boot_disks" { - for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] - content { - disk_image = secondary_boot_disks.value - mode = "CONTAINER_IMAGE_CACHE" - } - } - service_account = lookup( each.value, "service_account", @@ -812,21 +733,6 @@ resource "google_container_node_pool" "pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } - - dynamic "gpu_sharing_config" { - for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] - content { - gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") - max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) - } - } - } - } - - dynamic "advanced_machine_features" { - for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] - content { - threads_per_core = lookup(each.value, "threads_per_core", 0) } } @@ -970,13 +876,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "queued_provisioning" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - enabled = lookup(each.value, "queued_provisioning", null) - } - } - node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -993,12 +892,6 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } - dynamic "reservation_affinity" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - consume_reservation_type = "NO_RESERVATION" - } - } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -1042,12 +935,6 @@ resource "google_container_node_pool" "windows_pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") - dynamic "ephemeral_storage_local_ssd_config" { - for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] - content { - local_ssd_count = ephemeral_storage_local_ssd_config.value - } - } dynamic "ephemeral_storage_config" { for_each = lookup(each.value, "local_ssd_ephemeral_count", 0) > 0 ? [each.value.local_ssd_ephemeral_count] : [] content { @@ -1055,22 +942,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "local_nvme_ssd_block_config" { - for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] - content { - local_ssd_count = local_nvme_ssd_block_config.value - } - } - - # Supports a single secondary boot disk because `map(any)` must have the same values type. - dynamic "secondary_boot_disks" { - for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] - content { - disk_image = secondary_boot_disks.value - mode = "CONTAINER_IMAGE_CACHE" - } - } - service_account = lookup( each.value, "service_account", @@ -1097,21 +968,6 @@ resource "google_container_node_pool" "windows_pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } - - dynamic "gpu_sharing_config" { - for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] - content { - gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") - max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) - } - } - } - } - - dynamic "advanced_machine_features" { - for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] - content { - threads_per_core = lookup(each.value, "threads_per_core", 0) } } diff --git a/modules/beta-public-cluster-update-variant/main.tf b/modules/beta-public-cluster-update-variant/main.tf index 952588fb18..33dd316b35 100644 --- a/modules/beta-public-cluster-update-variant/main.tf +++ b/modules/beta-public-cluster-update-variant/main.tf @@ -94,7 +94,6 @@ locals { logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] - stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} @@ -190,13 +189,13 @@ locals { # BETA features cluster_istio_enabled = !local.cluster_output_istio_disabled + cluster_dns_cache_enabled = var.dns_cache cluster_telemetry_type_is_set = var.cluster_telemetry_type != null cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled # /BETA features - cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/modules/beta-public-cluster-update-variant/outputs.tf b/modules/beta-public-cluster-update-variant/outputs.tf index 29df3aad34..e388fb4406 100644 --- a/modules/beta-public-cluster-update-variant/outputs.tf +++ b/modules/beta-public-cluster-update-variant/outputs.tf @@ -175,11 +175,6 @@ output "mesh_certificates_config" { } -output "dns_cache_enabled" { - description = "Whether DNS Cache enabled" - value = local.cluster_dns_cache_enabled -} - output "cloudrun_enabled" { description = "Whether CloudRun enabled" value = local.cluster_cloudrun_enabled @@ -190,6 +185,11 @@ output "istio_enabled" { value = local.cluster_istio_enabled } +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} + output "pod_security_policy_enabled" { description = "Whether pod security policy is enabled" value = local.cluster_pod_security_policy_enabled diff --git a/modules/beta-public-cluster-update-variant/sa.tf b/modules/beta-public-cluster-update-variant/sa.tf index fe36dadd32..6f89899bee 100644 --- a/modules/beta-public-cluster-update-variant/sa.tf +++ b/modules/beta-public-cluster-update-variant/sa.tf @@ -52,20 +52,6 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/monitoring.metricWriter" - member = google_service_account.cluster_service_account[0].member -} - -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/stackdriver.resourceMetadata.writer" - member = google_service_account.cluster_service_account[0].member -} - resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key @@ -79,17 +65,3 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } - -resource "google_project_service_identity" "fleet_project" { - count = var.fleet_project_grant_service_agent ? 1 : 0 - provider = google-beta - project = var.fleet_project - service = "gkehub.googleapis.com" -} - -resource "google_project_iam_member" "service_agent" { - for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] - project = var.project_id - role = each.value - member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" -} diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index 5de0d1f40a..4a5d55dd6a 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -239,25 +239,17 @@ variable "enable_resource_consumption_export" { variable "cluster_autoscaling" { type = object({ - enabled = bool - autoscaling_profile = string - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) - image_type = optional(string) - strategy = optional(string) - max_surge = optional(number) - max_unavailable = optional(number) - node_pool_soak_duration = optional(string) - batch_soak_duration = optional(string) - batch_percentage = optional(number) - batch_node_count = optional(number) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) }) default = { enabled = false @@ -271,7 +263,6 @@ variable "cluster_autoscaling" { auto_upgrade = true disk_size = 100 disk_type = "pd-standard" - image_type = "COS_CONTAINERD" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -309,12 +300,6 @@ variable "node_pools_oauth_scopes" { } } -variable "network_tags" { - description = "(Optional) - List of network tags applied to auto-provisioned node pools." - type = list(string) - default = [] -} - variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -535,7 +520,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -675,12 +660,6 @@ variable "gcs_fuse_csi_driver" { default = false } -variable "stateful_ha" { - type = bool - description = "Whether the Stateful HA Addon is enabled for this cluster." - default = false -} - variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -721,7 +700,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." default = [] } @@ -743,12 +722,6 @@ variable "config_connector" { default = false } -variable "enable_l4_ilb_subsetting" { - type = bool - description = "Enable L4 ILB Subsetting on the cluster" - default = false -} - variable "istio" { description = "(Beta) Enable Istio addon" type = bool @@ -785,6 +758,12 @@ variable "enable_pod_security_policy" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." @@ -814,9 +793,3 @@ variable "fleet_project" { type = string default = null } - -variable "fleet_project_grant_service_agent" { - description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." - type = bool - default = false -} diff --git a/modules/beta-public-cluster-update-variant/versions.tf b/modules/beta-public-cluster-update-variant/versions.tf index afe9612323..2a8c09d960 100644 --- a/modules/beta-public-cluster-update-variant/versions.tf +++ b/modules/beta-public-cluster-update-variant/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.25.0, < 6" + version = ">= 5.9.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.25.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster-update-variant/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster-update-variant/v30.1.0" } } diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 2d963a101d..1a54af89e2 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -56,30 +56,25 @@ module "gke" { node_pools = [ { - name = "default-node-pool" - machine_type = "e2-medium" - node_locations = "us-central1-b,us-central1-c" - min_count = 1 - max_count = 100 - local_ssd_count = 0 - spot = false - local_ssd_ephemeral_count = 0 - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS_CONTAINERD" - enable_gcfs = false - enable_gvnic = false - logging_variant = "DEFAULT" - auto_repair = true - auto_upgrade = true - service_account = "project-service-account@.iam.gserviceaccount.com" - preemptible = false - initial_node_count = 80 - accelerator_count = 1 - accelerator_type = "nvidia-l4" - gpu_driver_version = "LATEST" - gpu_sharing_strategy = "TIME_SHARING" - max_shared_clients_per_gpu = 2 + name = "default-node-pool" + machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = false + local_ssd_ephemeral_count = 0 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + enable_gcfs = false + enable_gvnic = false + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 }, ] @@ -148,7 +143,7 @@ Then perform the following commands on the root folder: | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                 = bool
    autoscaling_profile     = string
    min_cpu_cores           = number
    max_cpu_cores           = number
    min_memory_gb           = number
    max_memory_gb           = number
    gpu_resources           = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair             = bool
    auto_upgrade            = bool
    disk_size               = optional(number)
    disk_type               = optional(string)
    image_type              = optional(string)
    strategy                = optional(string)
    max_surge               = optional(number)
    max_unavailable         = optional(number)
    node_pool_soak_duration = optional(string)
    batch_soak_duration     = optional(string)
    batch_percentage        = optional(number)
    batch_node_count        = optional(number)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
    disk_size           = optional(number)
    disk_type           = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -186,7 +181,6 @@ Then perform the following commands on the root folder: | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | -| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | @@ -223,7 +217,6 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | @@ -244,14 +237,13 @@ Then perform the following commands on the root folder: | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | | shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | @@ -326,23 +318,19 @@ The node_pools variable takes the following parameters: | enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | -| gpu_driver_version | Mode for how the GPU driver is installed | null | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | -| local_ssd_ephemeral_storage_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | | local_ssd_ephemeral_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | -| local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | | strategy | The upgrade stragey to be used for upgrading the nodes. Valid values of state are: `SURGE` or `BLUE_GREEN` | "SURGE" | Optional | -| threads_per_core | Optional The number of threads per physical core. To disable simultaneous multithreading (SMT) set this to 1. If unset, the maximum number of threads supported per core by the underlying processor is assumed | null | Optional | | max_surge | The number of additional nodes that can be added to the node pool during an upgrade. Increasing max_surge raises the number of nodes that can be upgraded simultaneously. Can be set to 0 or greater. Only works with `SURGE` strategy. | 1 | Optional | | max_unavailable | The number of nodes that can be simultaneously unavailable during an upgrade. Increasing max_unavailable raises the number of nodes that can be upgraded in parallel. Can be set to 0 or greater. Only works with `SURGE` strategy. | 0 | Optional | | node_pool_soak_duration | Time needed after draining the entire blue pool. After this period, the blue pool will be cleaned up. By default, it is set to one hour (3600 seconds). The maximum length of the soak time is 7 days (604,800 seconds). Only works with `BLUE_GREEN` strategy. | "3600s" | Optional | @@ -366,8 +354,6 @@ The node_pools variable takes the following parameters: | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | -| secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | -| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index f32f49a10d..e2f88f7cb2 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -130,28 +130,7 @@ resource "google_container_cluster" "primary" { disk_size = lookup(var.cluster_autoscaling, "disk_size", 100) disk_type = lookup(var.cluster_autoscaling, "disk_type", "pd-standard") - upgrade_settings { - strategy = lookup(var.cluster_autoscaling, "strategy", "SURGE") - max_surge = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_surge", 0) : null - max_unavailable = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_unavailable", 0) : null - - dynamic "blue_green_settings" { - for_each = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "BLUE_GREEN" ? [1] : [] - content { - node_pool_soak_duration = lookup(var.cluster_autoscaling, "node_pool_soak_duration", null) - - standard_rollout_policy { - batch_soak_duration = lookup(var.cluster_autoscaling, "batch_soak_duration", null) - batch_percentage = lookup(var.cluster_autoscaling, "batch_percentage", null) - batch_node_count = lookup(var.cluster_autoscaling, "batch_node_count", null) - } - } - } - } - min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") - - image_type = lookup(var.cluster_autoscaling, "image_type", "COS_CONTAINERD") } } autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" @@ -210,15 +189,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] - content { - network_tags { - tags = var.network_tags - } - } - } - master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -277,14 +247,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "stateful_ha_config" { - for_each = local.stateful_ha_config - - content { - enabled = stateful_ha_config.value.enabled - } - } - config_connector_config { enabled = var.config_connector } @@ -523,8 +485,6 @@ resource "google_container_cluster" "primary" { } } } - - depends_on = [google_project_iam_member.service_agent] } /****************************************** Create Container Cluster node pools @@ -606,13 +566,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "queued_provisioning" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - enabled = lookup(each.value, "queued_provisioning", null) - } - } - node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -629,12 +582,6 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } - dynamic "reservation_affinity" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - consume_reservation_type = "NO_RESERVATION" - } - } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -678,12 +625,6 @@ resource "google_container_node_pool" "pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") - dynamic "ephemeral_storage_local_ssd_config" { - for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] - content { - local_ssd_count = ephemeral_storage_local_ssd_config.value - } - } dynamic "ephemeral_storage_config" { for_each = lookup(each.value, "local_ssd_ephemeral_count", 0) > 0 ? [each.value.local_ssd_ephemeral_count] : [] content { @@ -691,22 +632,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "local_nvme_ssd_block_config" { - for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] - content { - local_ssd_count = local_nvme_ssd_block_config.value - } - } - - # Supports a single secondary boot disk because `map(any)` must have the same values type. - dynamic "secondary_boot_disks" { - for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] - content { - disk_image = secondary_boot_disks.value - mode = "CONTAINER_IMAGE_CACHE" - } - } - service_account = lookup( each.value, "service_account", @@ -733,21 +658,6 @@ resource "google_container_node_pool" "pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } - - dynamic "gpu_sharing_config" { - for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] - content { - gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") - max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) - } - } - } - } - - dynamic "advanced_machine_features" { - for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] - content { - threads_per_core = lookup(each.value, "threads_per_core", 0) } } @@ -890,13 +800,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "queued_provisioning" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - enabled = lookup(each.value, "queued_provisioning", null) - } - } - node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -913,12 +816,6 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } - dynamic "reservation_affinity" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - consume_reservation_type = "NO_RESERVATION" - } - } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -962,12 +859,6 @@ resource "google_container_node_pool" "windows_pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") - dynamic "ephemeral_storage_local_ssd_config" { - for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] - content { - local_ssd_count = ephemeral_storage_local_ssd_config.value - } - } dynamic "ephemeral_storage_config" { for_each = lookup(each.value, "local_ssd_ephemeral_count", 0) > 0 ? [each.value.local_ssd_ephemeral_count] : [] content { @@ -975,22 +866,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "local_nvme_ssd_block_config" { - for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] - content { - local_ssd_count = local_nvme_ssd_block_config.value - } - } - - # Supports a single secondary boot disk because `map(any)` must have the same values type. - dynamic "secondary_boot_disks" { - for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] - content { - disk_image = secondary_boot_disks.value - mode = "CONTAINER_IMAGE_CACHE" - } - } - service_account = lookup( each.value, "service_account", @@ -1017,21 +892,6 @@ resource "google_container_node_pool" "windows_pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } - - dynamic "gpu_sharing_config" { - for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] - content { - gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") - max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) - } - } - } - } - - dynamic "advanced_machine_features" { - for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] - content { - threads_per_core = lookup(each.value, "threads_per_core", 0) } } diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 952588fb18..33dd316b35 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -94,7 +94,6 @@ locals { logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] - stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} @@ -190,13 +189,13 @@ locals { # BETA features cluster_istio_enabled = !local.cluster_output_istio_disabled + cluster_dns_cache_enabled = var.dns_cache cluster_telemetry_type_is_set = var.cluster_telemetry_type != null cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled # /BETA features - cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf index 29df3aad34..e388fb4406 100644 --- a/modules/beta-public-cluster/outputs.tf +++ b/modules/beta-public-cluster/outputs.tf @@ -175,11 +175,6 @@ output "mesh_certificates_config" { } -output "dns_cache_enabled" { - description = "Whether DNS Cache enabled" - value = local.cluster_dns_cache_enabled -} - output "cloudrun_enabled" { description = "Whether CloudRun enabled" value = local.cluster_cloudrun_enabled @@ -190,6 +185,11 @@ output "istio_enabled" { value = local.cluster_istio_enabled } +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} + output "pod_security_policy_enabled" { description = "Whether pod security policy is enabled" value = local.cluster_pod_security_policy_enabled diff --git a/modules/beta-public-cluster/sa.tf b/modules/beta-public-cluster/sa.tf index fe36dadd32..6f89899bee 100644 --- a/modules/beta-public-cluster/sa.tf +++ b/modules/beta-public-cluster/sa.tf @@ -52,20 +52,6 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/monitoring.metricWriter" - member = google_service_account.cluster_service_account[0].member -} - -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/stackdriver.resourceMetadata.writer" - member = google_service_account.cluster_service_account[0].member -} - resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key @@ -79,17 +65,3 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } - -resource "google_project_service_identity" "fleet_project" { - count = var.fleet_project_grant_service_agent ? 1 : 0 - provider = google-beta - project = var.fleet_project - service = "gkehub.googleapis.com" -} - -resource "google_project_iam_member" "service_agent" { - for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] - project = var.project_id - role = each.value - member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" -} diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index 5de0d1f40a..4a5d55dd6a 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -239,25 +239,17 @@ variable "enable_resource_consumption_export" { variable "cluster_autoscaling" { type = object({ - enabled = bool - autoscaling_profile = string - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) - image_type = optional(string) - strategy = optional(string) - max_surge = optional(number) - max_unavailable = optional(number) - node_pool_soak_duration = optional(string) - batch_soak_duration = optional(string) - batch_percentage = optional(number) - batch_node_count = optional(number) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) }) default = { enabled = false @@ -271,7 +263,6 @@ variable "cluster_autoscaling" { auto_upgrade = true disk_size = 100 disk_type = "pd-standard" - image_type = "COS_CONTAINERD" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -309,12 +300,6 @@ variable "node_pools_oauth_scopes" { } } -variable "network_tags" { - description = "(Optional) - List of network tags applied to auto-provisioned node pools." - type = list(string) - default = [] -} - variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -535,7 +520,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -675,12 +660,6 @@ variable "gcs_fuse_csi_driver" { default = false } -variable "stateful_ha" { - type = bool - description = "Whether the Stateful HA Addon is enabled for this cluster." - default = false -} - variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -721,7 +700,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." default = [] } @@ -743,12 +722,6 @@ variable "config_connector" { default = false } -variable "enable_l4_ilb_subsetting" { - type = bool - description = "Enable L4 ILB Subsetting on the cluster" - default = false -} - variable "istio" { description = "(Beta) Enable Istio addon" type = bool @@ -785,6 +758,12 @@ variable "enable_pod_security_policy" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." @@ -814,9 +793,3 @@ variable "fleet_project" { type = string default = null } - -variable "fleet_project_grant_service_agent" { - description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." - type = bool - default = false -} diff --git a/modules/beta-public-cluster/versions.tf b/modules/beta-public-cluster/versions.tf index 4bacb721c0..eaea9cbf80 100644 --- a/modules/beta-public-cluster/versions.tf +++ b/modules/beta-public-cluster/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.25.0, < 6" + version = ">= 5.9.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.25.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster/v30.1.0" } } diff --git a/modules/binary-authorization/main.tf b/modules/binary-authorization/main.tf index 8e09bb13eb..81cbb620c2 100644 --- a/modules/binary-authorization/main.tf +++ b/modules/binary-authorization/main.tf @@ -25,7 +25,7 @@ locals { module "project-services" { source = "terraform-google-modules/project-factory/google//modules/project_services" - version = "~> 15.0" + version = "~> 14.0" project_id = var.project_id activate_apis = local.required_enabled_apis diff --git a/modules/binary-authorization/versions.tf b/modules/binary-authorization/versions.tf index 66482b1b99..fea953ed24 100644 --- a/modules/binary-authorization/versions.tf +++ b/modules/binary-authorization/versions.tf @@ -28,6 +28,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:binary-authorization/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:binary-authorization/v30.1.0" } } diff --git a/modules/fleet-membership/versions.tf b/modules/fleet-membership/versions.tf index 7b774dcdbe..4962a916f8 100644 --- a/modules/fleet-membership/versions.tf +++ b/modules/fleet-membership/versions.tf @@ -30,6 +30,6 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v30.1.0" } } diff --git a/modules/hub-legacy/versions.tf b/modules/hub-legacy/versions.tf index 678714ab54..a497d17c02 100644 --- a/modules/hub-legacy/versions.tf +++ b/modules/hub-legacy/versions.tf @@ -28,6 +28,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v30.1.0" } } diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index b3238b0c22..5c94834af4 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -78,33 +78,27 @@ module "gke" { enable_private_endpoint = true enable_private_nodes = true master_ipv4_cidr_block = "10.0.0.0/28" - dns_cache = false node_pools = [ { - name = "default-node-pool" - machine_type = "e2-medium" - node_locations = "us-central1-b,us-central1-c" - min_count = 1 - max_count = 100 - local_ssd_count = 0 - spot = false - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS_CONTAINERD" - enable_gcfs = false - enable_gvnic = false - logging_variant = "DEFAULT" - auto_repair = true - auto_upgrade = true - service_account = "project-service-account@.iam.gserviceaccount.com" - preemptible = false - initial_node_count = 80 - accelerator_count = 1 - accelerator_type = "nvidia-l4" - gpu_driver_version = "LATEST" - gpu_sharing_strategy = "TIME_SHARING" - max_shared_clients_per_gpu = 2 + name = "default-node-pool" + machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = false + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + enable_gcfs = false + enable_gvnic = false + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 }, ] @@ -171,7 +165,7 @@ Then perform the following commands on the root folder: | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                 = bool
    autoscaling_profile     = string
    min_cpu_cores           = number
    max_cpu_cores           = number
    min_memory_gb           = number
    max_memory_gb           = number
    gpu_resources           = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair             = bool
    auto_upgrade            = bool
    disk_size               = optional(number)
    disk_type               = optional(string)
    image_type              = optional(string)
    strategy                = optional(string)
    max_surge               = optional(number)
    max_unavailable         = optional(number)
    node_pool_soak_duration = optional(string)
    batch_soak_duration     = optional(string)
    batch_percentage        = optional(number)
    batch_node_count        = optional(number)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
    disk_size           = optional(number)
    disk_type           = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -184,7 +178,7 @@ Then perform the following commands on the root folder: | datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | | deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | | description | The description of the cluster | `string` | `""` | no | | disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | @@ -193,11 +187,10 @@ Then perform the following commands on the root folder: | enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `false` | no | +| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | | enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | @@ -241,7 +234,6 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | @@ -261,14 +253,13 @@ Then perform the following commands on the root folder: | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | | shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | @@ -282,7 +273,6 @@ Then perform the following commands on the root folder: |------|-------------| | ca\_certificate | Cluster ca certificate (base64 encoded) | | cluster\_id | Cluster ID | -| dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | | fleet\_membership | Fleet membership (if registered) | | gateway\_api\_channel | The gateway api channel of this cluster. | @@ -334,22 +324,18 @@ The node_pools variable takes the following parameters: | enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | -| gpu_driver_version | Mode for how the GPU driver is installed | null | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | -| local_ssd_ephemeral_storage_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | -| local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | | strategy | The upgrade stragey to be used for upgrading the nodes. Valid values of state are: `SURGE` or `BLUE_GREEN` | "SURGE" | Optional | -| threads_per_core | Optional The number of threads per physical core. To disable simultaneous multithreading (SMT) set this to 1. If unset, the maximum number of threads supported per core by the underlying processor is assumed | null | Optional | | max_surge | The number of additional nodes that can be added to the node pool during an upgrade. Increasing max_surge raises the number of nodes that can be upgraded simultaneously. Can be set to 0 or greater. Only works with `SURGE` strategy. | 1 | Optional | | max_unavailable | The number of nodes that can be simultaneously unavailable during an upgrade. Increasing max_unavailable raises the number of nodes that can be upgraded in parallel. Can be set to 0 or greater. Only works with `SURGE` strategy. | 0 | Optional | | node_pool_soak_duration | Time needed after draining the entire blue pool. After this period, the blue pool will be cleaned up. By default, it is set to one hour (3600 seconds). The maximum length of the soak time is 7 days (604,800 seconds). Only works with `BLUE_GREEN` strategy. | "3600s" | Optional | @@ -369,8 +355,6 @@ The node_pools variable takes the following parameters: | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | -| secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | -| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 079ec28814..236d61117f 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -124,27 +124,6 @@ resource "google_container_cluster" "primary" { disk_size = lookup(var.cluster_autoscaling, "disk_size", 100) disk_type = lookup(var.cluster_autoscaling, "disk_type", "pd-standard") - upgrade_settings { - strategy = lookup(var.cluster_autoscaling, "strategy", "SURGE") - max_surge = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_surge", 0) : null - max_unavailable = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_unavailable", 0) : null - - dynamic "blue_green_settings" { - for_each = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "BLUE_GREEN" ? [1] : [] - content { - node_pool_soak_duration = lookup(var.cluster_autoscaling, "node_pool_soak_duration", null) - - standard_rollout_policy { - batch_soak_duration = lookup(var.cluster_autoscaling, "batch_soak_duration", null) - batch_percentage = lookup(var.cluster_autoscaling, "batch_percentage", null) - batch_node_count = lookup(var.cluster_autoscaling, "batch_node_count", null) - } - } - } - } - - - image_type = lookup(var.cluster_autoscaling, "image_type", "COS_CONTAINERD") } } autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" @@ -172,8 +151,6 @@ resource "google_container_cluster" "primary" { enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu - - enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -187,15 +164,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] - content { - network_tags { - tags = var.network_tags - } - } - } - master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -254,14 +222,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "stateful_ha_config" { - for_each = local.stateful_ha_config - - content { - enabled = stateful_ha_config.value.enabled - } - } - config_connector_config { enabled = var.config_connector } @@ -489,9 +449,6 @@ locals { "accelerator_count", "accelerator_type", "gpu_partition_size", - "gpu_driver_version", - "gpu_sharing_strategy", - "max_shared_clients_per_gpu", "enable_secure_boot", "enable_integrity_monitoring", "local_ssd_count", @@ -507,7 +464,6 @@ locals { "enable_gvnic", "enable_secure_boot", "boot_disk_kms_key", - "queued_provisioning", ] } @@ -638,13 +594,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "queued_provisioning" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - enabled = lookup(each.value, "queued_provisioning", null) - } - } - node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -661,12 +610,6 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } - dynamic "reservation_affinity" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - consume_reservation_type = "NO_RESERVATION" - } - } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -710,28 +653,6 @@ resource "google_container_node_pool" "pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") - dynamic "ephemeral_storage_local_ssd_config" { - for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] - content { - local_ssd_count = ephemeral_storage_local_ssd_config.value - } - } - - dynamic "local_nvme_ssd_block_config" { - for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] - content { - local_ssd_count = local_nvme_ssd_block_config.value - } - } - - # Supports a single secondary boot disk because `map(any)` must have the same values type. - dynamic "secondary_boot_disks" { - for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] - content { - disk_image = secondary_boot_disks.value - mode = "CONTAINER_IMAGE_CACHE" - } - } service_account = lookup( each.value, @@ -759,21 +680,6 @@ resource "google_container_node_pool" "pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } - - dynamic "gpu_sharing_config" { - for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] - content { - gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") - max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) - } - } - } - } - - dynamic "advanced_machine_features" { - for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] - content { - threads_per_core = lookup(each.value, "threads_per_core", 0) } } @@ -898,13 +804,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "queued_provisioning" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - enabled = lookup(each.value, "queued_provisioning", null) - } - } - node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -921,12 +820,6 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } - dynamic "reservation_affinity" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - consume_reservation_type = "NO_RESERVATION" - } - } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -970,28 +863,6 @@ resource "google_container_node_pool" "windows_pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") - dynamic "ephemeral_storage_local_ssd_config" { - for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] - content { - local_ssd_count = ephemeral_storage_local_ssd_config.value - } - } - - dynamic "local_nvme_ssd_block_config" { - for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] - content { - local_ssd_count = local_nvme_ssd_block_config.value - } - } - - # Supports a single secondary boot disk because `map(any)` must have the same values type. - dynamic "secondary_boot_disks" { - for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] - content { - disk_image = secondary_boot_disks.value - mode = "CONTAINER_IMAGE_CACHE" - } - } service_account = lookup( each.value, @@ -1019,21 +890,6 @@ resource "google_container_node_pool" "windows_pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } - - dynamic "gpu_sharing_config" { - for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] - content { - gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") - max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) - } - } - } - } - - dynamic "advanced_machine_features" { - for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] - content { - threads_per_core = lookup(each.value, "threads_per_core", 0) } } diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index 614babb49a..4c5d8bd613 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -94,7 +94,6 @@ locals { logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] - stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group @@ -172,7 +171,6 @@ locals { }] : [] - cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/modules/private-cluster-update-variant/outputs.tf b/modules/private-cluster-update-variant/outputs.tf index 8f35e71fe0..e1e27f7bba 100644 --- a/modules/private-cluster-update-variant/outputs.tf +++ b/modules/private-cluster-update-variant/outputs.tf @@ -185,11 +185,6 @@ output "peering_name" { value = local.cluster_peering_name } -output "dns_cache_enabled" { - description = "Whether DNS Cache enabled" - value = local.cluster_dns_cache_enabled -} - output "fleet_membership" { description = "Fleet membership (if registered)" value = local.fleet_membership diff --git a/modules/private-cluster-update-variant/sa.tf b/modules/private-cluster-update-variant/sa.tf index da1930c7b3..6f89899bee 100644 --- a/modules/private-cluster-update-variant/sa.tf +++ b/modules/private-cluster-update-variant/sa.tf @@ -52,20 +52,6 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/monitoring.metricWriter" - member = google_service_account.cluster_service_account[0].member -} - -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/stackdriver.resourceMetadata.writer" - member = google_service_account.cluster_service_account[0].member -} - resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index ccf65a8a13..2d7f8cc8c6 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -239,25 +239,17 @@ variable "enable_resource_consumption_export" { variable "cluster_autoscaling" { type = object({ - enabled = bool - autoscaling_profile = string - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) - image_type = optional(string) - strategy = optional(string) - max_surge = optional(number) - max_unavailable = optional(number) - node_pool_soak_duration = optional(string) - batch_soak_duration = optional(string) - batch_percentage = optional(number) - batch_node_count = optional(number) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) }) default = { enabled = false @@ -271,7 +263,6 @@ variable "cluster_autoscaling" { auto_upgrade = true disk_size = 100 disk_type = "pd-standard" - image_type = "COS_CONTAINERD" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -309,12 +300,6 @@ variable "node_pools_oauth_scopes" { } } -variable "network_tags" { - description = "(Optional) - List of network tags applied to auto-provisioned node pools." - type = list(string) - default = [] -} - variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -414,25 +399,25 @@ variable "cluster_resource_labels" { variable "deploy_using_private_endpoint" { type = bool - description = "A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." + description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." default = false } variable "enable_private_endpoint" { type = bool - description = "Whether the master's internal IP address is used as the cluster endpoint" + description = "(Beta) Whether the master's internal IP address is used as the cluster endpoint" default = false } variable "enable_private_nodes" { type = bool - description = "Whether nodes have internal IP addresses only" + description = "(Beta) Whether nodes have internal IP addresses only" default = false } variable "master_ipv4_cidr_block" { type = string - description = "The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters." + description = "(Beta) The IP range in CIDR notation to use for the hosted master network" default = "10.0.0.0/28" } @@ -541,7 +526,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -681,12 +666,6 @@ variable "gcs_fuse_csi_driver" { default = false } -variable "stateful_ha" { - type = bool - description = "Whether the Stateful HA Addon is enabled for this cluster." - default = false -} - variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -727,7 +706,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." default = [] } @@ -749,12 +728,6 @@ variable "config_connector" { default = false } -variable "enable_l4_ilb_subsetting" { - type = bool - description = "Enable L4 ILB Subsetting on the cluster" - default = false -} - variable "fleet_project" { description = "(Optional) Register the cluster with the fleet in this project." type = string diff --git a/modules/private-cluster-update-variant/versions.tf b/modules/private-cluster-update-variant/versions.tf index b20b93f068..c953d4c17a 100644 --- a/modules/private-cluster-update-variant/versions.tf +++ b/modules/private-cluster-update-variant/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.25.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -33,6 +33,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster-update-variant/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster-update-variant/v30.1.0" } } diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 615fb43db2..3d17b7e890 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -56,33 +56,27 @@ module "gke" { enable_private_endpoint = true enable_private_nodes = true master_ipv4_cidr_block = "10.0.0.0/28" - dns_cache = false node_pools = [ { - name = "default-node-pool" - machine_type = "e2-medium" - node_locations = "us-central1-b,us-central1-c" - min_count = 1 - max_count = 100 - local_ssd_count = 0 - spot = false - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS_CONTAINERD" - enable_gcfs = false - enable_gvnic = false - logging_variant = "DEFAULT" - auto_repair = true - auto_upgrade = true - service_account = "project-service-account@.iam.gserviceaccount.com" - preemptible = false - initial_node_count = 80 - accelerator_count = 1 - accelerator_type = "nvidia-l4" - gpu_driver_version = "LATEST" - gpu_sharing_strategy = "TIME_SHARING" - max_shared_clients_per_gpu = 2 + name = "default-node-pool" + machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = false + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + enable_gcfs = false + enable_gvnic = false + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 }, ] @@ -149,7 +143,7 @@ Then perform the following commands on the root folder: | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                 = bool
    autoscaling_profile     = string
    min_cpu_cores           = number
    max_cpu_cores           = number
    min_memory_gb           = number
    max_memory_gb           = number
    gpu_resources           = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair             = bool
    auto_upgrade            = bool
    disk_size               = optional(number)
    disk_type               = optional(string)
    image_type              = optional(string)
    strategy                = optional(string)
    max_surge               = optional(number)
    max_unavailable         = optional(number)
    node_pool_soak_duration = optional(string)
    batch_soak_duration     = optional(string)
    batch_percentage        = optional(number)
    batch_node_count        = optional(number)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
    disk_size           = optional(number)
    disk_type           = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -162,7 +156,7 @@ Then perform the following commands on the root folder: | datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | | deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | | description | The description of the cluster | `string` | `""` | no | | disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | @@ -171,11 +165,10 @@ Then perform the following commands on the root folder: | enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | -| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `false` | no | +| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | | enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | @@ -219,7 +212,6 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | @@ -239,14 +231,13 @@ Then perform the following commands on the root folder: | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | | shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | -| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | @@ -260,7 +251,6 @@ Then perform the following commands on the root folder: |------|-------------| | ca\_certificate | Cluster ca certificate (base64 encoded) | | cluster\_id | Cluster ID | -| dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | | fleet\_membership | Fleet membership (if registered) | | gateway\_api\_channel | The gateway api channel of this cluster. | @@ -312,22 +302,18 @@ The node_pools variable takes the following parameters: | enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | -| gpu_driver_version | Mode for how the GPU driver is installed | null | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | -| local_ssd_ephemeral_storage_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | -| local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | | strategy | The upgrade stragey to be used for upgrading the nodes. Valid values of state are: `SURGE` or `BLUE_GREEN` | "SURGE" | Optional | -| threads_per_core | Optional The number of threads per physical core. To disable simultaneous multithreading (SMT) set this to 1. If unset, the maximum number of threads supported per core by the underlying processor is assumed | null | Optional | | max_surge | The number of additional nodes that can be added to the node pool during an upgrade. Increasing max_surge raises the number of nodes that can be upgraded simultaneously. Can be set to 0 or greater. Only works with `SURGE` strategy. | 1 | Optional | | max_unavailable | The number of nodes that can be simultaneously unavailable during an upgrade. Increasing max_unavailable raises the number of nodes that can be upgraded in parallel. Can be set to 0 or greater. Only works with `SURGE` strategy. | 0 | Optional | | node_pool_soak_duration | Time needed after draining the entire blue pool. After this period, the blue pool will be cleaned up. By default, it is set to one hour (3600 seconds). The maximum length of the soak time is 7 days (604,800 seconds). Only works with `BLUE_GREEN` strategy. | "3600s" | Optional | @@ -347,8 +333,6 @@ The node_pools variable takes the following parameters: | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | -| secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | -| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index b74bb1fc5f..9c989b2365 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -124,27 +124,6 @@ resource "google_container_cluster" "primary" { disk_size = lookup(var.cluster_autoscaling, "disk_size", 100) disk_type = lookup(var.cluster_autoscaling, "disk_type", "pd-standard") - upgrade_settings { - strategy = lookup(var.cluster_autoscaling, "strategy", "SURGE") - max_surge = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_surge", 0) : null - max_unavailable = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_unavailable", 0) : null - - dynamic "blue_green_settings" { - for_each = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "BLUE_GREEN" ? [1] : [] - content { - node_pool_soak_duration = lookup(var.cluster_autoscaling, "node_pool_soak_duration", null) - - standard_rollout_policy { - batch_soak_duration = lookup(var.cluster_autoscaling, "batch_soak_duration", null) - batch_percentage = lookup(var.cluster_autoscaling, "batch_percentage", null) - batch_node_count = lookup(var.cluster_autoscaling, "batch_node_count", null) - } - } - } - } - - - image_type = lookup(var.cluster_autoscaling, "image_type", "COS_CONTAINERD") } } autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" @@ -172,8 +151,6 @@ resource "google_container_cluster" "primary" { enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu - - enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -187,15 +164,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] - content { - network_tags { - tags = var.network_tags - } - } - } - master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -254,14 +222,6 @@ resource "google_container_cluster" "primary" { } } - dynamic "stateful_ha_config" { - for_each = local.stateful_ha_config - - content { - enabled = stateful_ha_config.value.enabled - } - } - config_connector_config { enabled = var.config_connector } @@ -559,13 +519,6 @@ resource "google_container_node_pool" "pools" { } } - dynamic "queued_provisioning" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - enabled = lookup(each.value, "queued_provisioning", null) - } - } - node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -582,12 +535,6 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } - dynamic "reservation_affinity" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - consume_reservation_type = "NO_RESERVATION" - } - } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -631,28 +578,6 @@ resource "google_container_node_pool" "pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") - dynamic "ephemeral_storage_local_ssd_config" { - for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] - content { - local_ssd_count = ephemeral_storage_local_ssd_config.value - } - } - - dynamic "local_nvme_ssd_block_config" { - for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] - content { - local_ssd_count = local_nvme_ssd_block_config.value - } - } - - # Supports a single secondary boot disk because `map(any)` must have the same values type. - dynamic "secondary_boot_disks" { - for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] - content { - disk_image = secondary_boot_disks.value - mode = "CONTAINER_IMAGE_CACHE" - } - } service_account = lookup( each.value, @@ -680,21 +605,6 @@ resource "google_container_node_pool" "pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } - - dynamic "gpu_sharing_config" { - for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] - content { - gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") - max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) - } - } - } - } - - dynamic "advanced_machine_features" { - for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] - content { - threads_per_core = lookup(each.value, "threads_per_core", 0) } } @@ -818,13 +728,6 @@ resource "google_container_node_pool" "windows_pools" { } } - dynamic "queued_provisioning" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - enabled = lookup(each.value, "queued_provisioning", null) - } - } - node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -841,12 +744,6 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } - dynamic "reservation_affinity" { - for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] - content { - consume_reservation_type = "NO_RESERVATION" - } - } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -890,28 +787,6 @@ resource "google_container_node_pool" "windows_pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") - dynamic "ephemeral_storage_local_ssd_config" { - for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] - content { - local_ssd_count = ephemeral_storage_local_ssd_config.value - } - } - - dynamic "local_nvme_ssd_block_config" { - for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] - content { - local_ssd_count = local_nvme_ssd_block_config.value - } - } - - # Supports a single secondary boot disk because `map(any)` must have the same values type. - dynamic "secondary_boot_disks" { - for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] - content { - disk_image = secondary_boot_disks.value - mode = "CONTAINER_IMAGE_CACHE" - } - } service_account = lookup( each.value, @@ -939,21 +814,6 @@ resource "google_container_node_pool" "windows_pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } - - dynamic "gpu_sharing_config" { - for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] - content { - gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") - max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) - } - } - } - } - - dynamic "advanced_machine_features" { - for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] - content { - threads_per_core = lookup(each.value, "threads_per_core", 0) } } diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index 614babb49a..4c5d8bd613 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -94,7 +94,6 @@ locals { logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] - stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group @@ -172,7 +171,6 @@ locals { }] : [] - cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/modules/private-cluster/outputs.tf b/modules/private-cluster/outputs.tf index 8f35e71fe0..e1e27f7bba 100644 --- a/modules/private-cluster/outputs.tf +++ b/modules/private-cluster/outputs.tf @@ -185,11 +185,6 @@ output "peering_name" { value = local.cluster_peering_name } -output "dns_cache_enabled" { - description = "Whether DNS Cache enabled" - value = local.cluster_dns_cache_enabled -} - output "fleet_membership" { description = "Fleet membership (if registered)" value = local.fleet_membership diff --git a/modules/private-cluster/sa.tf b/modules/private-cluster/sa.tf index da1930c7b3..6f89899bee 100644 --- a/modules/private-cluster/sa.tf +++ b/modules/private-cluster/sa.tf @@ -52,20 +52,6 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/monitoring.metricWriter" - member = google_service_account.cluster_service_account[0].member -} - -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/stackdriver.resourceMetadata.writer" - member = google_service_account.cluster_service_account[0].member -} - resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index ccf65a8a13..2d7f8cc8c6 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -239,25 +239,17 @@ variable "enable_resource_consumption_export" { variable "cluster_autoscaling" { type = object({ - enabled = bool - autoscaling_profile = string - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) - image_type = optional(string) - strategy = optional(string) - max_surge = optional(number) - max_unavailable = optional(number) - node_pool_soak_duration = optional(string) - batch_soak_duration = optional(string) - batch_percentage = optional(number) - batch_node_count = optional(number) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) }) default = { enabled = false @@ -271,7 +263,6 @@ variable "cluster_autoscaling" { auto_upgrade = true disk_size = 100 disk_type = "pd-standard" - image_type = "COS_CONTAINERD" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -309,12 +300,6 @@ variable "node_pools_oauth_scopes" { } } -variable "network_tags" { - description = "(Optional) - List of network tags applied to auto-provisioned node pools." - type = list(string) - default = [] -} - variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -414,25 +399,25 @@ variable "cluster_resource_labels" { variable "deploy_using_private_endpoint" { type = bool - description = "A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." + description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." default = false } variable "enable_private_endpoint" { type = bool - description = "Whether the master's internal IP address is used as the cluster endpoint" + description = "(Beta) Whether the master's internal IP address is used as the cluster endpoint" default = false } variable "enable_private_nodes" { type = bool - description = "Whether nodes have internal IP addresses only" + description = "(Beta) Whether nodes have internal IP addresses only" default = false } variable "master_ipv4_cidr_block" { type = string - description = "The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters." + description = "(Beta) The IP range in CIDR notation to use for the hosted master network" default = "10.0.0.0/28" } @@ -541,7 +526,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -681,12 +666,6 @@ variable "gcs_fuse_csi_driver" { default = false } -variable "stateful_ha" { - type = bool - description = "Whether the Stateful HA Addon is enabled for this cluster." - default = false -} - variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -727,7 +706,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." default = [] } @@ -749,12 +728,6 @@ variable "config_connector" { default = false } -variable "enable_l4_ilb_subsetting" { - type = bool - description = "Enable L4 ILB Subsetting on the cluster" - default = false -} - variable "fleet_project" { description = "(Optional) Register the cluster with the fleet in this project." type = string diff --git a/modules/private-cluster/versions.tf b/modules/private-cluster/versions.tf index b349b6fc93..0cc68f07f8 100644 --- a/modules/private-cluster/versions.tf +++ b/modules/private-cluster/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.25.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -33,6 +33,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster/v30.1.0" } } diff --git a/modules/safer-cluster-update-variant/README.md b/modules/safer-cluster-update-variant/README.md index ee2e8bee69..8aa383cec1 100644 --- a/modules/safer-cluster-update-variant/README.md +++ b/modules/safer-cluster-update-variant/README.md @@ -249,7 +249,6 @@ For simplicity, we suggest using `roles/container.admin` and | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster | `string` | n/a | yes | | network | The VPC network to host the cluster in | `string` | n/a | yes | @@ -270,7 +269,7 @@ For simplicity, we suggest using `roles/container.admin` and | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE` | `string` | `null` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. | `string` | `null` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | diff --git a/modules/safer-cluster-update-variant/main.tf b/modules/safer-cluster-update-variant/main.tf index f79239f47d..3da128f44e 100644 --- a/modules/safer-cluster-update-variant/main.tf +++ b/modules/safer-cluster-update-variant/main.tf @@ -104,7 +104,6 @@ module "gke" { monitoring_service = var.monitoring_service monitoring_enable_managed_prometheus = var.monitoring_enable_managed_prometheus - monitoring_enabled_components = var.monitoring_enabled_components // We never use the default service account for the cluster. The default // project/editor permissions can create problems if nodes were to be ever diff --git a/modules/safer-cluster-update-variant/variables.tf b/modules/safer-cluster-update-variant/variables.tf index d6e72d35c4..2b51a97541 100644 --- a/modules/safer-cluster-update-variant/variables.tf +++ b/modules/safer-cluster-update-variant/variables.tf @@ -280,12 +280,6 @@ variable "monitoring_enable_managed_prometheus" { default = false } -variable "monitoring_enabled_components" { - type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." - default = [] -} - variable "grant_registry_access" { type = bool description = "Grants created cluster-specific service account storage.objectViewer role." @@ -476,7 +470,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`" + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`." type = string default = null } diff --git a/modules/safer-cluster-update-variant/versions.tf b/modules/safer-cluster-update-variant/versions.tf index f2270feaaa..e7f7ab6a9c 100644 --- a/modules/safer-cluster-update-variant/versions.tf +++ b/modules/safer-cluster-update-variant/versions.tf @@ -21,6 +21,6 @@ terraform { required_version = ">=1.3" provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster-update-variant/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster-update-variant/v30.1.0" } } diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index ee2e8bee69..8aa383cec1 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -249,7 +249,6 @@ For simplicity, we suggest using `roles/container.admin` and | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster | `string` | n/a | yes | | network | The VPC network to host the cluster in | `string` | n/a | yes | @@ -270,7 +269,7 @@ For simplicity, we suggest using `roles/container.admin` and | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE` | `string` | `null` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. | `string` | `null` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf index ac8b6d9409..67a3a54ed4 100644 --- a/modules/safer-cluster/main.tf +++ b/modules/safer-cluster/main.tf @@ -104,7 +104,6 @@ module "gke" { monitoring_service = var.monitoring_service monitoring_enable_managed_prometheus = var.monitoring_enable_managed_prometheus - monitoring_enabled_components = var.monitoring_enabled_components // We never use the default service account for the cluster. The default // project/editor permissions can create problems if nodes were to be ever diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf index d6e72d35c4..2b51a97541 100644 --- a/modules/safer-cluster/variables.tf +++ b/modules/safer-cluster/variables.tf @@ -280,12 +280,6 @@ variable "monitoring_enable_managed_prometheus" { default = false } -variable "monitoring_enabled_components" { - type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." - default = [] -} - variable "grant_registry_access" { type = bool description = "Grants created cluster-specific service account storage.objectViewer role." @@ -476,7 +470,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`" + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`." type = string default = null } diff --git a/modules/safer-cluster/versions.tf b/modules/safer-cluster/versions.tf index fc44a1e96d..648847290e 100644 --- a/modules/safer-cluster/versions.tf +++ b/modules/safer-cluster/versions.tf @@ -21,6 +21,6 @@ terraform { required_version = ">=1.3" provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster/v30.1.0" } } diff --git a/modules/services/main.tf b/modules/services/main.tf index 9ef548c06a..74dbc89e18 100644 --- a/modules/services/main.tf +++ b/modules/services/main.tf @@ -16,7 +16,7 @@ module "services" { source = "terraform-google-modules/project-factory/google//modules/project_services" - version = "~> 15.0" + version = "~> 14.0" project_id = var.project_id enable_apis = var.enable_apis diff --git a/modules/services/versions.tf b/modules/services/versions.tf index 8413aa45f8..2f0d9ef972 100644 --- a/modules/services/versions.tf +++ b/modules/services/versions.tf @@ -19,6 +19,6 @@ terraform { required_version = ">= 0.13.0" provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:services/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:services/v30.1.0" } } diff --git a/modules/workload-identity/README.md b/modules/workload-identity/README.md index d11a8dbb71..5024cc7d33 100644 --- a/modules/workload-identity/README.md +++ b/modules/workload-identity/README.md @@ -94,51 +94,6 @@ module "my-app-workload-identity" { If annotation is disabled (via `annotate_k8s_sa = false`), the existing Kubernetes service account must already bear the `"iam.gke.io/gcp-service-account"` annotation. -## Using with multiple clusters - -This module accommodates configurations involving multiple clusters within the kubernetes provider. - -To begin, initialize the kubernetes provider for each cluster with a unique alias, as demonstrated below: - -Initialize your `kubernetes` provider with an alias like the following: - -```hcl -provider "kubernetes" { - alias = "alias-for-your-cluster" - host = "https://your-cluster-host-url.com" - token = "your-cluster-token" - cluster_ca_certificate = base64decode("Your-Cluster-Certificate") -} -``` - -Ensure each cluster configuration has a distinct alias. Repeat this step for every cluster you intend to manage. - -In your module configuration, include the providers attribute to assign the appropriate provider alias: - -```hcl -module "workload_identity_for_cluster" { - source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity" - - providers = { - kubernetes = kubernetes.alias-for-your-cluster - } - - name = "service-account-name" - namespace = "desired-namespace" - // Other module configurations -} -``` - -This approach is required when managing multiple clusters. Omitting this step can lead to errors like the one shown below: - -```shell -Error: Get "http://localhost/api/v1/namespaces/default/serviceaccounts/your-service-account": dial tcp [::1]:80: connect: connection refused -│ -│ with module.your_workload_identity.kubernetes_service_account.main[0], -│ on .terraform/modules/your_workload_identity/modules/workload-identity/main.tf line 50, in resource "kubernetes_service_account" "main": -│ 50: resource "kubernetes_service_account" "main" { -``` - ## Inputs diff --git a/modules/workload-identity/versions.tf b/modules/workload-identity/versions.tf index 29ead8fb56..34aa4cb45c 100644 --- a/modules/workload-identity/versions.tf +++ b/modules/workload-identity/versions.tf @@ -30,6 +30,6 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:workload-identity/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:workload-identity/v30.1.0" } } diff --git a/outputs.tf b/outputs.tf index 7aa1c3be10..6ed32741f2 100644 --- a/outputs.tf +++ b/outputs.tf @@ -175,11 +175,6 @@ output "mesh_certificates_config" { } -output "dns_cache_enabled" { - description = "Whether DNS Cache enabled" - value = local.cluster_dns_cache_enabled -} - output "fleet_membership" { description = "Fleet membership (if registered)" value = local.fleet_membership diff --git a/sa.tf b/sa.tf index da1930c7b3..6f89899bee 100644 --- a/sa.tf +++ b/sa.tf @@ -52,20 +52,6 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } -resource "google_project_iam_member" "cluster_service_account-metric_writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/monitoring.metricWriter" - member = google_service_account.cluster_service_account[0].member -} - -resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { - count = var.create_service_account ? 1 : 0 - project = google_service_account.cluster_service_account[0].project - role = "roles/stackdriver.resourceMetadata.writer" - member = google_service_account.cluster_service_account[0].member -} - resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key diff --git a/test/integration/go.mod b/test/integration/go.mod index 66e00997e6..fdc5577793 100644 --- a/test/integration/go.mod +++ b/test/integration/go.mod @@ -2,13 +2,12 @@ module github.com/terraform-google-modules/terraform-google-kubernetes-engine/te go 1.21 -toolchain go1.22.3 +toolchain go1.21.5 require ( - github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.15.0 - github.com/gruntwork-io/terratest v0.46.15 - github.com/stretchr/testify v1.9.0 - github.com/tidwall/gjson v1.17.1 + github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.12.1 + github.com/gruntwork-io/terratest v0.46.11 + github.com/stretchr/testify v1.8.4 ) require ( @@ -24,7 +23,7 @@ require ( github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/boombuler/barcode v1.0.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/go-errors/errors v1.5.0 // indirect github.com/go-logr/logr v1.2.4 // indirect @@ -36,7 +35,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.6.0 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/google/uuid v1.3.1 // indirect @@ -45,14 +44,14 @@ require ( github.com/gruntwork-io/go-commons v0.17.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-getter v1.7.4 // indirect + github.com/hashicorp/go-getter v1.7.2 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f // indirect - github.com/hashicorp/hcl/v2 v2.20.1 // indirect - github.com/hashicorp/terraform-config-inspect v0.0.0-20240509232506-4708120f8f30 // indirect - github.com/hashicorp/terraform-json v0.22.1 // indirect + github.com/hashicorp/hcl/v2 v2.18.0 // indirect + github.com/hashicorp/terraform-config-inspect v0.0.0-20231204233900-a34142ec2a72 // indirect + github.com/hashicorp/terraform-json v0.17.1 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/jinzhu/copier v0.4.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -68,10 +67,11 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pquerna/otp v1.4.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/tidwall/gjson v1.17.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect @@ -79,18 +79,17 @@ require ( github.com/ulikunitz/xz v0.5.11 // indirect github.com/urfave/cli/v2 v2.25.7 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect - github.com/zclconf/go-cty v1.14.4 // indirect + github.com/zclconf/go-cty v1.14.0 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/mod v0.17.0 // indirect - golang.org/x/net v0.23.0 // indirect + golang.org/x/crypto v0.17.0 // indirect + golang.org/x/mod v0.15.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect golang.org/x/sync v0.4.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/term v0.18.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/term v0.15.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect - golang.org/x/tools v0.13.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.138.0 // indirect google.golang.org/appengine v1.6.8 // indirect @@ -98,7 +97,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/grpc v1.58.3 // indirect - google.golang.org/protobuf v1.33.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -106,10 +105,10 @@ require ( k8s.io/apimachinery v0.28.4 // indirect k8s.io/client-go v0.28.4 // indirect k8s.io/klog/v2 v2.100.1 // indirect - k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect + k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/kustomize/kyaml v0.17.0 // indirect + sigs.k8s.io/kustomize/kyaml v0.16.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/test/integration/go.sum b/test/integration/go.sum index ff32902444..f2743cd309 100644 --- a/test/integration/go.sum +++ b/test/integration/go.sum @@ -187,8 +187,8 @@ cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoIS dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.15.0 h1:zT+PND04OQ96kmnQorLFtuY4Ny3KZ4fqxdy9faxMS6A= -github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.15.0/go.mod h1:Srfs5phdVU1kwoRWedu4hngTkwN3OILEWgUgE+IVp6s= +github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.12.1 h1:4K4HIuu95U6A56orD4LAX+yabfUmIRaoUV0JJWOZPjE= +github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.12.1/go.mod h1:iipeDC0VhxKtIVOZyz0ofiJ2x6sh9+VYEbG3JKsPqPw= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= @@ -228,9 +228,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHH github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -322,9 +321,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= -github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -378,15 +376,15 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gruntwork-io/go-commons v0.17.1 h1:2KS9wAqrgeOTWj33DSHzDNJ1FCprptWdLFqej+wB8x0= github.com/gruntwork-io/go-commons v0.17.1/go.mod h1:S98JcR7irPD1bcruSvnqupg+WSJEJ6xaM89fpUZVISk= -github.com/gruntwork-io/terratest v0.46.15 h1:qfqjTFveymaqe7aAWn3LjlK0SwVGpRfoOut5ggNyfQ8= -github.com/gruntwork-io/terratest v0.46.15/go.mod h1:9bd22zAojjBBiYdsp+AR1iyl2iB6bRUVm2Yf1AFhfrA= +github.com/gruntwork-io/terratest v0.46.11 h1:1Z9G18I2FNuH87Ro0YtjW4NH9ky4GDpfzE7+ivkPeB8= +github.com/gruntwork-io/terratest v0.46.11/go.mod h1:DVZG/s7eP1u3KOQJJfE6n7FDriMWpDvnj85XIlZMEM8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-getter v1.7.4 h1:3yQjWuxICvSpYwqSayAdKRFcvBl1y/vogCxczWSmix0= -github.com/hashicorp/go-getter v1.7.4/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= +github.com/hashicorp/go-getter v1.7.2 h1:uJDtyXwEfalmp1PqdxuhZqrNkUyClZAhVeZYTArbqkg= +github.com/hashicorp/go-getter v1.7.2/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= @@ -397,12 +395,12 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f h1:UdxlrJz4JOnY8W+DbLISwf2B8WXEolNRA8BGCwI9jws= github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= -github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdxtc= -github.com/hashicorp/hcl/v2 v2.20.1/go.mod h1:TZDqQ4kNKCbh1iJp99FdPiUaVDDUPivbqxZulxDYqL4= -github.com/hashicorp/terraform-config-inspect v0.0.0-20240509232506-4708120f8f30 h1:0qwr2oZy9mIIJMWh7W9NTHLWGMbEF5KEQ+QqM9hym34= -github.com/hashicorp/terraform-config-inspect v0.0.0-20240509232506-4708120f8f30/go.mod h1:Gz/z9Hbn+4KSp8A2FBtNszfLSdT2Tn/uAKGuVqqWmDI= -github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7orfb5Ltvec= -github.com/hashicorp/terraform-json v0.22.1/go.mod h1:JbWSQCLFSXFFhg42T7l9iJwdGXBYV8fmmD6o/ML4p3A= +github.com/hashicorp/hcl/v2 v2.18.0 h1:wYnG7Lt31t2zYkcquwgKo6MWXzRUDIeIVU5naZwHLl8= +github.com/hashicorp/hcl/v2 v2.18.0/go.mod h1:ThLC89FV4p9MPW804KVbe/cEXoQ8NZEh+JtMeeGErHE= +github.com/hashicorp/terraform-config-inspect v0.0.0-20231204233900-a34142ec2a72 h1:nZ5gGjbe5o7XUu1d7j+Y5Ztcxlp+yaumTKH9i0D3wlg= +github.com/hashicorp/terraform-config-inspect v0.0.0-20231204233900-a34142ec2a72/go.mod h1:l8HcFPm9cQh6Q0KSWoYPiePqMvRFenybP1CH2MjKdlg= +github.com/hashicorp/terraform-json v0.17.1 h1:eMfvh/uWggKmY7Pmb3T85u86E2EQg6EQHgyRwf3RkyA= +github.com/hashicorp/terraform-json v0.17.1/go.mod h1:Huy6zt6euxaY9knPAFKjUITn8QxUFIe9VuSzb4zn/0o= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= @@ -432,6 +430,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -460,18 +460,19 @@ github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLyw github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/otp v1.4.0 h1:wZvl1TIVxKRThZIBiwOOHOGP/1+nZyWBil9Y2XNEDzg= github.com/pquerna/otp v1.4.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= -github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -486,11 +487,11 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= -github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= -github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= +github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -513,10 +514,8 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8= -github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= -github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b h1:FosyBZYxY34Wul7O/MSKey3txpPYyCqVO5ZyceuQJEI= -github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= +github.com/zclconf/go-cty v1.14.0 h1:/Xrd39K7DXbHzlisFP9c4pHao4yyf+/Ug9LEz+Y/yhc= +github.com/zclconf/go-cty v1.14.0/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -533,8 +532,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= -golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -571,8 +570,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= -golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -620,8 +619,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= -golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -728,14 +727,13 @@ golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= -golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1038,8 +1036,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -1071,8 +1069,8 @@ k8s.io/client-go v0.28.4 h1:Np5ocjlZcTrkyRJ3+T3PkXDpe4UpatQxj85+xjaD2wY= k8s.io/client-go v0.28.4/go.mod h1:0VDZFpgoZfelyP5Wqu0/r/TRYcLYuJ2U1KEeoaPa1N4= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= -k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f h1:eeEUOoGYWhOz7EyXqhlR2zHKNw2mNJ9vzJmub6YN6kk= +k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= @@ -1080,8 +1078,8 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/kustomize/kyaml v0.17.0 h1:G2bWs03V9Ur2PinHLzTUJ8Ded+30SzXZKiO92SRDs3c= -sigs.k8s.io/kustomize/kyaml v0.17.0/go.mod h1:6lxkYF1Cv9Ic8g/N7I86cvxNc5iinUo/P2vKsHNmpyE= +sigs.k8s.io/kustomize/kyaml v0.16.0 h1:6J33uKSoATlKZH16unr2XOhDI+otoe2sR3M8PDzW3K0= +sigs.k8s.io/kustomize/kyaml v0.16.0/go.mod h1:xOK/7i+vmE14N2FdFyugIshB8eF6ALpy7jI87Q2nRh4= sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb index e3da9d06b2..38ac3ef554 100644 --- a/test/integration/node_pool/controls/gcloud.rb +++ b/test/integration/node_pool/controls/gcloud.rb @@ -62,8 +62,8 @@ describe "node pools" do let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" || p['name'] =~ %r{^nap-.*} } } - it "has 4" do - expect(node_pools.count).to eq 4 + it "has 3" do + expect(node_pools.count).to eq 3 end describe "pool-01" do @@ -493,27 +493,6 @@ ) end end - - describe "pool-04" do - it "exists" do - expect(data['nodePools']).to include( - including( - "name" => "pool-04", - ) - ) - end - - it "has queued_provisioning enabled" do - expect(data['nodePools']).not_to include( - including( - "name" => "pool-04", - "queued_provisioning" => including( - "enabled" => true, - ), - ) - ) - end - end end end diff --git a/test/integration/simple_regional/testdata/TestSimpleRegional.json b/test/integration/simple_regional/testdata/TestSimpleRegional.json index 85c8e55db7..1c22283ddb 100644 --- a/test/integration/simple_regional/testdata/TestSimpleRegional.json +++ b/test/integration/simple_regional/testdata/TestSimpleRegional.json @@ -17,9 +17,6 @@ }, "networkPolicyConfig": { "disabled": true - }, - "statefulHaConfig": { - "enabled": true } }, "autopilot": {}, diff --git a/test/setup/main.tf b/test/setup/main.tf index f0b3e278a6..2e6f551f91 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -45,7 +45,7 @@ locals { module "gke-project-1" { source = "terraform-google-modules/project-factory/google" - version = "~> 15.0" + version = "~> 14.0" name = "ci-gke-${random_id.random_project_id_suffix.hex}" random_project_id = true @@ -69,7 +69,7 @@ module "gke-project-1" { module "gke-project-2" { source = "terraform-google-modules/project-factory/google" - version = "~> 15.0" + version = "~> 14.0" name = "ci-gke-${random_id.random_project_id_suffix.hex}" random_project_id = true @@ -91,7 +91,7 @@ module "gke-project-2" { # apis as documented https://cloud.google.com/service-mesh/docs/scripted-install/reference#setting_up_your_project module "gke-project-asm" { source = "terraform-google-modules/project-factory/google" - version = "~> 15.0" + version = "~> 14.0" name = "ci-gke-asm-${random_id.random_project_id_suffix.hex}" random_project_id = true diff --git a/variables.tf b/variables.tf index 95cf143d3a..2a02ef1556 100644 --- a/variables.tf +++ b/variables.tf @@ -239,25 +239,17 @@ variable "enable_resource_consumption_export" { variable "cluster_autoscaling" { type = object({ - enabled = bool - autoscaling_profile = string - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) - image_type = optional(string) - strategy = optional(string) - max_surge = optional(number) - max_unavailable = optional(number) - node_pool_soak_duration = optional(string) - batch_soak_duration = optional(string) - batch_percentage = optional(number) - batch_node_count = optional(number) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) }) default = { enabled = false @@ -271,7 +263,6 @@ variable "cluster_autoscaling" { auto_upgrade = true disk_size = 100 disk_type = "pd-standard" - image_type = "COS_CONTAINERD" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -309,12 +300,6 @@ variable "node_pools_oauth_scopes" { } } -variable "network_tags" { - description = "(Optional) - List of network tags applied to auto-provisioned node pools." - type = list(string) - default = [] -} - variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -511,7 +496,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -651,12 +636,6 @@ variable "gcs_fuse_csi_driver" { default = false } -variable "stateful_ha" { - type = bool - description = "Whether the Stateful HA Addon is enabled for this cluster." - default = false -} - variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -697,7 +676,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." default = [] } @@ -719,12 +698,6 @@ variable "config_connector" { default = false } -variable "enable_l4_ilb_subsetting" { - type = bool - description = "Enable L4 ILB Subsetting on the cluster" - default = false -} - variable "fleet_project" { description = "(Optional) Register the cluster with the fleet in this project." type = string diff --git a/versions.tf b/versions.tf index efb1764683..65c47efec4 100644 --- a/versions.tf +++ b/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.25.0, < 6" + version = ">= 5.9.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -33,6 +33,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine/v31.0.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine/v30.1.0" } } From 1922c8b985f14d9bd7151c08a66a1222c01c1a47 Mon Sep 17 00:00:00 2001 From: Christopher Markieta Date: Thu, 30 May 2024 12:20:15 -0400 Subject: [PATCH 3/7] Reapply "Merge branch 'master' into fix-observability-config" This reverts commit 0ccc8d0ed09b77e98c9de12fbcd73030707524a7. --- CHANGELOG.md | 55 +++++ Makefile | 2 +- README.md | 56 +++-- autogen/main/README.md | 51 ++-- autogen/main/cluster.tf.tmpl | 100 +++++++- autogen/main/main.tf.tmpl | 3 +- autogen/main/outputs.tf.tmpl | 10 +- autogen/main/sa.tf.tmpl | 30 +++ autogen/main/variables.tf.tmpl | 78 ++++-- autogen/main/versions.tf.tmpl | 10 +- autogen/safer-cluster/main.tf.tmpl | 1 + autogen/safer-cluster/variables.tf.tmpl | 8 +- autogen/safer-cluster/versions.tf.tmpl | 2 +- build/int.cloudbuild.yaml | 2 +- build/lint.cloudbuild.yaml | 2 +- cluster.tf | 140 +++++++++++ docs/upgrading_to_v31.0.md | 5 + .../acm-terraform-blog-part1/terraform/gke.tf | 2 +- .../acm-terraform-blog-part2/terraform/gke.tf | 2 +- .../acm-terraform-blog-part3/terraform/gke.tf | 2 +- examples/deploy_service/main.tf | 2 +- .../island_cluster_with_vm_router/README.md | 38 +++ .../island_cluster_with_vm_router/main.tf | 226 ++++++++++++++++++ .../island_cluster_with_vm_router/outputs.tf | 19 ++ .../island_cluster_with_vm_router/router.tf | 51 ++++ .../terraform.tfvars | 29 +++ .../island_cluster_with_vm_router/vars.tf | 64 +++++ .../island_cluster_with_vm_router/versions.tf | 28 +++ examples/node_pool/main.tf | 31 ++- examples/safer_cluster_iap_bastion/apis.tf | 2 +- examples/simple_autopilot_private/main.tf | 1 - .../main.tf | 1 - examples/simple_regional/main.tf | 1 + main.tf | 2 + modules/acm/versions.tf | 4 +- modules/asm/versions.tf | 4 +- modules/auth/versions.tf | 2 +- .../beta-autopilot-private-cluster/README.md | 14 +- .../beta-autopilot-private-cluster/cluster.tf | 3 + .../beta-autopilot-private-cluster/main.tf | 2 +- .../beta-autopilot-private-cluster/outputs.tf | 10 +- modules/beta-autopilot-private-cluster/sa.tf | 28 +++ .../variables.tf | 21 +- .../versions.tf | 6 +- .../beta-autopilot-public-cluster/README.md | 6 +- .../beta-autopilot-public-cluster/cluster.tf | 3 + modules/beta-autopilot-public-cluster/main.tf | 2 +- .../beta-autopilot-public-cluster/outputs.tf | 10 +- modules/beta-autopilot-public-cluster/sa.tf | 28 +++ .../variables.tf | 11 +- .../beta-autopilot-public-cluster/versions.tf | 6 +- .../README.md | 62 +++-- .../cluster.tf | 144 +++++++++++ .../main.tf | 3 +- .../outputs.tf | 10 +- .../beta-private-cluster-update-variant/sa.tf | 28 +++ .../variables.tf | 73 ++++-- .../versions.tf | 6 +- modules/beta-private-cluster/README.md | 62 +++-- modules/beta-private-cluster/cluster.tf | 140 +++++++++++ modules/beta-private-cluster/main.tf | 3 +- modules/beta-private-cluster/outputs.tf | 10 +- modules/beta-private-cluster/sa.tf | 28 +++ modules/beta-private-cluster/variables.tf | 73 ++++-- modules/beta-private-cluster/versions.tf | 6 +- .../README.md | 56 +++-- .../cluster.tf | 144 +++++++++++ .../main.tf | 3 +- .../outputs.tf | 10 +- .../beta-public-cluster-update-variant/sa.tf | 28 +++ .../variables.tf | 65 +++-- .../versions.tf | 6 +- modules/beta-public-cluster/README.md | 56 +++-- modules/beta-public-cluster/cluster.tf | 140 +++++++++++ modules/beta-public-cluster/main.tf | 3 +- modules/beta-public-cluster/outputs.tf | 10 +- modules/beta-public-cluster/sa.tf | 28 +++ modules/beta-public-cluster/variables.tf | 65 +++-- modules/beta-public-cluster/versions.tf | 6 +- modules/binary-authorization/main.tf | 2 +- modules/binary-authorization/versions.tf | 2 +- modules/fleet-membership/versions.tf | 2 +- modules/hub-legacy/versions.tf | 2 +- .../private-cluster-update-variant/README.md | 62 +++-- .../private-cluster-update-variant/cluster.tf | 144 +++++++++++ .../private-cluster-update-variant/main.tf | 2 + .../private-cluster-update-variant/outputs.tf | 5 + modules/private-cluster-update-variant/sa.tf | 14 ++ .../variables.tf | 61 +++-- .../versions.tf | 4 +- modules/private-cluster/README.md | 62 +++-- modules/private-cluster/cluster.tf | 140 +++++++++++ modules/private-cluster/main.tf | 2 + modules/private-cluster/outputs.tf | 5 + modules/private-cluster/sa.tf | 14 ++ modules/private-cluster/variables.tf | 61 +++-- modules/private-cluster/versions.tf | 4 +- .../safer-cluster-update-variant/README.md | 3 +- modules/safer-cluster-update-variant/main.tf | 1 + .../safer-cluster-update-variant/variables.tf | 8 +- .../safer-cluster-update-variant/versions.tf | 2 +- modules/safer-cluster/README.md | 3 +- modules/safer-cluster/main.tf | 1 + modules/safer-cluster/variables.tf | 8 +- modules/safer-cluster/versions.tf | 2 +- modules/services/main.tf | 2 +- modules/services/versions.tf | 2 +- modules/workload-identity/README.md | 45 ++++ modules/workload-identity/versions.tf | 2 +- outputs.tf | 5 + sa.tf | 14 ++ test/integration/go.mod | 43 ++-- test/integration/go.sum | 86 +++---- test/integration/node_pool/controls/gcloud.rb | 25 +- .../testdata/TestSimpleRegional.json | 3 + test/setup/main.tf | 6 +- variables.tf | 53 +++- versions.tf | 4 +- 118 files changed, 2858 insertions(+), 532 deletions(-) create mode 100644 docs/upgrading_to_v31.0.md create mode 100644 examples/island_cluster_with_vm_router/README.md create mode 100644 examples/island_cluster_with_vm_router/main.tf create mode 100644 examples/island_cluster_with_vm_router/outputs.tf create mode 100644 examples/island_cluster_with_vm_router/router.tf create mode 100644 examples/island_cluster_with_vm_router/terraform.tfvars create mode 100644 examples/island_cluster_with_vm_router/vars.tf create mode 100644 examples/island_cluster_with_vm_router/versions.tf diff --git a/CHANGELOG.md b/CHANGELOG.md index c2679604a6..906450b4e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,61 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Extending the adopted spec, each change should have a link to its corresponding pull request appended. +## [31.0.0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v30.3.0...v31.0.0) (2024-05-28) + + +### ⚠ BREAKING CHANGES + +* **TPG>=5.25.0:** bump for #1948 ([#1955](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1955)) +* **TPG>=5.21:** add queued_provisioning ([#1950](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1950)) +* **TPG>=5.25.0:** Add support for StatefulHA Addon ([#1948](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1948)) +* **deps:** Update Terraform terraform-google-modules/project-factory/google to v15 ([#1936](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1936)) + +### Features + +* Add local_ssd_ephemeral_count to default configuration ([#1944](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1944)) ([04ebd0c](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/04ebd0c7d3ad236abcfcd9a7b1c54d4c75b1ba28)) +* add missing dns_cache output in non beta ([#1864](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1864)) ([3d5cc9f](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/3d5cc9fc4fd817c35e089222a043e02358d64c1d)) +* Add option image_type in cluster_autoscaling block ([#1905](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1905)) ([2272164](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/22721644102a8b6b875439c80a6571266e6bf723)) +* Add secondary_boot_disks to node_pool configuration. ([#1946](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1946)) ([11bae67](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/11bae6791b07efcce29a0227e0b8b4d6c7070971)) +* add security posture VULNERABILITY_ENTERPRISE ([#1947](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1947)) ([c48c8ab](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/c48c8abe22b78e40b6f56d2940403d8b6e586367)) +* add support for gpu_sharing_config on nodepool ([#1874](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1874)) ([b57387c](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/b57387cc771f90425e5b001fce2b9963dba1b791)) +* Add threads_per_core setting for node config in node pools ([#1942](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1942)) ([e573ced](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/e573cedee0a82517410f9d607ce69be7189124ca)) +* Add upgrade_settings for NAP created node pools ([#1908](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1908)) ([c87bb41](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/c87bb41a4d13a06e8e4b7687e15bed63595ad50b)) +* **TPG>=5.21:** add queued_provisioning ([#1950](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1950)) ([c87333b](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/c87333bcaaf4e91f550816f48862e18657f5596e)) +* **TPG>=5.25.0:** Add support for StatefulHA Addon ([#1948](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1948)) ([acbb453](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/acbb453cc7128ab9784486b1cc1238b0e7616f34)) + + +### Bug Fixes + +* add gpu_driver_version to recreate list + docs ([#1913](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1913)) ([12cf40f](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/12cf40fcfeb341220f639138483897a47c76c6f9)) +* **deps:** Update Terraform terraform-google-modules/project-factory/google to v15 ([#1936](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1936)) ([d01e5bc](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/d01e5bce35c73dba751fcb76f26b1a8f268f0506)) +* **docs:** remove beta from docs from non beta arguments ([#1957](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1957)) ([3de63b3](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/3de63b3a5ce80c4bdcfa106b7254a64dc956d7bb)) +* dynamic block ephemeral_storage_local_ssd_config ([#1951](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1951)) ([26eb7c0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/26eb7c09f6ebb7873719c68caa2fafb92be061ce)) +* local nvme ssd count ([#1937](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1937)) ([dbd90e3](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/dbd90e3e556d74881fb3132457a9159d7fb81e6b)) +* **TPG>=5.25.0:** bump for [#1948](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1948) ([#1955](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1955)) ([b12c2e7](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/b12c2e73ad8f09092bd1566f26f10b988beb5286)) + +## [30.3.0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v30.2.0...v30.3.0) (2024-05-09) + + +### Features + +* add enable_l4_ilb_subsetting for GA TPG ([#1924](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1924)) ([da0476a](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/da0476af8c2911bbe6bbdea65af59421a4fcd61e)) +* adds local_nvme_ssd_block_config to beta-public-cluster ([#1912](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1912)) ([f7c2ed9](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/f7c2ed9102eca18b0d2233784616c60f81180d72)) +* network tags for autoprovisioned node pools ([#1920](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1920)) ([f864e8a](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/f864e8add729b799529f5ede187347ed5cebd5f9)) +* **safer-clusters:** add components selection for monitoring ([#1851](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1851)) ([15b472f](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/15b472fafc6b63afb0b52acec56cea8fd5c25cdf)) + + +### Bug Fixes + +* make master_ipv4_cidr_block optional for autopilot ([#1902](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1902)) ([af43bdd](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/af43bddeaa5902681c0012b7ea724937b161fad6)) + +## [30.2.0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v30.1.0...v30.2.0) (2024-03-08) + + +### Features + +* add cross project fleet service agent ([#1896](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/issues/1896)) ([59d36b9](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/commit/59d36b9c2ce27e2830f174afe8dd42416b664833)) + ## [30.1.0](https://github.com/terraform-google-modules/terraform-google-kubernetes-engine/compare/v30.0.0...v30.1.0) (2024-02-26) diff --git a/Makefile b/Makefile index f92db301e4..942ff8cb78 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ # Make will use bash instead of sh SHELL := /usr/bin/env bash -DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.19 +DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.21 DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools REGISTRY_URL := gcr.io/cloud-foundation-cicd DOCKER_BIN ?= docker diff --git a/README.md b/README.md index 522939ba79..d613063d65 100644 --- a/README.md +++ b/README.md @@ -50,27 +50,33 @@ module "gke" { network_policy = false horizontal_pod_autoscaling = true filestore_csi_driver = false + dns_cache = false node_pools = [ { - name = "default-node-pool" - machine_type = "e2-medium" - node_locations = "us-central1-b,us-central1-c" - min_count = 1 - max_count = 100 - local_ssd_count = 0 - spot = false - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS_CONTAINERD" - enable_gcfs = false - enable_gvnic = false - logging_variant = "DEFAULT" - auto_repair = true - auto_upgrade = true - service_account = "project-service-account@.iam.gserviceaccount.com" - preemptible = false - initial_node_count = 80 + name = "default-node-pool" + machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = false + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + enable_gcfs = false + enable_gvnic = false + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 + accelerator_count = 1 + accelerator_type = "nvidia-l4" + gpu_driver_version = "LATEST" + gpu_sharing_strategy = "TIME_SHARING" + max_shared_clients_per_gpu = 2 }, ] @@ -137,7 +143,7 @@ Then perform the following commands on the root folder: | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
    disk_size           = optional(number)
    disk_type           = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                 = bool
    autoscaling_profile     = string
    min_cpu_cores           = number
    max_cpu_cores           = number
    min_memory_gb           = number
    max_memory_gb           = number
    gpu_resources           = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair             = bool
    auto_upgrade            = bool
    disk_size               = optional(number)
    disk_type               = optional(string)
    image_type              = optional(string)
    strategy                = optional(string)
    max_surge               = optional(number)
    max_unavailable         = optional(number)
    node_pool_soak_duration = optional(string)
    batch_soak_duration     = optional(string)
    batch_percentage        = optional(number)
    batch_node_count        = optional(number)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -158,6 +164,7 @@ Then perform the following commands on the root folder: | enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | @@ -201,6 +208,7 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | @@ -220,13 +228,14 @@ Then perform the following commands on the root folder: | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | | shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | @@ -240,6 +249,7 @@ Then perform the following commands on the root folder: |------|-------------| | ca\_certificate | Cluster ca certificate (base64 encoded) | | cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | | fleet\_membership | Fleet membership (if registered) | | gateway\_api\_channel | The gateway api channel of this cluster. | @@ -289,18 +299,22 @@ The node_pools variable takes the following parameters: | enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | +| gpu_driver_version | Mode for how the GPU driver is installed | null | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | +| local_ssd_ephemeral_storage_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | +| local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | | strategy | The upgrade stragey to be used for upgrading the nodes. Valid values of state are: `SURGE` or `BLUE_GREEN` | "SURGE" | Optional | +| threads_per_core | Optional The number of threads per physical core. To disable simultaneous multithreading (SMT) set this to 1. If unset, the maximum number of threads supported per core by the underlying processor is assumed | null | Optional | | max_surge | The number of additional nodes that can be added to the node pool during an upgrade. Increasing max_surge raises the number of nodes that can be upgraded simultaneously. Can be set to 0 or greater. Only works with `SURGE` strategy. | 1 | Optional | | max_unavailable | The number of nodes that can be simultaneously unavailable during an upgrade. Increasing max_unavailable raises the number of nodes that can be upgraded in parallel. Can be set to 0 or greater. Only works with `SURGE` strategy. | 0 | Optional | | node_pool_soak_duration | Time needed after draining the entire blue pool. After this period, the blue pool will be cleaned up. By default, it is set to one hour (3600 seconds). The maximum length of the soak time is 7 days (604,800 seconds). Only works with `BLUE_GREEN` strategy. | "3600s" | Optional | @@ -320,6 +334,8 @@ The node_pools variable takes the following parameters: | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | +| secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | +| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/autogen/main/README.md b/autogen/main/README.md index 83227dc2f7..f47c6171d7 100644 --- a/autogen/main/README.md +++ b/autogen/main/README.md @@ -90,33 +90,38 @@ module "gke" { {% if beta_cluster and autopilot_cluster != true %} istio = true cloudrun = true - dns_cache = false {% endif %} + dns_cache = false {% if autopilot_cluster != true %} node_pools = [ { - name = "default-node-pool" - machine_type = "e2-medium" - node_locations = "us-central1-b,us-central1-c" - min_count = 1 - max_count = 100 - local_ssd_count = 0 - spot = false + name = "default-node-pool" + machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = false {% if beta_cluster %} - local_ssd_ephemeral_count = 0 + local_ssd_ephemeral_count = 0 {% endif %} - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS_CONTAINERD" - enable_gcfs = false - enable_gvnic = false - logging_variant = "DEFAULT" - auto_repair = true - auto_upgrade = true - service_account = "project-service-account@.iam.gserviceaccount.com" - preemptible = false - initial_node_count = 80 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + enable_gcfs = false + enable_gvnic = false + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 + accelerator_count = 1 + accelerator_type = "nvidia-l4" + gpu_driver_version = "LATEST" + gpu_sharing_strategy = "TIME_SHARING" + max_shared_clients_per_gpu = 2 }, ] @@ -205,21 +210,25 @@ The node_pools variable takes the following parameters: | enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | +| gpu_driver_version | Mode for how the GPU driver is installed | null | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | +| local_ssd_ephemeral_storage_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | {% if beta_cluster %} | local_ssd_ephemeral_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | {% endif %} +| local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | | strategy | The upgrade stragey to be used for upgrading the nodes. Valid values of state are: `SURGE` or `BLUE_GREEN` | "SURGE" | Optional | +| threads_per_core | Optional The number of threads per physical core. To disable simultaneous multithreading (SMT) set this to 1. If unset, the maximum number of threads supported per core by the underlying processor is assumed | null | Optional | | max_surge | The number of additional nodes that can be added to the node pool during an upgrade. Increasing max_surge raises the number of nodes that can be upgraded simultaneously. Can be set to 0 or greater. Only works with `SURGE` strategy. | 1 | Optional | | max_unavailable | The number of nodes that can be simultaneously unavailable during an upgrade. Increasing max_unavailable raises the number of nodes that can be upgraded in parallel. Can be set to 0 or greater. Only works with `SURGE` strategy. | 0 | Optional | | node_pool_soak_duration | Time needed after draining the entire blue pool. After this period, the blue pool will be cleaned up. By default, it is set to one hour (3600 seconds). The maximum length of the soak time is 7 days (604,800 seconds). Only works with `BLUE_GREEN` strategy. | "3600s" | Optional | @@ -249,6 +258,8 @@ The node_pools variable takes the following parameters: | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | +| secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | +| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 991550c45e..b3c21041dc 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -151,9 +151,30 @@ resource "google_container_cluster" "primary" { disk_size = lookup(var.cluster_autoscaling, "disk_size", 100) disk_type = lookup(var.cluster_autoscaling, "disk_type", "pd-standard") + upgrade_settings { + strategy = lookup(var.cluster_autoscaling, "strategy", "SURGE") + max_surge = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_surge", 0) : null + max_unavailable = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_unavailable", 0) : null + + dynamic "blue_green_settings" { + for_each = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "BLUE_GREEN" ? [1] : [] + content { + node_pool_soak_duration = lookup(var.cluster_autoscaling, "node_pool_soak_duration", null) + + standard_rollout_policy { + batch_soak_duration = lookup(var.cluster_autoscaling, "batch_soak_duration", null) + batch_percentage = lookup(var.cluster_autoscaling, "batch_percentage", null) + batch_node_count = lookup(var.cluster_autoscaling, "batch_node_count", null) + } + } + } + } + {% if beta_cluster %} min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") {% endif %} + + image_type = lookup(var.cluster_autoscaling, "image_type", "COS_CONTAINERD") } } autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" @@ -210,10 +231,10 @@ resource "google_container_cluster" "primary" { enabled = identity_service_config.value } } + {% endif %} enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting {% endif %} - {% endif %} {% if beta_cluster %} enable_fqdn_network_policy = var.enable_fqdn_network_policy {% endif %} @@ -243,6 +264,17 @@ resource "google_container_cluster" "primary" { } {% endif %} +{% if autopilot_cluster != true %} + dynamic "node_pool_auto_config" { + for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + content { + network_tags { + tags = var.network_tags + } + } + } +{% endif %} + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -302,6 +334,14 @@ resource "google_container_cluster" "primary" { } } + dynamic "stateful_ha_config" { + for_each = local.stateful_ha_config + + content { + enabled = stateful_ha_config.value.enabled + } + } + config_connector_config { enabled = var.config_connector } @@ -588,6 +628,10 @@ resource "google_container_cluster" "primary" { } } {% endif %} + {% if beta_cluster %} + + depends_on = [google_project_iam_member.service_agent] + {% endif %} } {% if autopilot_cluster != true %} /****************************************** @@ -602,6 +646,9 @@ locals { "accelerator_count", "accelerator_type", "gpu_partition_size", + "gpu_driver_version", + "gpu_sharing_strategy", + "max_shared_clients_per_gpu", "enable_secure_boot", "enable_integrity_monitoring", "local_ssd_count", @@ -617,6 +664,7 @@ locals { "enable_gvnic", "enable_secure_boot", "boot_disk_kms_key", + "queued_provisioning", ] } @@ -770,6 +818,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -786,6 +841,12 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -829,6 +890,12 @@ resource "google_container_node_pool" "windows_pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") + dynamic "ephemeral_storage_local_ssd_config" { + for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] + content { + local_ssd_count = ephemeral_storage_local_ssd_config.value + } + } {% if beta_cluster %} dynamic "ephemeral_storage_config" { for_each = lookup(each.value, "local_ssd_ephemeral_count",0) > 0 ? [each.value.local_ssd_ephemeral_count] : [] @@ -838,6 +905,22 @@ resource "google_container_node_pool" "windows_pools" { } {% endif %} + dynamic "local_nvme_ssd_block_config" { + for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] + content { + local_ssd_count = local_nvme_ssd_block_config.value + } + } + + # Supports a single secondary boot disk because `map(any)` must have the same values type. + dynamic "secondary_boot_disks" { + for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] + content { + disk_image = secondary_boot_disks.value + mode = "CONTAINER_IMAGE_CACHE" + } + } + service_account = lookup( each.value, "service_account", @@ -864,6 +947,21 @@ resource "google_container_node_pool" "windows_pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } + + dynamic "gpu_sharing_config" { + for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) + } + } + } + } + + dynamic "advanced_machine_features" { + for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] + content { + threads_per_core = lookup(each.value, "threads_per_core", 0) } } diff --git a/autogen/main/main.tf.tmpl b/autogen/main/main.tf.tmpl index ddece38d66..779206e579 100644 --- a/autogen/main/main.tf.tmpl +++ b/autogen/main/main.tf.tmpl @@ -109,6 +109,7 @@ locals { logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] + stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] {% endif %} {% if beta_cluster and autopilot_cluster != true %} cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { @@ -231,7 +232,6 @@ locals { {% if beta_cluster %} # BETA features cluster_istio_enabled = ! local.cluster_output_istio_disabled - cluster_dns_cache_enabled = var.dns_cache {% if autopilot_cluster != true %} cluster_telemetry_type_is_set = var.cluster_telemetry_type != null {% endif %} @@ -241,6 +241,7 @@ locals { # /BETA features {% endif %} + cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/autogen/main/outputs.tf.tmpl b/autogen/main/outputs.tf.tmpl index afedce7f55..3d5be22ea9 100644 --- a/autogen/main/outputs.tf.tmpl +++ b/autogen/main/outputs.tf.tmpl @@ -198,6 +198,11 @@ output "peering_name" { value = local.cluster_peering_name } {% endif %} + +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} {% if beta_cluster %} output "cloudrun_enabled" { @@ -214,11 +219,6 @@ output "istio_enabled" { value = local.cluster_istio_enabled } -output "dns_cache_enabled" { - description = "Whether DNS Cache enabled" - value = local.cluster_dns_cache_enabled -} - output "pod_security_policy_enabled" { description = "Whether pod security policy is enabled" value = local.cluster_pod_security_policy_enabled diff --git a/autogen/main/sa.tf.tmpl b/autogen/main/sa.tf.tmpl index 1ab198a8b5..f38e3a8687 100644 --- a/autogen/main/sa.tf.tmpl +++ b/autogen/main/sa.tf.tmpl @@ -52,6 +52,20 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } +resource "google_project_iam_member" "cluster_service_account-metric_writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/monitoring.metricWriter" + member = google_service_account.cluster_service_account[0].member +} + +resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/stackdriver.resourceMetadata.writer" + member = google_service_account.cluster_service_account[0].member +} + resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key @@ -65,3 +79,19 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } +{% if beta_cluster %} + +resource "google_project_service_identity" "fleet_project" { + count = var.fleet_project_grant_service_agent ? 1 : 0 + provider = google-beta + project = var.fleet_project + service = "gkehub.googleapis.com" +} + +resource "google_project_iam_member" "service_agent" { + for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] + project = var.project_id + role = each.value + member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" +} +{% endif %} diff --git a/autogen/main/variables.tf.tmpl b/autogen/main/variables.tf.tmpl index bc82d32fdb..1737faa53b 100644 --- a/autogen/main/variables.tf.tmpl +++ b/autogen/main/variables.tf.tmpl @@ -244,17 +244,25 @@ variable "enable_resource_consumption_export" { {% if autopilot_cluster != true %} variable "cluster_autoscaling" { type = object({ - enabled = bool - autoscaling_profile = string - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) + image_type = optional(string) + strategy = optional(string) + max_surge = optional(number) + max_unavailable = optional(number) + node_pool_soak_duration = optional(string) + batch_soak_duration = optional(string) + batch_percentage = optional(number) + batch_node_count = optional(number) }) default = { enabled = false @@ -268,6 +276,7 @@ variable "cluster_autoscaling" { auto_upgrade = true disk_size = 100 disk_type = "pd-standard" + image_type = "COS_CONTAINERD" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -306,13 +315,12 @@ variable "node_pools_oauth_scopes" { } {% endif %} -{% if autopilot_cluster %} variable "network_tags" { - description = "(Optional, Beta) - List of network tags applied to auto-provisioned node pools." + description = "(Optional) - List of network tags applied to auto-provisioned node pools." type = list(string) default = [] } -{% endif %} + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -423,26 +431,30 @@ variable "cluster_resource_labels" { variable "deploy_using_private_endpoint" { type = bool - description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." + description = "A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." default = false } variable "enable_private_endpoint" { type = bool - description = "(Beta) Whether the master's internal IP address is used as the cluster endpoint" + description = "Whether the master's internal IP address is used as the cluster endpoint" default = false } variable "enable_private_nodes" { type = bool - description = "(Beta) Whether nodes have internal IP addresses only" + description = "Whether nodes have internal IP addresses only" default = false } variable "master_ipv4_cidr_block" { type = string - description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + description = "The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters." + {% if autopilot_cluster == true%} + default = null + {% else %} default = "10.0.0.0/28" + {% endif %} } variable "master_global_access_enabled" { @@ -577,7 +589,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -722,6 +734,12 @@ variable "gcs_fuse_csi_driver" { default = false } +variable "stateful_ha" { + type = bool + description = "Whether the Stateful HA Addon is enabled for this cluster." + default = false +} + {% endif %} variable "timeouts" { type = map(string) @@ -764,7 +782,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -785,6 +803,12 @@ variable "config_connector" { description = "Whether ConfigConnector is enabled for this cluster." default = false } + +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} {% endif %} {% if beta_cluster %} {% if autopilot_cluster != true %} @@ -825,12 +849,6 @@ variable "enable_pod_security_policy" { default = false } -variable "enable_l4_ilb_subsetting" { - type = bool - description = "Enable L4 ILB Subsetting on the cluster" - default = false -} - variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." @@ -869,3 +887,11 @@ variable "fleet_project" { type = string default = null } +{% if beta_cluster %} + +variable "fleet_project_grant_service_agent" { + description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." + type = bool + default = false +} +{% endif %} diff --git a/autogen/main/versions.tf.tmpl b/autogen/main/versions.tf.tmpl index 707a3b2747..3e266f9fc4 100644 --- a/autogen/main/versions.tf.tmpl +++ b/autogen/main/versions.tf.tmpl @@ -24,11 +24,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.9.0, < 6" + version = ">= 5.25.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.9.0, < 6" + version = ">= 5.25.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -40,13 +40,13 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v31.0.0" } {% else %} required_providers { google = { source = "hashicorp/google" - version = ">= 5.9.0, < 6" + version = ">= 5.25.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -58,7 +58,7 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v31.0.0" } {% endif %} } diff --git a/autogen/safer-cluster/main.tf.tmpl b/autogen/safer-cluster/main.tf.tmpl index 3146875d47..5a156c7ccc 100644 --- a/autogen/safer-cluster/main.tf.tmpl +++ b/autogen/safer-cluster/main.tf.tmpl @@ -108,6 +108,7 @@ module "gke" { monitoring_service = var.monitoring_service monitoring_enable_managed_prometheus = var.monitoring_enable_managed_prometheus + monitoring_enabled_components = var.monitoring_enabled_components // We never use the default service account for the cluster. The default // project/editor permissions can create problems if nodes were to be ever diff --git a/autogen/safer-cluster/variables.tf.tmpl b/autogen/safer-cluster/variables.tf.tmpl index 0f21e99a4e..88da0f4adc 100644 --- a/autogen/safer-cluster/variables.tf.tmpl +++ b/autogen/safer-cluster/variables.tf.tmpl @@ -280,6 +280,12 @@ variable "monitoring_enable_managed_prometheus" { default = false } +variable "monitoring_enabled_components" { + type = list(string) + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + default = [] +} + variable "grant_registry_access" { type = bool description = "Grants created cluster-specific service account storage.objectViewer role." @@ -470,7 +476,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`" type = string default = null } diff --git a/autogen/safer-cluster/versions.tf.tmpl b/autogen/safer-cluster/versions.tf.tmpl index 542b84b2b6..3ad61d487a 100644 --- a/autogen/safer-cluster/versions.tf.tmpl +++ b/autogen/safer-cluster/versions.tf.tmpl @@ -23,6 +23,6 @@ terraform { required_version = ">=1.3" provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine{% if module_registry_name %}:{{ module_registry_name }}{% endif %}/v31.0.0" } } diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index dc1ebab01a..8dc74732aa 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -519,6 +519,6 @@ tags: - 'integration' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.19' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.21' options: machineType: 'N1_HIGHCPU_8' diff --git a/build/lint.cloudbuild.yaml b/build/lint.cloudbuild.yaml index d2d4c0c066..8a067a2f7f 100644 --- a/build/lint.cloudbuild.yaml +++ b/build/lint.cloudbuild.yaml @@ -22,7 +22,7 @@ tags: - 'lint' substitutions: _DOCKER_IMAGE_DEVELOPER_TOOLS: 'cft/developer-tools' - _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.19' + _DOCKER_TAG_VERSION_DEVELOPER_TOOLS: '1.21' options: machineType: 'N1_HIGHCPU_8' env: diff --git a/cluster.tf b/cluster.tf index e78456aa7f..b3b4b7655f 100644 --- a/cluster.tf +++ b/cluster.tf @@ -124,6 +124,27 @@ resource "google_container_cluster" "primary" { disk_size = lookup(var.cluster_autoscaling, "disk_size", 100) disk_type = lookup(var.cluster_autoscaling, "disk_type", "pd-standard") + upgrade_settings { + strategy = lookup(var.cluster_autoscaling, "strategy", "SURGE") + max_surge = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_surge", 0) : null + max_unavailable = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_unavailable", 0) : null + + dynamic "blue_green_settings" { + for_each = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "BLUE_GREEN" ? [1] : [] + content { + node_pool_soak_duration = lookup(var.cluster_autoscaling, "node_pool_soak_duration", null) + + standard_rollout_policy { + batch_soak_duration = lookup(var.cluster_autoscaling, "batch_soak_duration", null) + batch_percentage = lookup(var.cluster_autoscaling, "batch_percentage", null) + batch_node_count = lookup(var.cluster_autoscaling, "batch_node_count", null) + } + } + } + } + + + image_type = lookup(var.cluster_autoscaling, "image_type", "COS_CONTAINERD") } } autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" @@ -151,6 +172,8 @@ resource "google_container_cluster" "primary" { enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu + + enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -164,6 +187,15 @@ resource "google_container_cluster" "primary" { } } + dynamic "node_pool_auto_config" { + for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + content { + network_tags { + tags = var.network_tags + } + } + } + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -222,6 +254,14 @@ resource "google_container_cluster" "primary" { } } + dynamic "stateful_ha_config" { + for_each = local.stateful_ha_config + + content { + enabled = stateful_ha_config.value.enabled + } + } + config_connector_config { enabled = var.config_connector } @@ -500,6 +540,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -516,6 +563,12 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -559,6 +612,28 @@ resource "google_container_node_pool" "pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") + dynamic "ephemeral_storage_local_ssd_config" { + for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] + content { + local_ssd_count = ephemeral_storage_local_ssd_config.value + } + } + + dynamic "local_nvme_ssd_block_config" { + for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] + content { + local_ssd_count = local_nvme_ssd_block_config.value + } + } + + # Supports a single secondary boot disk because `map(any)` must have the same values type. + dynamic "secondary_boot_disks" { + for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] + content { + disk_image = secondary_boot_disks.value + mode = "CONTAINER_IMAGE_CACHE" + } + } service_account = lookup( each.value, @@ -586,6 +661,21 @@ resource "google_container_node_pool" "pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } + + dynamic "gpu_sharing_config" { + for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) + } + } + } + } + + dynamic "advanced_machine_features" { + for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] + content { + threads_per_core = lookup(each.value, "threads_per_core", 0) } } @@ -709,6 +799,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -725,6 +822,12 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -768,6 +871,28 @@ resource "google_container_node_pool" "windows_pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") + dynamic "ephemeral_storage_local_ssd_config" { + for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] + content { + local_ssd_count = ephemeral_storage_local_ssd_config.value + } + } + + dynamic "local_nvme_ssd_block_config" { + for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] + content { + local_ssd_count = local_nvme_ssd_block_config.value + } + } + + # Supports a single secondary boot disk because `map(any)` must have the same values type. + dynamic "secondary_boot_disks" { + for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] + content { + disk_image = secondary_boot_disks.value + mode = "CONTAINER_IMAGE_CACHE" + } + } service_account = lookup( each.value, @@ -795,6 +920,21 @@ resource "google_container_node_pool" "windows_pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } + + dynamic "gpu_sharing_config" { + for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) + } + } + } + } + + dynamic "advanced_machine_features" { + for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] + content { + threads_per_core = lookup(each.value, "threads_per_core", 0) } } diff --git a/docs/upgrading_to_v31.0.md b/docs/upgrading_to_v31.0.md new file mode 100644 index 0000000000..a3f5adc170 --- /dev/null +++ b/docs/upgrading_to_v31.0.md @@ -0,0 +1,5 @@ +# Upgrading to v31.0 +The v31.0 release of *kubernetes-engine* is a backwards incompatible release. + +### Google Cloud Platform Provider upgrade +The Terraform Kubernetes Engine Module now requires version 5.25 or higher of the Google Cloud Platform Providers. diff --git a/examples/acm-terraform-blog-part1/terraform/gke.tf b/examples/acm-terraform-blog-part1/terraform/gke.tf index 9c997717dc..45a627c467 100644 --- a/examples/acm-terraform-blog-part1/terraform/gke.tf +++ b/examples/acm-terraform-blog-part1/terraform/gke.tf @@ -16,7 +16,7 @@ module "enabled_google_apis" { source = "terraform-google-modules/project-factory/google//modules/project_services" - version = "~> 14.0" + version = "~> 15.0" project_id = var.project disable_services_on_destroy = false diff --git a/examples/acm-terraform-blog-part2/terraform/gke.tf b/examples/acm-terraform-blog-part2/terraform/gke.tf index 2c0d637463..816e2bde95 100644 --- a/examples/acm-terraform-blog-part2/terraform/gke.tf +++ b/examples/acm-terraform-blog-part2/terraform/gke.tf @@ -16,7 +16,7 @@ module "enabled_google_apis" { source = "terraform-google-modules/project-factory/google//modules/project_services" - version = "~> 14.0" + version = "~> 15.0" project_id = var.project disable_services_on_destroy = false diff --git a/examples/acm-terraform-blog-part3/terraform/gke.tf b/examples/acm-terraform-blog-part3/terraform/gke.tf index 9f43da79bc..6694582d56 100644 --- a/examples/acm-terraform-blog-part3/terraform/gke.tf +++ b/examples/acm-terraform-blog-part3/terraform/gke.tf @@ -16,7 +16,7 @@ module "enabled_google_apis" { source = "terraform-google-modules/project-factory/google//modules/project_services" - version = "~> 14.0" + version = "~> 15.0" project_id = var.project disable_services_on_destroy = false diff --git a/examples/deploy_service/main.tf b/examples/deploy_service/main.tf index a36e40b5fa..e825609a1f 100644 --- a/examples/deploy_service/main.tf +++ b/examples/deploy_service/main.tf @@ -55,7 +55,7 @@ resource "kubernetes_pod" "nginx-example" { spec { container { - image = "nginx:1.25.3" + image = "nginx:1.26.0" name = "nginx-example" } } diff --git a/examples/island_cluster_with_vm_router/README.md b/examples/island_cluster_with_vm_router/README.md new file mode 100644 index 0000000000..e36d8ed115 --- /dev/null +++ b/examples/island_cluster_with_vm_router/README.md @@ -0,0 +1,38 @@ +# GKE island cluster using VM as router + +This example provisions a cluster in an island VPC allowing reuse of the IP address space for multiple clusters in the same project. + +1. An appliance(VM as router) with multiple NICs is used to establish connectivity between the island VPC and the existing network. +1. Outbound connections will go through the router. +1. For inbound connections, use Private Service Connect. + +## Deploy + +1. Update `project_id`, `cluster_name` and `primary_subnet` values in `terraform.tfvars`, and update other variables as needed. +1. Run `terraform apply`. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| cluster\_name | n/a | `string` | n/a | yes | +| master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | n/a | yes | +| node\_locations | n/a | `list(string)` | n/a | yes | +| primary\_net\_cidrs | n/a | `list(string)` | n/a | yes | +| primary\_subnet | n/a | `string` | n/a | yes | +| project\_id | n/a | `string` | n/a | yes | +| proxy\_subnet\_cidr | n/a | `string` | n/a | yes | +| psc\_subnet\_cidr | n/a | `string` | n/a | yes | +| region | n/a | `string` | n/a | yes | +| router\_machine\_type | n/a | `string` | n/a | yes | +| secondary\_ranges | n/a | `map(string)` | n/a | yes | +| subnet\_cidr | n/a | `string` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| cluster\_id | n/a | + + diff --git a/examples/island_cluster_with_vm_router/main.tf b/examples/island_cluster_with_vm_router/main.tf new file mode 100644 index 0000000000..86ce138888 --- /dev/null +++ b/examples/island_cluster_with_vm_router/main.tf @@ -0,0 +1,226 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_id" "rand" { + byte_length = 4 +} + +resource "google_service_account" "gke-sa" { + account_id = "gke-sa-${random_id.rand.hex}" + project = var.project_id +} + +module "net" { + source = "terraform-google-modules/network/google" + version = "~> 9.0" + + network_name = "gke-net-${random_id.rand.hex}" + routing_mode = "GLOBAL" + project_id = var.project_id + delete_default_internet_gateway_routes = true + + subnets = [ + { + subnet_name = "${var.cluster_name}-${var.region}-snet" + subnet_ip = var.subnet_cidr + subnet_region = var.region + subnet_private_access = "true" + }, + { + subnet_name = "${var.cluster_name}-${var.region}-proxy-snet" + subnet_ip = var.proxy_subnet_cidr + subnet_region = var.region + purpose = "REGIONAL_MANAGED_PROXY" + role = "ACTIVE" + }, + { + subnet_name = "${var.cluster_name}-${var.region}-psc-snet" + subnet_ip = var.psc_subnet_cidr + subnet_region = var.region + subnet_private_access = "true" + purpose = "PRIVATE_SERVICE_CONNECT" + } + ] + + secondary_ranges = { + "${var.cluster_name}-${var.region}-snet" = [ + { + range_name = "${var.cluster_name}-${var.region}-snet-pods" + ip_cidr_range = var.secondary_ranges["pods"] + }, + { + range_name = "${var.cluster_name}-${var.region}-snet-services" + ip_cidr_range = var.secondary_ranges["services"] + }, + ] + } + + routes = flatten([ + [for k, v in var.primary_net_cidrs : + { + name = "${var.cluster_name}-egress-gke-${k}" + description = "egress through the router for range ${v}" + destination_range = v + tags = "gke-${random_id.rand.hex}" + next_hop_instance = google_compute_instance.vm.self_link + priority = 100 + } + ], + [ + { + name = "${var.cluster_name}-default-igw" + description = "internet through the router" + destination_range = "0.0.0.0/0" + tags = "gke-${random_id.rand.hex}" + next_hop_instance = google_compute_instance.vm.self_link + priority = 100 + } + ] + ]) + + firewall_rules = [ + { + name = "${var.cluster_name}-iap" + direction = "INGRESS" + allow = [ + { + protocol = "TCP" + ports = ["22"] + } + ] + ranges = ["35.235.240.0/20"] + }, + { + name = "${var.cluster_name}-tcp-primary" + direction = "INGRESS" + allow = [ + { + protocol = "TCP" + } + ] + ranges = [ + var.subnet_cidr, + var.secondary_ranges["pods"] + ] + }, + { + name = "${var.cluster_name}-allow-psc" + direction = "INGRESS" + allow = [ + { + protocol = "TCP" + } + ] + ranges = [var.psc_subnet_cidr] + target_service_accounts = [google_service_account.gke-sa.email] + }, + { + name = "${var.cluster_name}-allow-proxy" + direction = "INGRESS" + allow = [ + { + protocol = "TCP" + } + ] + ranges = [var.proxy_subnet_cidr] + target_service_accounts = [google_service_account.gke-sa.email] + }, + ] +} + +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google//modules/beta-private-cluster" + version = "~> 30.0" + + depends_on = [google_compute_instance.vm] + + name = var.cluster_name + project_id = var.project_id + region = var.region + release_channel = "RAPID" + zones = var.node_locations + network = module.net.network_name + subnetwork = "${var.cluster_name}-${var.region}-snet" + ip_range_pods = "${var.cluster_name}-${var.region}-snet-pods" + ip_range_services = "${var.cluster_name}-${var.region}-snet-services" + enable_private_endpoint = true + enable_private_nodes = true + datapath_provider = "ADVANCED_DATAPATH" + monitoring_enable_managed_prometheus = false + enable_shielded_nodes = true + master_global_access_enabled = false + master_ipv4_cidr_block = var.secondary_ranges["master_cidr"] + master_authorized_networks = var.master_authorized_networks + deletion_protection = false + remove_default_node_pool = true + disable_default_snat = true + gateway_api_channel = "CHANNEL_STANDARD" + + node_pools = [ + { + name = "default" + machine_type = "e2-highcpu-2" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = true + local_ssd_ephemeral_count = 0 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = google_service_account.gke-sa.email + initial_node_count = 1 + enable_secure_boot = true + }, + ] + + node_pools_tags = { + all = ["gke-${random_id.rand.hex}"] + } + + node_pools_oauth_scopes = { + all = [ + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/monitoring", + ] + } + + timeouts = { + create = "15m" + update = "15m" + delete = "15m" + } +} + +resource "google_gke_hub_membership" "primary" { + provider = google-beta + + project = var.project_id + membership_id = "${var.project_id}-${module.gke.name}" + location = var.region + + endpoint { + gke_cluster { + resource_link = "//container.googleapis.com/${module.gke.cluster_id}" + } + } + authority { + issuer = "https://container.googleapis.com/v1/${module.gke.cluster_id}" + } +} diff --git a/examples/island_cluster_with_vm_router/outputs.tf b/examples/island_cluster_with_vm_router/outputs.tf new file mode 100644 index 0000000000..640692ed75 --- /dev/null +++ b/examples/island_cluster_with_vm_router/outputs.tf @@ -0,0 +1,19 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "cluster_id" { + value = module.gke.cluster_id +} diff --git a/examples/island_cluster_with_vm_router/router.tf b/examples/island_cluster_with_vm_router/router.tf new file mode 100644 index 0000000000..aee86f74cf --- /dev/null +++ b/examples/island_cluster_with_vm_router/router.tf @@ -0,0 +1,51 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "google_compute_instance" "vm" { + project = var.project_id + zone = var.node_locations[0] + name = "${var.cluster_name}-router-${random_id.rand.hex}" + machine_type = var.router_machine_type + allow_stopping_for_update = true + boot_disk { + initialize_params { + image = "debian-cloud/debian-12" + } + } + can_ip_forward = true + shielded_instance_config { + enable_secure_boot = true + } + network_interface { + subnetwork = var.primary_subnet + } + network_interface { + subnetwork = module.net.subnets["${var.region}/${var.cluster_name}-${var.region}-snet"]["self_link"] + } + metadata_startup_script = <<-EOT + #!/bin/bash + set -ex + sudo apt-get update + echo "net.ipv4.ip_forward = 1" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p + sudo iptables -A FORWARD -i ens5 -o ens4 -j ACCEPT + sudo iptables -A FORWARD -i ens4 -o ens5 -m state --state ESTABLISHED,RELATED -j ACCEPT + GWY_URL="http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/1/gateway" + GWY_IP=$(curl $${GWY_URL} -H "Metadata-Flavor: Google") + sudo ip route add ${var.secondary_ranges["pods"]} via $${GWY_IP} dev ens5 + sudo iptables -t nat -A POSTROUTING -o ens4 -s 0.0.0.0/0 -j MASQUERADE + EOT +} diff --git a/examples/island_cluster_with_vm_router/terraform.tfvars b/examples/island_cluster_with_vm_router/terraform.tfvars new file mode 100644 index 0000000000..3e9b74c54b --- /dev/null +++ b/examples/island_cluster_with_vm_router/terraform.tfvars @@ -0,0 +1,29 @@ +project_id = "" +region = "us-central1" +cluster_name = "gke-island-cluster-test" +node_locations = [ + "us-central1-a", + "us-central1-b", + "us-central1-f" +] +subnet_cidr = "100.64.0.0/20" +router_machine_type = "n2-highcpu-4" +primary_subnet = "projects//regions//subnetworks/" +secondary_ranges = { + pods = "100.64.64.0/18" + services = "100.64.128.0/20" + master_cidr = "100.64.144.0/28" +} +proxy_subnet_cidr = "100.64.168.0/24" +psc_subnet_cidr = "100.64.192.0/24" +master_authorized_networks = [ + { + cidr_block = "100.64.0.0/10" + display_name = "cluster net" + } +] +primary_net_cidrs = [ + "10.0.0.0/8", + "192.168.0.0/16", + "172.16.0.0/12" +] diff --git a/examples/island_cluster_with_vm_router/vars.tf b/examples/island_cluster_with_vm_router/vars.tf new file mode 100644 index 0000000000..37cf62d182 --- /dev/null +++ b/examples/island_cluster_with_vm_router/vars.tf @@ -0,0 +1,64 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + type = string +} + +variable "region" { + type = string +} + +variable "cluster_name" { + type = string +} + +variable "node_locations" { + type = list(string) +} + +variable "subnet_cidr" { + type = string +} + +variable "psc_subnet_cidr" { + type = string +} + +variable "proxy_subnet_cidr" { + type = string +} + +variable "secondary_ranges" { + type = map(string) +} + +variable "master_authorized_networks" { + type = list(object({ cidr_block = string, display_name = string })) + description = "List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists)." +} + +variable "primary_subnet" { + type = string +} + +variable "primary_net_cidrs" { + type = list(string) +} + +variable "router_machine_type" { + type = string +} diff --git a/examples/island_cluster_with_vm_router/versions.tf b/examples/island_cluster_with_vm_router/versions.tf new file mode 100644 index 0000000000..7e297ed82e --- /dev/null +++ b/examples/island_cluster_with_vm_router/versions.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_version = ">= 1.6" + + required_providers { + google = { + source = "hashicorp/google" + } + google-beta = { + source = "hashicorp/google-beta" + } + } +} diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index cea54d19b6..7626216fb7 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -66,18 +66,25 @@ module "gke" { service_account = var.compute_engine_service_account }, { - name = "pool-03" - machine_type = "n1-standard-2" - node_locations = "${var.region}-b,${var.region}-c" - autoscaling = false - node_count = 2 - disk_type = "pd-standard" - auto_upgrade = true - service_account = var.compute_engine_service_account - pod_range = "test" - sandbox_enabled = true - cpu_manager_policy = "static" - cpu_cfs_quota = true + name = "pool-03" + machine_type = "n1-standard-2" + node_locations = "${var.region}-b,${var.region}-c" + autoscaling = false + node_count = 2 + disk_type = "pd-standard" + auto_upgrade = true + service_account = var.compute_engine_service_account + pod_range = "test" + sandbox_enabled = true + cpu_manager_policy = "static" + cpu_cfs_quota = true + local_ssd_ephemeral_count = 2 + }, + { + name = "pool-04" + min_count = 0 + service_account = var.compute_engine_service_account + queued_provisioning = true }, ] diff --git a/examples/safer_cluster_iap_bastion/apis.tf b/examples/safer_cluster_iap_bastion/apis.tf index 8665a9b3d1..f72733494f 100644 --- a/examples/safer_cluster_iap_bastion/apis.tf +++ b/examples/safer_cluster_iap_bastion/apis.tf @@ -16,7 +16,7 @@ module "enabled_google_apis" { source = "terraform-google-modules/project-factory/google//modules/project_services" - version = "~> 14.0" + version = "~> 15.0" project_id = var.project_id disable_services_on_destroy = false diff --git a/examples/simple_autopilot_private/main.tf b/examples/simple_autopilot_private/main.tf index ee628d8459..125ba24706 100644 --- a/examples/simple_autopilot_private/main.tf +++ b/examples/simple_autopilot_private/main.tf @@ -49,7 +49,6 @@ module "gke" { enable_vertical_pod_autoscaling = true enable_private_endpoint = true enable_private_nodes = true - master_ipv4_cidr_block = "172.16.0.0/28" network_tags = [local.cluster_type] deletion_protection = false diff --git a/examples/simple_autopilot_private_non_default_sa/main.tf b/examples/simple_autopilot_private_non_default_sa/main.tf index 1da730e987..b9ff6e92a5 100644 --- a/examples/simple_autopilot_private_non_default_sa/main.tf +++ b/examples/simple_autopilot_private_non_default_sa/main.tf @@ -49,7 +49,6 @@ module "gke" { enable_vertical_pod_autoscaling = true enable_private_endpoint = true enable_private_nodes = true - master_ipv4_cidr_block = "172.16.0.0/28" deletion_protection = false master_authorized_networks = [ diff --git a/examples/simple_regional/main.tf b/examples/simple_regional/main.tf index 11a34fa0c6..5fc839611d 100644 --- a/examples/simple_regional/main.tf +++ b/examples/simple_regional/main.tf @@ -45,4 +45,5 @@ module "gke" { gcs_fuse_csi_driver = true fleet_project = var.project_id deletion_protection = false + stateful_ha = true } diff --git a/main.tf b/main.tf index af82fe3acb..ee6350cf77 100644 --- a/main.tf +++ b/main.tf @@ -94,6 +94,7 @@ locals { logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] + stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group @@ -170,6 +171,7 @@ locals { }] : [] + cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/modules/acm/versions.tf b/modules/acm/versions.tf index 557f677e42..a9babfe8f8 100644 --- a/modules/acm/versions.tf +++ b/modules/acm/versions.tf @@ -19,11 +19,11 @@ terraform { required_version = ">= 0.13.0" provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v31.0.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:acm/v31.0.0" } required_providers { diff --git a/modules/asm/versions.tf b/modules/asm/versions.tf index 161403d4f9..d45b12a96f 100644 --- a/modules/asm/versions.tf +++ b/modules/asm/versions.tf @@ -36,10 +36,10 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v31.0.0" } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:asm/v31.0.0" } } diff --git a/modules/auth/versions.tf b/modules/auth/versions.tf index 87862337c5..bc3a4c4d0c 100644 --- a/modules/auth/versions.tf +++ b/modules/auth/versions.tf @@ -26,6 +26,6 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:auth/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:auth/v31.0.0" } } diff --git a/modules/beta-autopilot-private-cluster/README.md b/modules/beta-autopilot-private-cluster/README.md index ff40b31c79..5628cde23f 100644 --- a/modules/beta-autopilot-private-cluster/README.md +++ b/modules/beta-autopilot-private-cluster/README.md @@ -54,6 +54,7 @@ module "gke" { enable_private_endpoint = true enable_private_nodes = true master_ipv4_cidr_block = "10.0.0.0/28" + dns_cache = false } ``` @@ -83,7 +84,7 @@ Then perform the following commands on the root folder: | create\_service\_account | Defines if service account specified to run nodes should be created. | `bool` | `true` | no | | database\_encryption | Application-layer Secrets Encryption settings. The object format is {state = string, key\_name = string}. Valid values of state are: "ENCRYPTED"; "DECRYPTED". key\_name is the name of a CloudKMS key. | `list(object({ state = string, key_name = string }))` | 
[
  {
    "key_name": "",
    "state": "DECRYPTED"
  }
]
| no | | deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | | description | The description of the cluster | `string` | `""` | no | | disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | | dns\_cache | The status of the NodeLocal DNSCache addon. | `bool` | `true` | no | @@ -91,14 +92,15 @@ Then perform the following commands on the root folder: | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | | enable\_fqdn\_network\_policy | Enable FQDN Network Policies on the cluster | `bool` | `null` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | +| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | | enable\_vertical\_pod\_autoscaling | Vertical Pod Autoscaling automatically adjusts the resources of pods controlled by it | `bool` | `true` | no | | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | @@ -116,11 +118,11 @@ Then perform the following commands on the root folder: | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `null` | no | | name | The name of the cluster (required) | `string` | n/a | yes | | network | The VPC network to host the cluster in (required) | `string` | n/a | yes | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional, Beta) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | | notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | | project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | @@ -130,7 +132,7 @@ Then perform the following commands on the root folder: | release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | diff --git a/modules/beta-autopilot-private-cluster/cluster.tf b/modules/beta-autopilot-private-cluster/cluster.tf index ae074625a1..397c7755b9 100644 --- a/modules/beta-autopilot-private-cluster/cluster.tf +++ b/modules/beta-autopilot-private-cluster/cluster.tf @@ -107,6 +107,7 @@ resource "google_container_cluster" "primary" { } } + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -268,4 +269,6 @@ resource "google_container_cluster" "primary" { topic = var.notification_config_topic } } + + depends_on = [google_project_iam_member.service_agent] } diff --git a/modules/beta-autopilot-private-cluster/main.tf b/modules/beta-autopilot-private-cluster/main.tf index f0d22b5840..3354fc71a1 100644 --- a/modules/beta-autopilot-private-cluster/main.tf +++ b/modules/beta-autopilot-private-cluster/main.tf @@ -127,12 +127,12 @@ locals { # BETA features cluster_istio_enabled = !local.cluster_output_istio_disabled - cluster_dns_cache_enabled = var.dns_cache cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled # /BETA features + cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/modules/beta-autopilot-private-cluster/outputs.tf b/modules/beta-autopilot-private-cluster/outputs.tf index f2888c0aa1..beb7a8c030 100644 --- a/modules/beta-autopilot-private-cluster/outputs.tf +++ b/modules/beta-autopilot-private-cluster/outputs.tf @@ -159,6 +159,11 @@ output "peering_name" { value = local.cluster_peering_name } +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} + output "cloudrun_enabled" { description = "Whether CloudRun enabled" value = false @@ -169,11 +174,6 @@ output "istio_enabled" { value = local.cluster_istio_enabled } -output "dns_cache_enabled" { - description = "Whether DNS Cache enabled" - value = local.cluster_dns_cache_enabled -} - output "pod_security_policy_enabled" { description = "Whether pod security policy is enabled" value = local.cluster_pod_security_policy_enabled diff --git a/modules/beta-autopilot-private-cluster/sa.tf b/modules/beta-autopilot-private-cluster/sa.tf index 6f89899bee..fe36dadd32 100644 --- a/modules/beta-autopilot-private-cluster/sa.tf +++ b/modules/beta-autopilot-private-cluster/sa.tf @@ -52,6 +52,20 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } +resource "google_project_iam_member" "cluster_service_account-metric_writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/monitoring.metricWriter" + member = google_service_account.cluster_service_account[0].member +} + +resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/stackdriver.resourceMetadata.writer" + member = google_service_account.cluster_service_account[0].member +} + resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key @@ -65,3 +79,17 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } + +resource "google_project_service_identity" "fleet_project" { + count = var.fleet_project_grant_service_agent ? 1 : 0 + provider = google-beta + project = var.fleet_project + service = "gkehub.googleapis.com" +} + +resource "google_project_iam_member" "service_agent" { + for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] + project = var.project_id + role = each.value + member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" +} diff --git a/modules/beta-autopilot-private-cluster/variables.tf b/modules/beta-autopilot-private-cluster/variables.tf index 35325d6774..9e5c223a81 100644 --- a/modules/beta-autopilot-private-cluster/variables.tf +++ b/modules/beta-autopilot-private-cluster/variables.tf @@ -174,10 +174,11 @@ variable "enable_resource_consumption_export" { variable "network_tags" { - description = "(Optional, Beta) - List of network tags applied to auto-provisioned node pools." + description = "(Optional) - List of network tags applied to auto-provisioned node pools." type = list(string) default = [] } + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -265,26 +266,26 @@ variable "cluster_resource_labels" { variable "deploy_using_private_endpoint" { type = bool - description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." + description = "A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." default = false } variable "enable_private_endpoint" { type = bool - description = "(Beta) Whether the master's internal IP address is used as the cluster endpoint" + description = "Whether the master's internal IP address is used as the cluster endpoint" default = false } variable "enable_private_nodes" { type = bool - description = "(Beta) Whether nodes have internal IP addresses only" + description = "Whether nodes have internal IP addresses only" default = false } variable "master_ipv4_cidr_block" { type = string - description = "(Beta) The IP range in CIDR notation to use for the hosted master network" - default = "10.0.0.0/28" + description = "The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters." + default = null } variable "master_global_access_enabled" { @@ -405,7 +406,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -466,3 +467,9 @@ variable "fleet_project" { type = string default = null } + +variable "fleet_project_grant_service_agent" { + description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." + type = bool + default = false +} diff --git a/modules/beta-autopilot-private-cluster/versions.tf b/modules/beta-autopilot-private-cluster/versions.tf index d80b7f0fce..ef38f35edc 100644 --- a/modules/beta-autopilot-private-cluster/versions.tf +++ b/modules/beta-autopilot-private-cluster/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.9.0, < 6" + version = ">= 5.25.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.9.0, < 6" + version = ">= 5.25.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-private-cluster/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-private-cluster/v31.0.0" } } diff --git a/modules/beta-autopilot-public-cluster/README.md b/modules/beta-autopilot-public-cluster/README.md index 198fde422e..0e3c169dff 100644 --- a/modules/beta-autopilot-public-cluster/README.md +++ b/modules/beta-autopilot-public-cluster/README.md @@ -48,6 +48,7 @@ module "gke" { ip_range_services = "us-central1-01-gke-01-services" horizontal_pod_autoscaling = true filestore_csi_driver = false + dns_cache = false } ``` @@ -90,6 +91,7 @@ Then perform the following commands on the root folder: | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | grant\_registry\_access | Grants created cluster-specific service account storage.objectViewer and artifactregistry.reader roles. | `bool` | `false` | no | | horizontal\_pod\_autoscaling | Enable horizontal pod autoscaling addon | `bool` | `true` | no | @@ -109,7 +111,7 @@ Then perform the following commands on the root folder: | name | The name of the cluster (required) | `string` | n/a | yes | | network | The VPC network to host the cluster in (required) | `string` | n/a | yes | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | -| network\_tags | (Optional, Beta) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | non\_masquerade\_cidrs | List of strings in CIDR notation that specify the IP address ranges that do not use IP masquerading. | `list(string)` | 
[
  "10.0.0.0/8",
  "172.16.0.0/12",
  "192.168.0.0/16"
]
| no | | notification\_config\_topic | The desired Pub/Sub topic to which notifications will be sent by GKE. Format is projects/{project}/topics/{topic}. | `string` | `""` | no | | project\_id | The project ID to host the cluster in (required) | `string` | n/a | yes | @@ -119,7 +121,7 @@ Then perform the following commands on the root folder: | release\_channel | The release channel of this cluster. Accepted values are `UNSPECIFIED`, `RAPID`, `REGULAR` and `STABLE`. Defaults to `REGULAR`. | `string` | `"REGULAR"` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | diff --git a/modules/beta-autopilot-public-cluster/cluster.tf b/modules/beta-autopilot-public-cluster/cluster.tf index 39b4c57bbe..01ea54baf4 100644 --- a/modules/beta-autopilot-public-cluster/cluster.tf +++ b/modules/beta-autopilot-public-cluster/cluster.tf @@ -107,6 +107,7 @@ resource "google_container_cluster" "primary" { } } + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -249,4 +250,6 @@ resource "google_container_cluster" "primary" { topic = var.notification_config_topic } } + + depends_on = [google_project_iam_member.service_agent] } diff --git a/modules/beta-autopilot-public-cluster/main.tf b/modules/beta-autopilot-public-cluster/main.tf index 24bcb36d3d..9df7f76c14 100644 --- a/modules/beta-autopilot-public-cluster/main.tf +++ b/modules/beta-autopilot-public-cluster/main.tf @@ -126,12 +126,12 @@ locals { # BETA features cluster_istio_enabled = !local.cluster_output_istio_disabled - cluster_dns_cache_enabled = var.dns_cache cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled # /BETA features + cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/modules/beta-autopilot-public-cluster/outputs.tf b/modules/beta-autopilot-public-cluster/outputs.tf index 9b26f835f8..17e634faf2 100644 --- a/modules/beta-autopilot-public-cluster/outputs.tf +++ b/modules/beta-autopilot-public-cluster/outputs.tf @@ -149,6 +149,11 @@ output "tpu_ipv4_cidr_block" { +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} + output "cloudrun_enabled" { description = "Whether CloudRun enabled" value = false @@ -159,11 +164,6 @@ output "istio_enabled" { value = local.cluster_istio_enabled } -output "dns_cache_enabled" { - description = "Whether DNS Cache enabled" - value = local.cluster_dns_cache_enabled -} - output "pod_security_policy_enabled" { description = "Whether pod security policy is enabled" value = local.cluster_pod_security_policy_enabled diff --git a/modules/beta-autopilot-public-cluster/sa.tf b/modules/beta-autopilot-public-cluster/sa.tf index 6f89899bee..fe36dadd32 100644 --- a/modules/beta-autopilot-public-cluster/sa.tf +++ b/modules/beta-autopilot-public-cluster/sa.tf @@ -52,6 +52,20 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } +resource "google_project_iam_member" "cluster_service_account-metric_writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/monitoring.metricWriter" + member = google_service_account.cluster_service_account[0].member +} + +resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/stackdriver.resourceMetadata.writer" + member = google_service_account.cluster_service_account[0].member +} + resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key @@ -65,3 +79,17 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } + +resource "google_project_service_identity" "fleet_project" { + count = var.fleet_project_grant_service_agent ? 1 : 0 + provider = google-beta + project = var.fleet_project + service = "gkehub.googleapis.com" +} + +resource "google_project_iam_member" "service_agent" { + for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] + project = var.project_id + role = each.value + member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" +} diff --git a/modules/beta-autopilot-public-cluster/variables.tf b/modules/beta-autopilot-public-cluster/variables.tf index 0d3c5cdd5c..0ecca7a1a3 100644 --- a/modules/beta-autopilot-public-cluster/variables.tf +++ b/modules/beta-autopilot-public-cluster/variables.tf @@ -174,10 +174,11 @@ variable "enable_resource_consumption_export" { variable "network_tags" { - description = "(Optional, Beta) - List of network tags applied to auto-provisioned node pools." + description = "(Optional) - List of network tags applied to auto-provisioned node pools." type = list(string) default = [] } + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -375,7 +376,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -436,3 +437,9 @@ variable "fleet_project" { type = string default = null } + +variable "fleet_project_grant_service_agent" { + description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." + type = bool + default = false +} diff --git a/modules/beta-autopilot-public-cluster/versions.tf b/modules/beta-autopilot-public-cluster/versions.tf index ad536dbc26..bc43de8446 100644 --- a/modules/beta-autopilot-public-cluster/versions.tf +++ b/modules/beta-autopilot-public-cluster/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.9.0, < 6" + version = ">= 5.25.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.9.0, < 6" + version = ">= 5.25.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-public-cluster/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-autopilot-public-cluster/v31.0.0" } } diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index 59479ea2bd..fffa95b587 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -84,25 +84,30 @@ module "gke" { node_pools = [ { - name = "default-node-pool" - machine_type = "e2-medium" - node_locations = "us-central1-b,us-central1-c" - min_count = 1 - max_count = 100 - local_ssd_count = 0 - spot = false - local_ssd_ephemeral_count = 0 - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS_CONTAINERD" - enable_gcfs = false - enable_gvnic = false - logging_variant = "DEFAULT" - auto_repair = true - auto_upgrade = true - service_account = "project-service-account@.iam.gserviceaccount.com" - preemptible = false - initial_node_count = 80 + name = "default-node-pool" + machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = false + local_ssd_ephemeral_count = 0 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + enable_gcfs = false + enable_gvnic = false + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 + accelerator_count = 1 + accelerator_type = "nvidia-l4" + gpu_driver_version = "LATEST" + gpu_sharing_strategy = "TIME_SHARING" + max_shared_clients_per_gpu = 2 }, ] @@ -171,7 +176,7 @@ Then perform the following commands on the root folder: | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
    disk_size           = optional(number)
    disk_type           = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                 = bool
    autoscaling_profile     = string
    min_cpu_cores           = number
    max_cpu_cores           = number
    min_memory_gb           = number
    max_memory_gb           = number
    gpu_resources           = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair             = bool
    auto_upgrade            = bool
    disk_size               = optional(number)
    disk_type               = optional(string)
    image_type              = optional(string)
    strategy                = optional(string)
    max_surge               = optional(number)
    max_unavailable         = optional(number)
    node_pool_soak_duration = optional(string)
    batch_soak_duration     = optional(string)
    batch_percentage        = optional(number)
    batch_node_count        = optional(number)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -185,7 +190,7 @@ Then perform the following commands on the root folder: | datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | | deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | | description | The description of the cluster | `string` | `""` | no | | disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | @@ -202,8 +207,8 @@ Then perform the following commands on the root folder: | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | | enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | -| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | +| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | | enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | @@ -212,6 +217,7 @@ Then perform the following commands on the root folder: | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | @@ -250,6 +256,7 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | @@ -270,13 +277,14 @@ Then perform the following commands on the root folder: | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | | shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | @@ -353,19 +361,23 @@ The node_pools variable takes the following parameters: | enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | +| gpu_driver_version | Mode for how the GPU driver is installed | null | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | +| local_ssd_ephemeral_storage_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | | local_ssd_ephemeral_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | +| local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | | strategy | The upgrade stragey to be used for upgrading the nodes. Valid values of state are: `SURGE` or `BLUE_GREEN` | "SURGE" | Optional | +| threads_per_core | Optional The number of threads per physical core. To disable simultaneous multithreading (SMT) set this to 1. If unset, the maximum number of threads supported per core by the underlying processor is assumed | null | Optional | | max_surge | The number of additional nodes that can be added to the node pool during an upgrade. Increasing max_surge raises the number of nodes that can be upgraded simultaneously. Can be set to 0 or greater. Only works with `SURGE` strategy. | 1 | Optional | | max_unavailable | The number of nodes that can be simultaneously unavailable during an upgrade. Increasing max_unavailable raises the number of nodes that can be upgraded in parallel. Can be set to 0 or greater. Only works with `SURGE` strategy. | 0 | Optional | | node_pool_soak_duration | Time needed after draining the entire blue pool. After this period, the blue pool will be cleaned up. By default, it is set to one hour (3600 seconds). The maximum length of the soak time is 7 days (604,800 seconds). Only works with `BLUE_GREEN` strategy. | "3600s" | Optional | @@ -388,6 +400,8 @@ The node_pools variable takes the following parameters: | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | +| secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | +| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 286a889dc7..99f22565cc 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -130,7 +130,28 @@ resource "google_container_cluster" "primary" { disk_size = lookup(var.cluster_autoscaling, "disk_size", 100) disk_type = lookup(var.cluster_autoscaling, "disk_type", "pd-standard") + upgrade_settings { + strategy = lookup(var.cluster_autoscaling, "strategy", "SURGE") + max_surge = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_surge", 0) : null + max_unavailable = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_unavailable", 0) : null + + dynamic "blue_green_settings" { + for_each = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "BLUE_GREEN" ? [1] : [] + content { + node_pool_soak_duration = lookup(var.cluster_autoscaling, "node_pool_soak_duration", null) + + standard_rollout_policy { + batch_soak_duration = lookup(var.cluster_autoscaling, "batch_soak_duration", null) + batch_percentage = lookup(var.cluster_autoscaling, "batch_percentage", null) + batch_node_count = lookup(var.cluster_autoscaling, "batch_node_count", null) + } + } + } + } + min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + + image_type = lookup(var.cluster_autoscaling, "image_type", "COS_CONTAINERD") } } autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" @@ -189,6 +210,15 @@ resource "google_container_cluster" "primary" { } } + dynamic "node_pool_auto_config" { + for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + content { + network_tags { + tags = var.network_tags + } + } + } + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -247,6 +277,14 @@ resource "google_container_cluster" "primary" { } } + dynamic "stateful_ha_config" { + for_each = local.stateful_ha_config + + content { + enabled = stateful_ha_config.value.enabled + } + } + config_connector_config { enabled = var.config_connector } @@ -504,6 +542,8 @@ resource "google_container_cluster" "primary" { } } } + + depends_on = [google_project_iam_member.service_agent] } /****************************************** Create Container Cluster node pools @@ -515,6 +555,9 @@ locals { "accelerator_count", "accelerator_type", "gpu_partition_size", + "gpu_driver_version", + "gpu_sharing_strategy", + "max_shared_clients_per_gpu", "enable_secure_boot", "enable_integrity_monitoring", "local_ssd_count", @@ -530,6 +573,7 @@ locals { "enable_gvnic", "enable_secure_boot", "boot_disk_kms_key", + "queued_provisioning", ] } @@ -660,6 +704,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -676,6 +727,12 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -719,6 +776,12 @@ resource "google_container_node_pool" "pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") + dynamic "ephemeral_storage_local_ssd_config" { + for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] + content { + local_ssd_count = ephemeral_storage_local_ssd_config.value + } + } dynamic "ephemeral_storage_config" { for_each = lookup(each.value, "local_ssd_ephemeral_count", 0) > 0 ? [each.value.local_ssd_ephemeral_count] : [] content { @@ -726,6 +789,22 @@ resource "google_container_node_pool" "pools" { } } + dynamic "local_nvme_ssd_block_config" { + for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] + content { + local_ssd_count = local_nvme_ssd_block_config.value + } + } + + # Supports a single secondary boot disk because `map(any)` must have the same values type. + dynamic "secondary_boot_disks" { + for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] + content { + disk_image = secondary_boot_disks.value + mode = "CONTAINER_IMAGE_CACHE" + } + } + service_account = lookup( each.value, "service_account", @@ -752,6 +831,21 @@ resource "google_container_node_pool" "pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } + + dynamic "gpu_sharing_config" { + for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) + } + } + } + } + + dynamic "advanced_machine_features" { + for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] + content { + threads_per_core = lookup(each.value, "threads_per_core", 0) } } @@ -895,6 +989,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -911,6 +1012,12 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -954,6 +1061,12 @@ resource "google_container_node_pool" "windows_pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") + dynamic "ephemeral_storage_local_ssd_config" { + for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] + content { + local_ssd_count = ephemeral_storage_local_ssd_config.value + } + } dynamic "ephemeral_storage_config" { for_each = lookup(each.value, "local_ssd_ephemeral_count", 0) > 0 ? [each.value.local_ssd_ephemeral_count] : [] content { @@ -961,6 +1074,22 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "local_nvme_ssd_block_config" { + for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] + content { + local_ssd_count = local_nvme_ssd_block_config.value + } + } + + # Supports a single secondary boot disk because `map(any)` must have the same values type. + dynamic "secondary_boot_disks" { + for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] + content { + disk_image = secondary_boot_disks.value + mode = "CONTAINER_IMAGE_CACHE" + } + } + service_account = lookup( each.value, "service_account", @@ -987,6 +1116,21 @@ resource "google_container_node_pool" "windows_pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } + + dynamic "gpu_sharing_config" { + for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) + } + } + } + } + + dynamic "advanced_machine_features" { + for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] + content { + threads_per_core = lookup(each.value, "threads_per_core", 0) } } diff --git a/modules/beta-private-cluster-update-variant/main.tf b/modules/beta-private-cluster-update-variant/main.tf index c59479b579..2e05bedeeb 100644 --- a/modules/beta-private-cluster-update-variant/main.tf +++ b/modules/beta-private-cluster-update-variant/main.tf @@ -94,6 +94,7 @@ locals { logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] + stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} @@ -190,13 +191,13 @@ locals { # BETA features cluster_istio_enabled = !local.cluster_output_istio_disabled - cluster_dns_cache_enabled = var.dns_cache cluster_telemetry_type_is_set = var.cluster_telemetry_type != null cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled # /BETA features + cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/modules/beta-private-cluster-update-variant/outputs.tf b/modules/beta-private-cluster-update-variant/outputs.tf index 2d8e768f7c..3209aca8ae 100644 --- a/modules/beta-private-cluster-update-variant/outputs.tf +++ b/modules/beta-private-cluster-update-variant/outputs.tf @@ -185,6 +185,11 @@ output "peering_name" { value = local.cluster_peering_name } +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} + output "cloudrun_enabled" { description = "Whether CloudRun enabled" value = local.cluster_cloudrun_enabled @@ -195,11 +200,6 @@ output "istio_enabled" { value = local.cluster_istio_enabled } -output "dns_cache_enabled" { - description = "Whether DNS Cache enabled" - value = local.cluster_dns_cache_enabled -} - output "pod_security_policy_enabled" { description = "Whether pod security policy is enabled" value = local.cluster_pod_security_policy_enabled diff --git a/modules/beta-private-cluster-update-variant/sa.tf b/modules/beta-private-cluster-update-variant/sa.tf index 6f89899bee..fe36dadd32 100644 --- a/modules/beta-private-cluster-update-variant/sa.tf +++ b/modules/beta-private-cluster-update-variant/sa.tf @@ -52,6 +52,20 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } +resource "google_project_iam_member" "cluster_service_account-metric_writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/monitoring.metricWriter" + member = google_service_account.cluster_service_account[0].member +} + +resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/stackdriver.resourceMetadata.writer" + member = google_service_account.cluster_service_account[0].member +} + resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key @@ -65,3 +79,17 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } + +resource "google_project_service_identity" "fleet_project" { + count = var.fleet_project_grant_service_agent ? 1 : 0 + provider = google-beta + project = var.fleet_project + service = "gkehub.googleapis.com" +} + +resource "google_project_iam_member" "service_agent" { + for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] + project = var.project_id + role = each.value + member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" +} diff --git a/modules/beta-private-cluster-update-variant/variables.tf b/modules/beta-private-cluster-update-variant/variables.tf index 3d13d63ac5..64f685a34d 100644 --- a/modules/beta-private-cluster-update-variant/variables.tf +++ b/modules/beta-private-cluster-update-variant/variables.tf @@ -239,17 +239,25 @@ variable "enable_resource_consumption_export" { variable "cluster_autoscaling" { type = object({ - enabled = bool - autoscaling_profile = string - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) + image_type = optional(string) + strategy = optional(string) + max_surge = optional(number) + max_unavailable = optional(number) + node_pool_soak_duration = optional(string) + batch_soak_duration = optional(string) + batch_percentage = optional(number) + batch_node_count = optional(number) }) default = { enabled = false @@ -263,6 +271,7 @@ variable "cluster_autoscaling" { auto_upgrade = true disk_size = 100 disk_type = "pd-standard" + image_type = "COS_CONTAINERD" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -300,6 +309,12 @@ variable "node_pools_oauth_scopes" { } } +variable "network_tags" { + description = "(Optional) - List of network tags applied to auto-provisioned node pools." + type = list(string) + default = [] +} + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -405,25 +420,25 @@ variable "cluster_resource_labels" { variable "deploy_using_private_endpoint" { type = bool - description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." + description = "A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." default = false } variable "enable_private_endpoint" { type = bool - description = "(Beta) Whether the master's internal IP address is used as the cluster endpoint" + description = "Whether the master's internal IP address is used as the cluster endpoint" default = false } variable "enable_private_nodes" { type = bool - description = "(Beta) Whether nodes have internal IP addresses only" + description = "Whether nodes have internal IP addresses only" default = false } variable "master_ipv4_cidr_block" { type = string - description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + description = "The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters." default = "10.0.0.0/28" } @@ -550,7 +565,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -690,6 +705,12 @@ variable "gcs_fuse_csi_driver" { default = false } +variable "stateful_ha" { + type = bool + description = "Whether the Stateful HA Addon is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -730,7 +751,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -752,6 +773,12 @@ variable "config_connector" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "istio" { description = "(Beta) Enable Istio addon" type = bool @@ -788,12 +815,6 @@ variable "enable_pod_security_policy" { default = false } -variable "enable_l4_ilb_subsetting" { - type = bool - description = "Enable L4 ILB Subsetting on the cluster" - default = false -} - variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." @@ -823,3 +844,9 @@ variable "fleet_project" { type = string default = null } + +variable "fleet_project_grant_service_agent" { + description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." + type = bool + default = false +} diff --git a/modules/beta-private-cluster-update-variant/versions.tf b/modules/beta-private-cluster-update-variant/versions.tf index 7fc2de48e4..d9385ca73d 100644 --- a/modules/beta-private-cluster-update-variant/versions.tf +++ b/modules/beta-private-cluster-update-variant/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.9.0, < 6" + version = ">= 5.25.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.9.0, < 6" + version = ">= 5.25.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster-update-variant/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster-update-variant/v31.0.0" } } diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index dce2535e3e..6690b071d2 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -62,25 +62,30 @@ module "gke" { node_pools = [ { - name = "default-node-pool" - machine_type = "e2-medium" - node_locations = "us-central1-b,us-central1-c" - min_count = 1 - max_count = 100 - local_ssd_count = 0 - spot = false - local_ssd_ephemeral_count = 0 - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS_CONTAINERD" - enable_gcfs = false - enable_gvnic = false - logging_variant = "DEFAULT" - auto_repair = true - auto_upgrade = true - service_account = "project-service-account@.iam.gserviceaccount.com" - preemptible = false - initial_node_count = 80 + name = "default-node-pool" + machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = false + local_ssd_ephemeral_count = 0 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + enable_gcfs = false + enable_gvnic = false + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 + accelerator_count = 1 + accelerator_type = "nvidia-l4" + gpu_driver_version = "LATEST" + gpu_sharing_strategy = "TIME_SHARING" + max_shared_clients_per_gpu = 2 }, ] @@ -149,7 +154,7 @@ Then perform the following commands on the root folder: | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
    disk_size           = optional(number)
    disk_type           = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                 = bool
    autoscaling_profile     = string
    min_cpu_cores           = number
    max_cpu_cores           = number
    min_memory_gb           = number
    max_memory_gb           = number
    gpu_resources           = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair             = bool
    auto_upgrade            = bool
    disk_size               = optional(number)
    disk_type               = optional(string)
    image_type              = optional(string)
    strategy                = optional(string)
    max_surge               = optional(number)
    max_unavailable         = optional(number)
    node_pool_soak_duration = optional(string)
    batch_soak_duration     = optional(string)
    batch_percentage        = optional(number)
    batch_node_count        = optional(number)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -163,7 +168,7 @@ Then perform the following commands on the root folder: | datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | | deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | | description | The description of the cluster | `string` | `""` | no | | disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | @@ -180,8 +185,8 @@ Then perform the following commands on the root folder: | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | | enable\_pod\_security\_policy | enabled - Enable the PodSecurityPolicy controller for this cluster. If enabled, pods must be valid under a PodSecurityPolicy to be created. Pod Security Policy was removed from GKE clusters with version >= 1.25.0. | `bool` | `false` | no | -| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | +| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | | enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | @@ -190,6 +195,7 @@ Then perform the following commands on the root folder: | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | @@ -228,6 +234,7 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | @@ -248,13 +255,14 @@ Then perform the following commands on the root folder: | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | | shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | @@ -331,19 +339,23 @@ The node_pools variable takes the following parameters: | enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | +| gpu_driver_version | Mode for how the GPU driver is installed | null | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | +| local_ssd_ephemeral_storage_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | | local_ssd_ephemeral_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | +| local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | | strategy | The upgrade stragey to be used for upgrading the nodes. Valid values of state are: `SURGE` or `BLUE_GREEN` | "SURGE" | Optional | +| threads_per_core | Optional The number of threads per physical core. To disable simultaneous multithreading (SMT) set this to 1. If unset, the maximum number of threads supported per core by the underlying processor is assumed | null | Optional | | max_surge | The number of additional nodes that can be added to the node pool during an upgrade. Increasing max_surge raises the number of nodes that can be upgraded simultaneously. Can be set to 0 or greater. Only works with `SURGE` strategy. | 1 | Optional | | max_unavailable | The number of nodes that can be simultaneously unavailable during an upgrade. Increasing max_unavailable raises the number of nodes that can be upgraded in parallel. Can be set to 0 or greater. Only works with `SURGE` strategy. | 0 | Optional | | node_pool_soak_duration | Time needed after draining the entire blue pool. After this period, the blue pool will be cleaned up. By default, it is set to one hour (3600 seconds). The maximum length of the soak time is 7 days (604,800 seconds). Only works with `BLUE_GREEN` strategy. | "3600s" | Optional | @@ -366,6 +378,8 @@ The node_pools variable takes the following parameters: | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | +| secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | +| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index aed3ab4152..de7b0c9caa 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -130,7 +130,28 @@ resource "google_container_cluster" "primary" { disk_size = lookup(var.cluster_autoscaling, "disk_size", 100) disk_type = lookup(var.cluster_autoscaling, "disk_type", "pd-standard") + upgrade_settings { + strategy = lookup(var.cluster_autoscaling, "strategy", "SURGE") + max_surge = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_surge", 0) : null + max_unavailable = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_unavailable", 0) : null + + dynamic "blue_green_settings" { + for_each = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "BLUE_GREEN" ? [1] : [] + content { + node_pool_soak_duration = lookup(var.cluster_autoscaling, "node_pool_soak_duration", null) + + standard_rollout_policy { + batch_soak_duration = lookup(var.cluster_autoscaling, "batch_soak_duration", null) + batch_percentage = lookup(var.cluster_autoscaling, "batch_percentage", null) + batch_node_count = lookup(var.cluster_autoscaling, "batch_node_count", null) + } + } + } + } + min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + + image_type = lookup(var.cluster_autoscaling, "image_type", "COS_CONTAINERD") } } autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" @@ -189,6 +210,15 @@ resource "google_container_cluster" "primary" { } } + dynamic "node_pool_auto_config" { + for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + content { + network_tags { + tags = var.network_tags + } + } + } + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -247,6 +277,14 @@ resource "google_container_cluster" "primary" { } } + dynamic "stateful_ha_config" { + for_each = local.stateful_ha_config + + content { + enabled = stateful_ha_config.value.enabled + } + } + config_connector_config { enabled = var.config_connector } @@ -504,6 +542,8 @@ resource "google_container_cluster" "primary" { } } } + + depends_on = [google_project_iam_member.service_agent] } /****************************************** Create Container Cluster node pools @@ -585,6 +625,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -601,6 +648,12 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -644,6 +697,12 @@ resource "google_container_node_pool" "pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") + dynamic "ephemeral_storage_local_ssd_config" { + for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] + content { + local_ssd_count = ephemeral_storage_local_ssd_config.value + } + } dynamic "ephemeral_storage_config" { for_each = lookup(each.value, "local_ssd_ephemeral_count", 0) > 0 ? [each.value.local_ssd_ephemeral_count] : [] content { @@ -651,6 +710,22 @@ resource "google_container_node_pool" "pools" { } } + dynamic "local_nvme_ssd_block_config" { + for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] + content { + local_ssd_count = local_nvme_ssd_block_config.value + } + } + + # Supports a single secondary boot disk because `map(any)` must have the same values type. + dynamic "secondary_boot_disks" { + for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] + content { + disk_image = secondary_boot_disks.value + mode = "CONTAINER_IMAGE_CACHE" + } + } + service_account = lookup( each.value, "service_account", @@ -677,6 +752,21 @@ resource "google_container_node_pool" "pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } + + dynamic "gpu_sharing_config" { + for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) + } + } + } + } + + dynamic "advanced_machine_features" { + for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] + content { + threads_per_core = lookup(each.value, "threads_per_core", 0) } } @@ -819,6 +909,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -835,6 +932,12 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -878,6 +981,12 @@ resource "google_container_node_pool" "windows_pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") + dynamic "ephemeral_storage_local_ssd_config" { + for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] + content { + local_ssd_count = ephemeral_storage_local_ssd_config.value + } + } dynamic "ephemeral_storage_config" { for_each = lookup(each.value, "local_ssd_ephemeral_count", 0) > 0 ? [each.value.local_ssd_ephemeral_count] : [] content { @@ -885,6 +994,22 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "local_nvme_ssd_block_config" { + for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] + content { + local_ssd_count = local_nvme_ssd_block_config.value + } + } + + # Supports a single secondary boot disk because `map(any)` must have the same values type. + dynamic "secondary_boot_disks" { + for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] + content { + disk_image = secondary_boot_disks.value + mode = "CONTAINER_IMAGE_CACHE" + } + } + service_account = lookup( each.value, "service_account", @@ -911,6 +1036,21 @@ resource "google_container_node_pool" "windows_pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } + + dynamic "gpu_sharing_config" { + for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) + } + } + } + } + + dynamic "advanced_machine_features" { + for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] + content { + threads_per_core = lookup(each.value, "threads_per_core", 0) } } diff --git a/modules/beta-private-cluster/main.tf b/modules/beta-private-cluster/main.tf index c59479b579..2e05bedeeb 100644 --- a/modules/beta-private-cluster/main.tf +++ b/modules/beta-private-cluster/main.tf @@ -94,6 +94,7 @@ locals { logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] + stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} @@ -190,13 +191,13 @@ locals { # BETA features cluster_istio_enabled = !local.cluster_output_istio_disabled - cluster_dns_cache_enabled = var.dns_cache cluster_telemetry_type_is_set = var.cluster_telemetry_type != null cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled # /BETA features + cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/modules/beta-private-cluster/outputs.tf b/modules/beta-private-cluster/outputs.tf index 2d8e768f7c..3209aca8ae 100644 --- a/modules/beta-private-cluster/outputs.tf +++ b/modules/beta-private-cluster/outputs.tf @@ -185,6 +185,11 @@ output "peering_name" { value = local.cluster_peering_name } +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} + output "cloudrun_enabled" { description = "Whether CloudRun enabled" value = local.cluster_cloudrun_enabled @@ -195,11 +200,6 @@ output "istio_enabled" { value = local.cluster_istio_enabled } -output "dns_cache_enabled" { - description = "Whether DNS Cache enabled" - value = local.cluster_dns_cache_enabled -} - output "pod_security_policy_enabled" { description = "Whether pod security policy is enabled" value = local.cluster_pod_security_policy_enabled diff --git a/modules/beta-private-cluster/sa.tf b/modules/beta-private-cluster/sa.tf index 6f89899bee..fe36dadd32 100644 --- a/modules/beta-private-cluster/sa.tf +++ b/modules/beta-private-cluster/sa.tf @@ -52,6 +52,20 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } +resource "google_project_iam_member" "cluster_service_account-metric_writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/monitoring.metricWriter" + member = google_service_account.cluster_service_account[0].member +} + +resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/stackdriver.resourceMetadata.writer" + member = google_service_account.cluster_service_account[0].member +} + resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key @@ -65,3 +79,17 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } + +resource "google_project_service_identity" "fleet_project" { + count = var.fleet_project_grant_service_agent ? 1 : 0 + provider = google-beta + project = var.fleet_project + service = "gkehub.googleapis.com" +} + +resource "google_project_iam_member" "service_agent" { + for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] + project = var.project_id + role = each.value + member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" +} diff --git a/modules/beta-private-cluster/variables.tf b/modules/beta-private-cluster/variables.tf index 3d13d63ac5..64f685a34d 100644 --- a/modules/beta-private-cluster/variables.tf +++ b/modules/beta-private-cluster/variables.tf @@ -239,17 +239,25 @@ variable "enable_resource_consumption_export" { variable "cluster_autoscaling" { type = object({ - enabled = bool - autoscaling_profile = string - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) + image_type = optional(string) + strategy = optional(string) + max_surge = optional(number) + max_unavailable = optional(number) + node_pool_soak_duration = optional(string) + batch_soak_duration = optional(string) + batch_percentage = optional(number) + batch_node_count = optional(number) }) default = { enabled = false @@ -263,6 +271,7 @@ variable "cluster_autoscaling" { auto_upgrade = true disk_size = 100 disk_type = "pd-standard" + image_type = "COS_CONTAINERD" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -300,6 +309,12 @@ variable "node_pools_oauth_scopes" { } } +variable "network_tags" { + description = "(Optional) - List of network tags applied to auto-provisioned node pools." + type = list(string) + default = [] +} + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -405,25 +420,25 @@ variable "cluster_resource_labels" { variable "deploy_using_private_endpoint" { type = bool - description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." + description = "A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." default = false } variable "enable_private_endpoint" { type = bool - description = "(Beta) Whether the master's internal IP address is used as the cluster endpoint" + description = "Whether the master's internal IP address is used as the cluster endpoint" default = false } variable "enable_private_nodes" { type = bool - description = "(Beta) Whether nodes have internal IP addresses only" + description = "Whether nodes have internal IP addresses only" default = false } variable "master_ipv4_cidr_block" { type = string - description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + description = "The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters." default = "10.0.0.0/28" } @@ -550,7 +565,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -690,6 +705,12 @@ variable "gcs_fuse_csi_driver" { default = false } +variable "stateful_ha" { + type = bool + description = "Whether the Stateful HA Addon is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -730,7 +751,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -752,6 +773,12 @@ variable "config_connector" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "istio" { description = "(Beta) Enable Istio addon" type = bool @@ -788,12 +815,6 @@ variable "enable_pod_security_policy" { default = false } -variable "enable_l4_ilb_subsetting" { - type = bool - description = "Enable L4 ILB Subsetting on the cluster" - default = false -} - variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." @@ -823,3 +844,9 @@ variable "fleet_project" { type = string default = null } + +variable "fleet_project_grant_service_agent" { + description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." + type = bool + default = false +} diff --git a/modules/beta-private-cluster/versions.tf b/modules/beta-private-cluster/versions.tf index ded29b9a0e..7184582867 100644 --- a/modules/beta-private-cluster/versions.tf +++ b/modules/beta-private-cluster/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.9.0, < 6" + version = ">= 5.25.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.9.0, < 6" + version = ">= 5.25.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-private-cluster/v31.0.0" } } diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 4bf4563bbc..8717517d0d 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -78,25 +78,30 @@ module "gke" { node_pools = [ { - name = "default-node-pool" - machine_type = "e2-medium" - node_locations = "us-central1-b,us-central1-c" - min_count = 1 - max_count = 100 - local_ssd_count = 0 - spot = false - local_ssd_ephemeral_count = 0 - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS_CONTAINERD" - enable_gcfs = false - enable_gvnic = false - logging_variant = "DEFAULT" - auto_repair = true - auto_upgrade = true - service_account = "project-service-account@.iam.gserviceaccount.com" - preemptible = false - initial_node_count = 80 + name = "default-node-pool" + machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = false + local_ssd_ephemeral_count = 0 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + enable_gcfs = false + enable_gvnic = false + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 + accelerator_count = 1 + accelerator_type = "nvidia-l4" + gpu_driver_version = "LATEST" + gpu_sharing_strategy = "TIME_SHARING" + max_shared_clients_per_gpu = 2 }, ] @@ -165,7 +170,7 @@ Then perform the following commands on the root folder: | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
    disk_size           = optional(number)
    disk_type           = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                 = bool
    autoscaling_profile     = string
    min_cpu_cores           = number
    max_cpu_cores           = number
    min_memory_gb           = number
    max_memory_gb           = number
    gpu_resources           = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair             = bool
    auto_upgrade            = bool
    disk_size               = optional(number)
    disk_type               = optional(string)
    image_type              = optional(string)
    strategy                = optional(string)
    max_surge               = optional(number)
    max_unavailable         = optional(number)
    node_pool_soak_duration = optional(string)
    batch_soak_duration     = optional(string)
    batch_percentage        = optional(number)
    batch_node_count        = optional(number)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -203,6 +208,7 @@ Then perform the following commands on the root folder: | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | @@ -239,6 +245,7 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | @@ -259,13 +266,14 @@ Then perform the following commands on the root folder: | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | | shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | @@ -340,19 +348,23 @@ The node_pools variable takes the following parameters: | enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | +| gpu_driver_version | Mode for how the GPU driver is installed | null | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | +| local_ssd_ephemeral_storage_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | | local_ssd_ephemeral_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | +| local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | | strategy | The upgrade stragey to be used for upgrading the nodes. Valid values of state are: `SURGE` or `BLUE_GREEN` | "SURGE" | Optional | +| threads_per_core | Optional The number of threads per physical core. To disable simultaneous multithreading (SMT) set this to 1. If unset, the maximum number of threads supported per core by the underlying processor is assumed | null | Optional | | max_surge | The number of additional nodes that can be added to the node pool during an upgrade. Increasing max_surge raises the number of nodes that can be upgraded simultaneously. Can be set to 0 or greater. Only works with `SURGE` strategy. | 1 | Optional | | max_unavailable | The number of nodes that can be simultaneously unavailable during an upgrade. Increasing max_unavailable raises the number of nodes that can be upgraded in parallel. Can be set to 0 or greater. Only works with `SURGE` strategy. | 0 | Optional | | node_pool_soak_duration | Time needed after draining the entire blue pool. After this period, the blue pool will be cleaned up. By default, it is set to one hour (3600 seconds). The maximum length of the soak time is 7 days (604,800 seconds). Only works with `BLUE_GREEN` strategy. | "3600s" | Optional | @@ -376,6 +388,8 @@ The node_pools variable takes the following parameters: | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | +| secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | +| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index eb4a2f7070..742c0b5ced 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -130,7 +130,28 @@ resource "google_container_cluster" "primary" { disk_size = lookup(var.cluster_autoscaling, "disk_size", 100) disk_type = lookup(var.cluster_autoscaling, "disk_type", "pd-standard") + upgrade_settings { + strategy = lookup(var.cluster_autoscaling, "strategy", "SURGE") + max_surge = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_surge", 0) : null + max_unavailable = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_unavailable", 0) : null + + dynamic "blue_green_settings" { + for_each = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "BLUE_GREEN" ? [1] : [] + content { + node_pool_soak_duration = lookup(var.cluster_autoscaling, "node_pool_soak_duration", null) + + standard_rollout_policy { + batch_soak_duration = lookup(var.cluster_autoscaling, "batch_soak_duration", null) + batch_percentage = lookup(var.cluster_autoscaling, "batch_percentage", null) + batch_node_count = lookup(var.cluster_autoscaling, "batch_node_count", null) + } + } + } + } + min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + + image_type = lookup(var.cluster_autoscaling, "image_type", "COS_CONTAINERD") } } autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" @@ -189,6 +210,15 @@ resource "google_container_cluster" "primary" { } } + dynamic "node_pool_auto_config" { + for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + content { + network_tags { + tags = var.network_tags + } + } + } + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -247,6 +277,14 @@ resource "google_container_cluster" "primary" { } } + dynamic "stateful_ha_config" { + for_each = local.stateful_ha_config + + content { + enabled = stateful_ha_config.value.enabled + } + } + config_connector_config { enabled = var.config_connector } @@ -485,6 +523,8 @@ resource "google_container_cluster" "primary" { } } } + + depends_on = [google_project_iam_member.service_agent] } /****************************************** Create Container Cluster node pools @@ -496,6 +536,9 @@ locals { "accelerator_count", "accelerator_type", "gpu_partition_size", + "gpu_driver_version", + "gpu_sharing_strategy", + "max_shared_clients_per_gpu", "enable_secure_boot", "enable_integrity_monitoring", "local_ssd_count", @@ -511,6 +554,7 @@ locals { "enable_gvnic", "enable_secure_boot", "boot_disk_kms_key", + "queued_provisioning", ] } @@ -641,6 +685,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -657,6 +708,12 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -700,6 +757,12 @@ resource "google_container_node_pool" "pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") + dynamic "ephemeral_storage_local_ssd_config" { + for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] + content { + local_ssd_count = ephemeral_storage_local_ssd_config.value + } + } dynamic "ephemeral_storage_config" { for_each = lookup(each.value, "local_ssd_ephemeral_count", 0) > 0 ? [each.value.local_ssd_ephemeral_count] : [] content { @@ -707,6 +770,22 @@ resource "google_container_node_pool" "pools" { } } + dynamic "local_nvme_ssd_block_config" { + for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] + content { + local_ssd_count = local_nvme_ssd_block_config.value + } + } + + # Supports a single secondary boot disk because `map(any)` must have the same values type. + dynamic "secondary_boot_disks" { + for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] + content { + disk_image = secondary_boot_disks.value + mode = "CONTAINER_IMAGE_CACHE" + } + } + service_account = lookup( each.value, "service_account", @@ -733,6 +812,21 @@ resource "google_container_node_pool" "pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } + + dynamic "gpu_sharing_config" { + for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) + } + } + } + } + + dynamic "advanced_machine_features" { + for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] + content { + threads_per_core = lookup(each.value, "threads_per_core", 0) } } @@ -876,6 +970,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -892,6 +993,12 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -935,6 +1042,12 @@ resource "google_container_node_pool" "windows_pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") + dynamic "ephemeral_storage_local_ssd_config" { + for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] + content { + local_ssd_count = ephemeral_storage_local_ssd_config.value + } + } dynamic "ephemeral_storage_config" { for_each = lookup(each.value, "local_ssd_ephemeral_count", 0) > 0 ? [each.value.local_ssd_ephemeral_count] : [] content { @@ -942,6 +1055,22 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "local_nvme_ssd_block_config" { + for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] + content { + local_ssd_count = local_nvme_ssd_block_config.value + } + } + + # Supports a single secondary boot disk because `map(any)` must have the same values type. + dynamic "secondary_boot_disks" { + for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] + content { + disk_image = secondary_boot_disks.value + mode = "CONTAINER_IMAGE_CACHE" + } + } + service_account = lookup( each.value, "service_account", @@ -968,6 +1097,21 @@ resource "google_container_node_pool" "windows_pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } + + dynamic "gpu_sharing_config" { + for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) + } + } + } + } + + dynamic "advanced_machine_features" { + for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] + content { + threads_per_core = lookup(each.value, "threads_per_core", 0) } } diff --git a/modules/beta-public-cluster-update-variant/main.tf b/modules/beta-public-cluster-update-variant/main.tf index 33dd316b35..952588fb18 100644 --- a/modules/beta-public-cluster-update-variant/main.tf +++ b/modules/beta-public-cluster-update-variant/main.tf @@ -94,6 +94,7 @@ locals { logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] + stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} @@ -189,13 +190,13 @@ locals { # BETA features cluster_istio_enabled = !local.cluster_output_istio_disabled - cluster_dns_cache_enabled = var.dns_cache cluster_telemetry_type_is_set = var.cluster_telemetry_type != null cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled # /BETA features + cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/modules/beta-public-cluster-update-variant/outputs.tf b/modules/beta-public-cluster-update-variant/outputs.tf index e388fb4406..29df3aad34 100644 --- a/modules/beta-public-cluster-update-variant/outputs.tf +++ b/modules/beta-public-cluster-update-variant/outputs.tf @@ -175,6 +175,11 @@ output "mesh_certificates_config" { } +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} + output "cloudrun_enabled" { description = "Whether CloudRun enabled" value = local.cluster_cloudrun_enabled @@ -185,11 +190,6 @@ output "istio_enabled" { value = local.cluster_istio_enabled } -output "dns_cache_enabled" { - description = "Whether DNS Cache enabled" - value = local.cluster_dns_cache_enabled -} - output "pod_security_policy_enabled" { description = "Whether pod security policy is enabled" value = local.cluster_pod_security_policy_enabled diff --git a/modules/beta-public-cluster-update-variant/sa.tf b/modules/beta-public-cluster-update-variant/sa.tf index 6f89899bee..fe36dadd32 100644 --- a/modules/beta-public-cluster-update-variant/sa.tf +++ b/modules/beta-public-cluster-update-variant/sa.tf @@ -52,6 +52,20 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } +resource "google_project_iam_member" "cluster_service_account-metric_writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/monitoring.metricWriter" + member = google_service_account.cluster_service_account[0].member +} + +resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/stackdriver.resourceMetadata.writer" + member = google_service_account.cluster_service_account[0].member +} + resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key @@ -65,3 +79,17 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } + +resource "google_project_service_identity" "fleet_project" { + count = var.fleet_project_grant_service_agent ? 1 : 0 + provider = google-beta + project = var.fleet_project + service = "gkehub.googleapis.com" +} + +resource "google_project_iam_member" "service_agent" { + for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] + project = var.project_id + role = each.value + member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" +} diff --git a/modules/beta-public-cluster-update-variant/variables.tf b/modules/beta-public-cluster-update-variant/variables.tf index 4a5d55dd6a..5de0d1f40a 100644 --- a/modules/beta-public-cluster-update-variant/variables.tf +++ b/modules/beta-public-cluster-update-variant/variables.tf @@ -239,17 +239,25 @@ variable "enable_resource_consumption_export" { variable "cluster_autoscaling" { type = object({ - enabled = bool - autoscaling_profile = string - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) + image_type = optional(string) + strategy = optional(string) + max_surge = optional(number) + max_unavailable = optional(number) + node_pool_soak_duration = optional(string) + batch_soak_duration = optional(string) + batch_percentage = optional(number) + batch_node_count = optional(number) }) default = { enabled = false @@ -263,6 +271,7 @@ variable "cluster_autoscaling" { auto_upgrade = true disk_size = 100 disk_type = "pd-standard" + image_type = "COS_CONTAINERD" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -300,6 +309,12 @@ variable "node_pools_oauth_scopes" { } } +variable "network_tags" { + description = "(Optional) - List of network tags applied to auto-provisioned node pools." + type = list(string) + default = [] +} + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -520,7 +535,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -660,6 +675,12 @@ variable "gcs_fuse_csi_driver" { default = false } +variable "stateful_ha" { + type = bool + description = "Whether the Stateful HA Addon is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -700,7 +721,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -722,6 +743,12 @@ variable "config_connector" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "istio" { description = "(Beta) Enable Istio addon" type = bool @@ -758,12 +785,6 @@ variable "enable_pod_security_policy" { default = false } -variable "enable_l4_ilb_subsetting" { - type = bool - description = "Enable L4 ILB Subsetting on the cluster" - default = false -} - variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." @@ -793,3 +814,9 @@ variable "fleet_project" { type = string default = null } + +variable "fleet_project_grant_service_agent" { + description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." + type = bool + default = false +} diff --git a/modules/beta-public-cluster-update-variant/versions.tf b/modules/beta-public-cluster-update-variant/versions.tf index 2a8c09d960..afe9612323 100644 --- a/modules/beta-public-cluster-update-variant/versions.tf +++ b/modules/beta-public-cluster-update-variant/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.9.0, < 6" + version = ">= 5.25.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.9.0, < 6" + version = ">= 5.25.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster-update-variant/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster-update-variant/v31.0.0" } } diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 1a54af89e2..2d963a101d 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -56,25 +56,30 @@ module "gke" { node_pools = [ { - name = "default-node-pool" - machine_type = "e2-medium" - node_locations = "us-central1-b,us-central1-c" - min_count = 1 - max_count = 100 - local_ssd_count = 0 - spot = false - local_ssd_ephemeral_count = 0 - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS_CONTAINERD" - enable_gcfs = false - enable_gvnic = false - logging_variant = "DEFAULT" - auto_repair = true - auto_upgrade = true - service_account = "project-service-account@.iam.gserviceaccount.com" - preemptible = false - initial_node_count = 80 + name = "default-node-pool" + machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = false + local_ssd_ephemeral_count = 0 + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + enable_gcfs = false + enable_gvnic = false + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 + accelerator_count = 1 + accelerator_type = "nvidia-l4" + gpu_driver_version = "LATEST" + gpu_sharing_strategy = "TIME_SHARING" + max_shared_clients_per_gpu = 2 }, ] @@ -143,7 +148,7 @@ Then perform the following commands on the root folder: | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | | cloudrun | (Beta) Enable CloudRun addon | `bool` | `false` | no | | cloudrun\_load\_balancer\_type | (Beta) Configure the Cloud Run load balancer type. External by default. Set to `LOAD_BALANCER_TYPE_INTERNAL` to configure as an internal load balancer. | `string` | `""` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
    disk_size           = optional(number)
    disk_type           = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                 = bool
    autoscaling_profile     = string
    min_cpu_cores           = number
    max_cpu_cores           = number
    min_memory_gb           = number
    max_memory_gb           = number
    gpu_resources           = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair             = bool
    auto_upgrade            = bool
    disk_size               = optional(number)
    disk_type               = optional(string)
    image_type              = optional(string)
    strategy                = optional(string)
    max_surge               = optional(number)
    max_unavailable         = optional(number)
    node_pool_soak_duration = optional(string)
    batch_soak_duration     = optional(string)
    batch_percentage        = optional(number)
    batch_node_count        = optional(number)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -181,6 +186,7 @@ Then perform the following commands on the root folder: | firewall\_inbound\_ports | List of TCP ports for admission/webhook controllers. Either flag `add_master_webhook_firewall_rules` or `add_cluster_firewall_rules` (also adds egress rules) must be set to `true` for inbound-ports firewall rules to be applied. | `list(string)` | 
[
  "8443",
  "9443",
  "15017"
]
| no | | firewall\_priority | Priority rule for firewall rules | `number` | `1000` | no | | fleet\_project | (Optional) Register the cluster with the fleet in this project. | `string` | `null` | no | +| fleet\_project\_grant\_service\_agent | (Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles. | `bool` | `false` | no | | gateway\_api\_channel | The gateway api channel of this cluster. Accepted values are `CHANNEL_STANDARD` and `CHANNEL_DISABLED`. | `string` | `null` | no | | gce\_pd\_csi\_driver | Whether this cluster should enable the Google Compute Engine Persistent Disk Container Storage Interface (CSI) Driver. | `bool` | `true` | no | | gcs\_fuse\_csi\_driver | Whether GCE FUSE CSI driver is enabled for this cluster. | `bool` | `false` | no | @@ -217,6 +223,7 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | @@ -237,13 +244,14 @@ Then perform the following commands on the root folder: | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | | shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | @@ -318,19 +326,23 @@ The node_pools variable takes the following parameters: | enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | +| gpu_driver_version | Mode for how the GPU driver is installed | null | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | +| local_ssd_ephemeral_storage_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | | local_ssd_ephemeral_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | +| local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | | strategy | The upgrade stragey to be used for upgrading the nodes. Valid values of state are: `SURGE` or `BLUE_GREEN` | "SURGE" | Optional | +| threads_per_core | Optional The number of threads per physical core. To disable simultaneous multithreading (SMT) set this to 1. If unset, the maximum number of threads supported per core by the underlying processor is assumed | null | Optional | | max_surge | The number of additional nodes that can be added to the node pool during an upgrade. Increasing max_surge raises the number of nodes that can be upgraded simultaneously. Can be set to 0 or greater. Only works with `SURGE` strategy. | 1 | Optional | | max_unavailable | The number of nodes that can be simultaneously unavailable during an upgrade. Increasing max_unavailable raises the number of nodes that can be upgraded in parallel. Can be set to 0 or greater. Only works with `SURGE` strategy. | 0 | Optional | | node_pool_soak_duration | Time needed after draining the entire blue pool. After this period, the blue pool will be cleaned up. By default, it is set to one hour (3600 seconds). The maximum length of the soak time is 7 days (604,800 seconds). Only works with `BLUE_GREEN` strategy. | "3600s" | Optional | @@ -354,6 +366,8 @@ The node_pools variable takes the following parameters: | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | +| secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | +| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index e2f88f7cb2..f32f49a10d 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -130,7 +130,28 @@ resource "google_container_cluster" "primary" { disk_size = lookup(var.cluster_autoscaling, "disk_size", 100) disk_type = lookup(var.cluster_autoscaling, "disk_type", "pd-standard") + upgrade_settings { + strategy = lookup(var.cluster_autoscaling, "strategy", "SURGE") + max_surge = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_surge", 0) : null + max_unavailable = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_unavailable", 0) : null + + dynamic "blue_green_settings" { + for_each = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "BLUE_GREEN" ? [1] : [] + content { + node_pool_soak_duration = lookup(var.cluster_autoscaling, "node_pool_soak_duration", null) + + standard_rollout_policy { + batch_soak_duration = lookup(var.cluster_autoscaling, "batch_soak_duration", null) + batch_percentage = lookup(var.cluster_autoscaling, "batch_percentage", null) + batch_node_count = lookup(var.cluster_autoscaling, "batch_node_count", null) + } + } + } + } + min_cpu_platform = lookup(var.node_pools[0], "min_cpu_platform", "") + + image_type = lookup(var.cluster_autoscaling, "image_type", "COS_CONTAINERD") } } autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" @@ -189,6 +210,15 @@ resource "google_container_cluster" "primary" { } } + dynamic "node_pool_auto_config" { + for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + content { + network_tags { + tags = var.network_tags + } + } + } + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -247,6 +277,14 @@ resource "google_container_cluster" "primary" { } } + dynamic "stateful_ha_config" { + for_each = local.stateful_ha_config + + content { + enabled = stateful_ha_config.value.enabled + } + } + config_connector_config { enabled = var.config_connector } @@ -485,6 +523,8 @@ resource "google_container_cluster" "primary" { } } } + + depends_on = [google_project_iam_member.service_agent] } /****************************************** Create Container Cluster node pools @@ -566,6 +606,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -582,6 +629,12 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -625,6 +678,12 @@ resource "google_container_node_pool" "pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") + dynamic "ephemeral_storage_local_ssd_config" { + for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] + content { + local_ssd_count = ephemeral_storage_local_ssd_config.value + } + } dynamic "ephemeral_storage_config" { for_each = lookup(each.value, "local_ssd_ephemeral_count", 0) > 0 ? [each.value.local_ssd_ephemeral_count] : [] content { @@ -632,6 +691,22 @@ resource "google_container_node_pool" "pools" { } } + dynamic "local_nvme_ssd_block_config" { + for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] + content { + local_ssd_count = local_nvme_ssd_block_config.value + } + } + + # Supports a single secondary boot disk because `map(any)` must have the same values type. + dynamic "secondary_boot_disks" { + for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] + content { + disk_image = secondary_boot_disks.value + mode = "CONTAINER_IMAGE_CACHE" + } + } + service_account = lookup( each.value, "service_account", @@ -658,6 +733,21 @@ resource "google_container_node_pool" "pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } + + dynamic "gpu_sharing_config" { + for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) + } + } + } + } + + dynamic "advanced_machine_features" { + for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] + content { + threads_per_core = lookup(each.value, "threads_per_core", 0) } } @@ -800,6 +890,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -816,6 +913,12 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -859,6 +962,12 @@ resource "google_container_node_pool" "windows_pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") + dynamic "ephemeral_storage_local_ssd_config" { + for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] + content { + local_ssd_count = ephemeral_storage_local_ssd_config.value + } + } dynamic "ephemeral_storage_config" { for_each = lookup(each.value, "local_ssd_ephemeral_count", 0) > 0 ? [each.value.local_ssd_ephemeral_count] : [] content { @@ -866,6 +975,22 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "local_nvme_ssd_block_config" { + for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] + content { + local_ssd_count = local_nvme_ssd_block_config.value + } + } + + # Supports a single secondary boot disk because `map(any)` must have the same values type. + dynamic "secondary_boot_disks" { + for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] + content { + disk_image = secondary_boot_disks.value + mode = "CONTAINER_IMAGE_CACHE" + } + } + service_account = lookup( each.value, "service_account", @@ -892,6 +1017,21 @@ resource "google_container_node_pool" "windows_pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } + + dynamic "gpu_sharing_config" { + for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) + } + } + } + } + + dynamic "advanced_machine_features" { + for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] + content { + threads_per_core = lookup(each.value, "threads_per_core", 0) } } diff --git a/modules/beta-public-cluster/main.tf b/modules/beta-public-cluster/main.tf index 33dd316b35..952588fb18 100644 --- a/modules/beta-public-cluster/main.tf +++ b/modules/beta-public-cluster/main.tf @@ -94,6 +94,7 @@ locals { logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] + stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] cluster_cloudrun_config_load_balancer_config = (var.cloudrun && var.cloudrun_load_balancer_type != "") ? { load_balancer_type = var.cloudrun_load_balancer_type } : {} @@ -189,13 +190,13 @@ locals { # BETA features cluster_istio_enabled = !local.cluster_output_istio_disabled - cluster_dns_cache_enabled = var.dns_cache cluster_telemetry_type_is_set = var.cluster_telemetry_type != null cluster_pod_security_policy_enabled = local.cluster_output_pod_security_policy_enabled cluster_intranode_visibility_enabled = local.cluster_output_intranode_visbility_enabled # /BETA features + cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/modules/beta-public-cluster/outputs.tf b/modules/beta-public-cluster/outputs.tf index e388fb4406..29df3aad34 100644 --- a/modules/beta-public-cluster/outputs.tf +++ b/modules/beta-public-cluster/outputs.tf @@ -175,6 +175,11 @@ output "mesh_certificates_config" { } +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} + output "cloudrun_enabled" { description = "Whether CloudRun enabled" value = local.cluster_cloudrun_enabled @@ -185,11 +190,6 @@ output "istio_enabled" { value = local.cluster_istio_enabled } -output "dns_cache_enabled" { - description = "Whether DNS Cache enabled" - value = local.cluster_dns_cache_enabled -} - output "pod_security_policy_enabled" { description = "Whether pod security policy is enabled" value = local.cluster_pod_security_policy_enabled diff --git a/modules/beta-public-cluster/sa.tf b/modules/beta-public-cluster/sa.tf index 6f89899bee..fe36dadd32 100644 --- a/modules/beta-public-cluster/sa.tf +++ b/modules/beta-public-cluster/sa.tf @@ -52,6 +52,20 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } +resource "google_project_iam_member" "cluster_service_account-metric_writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/monitoring.metricWriter" + member = google_service_account.cluster_service_account[0].member +} + +resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/stackdriver.resourceMetadata.writer" + member = google_service_account.cluster_service_account[0].member +} + resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key @@ -65,3 +79,17 @@ resource "google_project_iam_member" "cluster_service_account-artifact-registry" role = "roles/artifactregistry.reader" member = "serviceAccount:${google_service_account.cluster_service_account[0].email}" } + +resource "google_project_service_identity" "fleet_project" { + count = var.fleet_project_grant_service_agent ? 1 : 0 + provider = google-beta + project = var.fleet_project + service = "gkehub.googleapis.com" +} + +resource "google_project_iam_member" "service_agent" { + for_each = var.fleet_project_grant_service_agent ? toset(["roles/gkehub.serviceAgent", "roles/gkehub.crossProjectServiceAgent"]) : [] + project = var.project_id + role = each.value + member = "serviceAccount:${google_project_service_identity.fleet_project[0].email}" +} diff --git a/modules/beta-public-cluster/variables.tf b/modules/beta-public-cluster/variables.tf index 4a5d55dd6a..5de0d1f40a 100644 --- a/modules/beta-public-cluster/variables.tf +++ b/modules/beta-public-cluster/variables.tf @@ -239,17 +239,25 @@ variable "enable_resource_consumption_export" { variable "cluster_autoscaling" { type = object({ - enabled = bool - autoscaling_profile = string - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) + image_type = optional(string) + strategy = optional(string) + max_surge = optional(number) + max_unavailable = optional(number) + node_pool_soak_duration = optional(string) + batch_soak_duration = optional(string) + batch_percentage = optional(number) + batch_node_count = optional(number) }) default = { enabled = false @@ -263,6 +271,7 @@ variable "cluster_autoscaling" { auto_upgrade = true disk_size = 100 disk_type = "pd-standard" + image_type = "COS_CONTAINERD" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -300,6 +309,12 @@ variable "node_pools_oauth_scopes" { } } +variable "network_tags" { + description = "(Optional) - List of network tags applied to auto-provisioned node pools." + type = list(string) + default = [] +} + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -520,7 +535,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -660,6 +675,12 @@ variable "gcs_fuse_csi_driver" { default = false } +variable "stateful_ha" { + type = bool + description = "Whether the Stateful HA Addon is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -700,7 +721,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -722,6 +743,12 @@ variable "config_connector" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "istio" { description = "(Beta) Enable Istio addon" type = bool @@ -758,12 +785,6 @@ variable "enable_pod_security_policy" { default = false } -variable "enable_l4_ilb_subsetting" { - type = bool - description = "Enable L4 ILB Subsetting on the cluster" - default = false -} - variable "sandbox_enabled" { type = bool description = "(Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it)." @@ -793,3 +814,9 @@ variable "fleet_project" { type = string default = null } + +variable "fleet_project_grant_service_agent" { + description = "(Optional) Grant the fleet project service identity the `roles/gkehub.serviceAgent` and `roles/gkehub.crossProjectServiceAgent` roles." + type = bool + default = false +} diff --git a/modules/beta-public-cluster/versions.tf b/modules/beta-public-cluster/versions.tf index eaea9cbf80..4bacb721c0 100644 --- a/modules/beta-public-cluster/versions.tf +++ b/modules/beta-public-cluster/versions.tf @@ -21,11 +21,11 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.9.0, < 6" + version = ">= 5.25.0, < 6" } google-beta = { source = "hashicorp/google-beta" - version = ">= 5.9.0, < 6" + version = ">= 5.25.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -37,6 +37,6 @@ terraform { } } provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:beta-public-cluster/v31.0.0" } } diff --git a/modules/binary-authorization/main.tf b/modules/binary-authorization/main.tf index 81cbb620c2..8e09bb13eb 100644 --- a/modules/binary-authorization/main.tf +++ b/modules/binary-authorization/main.tf @@ -25,7 +25,7 @@ locals { module "project-services" { source = "terraform-google-modules/project-factory/google//modules/project_services" - version = "~> 14.0" + version = "~> 15.0" project_id = var.project_id activate_apis = local.required_enabled_apis diff --git a/modules/binary-authorization/versions.tf b/modules/binary-authorization/versions.tf index fea953ed24..66482b1b99 100644 --- a/modules/binary-authorization/versions.tf +++ b/modules/binary-authorization/versions.tf @@ -28,6 +28,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:binary-authorization/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:binary-authorization/v31.0.0" } } diff --git a/modules/fleet-membership/versions.tf b/modules/fleet-membership/versions.tf index 4962a916f8..7b774dcdbe 100644 --- a/modules/fleet-membership/versions.tf +++ b/modules/fleet-membership/versions.tf @@ -30,6 +30,6 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v31.0.0" } } diff --git a/modules/hub-legacy/versions.tf b/modules/hub-legacy/versions.tf index a497d17c02..678714ab54 100644 --- a/modules/hub-legacy/versions.tf +++ b/modules/hub-legacy/versions.tf @@ -28,6 +28,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:hub/v31.0.0" } } diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index 5c94834af4..b3238b0c22 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -78,27 +78,33 @@ module "gke" { enable_private_endpoint = true enable_private_nodes = true master_ipv4_cidr_block = "10.0.0.0/28" + dns_cache = false node_pools = [ { - name = "default-node-pool" - machine_type = "e2-medium" - node_locations = "us-central1-b,us-central1-c" - min_count = 1 - max_count = 100 - local_ssd_count = 0 - spot = false - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS_CONTAINERD" - enable_gcfs = false - enable_gvnic = false - logging_variant = "DEFAULT" - auto_repair = true - auto_upgrade = true - service_account = "project-service-account@.iam.gserviceaccount.com" - preemptible = false - initial_node_count = 80 + name = "default-node-pool" + machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = false + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + enable_gcfs = false + enable_gvnic = false + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 + accelerator_count = 1 + accelerator_type = "nvidia-l4" + gpu_driver_version = "LATEST" + gpu_sharing_strategy = "TIME_SHARING" + max_shared_clients_per_gpu = 2 }, ] @@ -165,7 +171,7 @@ Then perform the following commands on the root folder: | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
    disk_size           = optional(number)
    disk_type           = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                 = bool
    autoscaling_profile     = string
    min_cpu_cores           = number
    max_cpu_cores           = number
    min_memory_gb           = number
    max_memory_gb           = number
    gpu_resources           = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair             = bool
    auto_upgrade            = bool
    disk_size               = optional(number)
    disk_type               = optional(string)
    image_type              = optional(string)
    strategy                = optional(string)
    max_surge               = optional(number)
    max_unavailable         = optional(number)
    node_pool_soak_duration = optional(string)
    batch_soak_duration     = optional(string)
    batch_percentage        = optional(number)
    batch_node_count        = optional(number)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -178,7 +184,7 @@ Then perform the following commands on the root folder: | datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | | deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | | description | The description of the cluster | `string` | `""` | no | | disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | @@ -187,10 +193,11 @@ Then perform the following commands on the root folder: | enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | +| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | | enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | @@ -234,6 +241,7 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | @@ -253,13 +261,14 @@ Then perform the following commands on the root folder: | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | | shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | @@ -273,6 +282,7 @@ Then perform the following commands on the root folder: |------|-------------| | ca\_certificate | Cluster ca certificate (base64 encoded) | | cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | | fleet\_membership | Fleet membership (if registered) | | gateway\_api\_channel | The gateway api channel of this cluster. | @@ -324,18 +334,22 @@ The node_pools variable takes the following parameters: | enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | +| gpu_driver_version | Mode for how the GPU driver is installed | null | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | +| local_ssd_ephemeral_storage_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | +| local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | | strategy | The upgrade stragey to be used for upgrading the nodes. Valid values of state are: `SURGE` or `BLUE_GREEN` | "SURGE" | Optional | +| threads_per_core | Optional The number of threads per physical core. To disable simultaneous multithreading (SMT) set this to 1. If unset, the maximum number of threads supported per core by the underlying processor is assumed | null | Optional | | max_surge | The number of additional nodes that can be added to the node pool during an upgrade. Increasing max_surge raises the number of nodes that can be upgraded simultaneously. Can be set to 0 or greater. Only works with `SURGE` strategy. | 1 | Optional | | max_unavailable | The number of nodes that can be simultaneously unavailable during an upgrade. Increasing max_unavailable raises the number of nodes that can be upgraded in parallel. Can be set to 0 or greater. Only works with `SURGE` strategy. | 0 | Optional | | node_pool_soak_duration | Time needed after draining the entire blue pool. After this period, the blue pool will be cleaned up. By default, it is set to one hour (3600 seconds). The maximum length of the soak time is 7 days (604,800 seconds). Only works with `BLUE_GREEN` strategy. | "3600s" | Optional | @@ -355,6 +369,8 @@ The node_pools variable takes the following parameters: | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | +| secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | +| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 236d61117f..079ec28814 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -124,6 +124,27 @@ resource "google_container_cluster" "primary" { disk_size = lookup(var.cluster_autoscaling, "disk_size", 100) disk_type = lookup(var.cluster_autoscaling, "disk_type", "pd-standard") + upgrade_settings { + strategy = lookup(var.cluster_autoscaling, "strategy", "SURGE") + max_surge = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_surge", 0) : null + max_unavailable = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_unavailable", 0) : null + + dynamic "blue_green_settings" { + for_each = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "BLUE_GREEN" ? [1] : [] + content { + node_pool_soak_duration = lookup(var.cluster_autoscaling, "node_pool_soak_duration", null) + + standard_rollout_policy { + batch_soak_duration = lookup(var.cluster_autoscaling, "batch_soak_duration", null) + batch_percentage = lookup(var.cluster_autoscaling, "batch_percentage", null) + batch_node_count = lookup(var.cluster_autoscaling, "batch_node_count", null) + } + } + } + } + + + image_type = lookup(var.cluster_autoscaling, "image_type", "COS_CONTAINERD") } } autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" @@ -151,6 +172,8 @@ resource "google_container_cluster" "primary" { enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu + + enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -164,6 +187,15 @@ resource "google_container_cluster" "primary" { } } + dynamic "node_pool_auto_config" { + for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + content { + network_tags { + tags = var.network_tags + } + } + } + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -222,6 +254,14 @@ resource "google_container_cluster" "primary" { } } + dynamic "stateful_ha_config" { + for_each = local.stateful_ha_config + + content { + enabled = stateful_ha_config.value.enabled + } + } + config_connector_config { enabled = var.config_connector } @@ -449,6 +489,9 @@ locals { "accelerator_count", "accelerator_type", "gpu_partition_size", + "gpu_driver_version", + "gpu_sharing_strategy", + "max_shared_clients_per_gpu", "enable_secure_boot", "enable_integrity_monitoring", "local_ssd_count", @@ -464,6 +507,7 @@ locals { "enable_gvnic", "enable_secure_boot", "boot_disk_kms_key", + "queued_provisioning", ] } @@ -594,6 +638,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -610,6 +661,12 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -653,6 +710,28 @@ resource "google_container_node_pool" "pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") + dynamic "ephemeral_storage_local_ssd_config" { + for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] + content { + local_ssd_count = ephemeral_storage_local_ssd_config.value + } + } + + dynamic "local_nvme_ssd_block_config" { + for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] + content { + local_ssd_count = local_nvme_ssd_block_config.value + } + } + + # Supports a single secondary boot disk because `map(any)` must have the same values type. + dynamic "secondary_boot_disks" { + for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] + content { + disk_image = secondary_boot_disks.value + mode = "CONTAINER_IMAGE_CACHE" + } + } service_account = lookup( each.value, @@ -680,6 +759,21 @@ resource "google_container_node_pool" "pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } + + dynamic "gpu_sharing_config" { + for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) + } + } + } + } + + dynamic "advanced_machine_features" { + for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] + content { + threads_per_core = lookup(each.value, "threads_per_core", 0) } } @@ -804,6 +898,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -820,6 +921,12 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -863,6 +970,28 @@ resource "google_container_node_pool" "windows_pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") + dynamic "ephemeral_storage_local_ssd_config" { + for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] + content { + local_ssd_count = ephemeral_storage_local_ssd_config.value + } + } + + dynamic "local_nvme_ssd_block_config" { + for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] + content { + local_ssd_count = local_nvme_ssd_block_config.value + } + } + + # Supports a single secondary boot disk because `map(any)` must have the same values type. + dynamic "secondary_boot_disks" { + for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] + content { + disk_image = secondary_boot_disks.value + mode = "CONTAINER_IMAGE_CACHE" + } + } service_account = lookup( each.value, @@ -890,6 +1019,21 @@ resource "google_container_node_pool" "windows_pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } + + dynamic "gpu_sharing_config" { + for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) + } + } + } + } + + dynamic "advanced_machine_features" { + for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] + content { + threads_per_core = lookup(each.value, "threads_per_core", 0) } } diff --git a/modules/private-cluster-update-variant/main.tf b/modules/private-cluster-update-variant/main.tf index 4c5d8bd613..614babb49a 100644 --- a/modules/private-cluster-update-variant/main.tf +++ b/modules/private-cluster-update-variant/main.tf @@ -94,6 +94,7 @@ locals { logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] + stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group @@ -171,6 +172,7 @@ locals { }] : [] + cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/modules/private-cluster-update-variant/outputs.tf b/modules/private-cluster-update-variant/outputs.tf index e1e27f7bba..8f35e71fe0 100644 --- a/modules/private-cluster-update-variant/outputs.tf +++ b/modules/private-cluster-update-variant/outputs.tf @@ -185,6 +185,11 @@ output "peering_name" { value = local.cluster_peering_name } +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} + output "fleet_membership" { description = "Fleet membership (if registered)" value = local.fleet_membership diff --git a/modules/private-cluster-update-variant/sa.tf b/modules/private-cluster-update-variant/sa.tf index 6f89899bee..da1930c7b3 100644 --- a/modules/private-cluster-update-variant/sa.tf +++ b/modules/private-cluster-update-variant/sa.tf @@ -52,6 +52,20 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } +resource "google_project_iam_member" "cluster_service_account-metric_writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/monitoring.metricWriter" + member = google_service_account.cluster_service_account[0].member +} + +resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/stackdriver.resourceMetadata.writer" + member = google_service_account.cluster_service_account[0].member +} + resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key diff --git a/modules/private-cluster-update-variant/variables.tf b/modules/private-cluster-update-variant/variables.tf index 2d7f8cc8c6..ccf65a8a13 100644 --- a/modules/private-cluster-update-variant/variables.tf +++ b/modules/private-cluster-update-variant/variables.tf @@ -239,17 +239,25 @@ variable "enable_resource_consumption_export" { variable "cluster_autoscaling" { type = object({ - enabled = bool - autoscaling_profile = string - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) + image_type = optional(string) + strategy = optional(string) + max_surge = optional(number) + max_unavailable = optional(number) + node_pool_soak_duration = optional(string) + batch_soak_duration = optional(string) + batch_percentage = optional(number) + batch_node_count = optional(number) }) default = { enabled = false @@ -263,6 +271,7 @@ variable "cluster_autoscaling" { auto_upgrade = true disk_size = 100 disk_type = "pd-standard" + image_type = "COS_CONTAINERD" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -300,6 +309,12 @@ variable "node_pools_oauth_scopes" { } } +variable "network_tags" { + description = "(Optional) - List of network tags applied to auto-provisioned node pools." + type = list(string) + default = [] +} + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -399,25 +414,25 @@ variable "cluster_resource_labels" { variable "deploy_using_private_endpoint" { type = bool - description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." + description = "A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." default = false } variable "enable_private_endpoint" { type = bool - description = "(Beta) Whether the master's internal IP address is used as the cluster endpoint" + description = "Whether the master's internal IP address is used as the cluster endpoint" default = false } variable "enable_private_nodes" { type = bool - description = "(Beta) Whether nodes have internal IP addresses only" + description = "Whether nodes have internal IP addresses only" default = false } variable "master_ipv4_cidr_block" { type = string - description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + description = "The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters." default = "10.0.0.0/28" } @@ -526,7 +541,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -666,6 +681,12 @@ variable "gcs_fuse_csi_driver" { default = false } +variable "stateful_ha" { + type = bool + description = "Whether the Stateful HA Addon is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -706,7 +727,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -728,6 +749,12 @@ variable "config_connector" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "fleet_project" { description = "(Optional) Register the cluster with the fleet in this project." type = string diff --git a/modules/private-cluster-update-variant/versions.tf b/modules/private-cluster-update-variant/versions.tf index c953d4c17a..b20b93f068 100644 --- a/modules/private-cluster-update-variant/versions.tf +++ b/modules/private-cluster-update-variant/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.9.0, < 6" + version = ">= 5.25.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -33,6 +33,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster-update-variant/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster-update-variant/v31.0.0" } } diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 3d17b7e890..615fb43db2 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -56,27 +56,33 @@ module "gke" { enable_private_endpoint = true enable_private_nodes = true master_ipv4_cidr_block = "10.0.0.0/28" + dns_cache = false node_pools = [ { - name = "default-node-pool" - machine_type = "e2-medium" - node_locations = "us-central1-b,us-central1-c" - min_count = 1 - max_count = 100 - local_ssd_count = 0 - spot = false - disk_size_gb = 100 - disk_type = "pd-standard" - image_type = "COS_CONTAINERD" - enable_gcfs = false - enable_gvnic = false - logging_variant = "DEFAULT" - auto_repair = true - auto_upgrade = true - service_account = "project-service-account@.iam.gserviceaccount.com" - preemptible = false - initial_node_count = 80 + name = "default-node-pool" + machine_type = "e2-medium" + node_locations = "us-central1-b,us-central1-c" + min_count = 1 + max_count = 100 + local_ssd_count = 0 + spot = false + disk_size_gb = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + enable_gcfs = false + enable_gvnic = false + logging_variant = "DEFAULT" + auto_repair = true + auto_upgrade = true + service_account = "project-service-account@.iam.gserviceaccount.com" + preemptible = false + initial_node_count = 80 + accelerator_count = 1 + accelerator_type = "nvidia-l4" + gpu_driver_version = "LATEST" + gpu_sharing_strategy = "TIME_SHARING" + max_shared_clients_per_gpu = 2 }, ] @@ -143,7 +149,7 @@ Then perform the following commands on the root folder: | add\_shadow\_firewall\_rules | Create GKE shadow firewall (the same as default firewall rules with firewall logs enabled). | `bool` | `false` | no | | additional\_ip\_range\_pods | List of _names_ of the additional secondary subnet ip ranges to use for pods | `list(string)` | `[]` | no | | authenticator\_security\_group | The name of the RBAC security group for use with Google security groups in Kubernetes RBAC. Group name must be in format gke-security-groups@yourdomain.com | `string` | `null` | no | -| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled             = bool
    autoscaling_profile = string
    min_cpu_cores       = number
    max_cpu_cores       = number
    min_memory_gb       = number
    max_memory_gb       = number
    gpu_resources       = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair         = bool
    auto_upgrade        = bool
    disk_size           = optional(number)
    disk_type           = optional(string)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | +| cluster\_autoscaling | Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling) | 
object({
    enabled                 = bool
    autoscaling_profile     = string
    min_cpu_cores           = number
    max_cpu_cores           = number
    min_memory_gb           = number
    max_memory_gb           = number
    gpu_resources           = list(object({ resource_type = string, minimum = number, maximum = number }))
    auto_repair             = bool
    auto_upgrade            = bool
    disk_size               = optional(number)
    disk_type               = optional(string)
    image_type              = optional(string)
    strategy                = optional(string)
    max_surge               = optional(number)
    max_unavailable         = optional(number)
    node_pool_soak_duration = optional(string)
    batch_soak_duration     = optional(string)
    batch_percentage        = optional(number)
    batch_node_count        = optional(number)
  })
| 
{
  "auto_repair": true,
  "auto_upgrade": true,
  "autoscaling_profile": "BALANCED",
  "disk_size": 100,
  "disk_type": "pd-standard",
  "enabled": false,
  "gpu_resources": [],
  "image_type": "COS_CONTAINERD",
  "max_cpu_cores": 0,
  "max_memory_gb": 0,
  "min_cpu_cores": 0,
  "min_memory_gb": 0
}
| no | | cluster\_dns\_domain | The suffix used for all cluster service records. | `string` | `""` | no | | cluster\_dns\_provider | Which in-cluster DNS provider should be used. PROVIDER\_UNSPECIFIED (default) or PLATFORM\_DEFAULT or CLOUD\_DNS. | `string` | `"PROVIDER_UNSPECIFIED"` | no | | cluster\_dns\_scope | The scope of access to cluster DNS records. DNS\_SCOPE\_UNSPECIFIED (default) or CLUSTER\_SCOPE or VPC\_SCOPE. | `string` | `"DNS_SCOPE_UNSPECIFIED"` | no | @@ -156,7 +162,7 @@ Then perform the following commands on the root folder: | datapath\_provider | The desired datapath provider for this cluster. By default, `DATAPATH_PROVIDER_UNSPECIFIED` enables the IPTables-based kube-proxy implementation. `ADVANCED_DATAPATH` enables Dataplane-V2 feature. | `string` | `"DATAPATH_PROVIDER_UNSPECIFIED"` | no | | default\_max\_pods\_per\_node | The maximum number of pods to schedule per node | `number` | `110` | no | | deletion\_protection | Whether or not to allow Terraform to destroy the cluster. | `bool` | `true` | no | -| deploy\_using\_private\_endpoint | (Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | +| deploy\_using\_private\_endpoint | A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment. | `bool` | `false` | no | | description | The description of the cluster | `string` | `""` | no | | disable\_default\_snat | Whether to disable the default SNAT to support the private use of public IP addresses | `bool` | `false` | no | | disable\_legacy\_metadata\_endpoints | Disable the /0.1/ and /v1beta1/ metadata server endpoints on the node. Changing this value will cause all node pools to be recreated. | `bool` | `true` | no | @@ -165,10 +171,11 @@ Then perform the following commands on the root folder: | enable\_confidential\_nodes | An optional flag to enable confidential node config. | `bool` | `false` | no | | enable\_cost\_allocation | Enables Cost Allocation Feature and the cluster name and namespace of your GKE workloads appear in the labels field of the billing export to BigQuery | `bool` | `false` | no | | enable\_kubernetes\_alpha | Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. | `bool` | `false` | no | +| enable\_l4\_ilb\_subsetting | Enable L4 ILB Subsetting on the cluster | `bool` | `false` | no | | enable\_mesh\_certificates | Controls the issuance of workload mTLS certificates. When enabled the GKE Workload Identity Certificates controller and node agent will be deployed in the cluster. Requires Workload Identity. | `bool` | `false` | no | | enable\_network\_egress\_export | Whether to enable network egress metering for this cluster. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `bool` | `false` | no | -| enable\_private\_endpoint | (Beta) Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | -| enable\_private\_nodes | (Beta) Whether nodes have internal IP addresses only | `bool` | `false` | no | +| enable\_private\_endpoint | Whether the master's internal IP address is used as the cluster endpoint | `bool` | `false` | no | +| enable\_private\_nodes | Whether nodes have internal IP addresses only | `bool` | `false` | no | | enable\_resource\_consumption\_export | Whether to enable resource consumption metering on this cluster. When enabled, a table will be created in the resource export BigQuery dataset to store resource consumption data. The resulting table can be joined with the resource usage table or with BigQuery billing export. | `bool` | `true` | no | | enable\_shielded\_nodes | Enable Shielded Nodes features on all nodes in this cluster | `bool` | `true` | no | | enable\_tpu | Enable Cloud TPU resources in the cluster. WARNING: changing this after cluster creation is destructive! | `bool` | `false` | no | @@ -212,6 +219,7 @@ Then perform the following commands on the root folder: | network\_policy | Enable network policy addon | `bool` | `false` | no | | network\_policy\_provider | The network policy provider. | `string` | `"CALICO"` | no | | network\_project\_id | The project ID of the shared VPC's host (for shared vpc support) | `string` | `""` | no | +| network\_tags | (Optional) - List of network tags applied to auto-provisioned node pools. | `list(string)` | `[]` | no | | node\_metadata | Specifies how node metadata is exposed to the workload running on the node | `string` | `"GKE_METADATA"` | no | | node\_pools | List of maps containing node pools | `list(map(any))` | 
[
  {
    "name": "default-node-pool"
  }
]
| no | | node\_pools\_labels | Map of maps containing node labels by node-pool name | `map(map(string))` | 
{
  "all": {},
  "default-node-pool": {}
}
| no | @@ -231,13 +239,14 @@ Then perform the following commands on the root folder: | remove\_default\_node\_pool | Remove default node pool while setting up the cluster | `bool` | `false` | no | | resource\_usage\_export\_dataset\_id | The ID of a BigQuery Dataset for using BigQuery as the destination of resource usage export. | `string` | `""` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`. | `string` | `"VULNERABILITY_DISABLED"` | no | | service\_account | The service account to run nodes as if not overridden in `node_pools`. The create\_service\_account variable default value (true) will cause a cluster-specific service account to be created. This service account should already exists and it will be used by the node pools. If you wish to only override the service account name, you can use service\_account\_name variable. | `string` | `""` | no | | service\_account\_name | The name of the service account that will be created if create\_service\_account is true. If you wish to use an existing service account, use service\_account variable. | `string` | `""` | no | | service\_external\_ips | Whether external ips specified by a service will be allowed in this cluster | `bool` | `false` | no | | shadow\_firewall\_rules\_log\_config | The log\_config for shadow firewall rules. You can set this variable to `null` to disable logging. | 
object({
    metadata = string
  })
| 
{
  "metadata": "INCLUDE_ALL_METADATA"
}
| no | | shadow\_firewall\_rules\_priority | The firewall priority of GKE shadow firewall rules. The priority should be less than default firewall, which is 1000. | `number` | `999` | no | | stack\_type | The stack type to use for this cluster. Either `IPV4` or `IPV4_IPV6`. Defaults to `IPV4`. | `string` | `"IPV4"` | no | +| stateful\_ha | Whether the Stateful HA Addon is enabled for this cluster. | `bool` | `false` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in (required) | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | @@ -251,6 +260,7 @@ Then perform the following commands on the root folder: |------|-------------| | ca\_certificate | Cluster ca certificate (base64 encoded) | | cluster\_id | Cluster ID | +| dns\_cache\_enabled | Whether DNS Cache enabled | | endpoint | Cluster endpoint | | fleet\_membership | Fleet membership (if registered) | | gateway\_api\_channel | The gateway api channel of this cluster. | @@ -302,18 +312,22 @@ The node_pools variable takes the following parameters: | enable_gvnic | gVNIC (GVE) is an alternative to the virtIO-based ethernet driver. Needs a Container-Optimized OS node image. | false | Optional | | enable_integrity_monitoring | Enables monitoring and attestation of the boot integrity of the instance. The attestation is performed against the integrity policy baseline. This baseline is initially derived from the implicitly trusted boot image when the instance is created. | true | Optional | | enable_secure_boot | Secure Boot helps ensure that the system only runs authentic software by verifying the digital signature of all boot components, and halting the boot process if signature verification fails. | false | Optional | +| gpu_driver_version | Mode for how the GPU driver is installed | null | Optional | | gpu_partition_size | Size of partitions to create on the GPU | null | Optional | | image_type | The image type to use for this node. Note that changing the image type will delete and recreate all nodes in the node pool | COS_CONTAINERD | Optional | | initial_node_count | The initial number of nodes for the pool. In regional or multi-zonal clusters, this is the number of nodes per zone. Changing this will force recreation of the resource. Defaults to the value of min_count | " " | Optional | | key | The key required for the taint | | Required | | logging_variant | The type of logging agent that is deployed by default for newly created node pools in the cluster. Valid values include DEFAULT and MAX_THROUGHPUT. | DEFAULT | Optional | | local_ssd_count | The amount of local SSD disks that will be attached to each cluster node and may be used as a `hostpath` volume or a `local` PersistentVolume. | 0 | Optional | +| local_ssd_ephemeral_storage_count | The amount of local SSD disks that will be attached to each cluster node and assigned as scratch space as an `emptyDir` volume. If unspecified, ephemeral storage is backed by the cluster node boot disk. | 0 | Optional | +| local_nvme_ssd_count | Number of raw-block local NVMe SSD disks to be attached to the node.Each local SSD is 375 GB in size. If zero, it means no raw-block local NVMe SSD disks to be attached to the node. | 0 | Optional | | machine_type | The name of a Google Compute Engine machine type | e2-medium | Optional | | min_cpu_platform | Minimum CPU platform to be used by the nodes in the pool. The nodes may be scheduled on the specified or newer CPU platform. | " " | Optional | | max_count | Maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with total limits. | 100 | Optional | | total_max_count | Total maximum number of nodes in the NodePool. Must be >= min_count. Cannot be used with per zone limits. | null | Optional | | max_pods_per_node | The maximum number of pods per node in this cluster | null | Optional | | strategy | The upgrade stragey to be used for upgrading the nodes. Valid values of state are: `SURGE` or `BLUE_GREEN` | "SURGE" | Optional | +| threads_per_core | Optional The number of threads per physical core. To disable simultaneous multithreading (SMT) set this to 1. If unset, the maximum number of threads supported per core by the underlying processor is assumed | null | Optional | | max_surge | The number of additional nodes that can be added to the node pool during an upgrade. Increasing max_surge raises the number of nodes that can be upgraded simultaneously. Can be set to 0 or greater. Only works with `SURGE` strategy. | 1 | Optional | | max_unavailable | The number of nodes that can be simultaneously unavailable during an upgrade. Increasing max_unavailable raises the number of nodes that can be upgraded in parallel. Can be set to 0 or greater. Only works with `SURGE` strategy. | 0 | Optional | | node_pool_soak_duration | Time needed after draining the entire blue pool. After this period, the blue pool will be cleaned up. By default, it is set to one hour (3600 seconds). The maximum length of the soak time is 7 days (604,800 seconds). Only works with `BLUE_GREEN` strategy. | "3600s" | Optional | @@ -333,6 +347,8 @@ The node_pools variable takes the following parameters: | value | The value for the taint | | Required | | version | The Kubernetes version for the nodes in this pool. Should only be set if auto_upgrade is false | " " | Optional | | location_policy | [Location policy](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_node_pool#location_policy) specifies the algorithm used when scaling-up the node pool. Location policy is supported only in 1.24.1+ clusters. | " " | Optional | +| secondary_boot_disk | Image of a secondary boot disk to preload container images and data on new nodes. For detail see [documentation](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/container_cluster#nested_secondary_boot_disks). `gcfs_config` must be `enabled=true` for this feature to work. | | Optional | +| queued_provisioning | Makes nodes obtainable through the ProvisioningRequest API exclusively. | | Optional | ## windows_node_pools variable The windows_node_pools variable takes the same parameters as [node_pools](#node\_pools-variable) but is reserved for provisioning Windows based node pools only. This variable is introduced to satisfy a [specific requirement](https://cloud.google.com/kubernetes-engine/docs/how-to/creating-a-cluster-windows#create_a_cluster_and_node_pools) for the presence of at least one linux based node pool in the cluster before a windows based node pool can be created. diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 9c989b2365..b74bb1fc5f 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -124,6 +124,27 @@ resource "google_container_cluster" "primary" { disk_size = lookup(var.cluster_autoscaling, "disk_size", 100) disk_type = lookup(var.cluster_autoscaling, "disk_type", "pd-standard") + upgrade_settings { + strategy = lookup(var.cluster_autoscaling, "strategy", "SURGE") + max_surge = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_surge", 0) : null + max_unavailable = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "SURGE" ? lookup(var.cluster_autoscaling, "max_unavailable", 0) : null + + dynamic "blue_green_settings" { + for_each = lookup(var.cluster_autoscaling, "strategy", "SURGE") == "BLUE_GREEN" ? [1] : [] + content { + node_pool_soak_duration = lookup(var.cluster_autoscaling, "node_pool_soak_duration", null) + + standard_rollout_policy { + batch_soak_duration = lookup(var.cluster_autoscaling, "batch_soak_duration", null) + batch_percentage = lookup(var.cluster_autoscaling, "batch_percentage", null) + batch_node_count = lookup(var.cluster_autoscaling, "batch_node_count", null) + } + } + } + } + + + image_type = lookup(var.cluster_autoscaling, "image_type", "COS_CONTAINERD") } } autoscaling_profile = var.cluster_autoscaling.autoscaling_profile != null ? var.cluster_autoscaling.autoscaling_profile : "BALANCED" @@ -151,6 +172,8 @@ resource "google_container_cluster" "primary" { enable_kubernetes_alpha = var.enable_kubernetes_alpha enable_tpu = var.enable_tpu + + enable_l4_ilb_subsetting = var.enable_l4_ilb_subsetting dynamic "master_authorized_networks_config" { for_each = local.master_authorized_networks_config content { @@ -164,6 +187,15 @@ resource "google_container_cluster" "primary" { } } + dynamic "node_pool_auto_config" { + for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + content { + network_tags { + tags = var.network_tags + } + } + } + master_auth { client_certificate_config { issue_client_certificate = var.issue_client_certificate @@ -222,6 +254,14 @@ resource "google_container_cluster" "primary" { } } + dynamic "stateful_ha_config" { + for_each = local.stateful_ha_config + + content { + enabled = stateful_ha_config.value.enabled + } + } + config_connector_config { enabled = var.config_connector } @@ -519,6 +559,13 @@ resource "google_container_node_pool" "pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -535,6 +582,12 @@ resource "google_container_node_pool" "pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -578,6 +631,28 @@ resource "google_container_node_pool" "pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") + dynamic "ephemeral_storage_local_ssd_config" { + for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] + content { + local_ssd_count = ephemeral_storage_local_ssd_config.value + } + } + + dynamic "local_nvme_ssd_block_config" { + for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] + content { + local_ssd_count = local_nvme_ssd_block_config.value + } + } + + # Supports a single secondary boot disk because `map(any)` must have the same values type. + dynamic "secondary_boot_disks" { + for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] + content { + disk_image = secondary_boot_disks.value + mode = "CONTAINER_IMAGE_CACHE" + } + } service_account = lookup( each.value, @@ -605,6 +680,21 @@ resource "google_container_node_pool" "pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } + + dynamic "gpu_sharing_config" { + for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) + } + } + } + } + + dynamic "advanced_machine_features" { + for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] + content { + threads_per_core = lookup(each.value, "threads_per_core", 0) } } @@ -728,6 +818,13 @@ resource "google_container_node_pool" "windows_pools" { } } + dynamic "queued_provisioning" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + enabled = lookup(each.value, "queued_provisioning", null) + } + } + node_config { image_type = lookup(each.value, "image_type", "COS_CONTAINERD") machine_type = lookup(each.value, "machine_type", "e2-medium") @@ -744,6 +841,12 @@ resource "google_container_node_pool" "windows_pools" { enabled = gvnic.value } } + dynamic "reservation_affinity" { + for_each = lookup(each.value, "queued_provisioning", false) ? [true] : [] + content { + consume_reservation_type = "NO_RESERVATION" + } + } labels = merge( lookup(lookup(local.node_pools_labels, "default_values", {}), "cluster_name", true) ? { "cluster_name" = var.name } : {}, lookup(lookup(local.node_pools_labels, "default_values", {}), "node_pool", true) ? { "node_pool" = each.value["name"] } : {}, @@ -787,6 +890,28 @@ resource "google_container_node_pool" "windows_pools" { disk_size_gb = lookup(each.value, "disk_size_gb", 100) disk_type = lookup(each.value, "disk_type", "pd-standard") + dynamic "ephemeral_storage_local_ssd_config" { + for_each = lookup(each.value, "local_ssd_ephemeral_storage_count", 0) > 0 ? [each.value.local_ssd_ephemeral_storage_count] : [] + content { + local_ssd_count = ephemeral_storage_local_ssd_config.value + } + } + + dynamic "local_nvme_ssd_block_config" { + for_each = lookup(each.value, "local_nvme_ssd_count", 0) > 0 ? [each.value.local_nvme_ssd_count] : [] + content { + local_ssd_count = local_nvme_ssd_block_config.value + } + } + + # Supports a single secondary boot disk because `map(any)` must have the same values type. + dynamic "secondary_boot_disks" { + for_each = lookup(each.value, "secondary_boot_disk", "") != "" ? [each.value.secondary_boot_disk] : [] + content { + disk_image = secondary_boot_disks.value + mode = "CONTAINER_IMAGE_CACHE" + } + } service_account = lookup( each.value, @@ -814,6 +939,21 @@ resource "google_container_node_pool" "windows_pools" { gpu_driver_version = lookup(each.value, "gpu_driver_version", "") } } + + dynamic "gpu_sharing_config" { + for_each = lookup(each.value, "gpu_sharing_strategy", "") != "" ? [1] : [] + content { + gpu_sharing_strategy = lookup(each.value, "gpu_sharing_strategy", "") + max_shared_clients_per_gpu = lookup(each.value, "max_shared_clients_per_gpu", 2) + } + } + } + } + + dynamic "advanced_machine_features" { + for_each = lookup(each.value, "threads_per_core", 0) > 0 ? [1] : [] + content { + threads_per_core = lookup(each.value, "threads_per_core", 0) } } diff --git a/modules/private-cluster/main.tf b/modules/private-cluster/main.tf index 4c5d8bd613..614babb49a 100644 --- a/modules/private-cluster/main.tf +++ b/modules/private-cluster/main.tf @@ -94,6 +94,7 @@ locals { logmon_config_is_set = length(var.logging_enabled_components) > 0 || length(var.monitoring_enabled_components) > 0 || var.monitoring_enable_managed_prometheus != null || var.monitoring_enable_observability_metrics != null || var.monitoring_enable_observability_relay != null || var.monitoring_observability_metrics_relay_mode != null gke_backup_agent_config = var.gke_backup_agent_config ? [{ enabled = true }] : [{ enabled = false }] gcs_fuse_csi_driver_config = var.gcs_fuse_csi_driver ? [{ enabled = true }] : [] + stateful_ha_config = var.stateful_ha ? [{ enabled = true }] : [] cluster_authenticator_security_group = var.authenticator_security_group == null ? [] : [{ security_group = var.authenticator_security_group @@ -171,6 +172,7 @@ locals { }] : [] + cluster_dns_cache_enabled = var.dns_cache cluster_maintenance_window_is_recurring = var.maintenance_recurrence != "" && var.maintenance_end_time != "" ? [1] : [] cluster_maintenance_window_is_daily = length(local.cluster_maintenance_window_is_recurring) > 0 ? [] : [1] } diff --git a/modules/private-cluster/outputs.tf b/modules/private-cluster/outputs.tf index e1e27f7bba..8f35e71fe0 100644 --- a/modules/private-cluster/outputs.tf +++ b/modules/private-cluster/outputs.tf @@ -185,6 +185,11 @@ output "peering_name" { value = local.cluster_peering_name } +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} + output "fleet_membership" { description = "Fleet membership (if registered)" value = local.fleet_membership diff --git a/modules/private-cluster/sa.tf b/modules/private-cluster/sa.tf index 6f89899bee..da1930c7b3 100644 --- a/modules/private-cluster/sa.tf +++ b/modules/private-cluster/sa.tf @@ -52,6 +52,20 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } +resource "google_project_iam_member" "cluster_service_account-metric_writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/monitoring.metricWriter" + member = google_service_account.cluster_service_account[0].member +} + +resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/stackdriver.resourceMetadata.writer" + member = google_service_account.cluster_service_account[0].member +} + resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key diff --git a/modules/private-cluster/variables.tf b/modules/private-cluster/variables.tf index 2d7f8cc8c6..ccf65a8a13 100644 --- a/modules/private-cluster/variables.tf +++ b/modules/private-cluster/variables.tf @@ -239,17 +239,25 @@ variable "enable_resource_consumption_export" { variable "cluster_autoscaling" { type = object({ - enabled = bool - autoscaling_profile = string - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) + image_type = optional(string) + strategy = optional(string) + max_surge = optional(number) + max_unavailable = optional(number) + node_pool_soak_duration = optional(string) + batch_soak_duration = optional(string) + batch_percentage = optional(number) + batch_node_count = optional(number) }) default = { enabled = false @@ -263,6 +271,7 @@ variable "cluster_autoscaling" { auto_upgrade = true disk_size = 100 disk_type = "pd-standard" + image_type = "COS_CONTAINERD" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -300,6 +309,12 @@ variable "node_pools_oauth_scopes" { } } +variable "network_tags" { + description = "(Optional) - List of network tags applied to auto-provisioned node pools." + type = list(string) + default = [] +} + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -399,25 +414,25 @@ variable "cluster_resource_labels" { variable "deploy_using_private_endpoint" { type = bool - description = "(Beta) A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." + description = "A toggle for Terraform and kubectl to connect to the master's internal IP address during deployment." default = false } variable "enable_private_endpoint" { type = bool - description = "(Beta) Whether the master's internal IP address is used as the cluster endpoint" + description = "Whether the master's internal IP address is used as the cluster endpoint" default = false } variable "enable_private_nodes" { type = bool - description = "(Beta) Whether nodes have internal IP addresses only" + description = "Whether nodes have internal IP addresses only" default = false } variable "master_ipv4_cidr_block" { type = string - description = "(Beta) The IP range in CIDR notation to use for the hosted master network" + description = "The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters." default = "10.0.0.0/28" } @@ -526,7 +541,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -666,6 +681,12 @@ variable "gcs_fuse_csi_driver" { default = false } +variable "stateful_ha" { + type = bool + description = "Whether the Stateful HA Addon is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -706,7 +727,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -728,6 +749,12 @@ variable "config_connector" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "fleet_project" { description = "(Optional) Register the cluster with the fleet in this project." type = string diff --git a/modules/private-cluster/versions.tf b/modules/private-cluster/versions.tf index 0cc68f07f8..b349b6fc93 100644 --- a/modules/private-cluster/versions.tf +++ b/modules/private-cluster/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.9.0, < 6" + version = ">= 5.25.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -33,6 +33,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:private-cluster/v31.0.0" } } diff --git a/modules/safer-cluster-update-variant/README.md b/modules/safer-cluster-update-variant/README.md index 8aa383cec1..ee2e8bee69 100644 --- a/modules/safer-cluster-update-variant/README.md +++ b/modules/safer-cluster-update-variant/README.md @@ -249,6 +249,7 @@ For simplicity, we suggest using `roles/container.admin` and | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster | `string` | n/a | yes | | network | The VPC network to host the cluster in | `string` | n/a | yes | @@ -269,7 +270,7 @@ For simplicity, we suggest using `roles/container.admin` and | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. | `string` | `null` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE` | `string` | `null` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | diff --git a/modules/safer-cluster-update-variant/main.tf b/modules/safer-cluster-update-variant/main.tf index 3da128f44e..f79239f47d 100644 --- a/modules/safer-cluster-update-variant/main.tf +++ b/modules/safer-cluster-update-variant/main.tf @@ -104,6 +104,7 @@ module "gke" { monitoring_service = var.monitoring_service monitoring_enable_managed_prometheus = var.monitoring_enable_managed_prometheus + monitoring_enabled_components = var.monitoring_enabled_components // We never use the default service account for the cluster. The default // project/editor permissions can create problems if nodes were to be ever diff --git a/modules/safer-cluster-update-variant/variables.tf b/modules/safer-cluster-update-variant/variables.tf index 2b51a97541..d6e72d35c4 100644 --- a/modules/safer-cluster-update-variant/variables.tf +++ b/modules/safer-cluster-update-variant/variables.tf @@ -280,6 +280,12 @@ variable "monitoring_enable_managed_prometheus" { default = false } +variable "monitoring_enabled_components" { + type = list(string) + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + default = [] +} + variable "grant_registry_access" { type = bool description = "Grants created cluster-specific service account storage.objectViewer role." @@ -470,7 +476,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`" type = string default = null } diff --git a/modules/safer-cluster-update-variant/versions.tf b/modules/safer-cluster-update-variant/versions.tf index e7f7ab6a9c..f2270feaaa 100644 --- a/modules/safer-cluster-update-variant/versions.tf +++ b/modules/safer-cluster-update-variant/versions.tf @@ -21,6 +21,6 @@ terraform { required_version = ">=1.3" provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster-update-variant/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster-update-variant/v31.0.0" } } diff --git a/modules/safer-cluster/README.md b/modules/safer-cluster/README.md index 8aa383cec1..ee2e8bee69 100644 --- a/modules/safer-cluster/README.md +++ b/modules/safer-cluster/README.md @@ -249,6 +249,7 @@ For simplicity, we suggest using `roles/container.admin` and | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | (Beta) Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `false` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster | `string` | n/a | yes | | network | The VPC network to host the cluster in | `string` | n/a | yes | @@ -269,7 +270,7 @@ For simplicity, we suggest using `roles/container.admin` and | resource\_usage\_export\_dataset\_id | The dataset id for which network egress metering for this cluster will be enabled. If enabled, a daemonset will be created in the cluster to meter network egress traffic. | `string` | `""` | no | | sandbox\_enabled | (Beta) Enable GKE Sandbox (Do not forget to set `image_type` = `COS_CONTAINERD` to use it). | `bool` | `false` | no | | security\_posture\_mode | Security posture mode. Accepted values are `DISABLED` and `BASIC`. Defaults to `DISABLED`. | `string` | `"DISABLED"` | no | -| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. | `string` | `null` | no | +| security\_posture\_vulnerability\_mode | Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE` | `string` | `null` | no | | stub\_domains | Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server | `map(list(string))` | `{}` | no | | subnetwork | The subnetwork to host the cluster in | `string` | n/a | yes | | timeouts | Timeout for cluster operations. | `map(string)` | `{}` | no | diff --git a/modules/safer-cluster/main.tf b/modules/safer-cluster/main.tf index 67a3a54ed4..ac8b6d9409 100644 --- a/modules/safer-cluster/main.tf +++ b/modules/safer-cluster/main.tf @@ -104,6 +104,7 @@ module "gke" { monitoring_service = var.monitoring_service monitoring_enable_managed_prometheus = var.monitoring_enable_managed_prometheus + monitoring_enabled_components = var.monitoring_enabled_components // We never use the default service account for the cluster. The default // project/editor permissions can create problems if nodes were to be ever diff --git a/modules/safer-cluster/variables.tf b/modules/safer-cluster/variables.tf index 2b51a97541..d6e72d35c4 100644 --- a/modules/safer-cluster/variables.tf +++ b/modules/safer-cluster/variables.tf @@ -280,6 +280,12 @@ variable "monitoring_enable_managed_prometheus" { default = false } +variable "monitoring_enabled_components" { + type = list(string) + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." + default = [] +} + variable "grant_registry_access" { type = bool description = "Grants created cluster-specific service account storage.objectViewer role." @@ -470,7 +476,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`" type = string default = null } diff --git a/modules/safer-cluster/versions.tf b/modules/safer-cluster/versions.tf index 648847290e..fc44a1e96d 100644 --- a/modules/safer-cluster/versions.tf +++ b/modules/safer-cluster/versions.tf @@ -21,6 +21,6 @@ terraform { required_version = ">=1.3" provider_meta "google-beta" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:safer-cluster/v31.0.0" } } diff --git a/modules/services/main.tf b/modules/services/main.tf index 74dbc89e18..9ef548c06a 100644 --- a/modules/services/main.tf +++ b/modules/services/main.tf @@ -16,7 +16,7 @@ module "services" { source = "terraform-google-modules/project-factory/google//modules/project_services" - version = "~> 14.0" + version = "~> 15.0" project_id = var.project_id enable_apis = var.enable_apis diff --git a/modules/services/versions.tf b/modules/services/versions.tf index 2f0d9ef972..8413aa45f8 100644 --- a/modules/services/versions.tf +++ b/modules/services/versions.tf @@ -19,6 +19,6 @@ terraform { required_version = ">= 0.13.0" provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:services/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:services/v31.0.0" } } diff --git a/modules/workload-identity/README.md b/modules/workload-identity/README.md index 5024cc7d33..d11a8dbb71 100644 --- a/modules/workload-identity/README.md +++ b/modules/workload-identity/README.md @@ -94,6 +94,51 @@ module "my-app-workload-identity" { If annotation is disabled (via `annotate_k8s_sa = false`), the existing Kubernetes service account must already bear the `"iam.gke.io/gcp-service-account"` annotation. +## Using with multiple clusters + +This module accommodates configurations involving multiple clusters within the kubernetes provider. + +To begin, initialize the kubernetes provider for each cluster with a unique alias, as demonstrated below: + +Initialize your `kubernetes` provider with an alias like the following: + +```hcl +provider "kubernetes" { + alias = "alias-for-your-cluster" + host = "https://your-cluster-host-url.com" + token = "your-cluster-token" + cluster_ca_certificate = base64decode("Your-Cluster-Certificate") +} +``` + +Ensure each cluster configuration has a distinct alias. Repeat this step for every cluster you intend to manage. + +In your module configuration, include the providers attribute to assign the appropriate provider alias: + +```hcl +module "workload_identity_for_cluster" { + source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity" + + providers = { + kubernetes = kubernetes.alias-for-your-cluster + } + + name = "service-account-name" + namespace = "desired-namespace" + // Other module configurations +} +``` + +This approach is required when managing multiple clusters. Omitting this step can lead to errors like the one shown below: + +```shell +Error: Get "http://localhost/api/v1/namespaces/default/serviceaccounts/your-service-account": dial tcp [::1]:80: connect: connection refused +│ +│ with module.your_workload_identity.kubernetes_service_account.main[0], +│ on .terraform/modules/your_workload_identity/modules/workload-identity/main.tf line 50, in resource "kubernetes_service_account" "main": +│ 50: resource "kubernetes_service_account" "main" { +``` + ## Inputs diff --git a/modules/workload-identity/versions.tf b/modules/workload-identity/versions.tf index 34aa4cb45c..29ead8fb56 100644 --- a/modules/workload-identity/versions.tf +++ b/modules/workload-identity/versions.tf @@ -30,6 +30,6 @@ terraform { } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine:workload-identity/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine:workload-identity/v31.0.0" } } diff --git a/outputs.tf b/outputs.tf index 6ed32741f2..7aa1c3be10 100644 --- a/outputs.tf +++ b/outputs.tf @@ -175,6 +175,11 @@ output "mesh_certificates_config" { } +output "dns_cache_enabled" { + description = "Whether DNS Cache enabled" + value = local.cluster_dns_cache_enabled +} + output "fleet_membership" { description = "Fleet membership (if registered)" value = local.fleet_membership diff --git a/sa.tf b/sa.tf index 6f89899bee..da1930c7b3 100644 --- a/sa.tf +++ b/sa.tf @@ -52,6 +52,20 @@ resource "google_project_iam_member" "cluster_service_account-nodeService_accoun member = google_service_account.cluster_service_account[0].member } +resource "google_project_iam_member" "cluster_service_account-metric_writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/monitoring.metricWriter" + member = google_service_account.cluster_service_account[0].member +} + +resource "google_project_iam_member" "cluster_service_account-resourceMetadata-writer" { + count = var.create_service_account ? 1 : 0 + project = google_service_account.cluster_service_account[0].project + role = "roles/stackdriver.resourceMetadata.writer" + member = google_service_account.cluster_service_account[0].member +} + resource "google_project_iam_member" "cluster_service_account-gcr" { for_each = var.create_service_account && var.grant_registry_access ? toset(local.registry_projects_list) : [] project = each.key diff --git a/test/integration/go.mod b/test/integration/go.mod index fdc5577793..66e00997e6 100644 --- a/test/integration/go.mod +++ b/test/integration/go.mod @@ -2,12 +2,13 @@ module github.com/terraform-google-modules/terraform-google-kubernetes-engine/te go 1.21 -toolchain go1.21.5 +toolchain go1.22.3 require ( - github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.12.1 - github.com/gruntwork-io/terratest v0.46.11 - github.com/stretchr/testify v1.8.4 + github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.15.0 + github.com/gruntwork-io/terratest v0.46.15 + github.com/stretchr/testify v1.9.0 + github.com/tidwall/gjson v1.17.1 ) require ( @@ -23,7 +24,7 @@ require ( github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/boombuler/barcode v1.0.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/go-errors/errors v1.5.0 // indirect github.com/go-logr/logr v1.2.4 // indirect @@ -35,7 +36,7 @@ require ( github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/gnostic-models v0.6.8 // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/google/uuid v1.3.1 // indirect @@ -44,14 +45,14 @@ require ( github.com/gruntwork-io/go-commons v0.17.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-getter v1.7.2 // indirect + github.com/hashicorp/go-getter v1.7.4 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f // indirect - github.com/hashicorp/hcl/v2 v2.18.0 // indirect - github.com/hashicorp/terraform-config-inspect v0.0.0-20231204233900-a34142ec2a72 // indirect - github.com/hashicorp/terraform-json v0.17.1 // indirect + github.com/hashicorp/hcl/v2 v2.20.1 // indirect + github.com/hashicorp/terraform-config-inspect v0.0.0-20240509232506-4708120f8f30 // indirect + github.com/hashicorp/terraform-json v0.22.1 // indirect github.com/imdario/mergo v0.3.16 // indirect github.com/jinzhu/copier v0.4.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect @@ -67,11 +68,10 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/pquerna/otp v1.4.0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/tidwall/gjson v1.17.0 // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.1 // indirect github.com/tidwall/sjson v1.2.5 // indirect @@ -79,17 +79,18 @@ require ( github.com/ulikunitz/xz v0.5.11 // indirect github.com/urfave/cli/v2 v2.25.7 // indirect github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect - github.com/zclconf/go-cty v1.14.0 // indirect + github.com/zclconf/go-cty v1.14.4 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/crypto v0.17.0 // indirect - golang.org/x/mod v0.15.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/net v0.23.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect golang.org/x/sync v0.4.0 // indirect - golang.org/x/sys v0.16.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/term v0.18.0 // indirect golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.13.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.138.0 // indirect google.golang.org/appengine v1.6.8 // indirect @@ -97,7 +98,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230822172742-b8732ec3820d // indirect google.golang.org/grpc v1.58.3 // indirect - google.golang.org/protobuf v1.31.0 // indirect + google.golang.org/protobuf v1.33.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect @@ -105,10 +106,10 @@ require ( k8s.io/apimachinery v0.28.4 // indirect k8s.io/client-go v0.28.4 // indirect k8s.io/klog/v2 v2.100.1 // indirect - k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f // indirect + k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/kustomize/kyaml v0.16.0 // indirect + sigs.k8s.io/kustomize/kyaml v0.17.0 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect sigs.k8s.io/yaml v1.4.0 // indirect ) diff --git a/test/integration/go.sum b/test/integration/go.sum index f2743cd309..ff32902444 100644 --- a/test/integration/go.sum +++ b/test/integration/go.sum @@ -187,8 +187,8 @@ cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoIS dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.12.1 h1:4K4HIuu95U6A56orD4LAX+yabfUmIRaoUV0JJWOZPjE= -github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.12.1/go.mod h1:iipeDC0VhxKtIVOZyz0ofiJ2x6sh9+VYEbG3JKsPqPw= +github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.15.0 h1:zT+PND04OQ96kmnQorLFtuY4Ny3KZ4fqxdy9faxMS6A= +github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.15.0/go.mod h1:Srfs5phdVU1kwoRWedu4hngTkwN3OILEWgUgE+IVp6s= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= @@ -228,8 +228,9 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHH github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -321,8 +322,9 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -376,15 +378,15 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/gruntwork-io/go-commons v0.17.1 h1:2KS9wAqrgeOTWj33DSHzDNJ1FCprptWdLFqej+wB8x0= github.com/gruntwork-io/go-commons v0.17.1/go.mod h1:S98JcR7irPD1bcruSvnqupg+WSJEJ6xaM89fpUZVISk= -github.com/gruntwork-io/terratest v0.46.11 h1:1Z9G18I2FNuH87Ro0YtjW4NH9ky4GDpfzE7+ivkPeB8= -github.com/gruntwork-io/terratest v0.46.11/go.mod h1:DVZG/s7eP1u3KOQJJfE6n7FDriMWpDvnj85XIlZMEM8= +github.com/gruntwork-io/terratest v0.46.15 h1:qfqjTFveymaqe7aAWn3LjlK0SwVGpRfoOut5ggNyfQ8= +github.com/gruntwork-io/terratest v0.46.15/go.mod h1:9bd22zAojjBBiYdsp+AR1iyl2iB6bRUVm2Yf1AFhfrA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-getter v1.7.2 h1:uJDtyXwEfalmp1PqdxuhZqrNkUyClZAhVeZYTArbqkg= -github.com/hashicorp/go-getter v1.7.2/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= +github.com/hashicorp/go-getter v1.7.4 h1:3yQjWuxICvSpYwqSayAdKRFcvBl1y/vogCxczWSmix0= +github.com/hashicorp/go-getter v1.7.4/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= @@ -395,12 +397,12 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f h1:UdxlrJz4JOnY8W+DbLISwf2B8WXEolNRA8BGCwI9jws= github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f/go.mod h1:oZtUIOe8dh44I2q6ScRibXws4Ajl+d+nod3AaR9vL5w= -github.com/hashicorp/hcl/v2 v2.18.0 h1:wYnG7Lt31t2zYkcquwgKo6MWXzRUDIeIVU5naZwHLl8= -github.com/hashicorp/hcl/v2 v2.18.0/go.mod h1:ThLC89FV4p9MPW804KVbe/cEXoQ8NZEh+JtMeeGErHE= -github.com/hashicorp/terraform-config-inspect v0.0.0-20231204233900-a34142ec2a72 h1:nZ5gGjbe5o7XUu1d7j+Y5Ztcxlp+yaumTKH9i0D3wlg= -github.com/hashicorp/terraform-config-inspect v0.0.0-20231204233900-a34142ec2a72/go.mod h1:l8HcFPm9cQh6Q0KSWoYPiePqMvRFenybP1CH2MjKdlg= -github.com/hashicorp/terraform-json v0.17.1 h1:eMfvh/uWggKmY7Pmb3T85u86E2EQg6EQHgyRwf3RkyA= -github.com/hashicorp/terraform-json v0.17.1/go.mod h1:Huy6zt6euxaY9knPAFKjUITn8QxUFIe9VuSzb4zn/0o= +github.com/hashicorp/hcl/v2 v2.20.1 h1:M6hgdyz7HYt1UN9e61j+qKJBqR3orTWbI1HKBJEdxtc= +github.com/hashicorp/hcl/v2 v2.20.1/go.mod h1:TZDqQ4kNKCbh1iJp99FdPiUaVDDUPivbqxZulxDYqL4= +github.com/hashicorp/terraform-config-inspect v0.0.0-20240509232506-4708120f8f30 h1:0qwr2oZy9mIIJMWh7W9NTHLWGMbEF5KEQ+QqM9hym34= +github.com/hashicorp/terraform-config-inspect v0.0.0-20240509232506-4708120f8f30/go.mod h1:Gz/z9Hbn+4KSp8A2FBtNszfLSdT2Tn/uAKGuVqqWmDI= +github.com/hashicorp/terraform-json v0.22.1 h1:xft84GZR0QzjPVWs4lRUwvTcPnegqlyS7orfb5Ltvec= +github.com/hashicorp/terraform-json v0.22.1/go.mod h1:JbWSQCLFSXFFhg42T7l9iJwdGXBYV8fmmD6o/ML4p3A= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= @@ -430,8 +432,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= -github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= @@ -460,19 +460,18 @@ github.com/onsi/ginkgo/v2 v2.9.4/go.mod h1:gCQYp2Q+kSoIj7ykSVb9nskRSsR6PUj4AiLyw github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/otp v1.4.0 h1:wZvl1TIVxKRThZIBiwOOHOGP/1+nZyWBil9Y2XNEDzg= github.com/pquerna/otp v1.4.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= -github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= -github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= @@ -487,11 +486,11 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.17.0 h1:/Jocvlh98kcTfpN2+JzGQWQcqrPQwDrVEMApx/M5ZwM= -github.com/tidwall/gjson v1.17.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.17.1 h1:wlYEnwqAHgzmhNUFfw7Xalt2JzQvsMx2Se4PcoFCT/U= +github.com/tidwall/gjson v1.17.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= @@ -514,8 +513,10 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zclconf/go-cty v1.14.0 h1:/Xrd39K7DXbHzlisFP9c4pHao4yyf+/Ug9LEz+Y/yhc= -github.com/zclconf/go-cty v1.14.0/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zclconf/go-cty v1.14.4 h1:uXXczd9QDGsgu0i/QFR/hzI5NYCHLf6NQw/atrbnhq8= +github.com/zclconf/go-cty v1.14.4/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b h1:FosyBZYxY34Wul7O/MSKey3txpPYyCqVO5ZyceuQJEI= +github.com/zclconf/go-cty-debug v0.0.0-20191215020915-b22d67c1ba0b/go.mod h1:ZRKQfBXbGkpdV6QMzT3rU1kSTAnfu1dO8dPKjYprgj8= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -532,8 +533,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -570,8 +571,8 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.15.0 h1:SernR4v+D55NyBH2QiEQrlBAnj1ECL6AGrA5+dPaMY8= -golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -619,8 +620,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -727,13 +728,14 @@ golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1036,8 +1038,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= @@ -1069,8 +1071,8 @@ k8s.io/client-go v0.28.4 h1:Np5ocjlZcTrkyRJ3+T3PkXDpe4UpatQxj85+xjaD2wY= k8s.io/client-go v0.28.4/go.mod h1:0VDZFpgoZfelyP5Wqu0/r/TRYcLYuJ2U1KEeoaPa1N4= k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= -k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f h1:eeEUOoGYWhOz7EyXqhlR2zHKNw2mNJ9vzJmub6YN6kk= -k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= +k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= @@ -1078,8 +1080,8 @@ rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= -sigs.k8s.io/kustomize/kyaml v0.16.0 h1:6J33uKSoATlKZH16unr2XOhDI+otoe2sR3M8PDzW3K0= -sigs.k8s.io/kustomize/kyaml v0.16.0/go.mod h1:xOK/7i+vmE14N2FdFyugIshB8eF6ALpy7jI87Q2nRh4= +sigs.k8s.io/kustomize/kyaml v0.17.0 h1:G2bWs03V9Ur2PinHLzTUJ8Ded+30SzXZKiO92SRDs3c= +sigs.k8s.io/kustomize/kyaml v0.17.0/go.mod h1:6lxkYF1Cv9Ic8g/N7I86cvxNc5iinUo/P2vKsHNmpyE= sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= diff --git a/test/integration/node_pool/controls/gcloud.rb b/test/integration/node_pool/controls/gcloud.rb index 38ac3ef554..e3da9d06b2 100644 --- a/test/integration/node_pool/controls/gcloud.rb +++ b/test/integration/node_pool/controls/gcloud.rb @@ -62,8 +62,8 @@ describe "node pools" do let(:node_pools) { data['nodePools'].reject { |p| p['name'] == "default-pool" || p['name'] =~ %r{^nap-.*} } } - it "has 3" do - expect(node_pools.count).to eq 3 + it "has 4" do + expect(node_pools.count).to eq 4 end describe "pool-01" do @@ -493,6 +493,27 @@ ) end end + + describe "pool-04" do + it "exists" do + expect(data['nodePools']).to include( + including( + "name" => "pool-04", + ) + ) + end + + it "has queued_provisioning enabled" do + expect(data['nodePools']).not_to include( + including( + "name" => "pool-04", + "queued_provisioning" => including( + "enabled" => true, + ), + ) + ) + end + end end end diff --git a/test/integration/simple_regional/testdata/TestSimpleRegional.json b/test/integration/simple_regional/testdata/TestSimpleRegional.json index 1c22283ddb..85c8e55db7 100644 --- a/test/integration/simple_regional/testdata/TestSimpleRegional.json +++ b/test/integration/simple_regional/testdata/TestSimpleRegional.json @@ -17,6 +17,9 @@ }, "networkPolicyConfig": { "disabled": true + }, + "statefulHaConfig": { + "enabled": true } }, "autopilot": {}, diff --git a/test/setup/main.tf b/test/setup/main.tf index 2e6f551f91..f0b3e278a6 100644 --- a/test/setup/main.tf +++ b/test/setup/main.tf @@ -45,7 +45,7 @@ locals { module "gke-project-1" { source = "terraform-google-modules/project-factory/google" - version = "~> 14.0" + version = "~> 15.0" name = "ci-gke-${random_id.random_project_id_suffix.hex}" random_project_id = true @@ -69,7 +69,7 @@ module "gke-project-1" { module "gke-project-2" { source = "terraform-google-modules/project-factory/google" - version = "~> 14.0" + version = "~> 15.0" name = "ci-gke-${random_id.random_project_id_suffix.hex}" random_project_id = true @@ -91,7 +91,7 @@ module "gke-project-2" { # apis as documented https://cloud.google.com/service-mesh/docs/scripted-install/reference#setting_up_your_project module "gke-project-asm" { source = "terraform-google-modules/project-factory/google" - version = "~> 14.0" + version = "~> 15.0" name = "ci-gke-asm-${random_id.random_project_id_suffix.hex}" random_project_id = true diff --git a/variables.tf b/variables.tf index 2a02ef1556..95cf143d3a 100644 --- a/variables.tf +++ b/variables.tf @@ -239,17 +239,25 @@ variable "enable_resource_consumption_export" { variable "cluster_autoscaling" { type = object({ - enabled = bool - autoscaling_profile = string - min_cpu_cores = number - max_cpu_cores = number - min_memory_gb = number - max_memory_gb = number - gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) - auto_repair = bool - auto_upgrade = bool - disk_size = optional(number) - disk_type = optional(string) + enabled = bool + autoscaling_profile = string + min_cpu_cores = number + max_cpu_cores = number + min_memory_gb = number + max_memory_gb = number + gpu_resources = list(object({ resource_type = string, minimum = number, maximum = number })) + auto_repair = bool + auto_upgrade = bool + disk_size = optional(number) + disk_type = optional(string) + image_type = optional(string) + strategy = optional(string) + max_surge = optional(number) + max_unavailable = optional(number) + node_pool_soak_duration = optional(string) + batch_soak_duration = optional(string) + batch_percentage = optional(number) + batch_node_count = optional(number) }) default = { enabled = false @@ -263,6 +271,7 @@ variable "cluster_autoscaling" { auto_upgrade = true disk_size = 100 disk_type = "pd-standard" + image_type = "COS_CONTAINERD" } description = "Cluster autoscaling configuration. See [more details](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#clusterautoscaling)" } @@ -300,6 +309,12 @@ variable "node_pools_oauth_scopes" { } } +variable "network_tags" { + description = "(Optional) - List of network tags applied to auto-provisioned node pools." + type = list(string) + default = [] +} + variable "stub_domains" { type = map(list(string)) description = "Map of stub domains and their resolvers to forward DNS queries for a certain domain to an external DNS server" @@ -496,7 +511,7 @@ variable "security_posture_mode" { } variable "security_posture_vulnerability_mode" { - description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED` and `VULNERABILITY_BASIC`. Defaults to `VULNERABILITY_DISABLED`." + description = "Security posture vulnerability mode. Accepted values are `VULNERABILITY_DISABLED`, `VULNERABILITY_BASIC`, and `VULNERABILITY_ENTERPRISE`. Defaults to `VULNERABILITY_DISABLED`." type = string default = "VULNERABILITY_DISABLED" } @@ -636,6 +651,12 @@ variable "gcs_fuse_csi_driver" { default = false } +variable "stateful_ha" { + type = bool + description = "Whether the Stateful HA Addon is enabled for this cluster." + default = false +} + variable "timeouts" { type = map(string) description = "Timeout for cluster operations." @@ -676,7 +697,7 @@ variable "monitoring_observability_metrics_relay_mode" { variable "monitoring_enabled_components" { type = list(string) - description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration." + description = "List of services to monitor: SYSTEM_COMPONENTS, WORKLOADS. Empty list is default GKE configuration." default = [] } @@ -698,6 +719,12 @@ variable "config_connector" { default = false } +variable "enable_l4_ilb_subsetting" { + type = bool + description = "Enable L4 ILB Subsetting on the cluster" + default = false +} + variable "fleet_project" { description = "(Optional) Register the cluster with the fleet in this project." type = string diff --git a/versions.tf b/versions.tf index 65c47efec4..efb1764683 100644 --- a/versions.tf +++ b/versions.tf @@ -21,7 +21,7 @@ terraform { required_providers { google = { source = "hashicorp/google" - version = ">= 5.9.0, < 6" + version = ">= 5.25.0, < 6" } kubernetes = { source = "hashicorp/kubernetes" @@ -33,6 +33,6 @@ terraform { } } provider_meta "google" { - module_name = "blueprints/terraform/terraform-google-kubernetes-engine/v30.1.0" + module_name = "blueprints/terraform/terraform-google-kubernetes-engine/v31.0.0" } } From 7e207c3d60d196846c7633fd4ce1ae59cf82ebf9 Mon Sep 17 00:00:00 2001 From: Christopher Markieta Date: Thu, 30 May 2024 12:28:52 -0400 Subject: [PATCH 4/7] docs: :memo: Re-removing " (provider version >= 3.89.0)" from `monitoring_enabled_components`. --- README.md | 2 +- modules/beta-private-cluster-update-variant/README.md | 2 +- modules/beta-private-cluster/README.md | 2 +- modules/beta-public-cluster-update-variant/README.md | 2 +- modules/beta-public-cluster/README.md | 2 +- modules/private-cluster-update-variant/README.md | 2 +- modules/private-cluster/README.md | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index d613063d65..6e8970d3a7 100644 --- a/README.md +++ b/README.md @@ -200,7 +200,7 @@ Then perform the following commands on the root folder: | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `null` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index fffa95b587..f791803b96 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -248,7 +248,7 @@ Then perform the following commands on the root folder: | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `null` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 6690b071d2..998fb027a3 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -226,7 +226,7 @@ Then perform the following commands on the root folder: | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `null` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | diff --git a/modules/beta-public-cluster-update-variant/README.md b/modules/beta-public-cluster-update-variant/README.md index 8717517d0d..8860e184c2 100644 --- a/modules/beta-public-cluster-update-variant/README.md +++ b/modules/beta-public-cluster-update-variant/README.md @@ -237,7 +237,7 @@ Then perform the following commands on the root folder: | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `null` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | diff --git a/modules/beta-public-cluster/README.md b/modules/beta-public-cluster/README.md index 2d963a101d..a3d90ee316 100644 --- a/modules/beta-public-cluster/README.md +++ b/modules/beta-public-cluster/README.md @@ -215,7 +215,7 @@ Then perform the following commands on the root folder: | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `null` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index b3238b0c22..ae6b408f5f 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -233,7 +233,7 @@ Then perform the following commands on the root folder: | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `null` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 615fb43db2..9f08a57250 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -211,7 +211,7 @@ Then perform the following commands on the root folder: | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `null` | no | -| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS (provider version >= 3.89.0). Empty list is default GKE configuration. | `list(string)` | `[]` | no | +| monitoring\_enabled\_components | List of services to monitor: SYSTEM\_COMPONENTS, WORKLOADS. Empty list is default GKE configuration. | `list(string)` | `[]` | no | | monitoring\_observability\_metrics\_relay\_mode | Mode used to make advanced datapath metrics relay available. | `string` | `null` | no | | monitoring\_service | The monitoring service that the cluster should write metrics to. Automatically send metrics from pods in the cluster to the Google Cloud Monitoring API. VM metrics will be collected by Google Compute Engine regardless of this setting Available options include monitoring.googleapis.com, monitoring.googleapis.com/kubernetes (beta) and none | `string` | `"monitoring.googleapis.com/kubernetes"` | no | | name | The name of the cluster (required) | `string` | n/a | yes | From bad9eb82b6f17c6c7d29c6c1e632f48b3e66e4fb Mon Sep 17 00:00:00 2001 From: Christopher Markieta Date: Thu, 30 May 2024 12:36:28 -0400 Subject: [PATCH 5/7] docs: :memo: Revert `master_ipv4_cidr_block` description. --- modules/beta-private-cluster-update-variant/README.md | 2 +- modules/beta-private-cluster/README.md | 2 +- modules/private-cluster-update-variant/README.md | 2 +- modules/private-cluster/README.md | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/beta-private-cluster-update-variant/README.md b/modules/beta-private-cluster-update-variant/README.md index f791803b96..0f70c28c66 100644 --- a/modules/beta-private-cluster-update-variant/README.md +++ b/modules/beta-private-cluster-update-variant/README.md @@ -244,7 +244,7 @@ Then perform the following commands on the root folder: | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `null` | no | diff --git a/modules/beta-private-cluster/README.md b/modules/beta-private-cluster/README.md index 998fb027a3..cf3c1a4ee7 100644 --- a/modules/beta-private-cluster/README.md +++ b/modules/beta-private-cluster/README.md @@ -222,7 +222,7 @@ Then perform the following commands on the root folder: | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `null` | no | diff --git a/modules/private-cluster-update-variant/README.md b/modules/private-cluster-update-variant/README.md index ae6b408f5f..59ddc3822e 100644 --- a/modules/private-cluster-update-variant/README.md +++ b/modules/private-cluster-update-variant/README.md @@ -229,7 +229,7 @@ Then perform the following commands on the root folder: | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `null` | no | diff --git a/modules/private-cluster/README.md b/modules/private-cluster/README.md index 9f08a57250..ff6487131f 100644 --- a/modules/private-cluster/README.md +++ b/modules/private-cluster/README.md @@ -207,7 +207,7 @@ Then perform the following commands on the root folder: | maintenance\_start\_time | Time window specified for daily or recurring maintenance operations in RFC3339 format | `string` | `"05:00"` | no | | master\_authorized\_networks | List of master authorized networks. If none are provided, disallow external access (except the cluster node IPs, which GKE automatically whitelists). | `list(object({ cidr_block = string, display_name = string }))` | `[]` | no | | master\_global\_access\_enabled | Whether the cluster master is accessible globally (from any region) or only within the same region as the private endpoint. | `bool` | `true` | no | -| master\_ipv4\_cidr\_block | (Beta) The IP range in CIDR notation to use for the hosted master network | `string` | `"10.0.0.0/28"` | no | +| master\_ipv4\_cidr\_block | The IP range in CIDR notation to use for the hosted master network. Optional for Autopilot clusters. | `string` | `"10.0.0.0/28"` | no | | monitoring\_enable\_managed\_prometheus | Configuration for Managed Service for Prometheus. Whether or not the managed collection is enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_metrics | Whether or not the advanced datapath metrics are enabled. | `bool` | `null` | no | | monitoring\_enable\_observability\_relay | Whether or not the advanced datapath relay is enabled. | `bool` | `null` | no | From 83bf7f00b674f31fd2f746016a099ecb6b072ddc Mon Sep 17 00:00:00 2001 From: Christopher Markieta Date: Thu, 30 May 2024 12:52:12 -0400 Subject: [PATCH 6/7] style: :art: Add space and quotes to `managed_prometheus` --- autogen/main/cluster.tf.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index b3c21041dc..cb66591b88 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -121,9 +121,9 @@ resource "google_container_cluster" "primary" { {% endif %} content{ enable_components = var.monitoring_enabled_components - dynamic managed_prometheus { + dynamic "managed_prometheus" { for_each = var.monitoring_enable_managed_prometheus != null ? [1] : [] - content{ + content { enabled = var.monitoring_enable_managed_prometheus } } From 4920996ad234823d5b210194b423249daf907026 Mon Sep 17 00:00:00 2001 From: Christopher Markieta Date: Thu, 30 May 2024 23:49:48 -0400 Subject: [PATCH 7/7] test: Replace `enableRelay` with `enableMetrics` in tests including `advancedDatapathObservabilityConfig` --- test/integration/beta_cluster/testdata/TestBetaCluster.json | 2 +- .../simple_regional/testdata/TestSimpleRegional.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/beta_cluster/testdata/TestBetaCluster.json b/test/integration/beta_cluster/testdata/TestBetaCluster.json index dbc703d59b..80a604c9cd 100755 --- a/test/integration/beta_cluster/testdata/TestBetaCluster.json +++ b/test/integration/beta_cluster/testdata/TestBetaCluster.json @@ -100,7 +100,7 @@ }, "monitoringConfig": { "advancedDatapathObservabilityConfig": { - "enableRelay": false + "enableMetrics": false }, "componentConfig": { "enableComponents": [ diff --git a/test/integration/simple_regional/testdata/TestSimpleRegional.json b/test/integration/simple_regional/testdata/TestSimpleRegional.json index 85c8e55db7..990459fcbf 100644 --- a/test/integration/simple_regional/testdata/TestSimpleRegional.json +++ b/test/integration/simple_regional/testdata/TestSimpleRegional.json @@ -101,7 +101,7 @@ }, "monitoringConfig": { "advancedDatapathObservabilityConfig": { - "enableRelay": false + "enableMetrics": false }, "componentConfig": { "enableComponents": [