Merge branch 'master' into fast_socket

README.md

@@ -264,6 +264,7 @@ Then perform the following commands on the root folder:
 | cluster\_id | Cluster ID |
 | dns\_cache\_enabled | Whether DNS Cache enabled |
 | endpoint | Cluster endpoint |
+| endpoint\_dns | Cluster endpoint DNS |
 | fleet\_membership | Fleet membership (if registered) |
 | gateway\_api\_channel | The gateway api channel of this cluster. |
 | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |

autogen/main/cluster.tf.tmpl

@@ -637,6 +637,15 @@ resource "google_container_cluster" "primary" {
       }
     }
   }
+
+  dynamic "control_plane_endpoints_config" {
+    for_each = var.enable_private_endpoint && var.deploy_using_private_endpoint ? [1] : [0]
+    content {
+      dns_endpoint_config {
+        allow_external_traffic = var.deploy_using_private_endpoint
+      }
+    }
+  }
 {% endif %}
 
   {% if autopilot_cluster != true %}
@@ -723,8 +732,6 @@ resource "google_container_cluster" "primary" {
 {% if update_variant %}
 locals {
   force_node_pool_recreation_resources = [
-    "disk_size_gb",
-    "disk_type",
     "accelerator_count",
     "accelerator_type",
     "gpu_partition_size",
@@ -737,15 +744,13 @@ locals {
     {% if beta_cluster %}
     "local_ssd_ephemeral_count",
     {% endif %}
-    "machine_type",
     "placement_policy",
     "max_pods_per_node",
     "min_cpu_platform",
     "pod_range",
     "preemptible",
     "spot",
     "service_account",
-    "enable_gcfs",
     "enable_gvnic",
     "boot_disk_kms_key",
     "queued_provisioning",

autogen/main/outputs.tf.tmpl

@@ -76,6 +76,23 @@ output "endpoint" {
   ]
 }
 
+output "endpoint_dns" {
+  description = "Cluster endpoint DNS"
+  value       = google_container_cluster.primary.control_plane_endpoints_config[0].dns_endpoint_config[0].endpoint
+  depends_on = [
+    /* Nominally, the endpoint is populated as soon as it is known to Terraform.
+    * However, the cluster may not be in a usable state yet.  Therefore any
+    * resources dependent on the cluster being up will fail to deploy.  With
+    * this explicit dependency, dependent resources can wait for the cluster
+    * to be up.
+    */
+    google_container_cluster.primary,
+    {% if autopilot_cluster != true %}
+    google_container_node_pool.pools,
+    {% endif %}
+  ]
+}
+
 output "min_master_version" {
   description = "Minimum master kubernetes version"
   value       = local.cluster_min_master_version

autogen/main/versions.tf.tmpl

@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 Google LLC
+ * Copyright 2022-2024 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -24,33 +24,33 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 6.7.0, < 7"
+      version = ">= 6.11.0, < 7"
     }
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 6.7.0, < 7"
+      version = ">= 6.11.0, < 7"
     }
 {% elif beta_cluster and autopilot_cluster %}
   required_providers {
     google = {
       source = "hashicorp/google"
-      version = ">= 6.8.0, < 7"
+      version = ">= 6.11.0, < 7"
     }
     google-beta = {
       source = "hashicorp/google-beta"
-      version = ">= 6.8.0, < 7"
+      version = ">= 6.11.0, < 7"
     }
 {% elif autopilot_cluster  %}
   required_providers {
     google = {
       source = "hashicorp/google"
-      version = ">= 6.8.0, < 7"
+      version = ">= 6.11.0, < 7"
     }
 {% else %}
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 6.7.0, < 7"
+      version = ">= 6.11.0, < 7"
     }
 {% endif %}
     kubernetes = {

autogen/safer-cluster/outputs.tf.tmpl

@@ -52,6 +52,11 @@ output "endpoint" {
   value       = module.gke.endpoint
 }
 
+output "endpoint_dns" {
+  description = "Cluster endpoint DNS"
+  value       = module.gke.endpoint_dns
+}
+
 output "min_master_version" {
   description = "Minimum master kubernetes version"
   value       = module.gke.min_master_version

docs/upgrading_to_v35.0.md

@@ -6,3 +6,47 @@ The Terraform Kubernetes Engine Module now requires version 6 of the Google Clou
 
 ### Private Cluster Sub-Modules Endpoint Output
 The private cluster sub-modules now return the cluster's private endpoint for the `endpoint` output when the `enable_private_endpoint` argument is `true`, regardless of the `deploy_using_private_endpoint` argument value.
+
+## Update variant random ID keepers updated
+
+The v35.0 release updates the keepers for the update variant modules. This will force a recreation of the nodepools.
+
+To avoid this, it is possible to edit the remote state of the `random_id` resource to add the new attributes.
+
+1. Perform a `terraform plan` as normal, identifying the `random_id` resource(s) changing and the new/removed attributes
+```tf
+      ~ keepers     = { # forces replacement
+          - "disk_type"                   = "" -> null
+          - "disk_size_gb"                = "" -> null
+          - "machine_type"                = "" -> null
+          - "enable_gcfs"                 = "" -> null
+            # (19 unchanged elements hidden)
+        }
+        # (2 unchanged attributes hidden)
+    }
+```
+2. Pull the remote state locally: `terraform state pull > default.tfstate`
+3. Back up the original remote state: `cp default.tfstate original.tfstate`
+4. Edit the `random_id` resource(s) to add/remove the attributes from the `terraform plan` step
+```diff
+"attributes": {
+            "b64_std": "pool-02-vb4=",
+            "b64_url": "pool-02-vb4",
+            "byte_length": 2,
+            "dec": "pool-02-48574",
+            "hex": "pool-02-bdbe",
+            "id": "vb4",
+            "keepers": {
+            ...
+              "taints": "",
+-             "disk_size_gb": "",
+-             "enable_gcfs": "",
+-             "machine_type": "",
+-             "disk_type": "",
+            },
+            "prefix": "pool-02-"
+          }
+```
+1. Bump the serial number at the top
+2. Push the modified state to the remote `terraform state push default.tfstate`
+3. Confirm the `random_id` resource(s) no longer changes (or the corresponding `nodepool`) in a `terraform plan`

examples/safer_cluster_iap_bastion/README.md

@@ -60,6 +60,7 @@ To deploy this example:
 | ca\_certificate | Cluster ca certificate (base64 encoded) |
 | cluster\_name | Cluster name |
 | endpoint | Cluster endpoint |
+| endpoint\_dns | Cluster endpoint DNS |
 | get\_credentials\_command | gcloud get-credentials command to generate kubeconfig for the private cluster |
 | keyring | The name of the keyring. |
 | keyring\_resource | The location of the keyring. |

examples/safer_cluster_iap_bastion/bastion.tf

@@ -34,4 +34,6 @@ module "bastion" {
   startup_script = templatefile("${path.module}/templates/startup-script.tftpl", {})
   members        = var.bastion_members
   shielded_vm    = "false"
+
+  service_account_roles = ["roles/container.viewer"]
 }

examples/safer_cluster_iap_bastion/outputs.tf

@@ -35,6 +35,12 @@ output "endpoint" {
   value       = module.gke.endpoint
 }
 
+output "endpoint_dns" {
+  sensitive   = true
+  description = "Cluster endpoint DNS"
+  value       = module.gke.endpoint_dns
+}
+
 output "master_authorized_networks_config" {
   description = "Networks from which access to master is permitted"
   value       = module.gke.master_authorized_networks_config

examples/simple_regional_beta/main.tf

@@ -20,12 +20,6 @@ locals {
 
 data "google_client_config" "default" {}
 
-provider "kubernetes" {
-  host                   = "https://${module.gke.endpoint}"
-  token                  = data.google_client_config.default.access_token
-  cluster_ca_certificate = base64decode(module.gke.ca_certificate)
-}
-
 module "gke" {
   source  = "terraform-google-modules/kubernetes-engine/google//modules/beta-public-cluster"
   version = "~> 34.0"

examples/simple_regional_beta/versions.tf

@@ -23,8 +23,5 @@ terraform {
     google-beta = {
       source = "hashicorp/google-beta"
     }
-    kubernetes = {
-      source = "hashicorp/kubernetes"
-    }
   }
 }

modules/beta-autopilot-private-cluster/README.md

@@ -170,6 +170,7 @@ Then perform the following commands on the root folder:
 | cluster\_id | Cluster ID |
 | dns\_cache\_enabled | Whether DNS Cache enabled |
 | endpoint | Cluster endpoint |
+| endpoint\_dns | Cluster endpoint DNS |
 | fleet\_membership | Fleet membership (if registered) |
 | gateway\_api\_channel | The gateway api channel of this cluster. |
 | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |

modules/beta-autopilot-private-cluster/cluster.tf

@@ -326,6 +326,15 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  dynamic "control_plane_endpoints_config" {
+    for_each = var.enable_private_endpoint && var.deploy_using_private_endpoint ? [1] : [0]
+    content {
+      dns_endpoint_config {
+        allow_external_traffic = var.deploy_using_private_endpoint
+      }
+    }
+  }
+
 
   dynamic "database_encryption" {
     for_each = var.database_encryption

modules/beta-autopilot-private-cluster/outputs.tf

@@ -70,6 +70,20 @@ output "endpoint" {
   ]
 }
 
+output "endpoint_dns" {
+  description = "Cluster endpoint DNS"
+  value       = google_container_cluster.primary.control_plane_endpoints_config[0].dns_endpoint_config[0].endpoint
+  depends_on = [
+    /* Nominally, the endpoint is populated as soon as it is known to Terraform.
+    * However, the cluster may not be in a usable state yet.  Therefore any
+    * resources dependent on the cluster being up will fail to deploy.  With
+    * this explicit dependency, dependent resources can wait for the cluster
+    * to be up.
+    */
+    google_container_cluster.primary,
+  ]
+}
+
 output "min_master_version" {
   description = "Minimum master kubernetes version"
   value       = local.cluster_min_master_version

modules/beta-autopilot-private-cluster/versions.tf

@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 Google LLC
+ * Copyright 2022-2024 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,11 +21,11 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 6.8.0, < 7"
+      version = ">= 6.11.0, < 7"
     }
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 6.8.0, < 7"
+      version = ">= 6.11.0, < 7"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

modules/beta-autopilot-public-cluster/README.md

@@ -158,6 +158,7 @@ Then perform the following commands on the root folder:
 | cluster\_id | Cluster ID |
 | dns\_cache\_enabled | Whether DNS Cache enabled |
 | endpoint | Cluster endpoint |
+| endpoint\_dns | Cluster endpoint DNS |
 | fleet\_membership | Fleet membership (if registered) |
 | gateway\_api\_channel | The gateway api channel of this cluster. |
 | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |

modules/beta-autopilot-public-cluster/outputs.tf

@@ -70,6 +70,20 @@ output "endpoint" {
   ]
 }
 
+output "endpoint_dns" {
+  description = "Cluster endpoint DNS"
+  value       = google_container_cluster.primary.control_plane_endpoints_config[0].dns_endpoint_config[0].endpoint
+  depends_on = [
+    /* Nominally, the endpoint is populated as soon as it is known to Terraform.
+    * However, the cluster may not be in a usable state yet.  Therefore any
+    * resources dependent on the cluster being up will fail to deploy.  With
+    * this explicit dependency, dependent resources can wait for the cluster
+    * to be up.
+    */
+    google_container_cluster.primary,
+  ]
+}
+
 output "min_master_version" {
   description = "Minimum master kubernetes version"
   value       = local.cluster_min_master_version

modules/beta-autopilot-public-cluster/versions.tf

@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 Google LLC
+ * Copyright 2022-2024 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,11 +21,11 @@ terraform {
   required_providers {
     google = {
       source  = "hashicorp/google"
-      version = ">= 6.8.0, < 7"
+      version = ">= 6.11.0, < 7"
     }
     google-beta = {
       source  = "hashicorp/google-beta"
-      version = ">= 6.8.0, < 7"
+      version = ">= 6.11.0, < 7"
     }
     kubernetes = {
       source  = "hashicorp/kubernetes"

modules/beta-private-cluster-update-variant/README.md

@@ -314,6 +314,7 @@ Then perform the following commands on the root folder:
 | cluster\_id | Cluster ID |
 | dns\_cache\_enabled | Whether DNS Cache enabled |
 | endpoint | Cluster endpoint |
+| endpoint\_dns | Cluster endpoint DNS |
 | fleet\_membership | Fleet membership (if registered) |
 | gateway\_api\_channel | The gateway api channel of this cluster. |
 | horizontal\_pod\_autoscaling\_enabled | Whether horizontal pod autoscaling enabled |

modules/beta-private-cluster-update-variant/cluster.tf

@@ -553,6 +553,15 @@ resource "google_container_cluster" "primary" {
     }
   }
 
+  dynamic "control_plane_endpoints_config" {
+    for_each = var.enable_private_endpoint && var.deploy_using_private_endpoint ? [1] : [0]
+    content {
+      dns_endpoint_config {
+        allow_external_traffic = var.deploy_using_private_endpoint
+      }
+    }
+  }
+
   remove_default_node_pool = var.remove_default_node_pool
 
   dynamic "database_encryption" {
@@ -617,8 +626,6 @@ resource "google_container_cluster" "primary" {
  *****************************************/
 locals {
   force_node_pool_recreation_resources = [
-    "disk_size_gb",
-    "disk_type",
     "accelerator_count",
     "accelerator_type",
     "gpu_partition_size",
@@ -629,15 +636,13 @@ locals {
     "enable_integrity_monitoring",
     "local_ssd_count",
     "local_ssd_ephemeral_count",
-    "machine_type",
     "placement_policy",
     "max_pods_per_node",
     "min_cpu_platform",
     "pod_range",
     "preemptible",
     "spot",
     "service_account",
-    "enable_gcfs",
     "enable_gvnic",
     "boot_disk_kms_key",
     "queued_provisioning",