fix: add target tags to node_pool_auto_config for standard clusters (…

…#2118) Signed-off-by: William Yardley <[email protected]>
terraform-google-modules · Oct 9, 2024 · d5f66e8 · d5f66e8
1 parent 745b16a
commit d5f66e8
Show file tree

Hide file tree

Showing 21 changed files with 914 additions and 16 deletions.
diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl
@@ -282,10 +282,10 @@ resource "google_container_cluster" "primary" {
 
 {% if autopilot_cluster != true %}
   dynamic "node_pool_auto_config" {
-    for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : []
+    for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : []
     content {
       network_tags {
-        tags = var.network_tags
+        tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags
       }
     }
   }

diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml
@@ -134,6 +134,22 @@ steps:
     - verify simple-regional-private-local
   name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
   args: ['/bin/bash', '-c', 'cft test run TestSimpleRegionalPrivate --stage teardown --verbose']
+- id: apply simple-regional-cluster-autoscaling
+  waitFor:
+    - create-all
+    - destroy simple-regional-private-local
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleRegionalClusterAutoscaling --stage apply --verbose']
+- id: verify simple-regional-cluster-autoscaling
+  waitFor:
+    - apply simple-regional-cluster-autoscaling
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleRegionalClusterAutoscaling --stage verify --verbose']
+- id: destroy simple-regional-cluster-autoscaling
+  waitFor:
+    - verify simple-regional-cluster-autoscaling
+  name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
+  args: ['/bin/bash', '-c', 'cft test run TestSimpleRegionalClusterAutoscaling --stage teardown --verbose']
 - id: init simple-regional-with-kubeconfig-local
   waitFor:
     - create-all

diff --git a/cluster.tf b/cluster.tf
@@ -204,10 +204,10 @@ resource "google_container_cluster" "primary" {
   }
 
   dynamic "node_pool_auto_config" {
-    for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : []
+    for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : []
     content {
       network_tags {
-        tags = var.network_tags
+        tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags
       }
     }
   }

diff --git a/examples/simple_regional_cluster_autoscaling/README.md b/examples/simple_regional_cluster_autoscaling/README.md
@@ -0,0 +1,39 @@
+# Simple Regional Cluster with Node Autoscaling
+
+This example illustrates how to create a simple private cluster with cluster
+level node autoprovisioning.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no |
+| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | `any` | n/a | yes |
+| ip\_range\_pods | The secondary ip range to use for pods | `any` | n/a | yes |
+| ip\_range\_services | The secondary ip range to use for services | `any` | n/a | yes |
+| network | The VPC network to host the cluster in | `any` | n/a | yes |
+| project\_id | The project ID to host the cluster in | `any` | n/a | yes |
+| region | The region to host the cluster in | `any` | n/a | yes |
+| subnetwork | The subnetwork to host the cluster in | `any` | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| ca\_certificate | n/a |
+| client\_token | n/a |
+| cluster\_name | Cluster name |
+| ip\_range\_pods | The secondary IP range used for pods |
+| ip\_range\_services | The secondary IP range used for services |
+| kubernetes\_endpoint | n/a |
+| location | n/a |
+| master\_kubernetes\_version | The master Kubernetes version |
+| network | n/a |
+| project\_id | n/a |
+| region | n/a |
+| service\_account | The default service account used for running nodes. |
+| subnetwork | n/a |
+| zones | List of zones in which the cluster resides |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/examples/simple_regional_cluster_autoscaling/main.tf b/examples/simple_regional_cluster_autoscaling/main.tf
@@ -0,0 +1,70 @@
+/**
+ * Copyright 2018-2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  cluster_type = "simple-rgnl-cluster-autosc"
+}
+
+data "google_client_config" "default" {}
+
+provider "kubernetes" {
+  host                   = "https://${module.gke.endpoint}"
+  token                  = data.google_client_config.default.access_token
+  cluster_ca_certificate = base64decode(module.gke.ca_certificate)
+}
+
+module "gke" {
+  source  = "terraform-google-modules/kubernetes-engine/google"
+  version = "~> 33.0"
+
+  project_id                = var.project_id
+  name                      = "${local.cluster_type}-cluster${var.cluster_name_suffix}"
+  regional                  = true
+  region                    = var.region
+  network                   = var.network
+  subnetwork                = var.subnetwork
+  ip_range_pods             = var.ip_range_pods
+  ip_range_services         = var.ip_range_services
+  create_service_account    = false
+  service_account           = var.compute_engine_service_account
+  default_max_pods_per_node = 20
+  remove_default_node_pool  = true
+  deletion_protection       = false
+
+  add_cluster_firewall_rules = true
+  firewall_inbound_ports     = ["8443", "9443", "15017"]
+
+  # Just an example
+  network_tags = ["egress-internet"]
+
+  cluster_autoscaling = {
+    enabled             = true
+    autoscaling_profile = "OPTIMIZE_UTILIZATION"
+    min_cpu_cores       = 4
+    max_cpu_cores       = 86
+    min_memory_gb       = 16
+    max_memory_gb       = 256
+    disk_size           = 100
+    disk_type           = "pd-standard"
+    image_type          = "COS_CONTAINERD"
+    gpu_resources       = []
+    auto_repair         = true
+    auto_upgrade        = true
+    strategy            = "SURGE"
+    max_surge           = 1
+    max_unavailable     = 0
+  }
+}
diff --git a/examples/simple_regional_cluster_autoscaling/outputs.tf b/examples/simple_regional_cluster_autoscaling/outputs.tf
@@ -0,0 +1,36 @@
+/**
+ * Copyright 2018-2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "kubernetes_endpoint" {
+  sensitive = true
+  value     = module.gke.endpoint
+}
+
+output "client_token" {
+  sensitive = true
+  value     = base64encode(data.google_client_config.default.access_token)
+}
+
+output "ca_certificate" {
+  sensitive = true
+  value     = module.gke.ca_certificate
+}
+
+output "service_account" {
+  description = "The default service account used for running nodes."
+  value       = module.gke.service_account
+}
+
diff --git a/examples/simple_regional_cluster_autoscaling/test_outputs.tf b/examples/simple_regional_cluster_autoscaling/test_outputs.tf
@@ -0,0 +1,63 @@
+/**
+ * Copyright 2018-2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+// These outputs are used to test the module with kitchen-terraform
+// They do not need to be included in real-world uses of this module
+
+output "project_id" {
+  value = var.project_id
+}
+
+output "region" {
+  value = module.gke.region
+}
+
+output "cluster_name" {
+  description = "Cluster name"
+  value       = module.gke.name
+}
+
+output "network" {
+  value = var.network
+}
+
+output "subnetwork" {
+  value = var.subnetwork
+}
+
+output "location" {
+  value = module.gke.location
+}
+
+output "ip_range_pods" {
+  description = "The secondary IP range used for pods"
+  value       = var.ip_range_pods
+}
+
+output "ip_range_services" {
+  description = "The secondary IP range used for services"
+  value       = var.ip_range_services
+}
+
+output "zones" {
+  description = "List of zones in which the cluster resides"
+  value       = module.gke.zones
+}
+
+output "master_kubernetes_version" {
+  description = "The master Kubernetes version"
+  value       = module.gke.master_version
+}
diff --git a/examples/simple_regional_cluster_autoscaling/variables.tf b/examples/simple_regional_cluster_autoscaling/variables.tf
@@ -0,0 +1,49 @@
+/**
+ * Copyright 2018-2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "The project ID to host the cluster in"
+}
+
+variable "cluster_name_suffix" {
+  description = "A suffix to append to the default cluster name"
+  default     = ""
+}
+
+variable "region" {
+  description = "The region to host the cluster in"
+}
+
+variable "network" {
+  description = "The VPC network to host the cluster in"
+}
+
+variable "subnetwork" {
+  description = "The subnetwork to host the cluster in"
+}
+
+variable "ip_range_pods" {
+  description = "The secondary ip range to use for pods"
+}
+
+variable "ip_range_services" {
+  description = "The secondary ip range to use for services"
+}
+
+variable "compute_engine_service_account" {
+  description = "Service account to associate to the nodes in the cluster"
+}
+
diff --git a/examples/simple_regional_cluster_autoscaling/versions.tf b/examples/simple_regional_cluster_autoscaling/versions.tf
@@ -0,0 +1,27 @@
+/**
+ * Copyright 2021-2024 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+  required_providers {
+    google = {
+      source = "hashicorp/google"
+    }
+    kubernetes = {
+      source = "hashicorp/kubernetes"
+    }
+  }
+  required_version = ">= 0.13"
+}
diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf
@@ -225,10 +225,10 @@ resource "google_container_cluster" "primary" {
   }
 
   dynamic "node_pool_auto_config" {
-    for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : []
+    for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : []
     content {
       network_tags {
-        tags = var.network_tags
+        tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags
       }
     }
   }

diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf
@@ -225,10 +225,10 @@ resource "google_container_cluster" "primary" {
   }
 
   dynamic "node_pool_auto_config" {
-    for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : []
+    for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : []
     content {
       network_tags {
-        tags = var.network_tags
+        tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags
       }
     }
   }

diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf
@@ -225,10 +225,10 @@ resource "google_container_cluster" "primary" {
   }
 
   dynamic "node_pool_auto_config" {
-    for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : []
+    for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : []
     content {
       network_tags {
-        tags = var.network_tags
+        tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags
       }
     }
   }

diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf
@@ -225,10 +225,10 @@ resource "google_container_cluster" "primary" {
   }
 
   dynamic "node_pool_auto_config" {
-    for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : []
+    for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : []
     content {
       network_tags {
-        tags = var.network_tags
+        tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags
       }
     }
   }