diff --git a/autogen/main/cluster.tf.tmpl b/autogen/main/cluster.tf.tmpl index 596aef8236..a77c2666db 100644 --- a/autogen/main/cluster.tf.tmpl +++ b/autogen/main/cluster.tf.tmpl @@ -281,10 +281,10 @@ resource "google_container_cluster" "primary" { {% if autopilot_cluster != true %} dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags } } } diff --git a/cluster.tf b/cluster.tf index ad40edfcd6..078dbd667d 100644 --- a/cluster.tf +++ b/cluster.tf @@ -203,10 +203,10 @@ resource "google_container_cluster" "primary" { } dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags } } } diff --git a/examples/simple_regional_cluster_autoscaling/README.md b/examples/simple_regional_cluster_autoscaling/README.md new file mode 100644 index 0000000000..16a8ad5f9d --- /dev/null +++ b/examples/simple_regional_cluster_autoscaling/README.md @@ -0,0 +1,39 @@ +# Simple Regional Cluster with Node Autoscaling + +This example illustrates how to create a simple private cluster with cluster +level node autoprovisioning. + + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| cluster\_name\_suffix | A suffix to append to the default cluster name | `string` | `""` | no | +| compute\_engine\_service\_account | Service account to associate to the nodes in the cluster | `any` | n/a | yes | +| ip\_range\_pods | The secondary ip range to use for pods | `any` | n/a | yes | +| ip\_range\_services | The secondary ip range to use for services | `any` | n/a | yes | +| network | The VPC network to host the cluster in | `any` | n/a | yes | +| project\_id | The project ID to host the cluster in | `any` | n/a | yes | +| region | The region to host the cluster in | `any` | n/a | yes | +| subnetwork | The subnetwork to host the cluster in | `any` | n/a | yes | + +## Outputs + +| Name | Description | +|------|-------------| +| ca\_certificate | n/a | +| client\_token | n/a | +| cluster\_name | Cluster name | +| ip\_range\_pods | The secondary IP range used for pods | +| ip\_range\_services | The secondary IP range used for services | +| kubernetes\_endpoint | n/a | +| location | n/a | +| master\_kubernetes\_version | The master Kubernetes version | +| network | n/a | +| project\_id | n/a | +| region | n/a | +| service\_account | The default service account used for running nodes. | +| subnetwork | n/a | +| zones | List of zones in which the cluster resides | + + diff --git a/examples/simple_regional_cluster_autoscaling/main.tf b/examples/simple_regional_cluster_autoscaling/main.tf new file mode 100644 index 0000000000..9669572632 --- /dev/null +++ b/examples/simple_regional_cluster_autoscaling/main.tf @@ -0,0 +1,70 @@ +/** + * Copyright 2018-2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +locals { + cluster_type = "simple-rgnl-cluster-autosc" +} + +data "google_client_config" "default" {} + +provider "kubernetes" { + host = "https://${module.gke.endpoint}" + token = data.google_client_config.default.access_token + cluster_ca_certificate = base64decode(module.gke.ca_certificate) +} + +module "gke" { + source = "terraform-google-modules/kubernetes-engine/google" + version = "~> 33.0" + + project_id = var.project_id + name = "${local.cluster_type}-cluster${var.cluster_name_suffix}" + regional = true + region = var.region + network = var.network + subnetwork = var.subnetwork + ip_range_pods = var.ip_range_pods + ip_range_services = var.ip_range_services + create_service_account = false + service_account = var.compute_engine_service_account + default_max_pods_per_node = 20 + remove_default_node_pool = true + deletion_protection = false + + add_cluster_firewall_rules = true + firewall_inbound_ports = ["8443", "9443", "15017"] + + # Just an example + network_tags = ["egress-internet"] + + cluster_autoscaling = { + enabled = true + autoscaling_profile = "OPTIMIZE_UTILIZATION" + min_cpu_cores = 4 + max_cpu_cores = 86 + min_memory_gb = 16 + max_memory_gb = 256 + disk_size = 100 + disk_type = "pd-standard" + image_type = "COS_CONTAINERD" + gpu_resources = [] + auto_repair = true + auto_upgrade = true + strategy = "SURGE" + max_surge = 1 + max_unavailable = 0 + } +} diff --git a/examples/simple_regional_cluster_autoscaling/outputs.tf b/examples/simple_regional_cluster_autoscaling/outputs.tf new file mode 100644 index 0000000000..e527390d91 --- /dev/null +++ b/examples/simple_regional_cluster_autoscaling/outputs.tf @@ -0,0 +1,36 @@ +/** + * Copyright 2018-2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "kubernetes_endpoint" { + sensitive = true + value = module.gke.endpoint +} + +output "client_token" { + sensitive = true + value = base64encode(data.google_client_config.default.access_token) +} + +output "ca_certificate" { + sensitive = true + value = module.gke.ca_certificate +} + +output "service_account" { + description = "The default service account used for running nodes." + value = module.gke.service_account +} + diff --git a/examples/simple_regional_cluster_autoscaling/test_outputs.tf b/examples/simple_regional_cluster_autoscaling/test_outputs.tf new file mode 100644 index 0000000000..ea5f6c4e41 --- /dev/null +++ b/examples/simple_regional_cluster_autoscaling/test_outputs.tf @@ -0,0 +1,63 @@ +/** + * Copyright 2018-2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// These outputs are used to test the module with kitchen-terraform +// They do not need to be included in real-world uses of this module + +output "project_id" { + value = var.project_id +} + +output "region" { + value = module.gke.region +} + +output "cluster_name" { + description = "Cluster name" + value = module.gke.name +} + +output "network" { + value = var.network +} + +output "subnetwork" { + value = var.subnetwork +} + +output "location" { + value = module.gke.location +} + +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = var.ip_range_pods +} + +output "ip_range_services" { + description = "The secondary IP range used for services" + value = var.ip_range_services +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.gke.zones +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = module.gke.master_version +} diff --git a/examples/simple_regional_cluster_autoscaling/variables.tf b/examples/simple_regional_cluster_autoscaling/variables.tf new file mode 100644 index 0000000000..2ce217972e --- /dev/null +++ b/examples/simple_regional_cluster_autoscaling/variables.tf @@ -0,0 +1,49 @@ +/** + * Copyright 2018-2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_id" { + description = "The project ID to host the cluster in" +} + +variable "cluster_name_suffix" { + description = "A suffix to append to the default cluster name" + default = "" +} + +variable "region" { + description = "The region to host the cluster in" +} + +variable "network" { + description = "The VPC network to host the cluster in" +} + +variable "subnetwork" { + description = "The subnetwork to host the cluster in" +} + +variable "ip_range_pods" { + description = "The secondary ip range to use for pods" +} + +variable "ip_range_services" { + description = "The secondary ip range to use for services" +} + +variable "compute_engine_service_account" { + description = "Service account to associate to the nodes in the cluster" +} + diff --git a/examples/simple_regional_cluster_autoscaling/versions.tf b/examples/simple_regional_cluster_autoscaling/versions.tf new file mode 100644 index 0000000000..62771b4e77 --- /dev/null +++ b/examples/simple_regional_cluster_autoscaling/versions.tf @@ -0,0 +1,27 @@ +/** + * Copyright 2021-2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +terraform { + required_providers { + google = { + source = "hashicorp/google" + } + kubernetes = { + source = "hashicorp/kubernetes" + } + } + required_version = ">= 0.13" +} diff --git a/modules/beta-private-cluster-update-variant/cluster.tf b/modules/beta-private-cluster-update-variant/cluster.tf index 4ce2398e78..d0177bd76e 100644 --- a/modules/beta-private-cluster-update-variant/cluster.tf +++ b/modules/beta-private-cluster-update-variant/cluster.tf @@ -224,10 +224,10 @@ resource "google_container_cluster" "primary" { } dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags } } } diff --git a/modules/beta-private-cluster/cluster.tf b/modules/beta-private-cluster/cluster.tf index 76a2ea084d..3d2bb9f473 100644 --- a/modules/beta-private-cluster/cluster.tf +++ b/modules/beta-private-cluster/cluster.tf @@ -224,10 +224,10 @@ resource "google_container_cluster" "primary" { } dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags } } } diff --git a/modules/beta-public-cluster-update-variant/cluster.tf b/modules/beta-public-cluster-update-variant/cluster.tf index 0bd731d0f9..2b7ba9fc25 100644 --- a/modules/beta-public-cluster-update-variant/cluster.tf +++ b/modules/beta-public-cluster-update-variant/cluster.tf @@ -224,10 +224,10 @@ resource "google_container_cluster" "primary" { } dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags } } } diff --git a/modules/beta-public-cluster/cluster.tf b/modules/beta-public-cluster/cluster.tf index 234d71e718..39cf5f18a7 100644 --- a/modules/beta-public-cluster/cluster.tf +++ b/modules/beta-public-cluster/cluster.tf @@ -224,10 +224,10 @@ resource "google_container_cluster" "primary" { } dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags } } } diff --git a/modules/private-cluster-update-variant/cluster.tf b/modules/private-cluster-update-variant/cluster.tf index 2d2f5ca5f5..0431afdc8f 100644 --- a/modules/private-cluster-update-variant/cluster.tf +++ b/modules/private-cluster-update-variant/cluster.tf @@ -203,10 +203,10 @@ resource "google_container_cluster" "primary" { } dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags } } } diff --git a/modules/private-cluster/cluster.tf b/modules/private-cluster/cluster.tf index 5f3886a4ff..2d6486e70e 100644 --- a/modules/private-cluster/cluster.tf +++ b/modules/private-cluster/cluster.tf @@ -203,10 +203,10 @@ resource "google_container_cluster" "primary" { } dynamic "node_pool_auto_config" { - for_each = var.cluster_autoscaling.enabled && length(var.network_tags) > 0 ? [1] : [] + for_each = var.cluster_autoscaling.enabled && (length(var.network_tags) > 0 || var.add_cluster_firewall_rules) ? [1] : [] content { network_tags { - tags = var.network_tags + tags = var.add_cluster_firewall_rules ? (concat(var.network_tags, [local.cluster_network_tag])) : var.network_tags } } } diff --git a/test/fixtures/simple_regional_cluster_autoscaling/example.tf b/test/fixtures/simple_regional_cluster_autoscaling/example.tf new file mode 100644 index 0000000000..f7651de4fb --- /dev/null +++ b/test/fixtures/simple_regional_cluster_autoscaling/example.tf @@ -0,0 +1,28 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +module "example" { + source = "../../../examples/simple_regional_cluster_autoscaling" + + project_id = var.project_ids[0] + cluster_name_suffix = "-${random_string.suffix.result}" + region = "us-west1" + network = google_compute_network.main.name + subnetwork = google_compute_subnetwork.main.name + ip_range_pods = google_compute_subnetwork.main.secondary_ip_range[0].range_name + ip_range_services = google_compute_subnetwork.main.secondary_ip_range[1].range_name + compute_engine_service_account = var.compute_engine_service_accounts[0] +} diff --git a/test/fixtures/simple_regional_cluster_autoscaling/network.tf b/test/fixtures/simple_regional_cluster_autoscaling/network.tf new file mode 100644 index 0000000000..03e1e87b16 --- /dev/null +++ b/test/fixtures/simple_regional_cluster_autoscaling/network.tf @@ -0,0 +1,48 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +resource "random_string" "suffix" { + length = 4 + special = false + upper = false +} + +provider "google" { + project = var.project_ids[0] +} + +resource "google_compute_network" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + auto_create_subnetworks = false +} + +resource "google_compute_subnetwork" "main" { + name = "cft-gke-test-${random_string.suffix.result}" + ip_cidr_range = "10.0.0.0/17" + region = "us-west1" + network = google_compute_network.main.self_link + + secondary_ip_range { + range_name = "cft-gke-test-pods-${random_string.suffix.result}" + ip_cidr_range = "192.168.0.0/18" + } + + secondary_ip_range { + range_name = "cft-gke-test-services-${random_string.suffix.result}" + ip_cidr_range = "192.168.64.0/18" + } +} + diff --git a/test/fixtures/simple_regional_cluster_autoscaling/outputs.tf b/test/fixtures/simple_regional_cluster_autoscaling/outputs.tf new file mode 100644 index 0000000000..57b647ad8a --- /dev/null +++ b/test/fixtures/simple_regional_cluster_autoscaling/outputs.tf @@ -0,0 +1,85 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +output "project_id" { + value = module.example.project_id +} + +output "region" { + value = module.example.region +} + +output "cluster_name" { + description = "Cluster name" + value = module.example.cluster_name +} + +output "network" { + value = google_compute_network.main.name +} + +output "subnetwork" { + value = google_compute_subnetwork.main.name +} + +output "location" { + value = module.example.location +} + +output "ip_range_pods" { + description = "The secondary IP range used for pods" + value = google_compute_subnetwork.main.secondary_ip_range[0].range_name +} + +output "ip_range_services" { + description = "The secondary IP range used for services" + value = google_compute_subnetwork.main.secondary_ip_range[1].range_name +} + +output "zones" { + description = "List of zones in which the cluster resides" + value = module.example.zones +} + +output "master_kubernetes_version" { + description = "The master Kubernetes version" + value = module.example.master_kubernetes_version +} + +output "kubernetes_endpoint" { + sensitive = true + value = module.example.kubernetes_endpoint +} + +output "client_token" { + sensitive = true + value = module.example.client_token +} + +output "ca_certificate" { + description = "The cluster CA certificate" + value = module.example.ca_certificate + sensitive = true +} + +output "service_account" { + description = "The service account to default running nodes as if not overridden in `node_pools`." + value = module.example.service_account +} + +output "registry_project_ids" { + value = var.registry_project_ids +} diff --git a/test/fixtures/simple_regional_cluster_autoscaling/variables.tf b/test/fixtures/simple_regional_cluster_autoscaling/variables.tf new file mode 100644 index 0000000000..a5bf1908f1 --- /dev/null +++ b/test/fixtures/simple_regional_cluster_autoscaling/variables.tf @@ -0,0 +1,47 @@ +/** + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +variable "project_ids" { + type = list(string) + description = "The GCP projects to use for integration tests" +} + +variable "region" { + description = "The GCP region to create and test resources in" + default = "us-central1" +} + +variable "zones" { + type = list(string) + description = "The GCP zones to create and test resources in, for applicable tests" + default = ["us-central1-a", "us-central1-b", "us-central1-c"] +} + +variable "compute_engine_service_accounts" { + type = list(string) + description = "The email addresses of the service account to associate with the GKE cluster" +} + +variable "registry_project_ids" { + description = "Projects to use for granting access to GCR registries, if requested" + type = list(string) +} + +variable "kubernetes_version" { + type = string + description = "The Kubernetes version of the masters. If set to 'latest' it will pull latest available version in the selected region." + default = "latest" +} diff --git a/test/integration/simple_regional_cluster_autoscaling/simple_regional_cluster_autoscaling_test.go b/test/integration/simple_regional_cluster_autoscaling/simple_regional_cluster_autoscaling_test.go new file mode 100644 index 0000000000..2c12f84750 --- /dev/null +++ b/test/integration/simple_regional_cluster_autoscaling/simple_regional_cluster_autoscaling_test.go @@ -0,0 +1,68 @@ +// Copyright 2022-2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package simple_regional_cluster_autoscaling + +import ( + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" + "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" +) + +func TestSimpleRegionalClusterAutoscaling(t *testing.T) { + bpt := tft.NewTFBlueprintTest(t, + tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), + ) + + bpt.DefineVerify(func(assert *assert.Assertions) { + //Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token + // bpt.DefaultVerify(assert) + + projectId := bpt.GetStringOutput("project_id") + location := bpt.GetStringOutput("location") + clusterName := bpt.GetStringOutput("cluster_name") + serviceAccount := bpt.GetStringOutput("service_account") + + op := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId) + g := golden.NewOrUpdate(t, op.String(), + golden.WithSanitizer(golden.StringSanitizer(serviceAccount, "SERVICE_ACCOUNT")), + golden.WithSanitizer(golden.StringSanitizer(projectId, "PROJECT_ID")), + golden.WithSanitizer(golden.StringSanitizer(clusterName, "CLUSTER_NAME")), + ) + validateJSONPaths := []string{ + "status", + "location", + "privateClusterConfig.enablePrivateEndpoint", + "privateClusterConfig.enablePrivateNodes", + "addonsConfig", + "databaseEncryption", + "shieldedNodes", + "binaryAuthorization", + "nodePools.autoscaling", + "nodePools.config", + "nodePools.config.tags", + "nodePools.management", + } + for _, pth := range validateJSONPaths { + g.JSONEq(assert, op, pth) + } + + }) + + bpt.Test() +} diff --git a/test/integration/simple_regional_cluster_autoscaling/testdata/TestSimpleRegionalClusterAutoscaling.json b/test/integration/simple_regional_cluster_autoscaling/testdata/TestSimpleRegionalClusterAutoscaling.json new file mode 100644 index 0000000000..16d6297fdc --- /dev/null +++ b/test/integration/simple_regional_cluster_autoscaling/testdata/TestSimpleRegionalClusterAutoscaling.json @@ -0,0 +1,333 @@ +{ + "addonsConfig": { + "configConnectorConfig": {}, + "dnsCacheConfig": {}, + "gcePersistentDiskCsiDriverConfig": { + "enabled": true + }, + "gcpFilestoreCsiDriverConfig": {}, + "gcsFuseCsiDriverConfig": { + "enabled": true + }, + "gkeBackupAgentConfig": {}, + "horizontalPodAutoscaling": {}, + "httpLoadBalancing": {}, + "kubernetesDashboard": { + "disabled": true + }, + "networkPolicyConfig": { + "disabled": true + }, + "statefulHaConfig": { + "enabled": true + } + }, + "autopilot": {}, + "autoscaling": { + "autoscalingProfile": "BALANCED" + }, + "binaryAuthorization": { + "evaluationMode": "PROJECT_SINGLETON_POLICY_ENFORCE" + }, + "clusterIpv4Cidr": "192.168.0.0/18", + "costManagementConfig": { + "enabled": true + }, + "createTime": "2023-08-25T16:39:57+00:00", + "currentMasterVersion": "1.27.3-gke.100", + "currentNodeCount": 3, + "currentNodeVersion": "1.27.3-gke.100", + "databaseEncryption": { + "currentState": "CURRENT_STATE_DECRYPTED", + "state": "DECRYPTED" + }, + "defaultMaxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "endpoint": "35.238.117.51", + "etag": "a214a2ff-16ea-4e65-8c01-43e7a6cece82", + "id": "8e4011253bcb4fbc943f88ae797f124f0f001ed95cc94b229231d68b8a44e20b", + "initialClusterVersion": "1.27.3-gke.100", + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-a/instanceGroupManagers/gke-simple-regional-clus-default-pool-209983a6-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-c/instanceGroupManagers/gke-simple-regional-clus-default-pool-6094c28c-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-f/instanceGroupManagers/gke-simple-regional-clus-default-pool-a9225012-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-a/instanceGroupManagers/gke-simple-regional--default-node-poo-8fb4fd0f-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-c/instanceGroupManagers/gke-simple-regional--default-node-poo-24ffed9f-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-f/instanceGroupManagers/gke-simple-regional--default-node-poo-1d2dc357-grp" + ], + "ipAllocationPolicy": { + "clusterIpv4Cidr": "192.168.0.0/18", + "clusterIpv4CidrBlock": "192.168.0.0/18", + "clusterSecondaryRangeName": "cft-gke-test-pods-44kc", + "defaultPodIpv4RangeUtilization": 0.0469, + "podCidrOverprovisionConfig": {}, + "servicesIpv4Cidr": "192.168.64.0/18", + "servicesIpv4CidrBlock": "192.168.64.0/18", + "servicesSecondaryRangeName": "cft-gke-test-services-44kc", + "stackType": "IPV4", + "useIpAliases": true + }, + "labelFingerprint": "a9dc16a7", + "legacyAbac": {}, + "location": "us-west1", + "locations": [ + "us-west1-a", + "us-west1-c", + "us-west1-f" + ], + "loggingConfig": { + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS", + "WORKLOADS" + ] + } + }, + "loggingService": "logging.googleapis.com/kubernetes", + "maintenancePolicy": { + "resourceVersion": "ce912209", + "window": { + "dailyMaintenanceWindow": { + "duration": "PT4H0M0S", + "startTime": "05:00" + } + } + }, + "masterAuth": { + "clusterCaCertificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVMRENDQXBTZ0F3SUJBZ0lRQ0J5aEs4ZEtSMTNBeGIrTkxTM1FSREFOQmdrcWhraUc5dzBCQVFzRkFEQXYKTVMwd0t3WURWUVFERXlReE56VTNaR1l4WkMwMFpUY3dMVFF5TVdZdE9XWmpPUzB4TVRZelpXRXdNbUpsWmpRdwpJQmNOTWpNd09ESTFNVFV6T1RVM1doZ1BNakExTXpBNE1UY3hOak01TlRkYU1DOHhMVEFyQmdOVkJBTVRKREUzCk5UZGtaakZrTFRSbE56QXROREl4WmkwNVptTTVMVEV4TmpObFlUQXlZbVZtTkRDQ0FhSXdEUVlKS29aSWh2Y04KQVFFQkJRQURnZ0dQQURDQ0FZb0NnZ0dCQUtDYW1KRWwvT04rUUxIWGh0dVVaVVQ5VHFYMnNoeTZtL2lvQ3d0dgp1TG9LVTYzQzRvS0N1ajQzbXRYMVFTaUFRLzloNHVqV0xucW9PUzdpd2FrWnZHMHpBdjJrZmNuVUtXQnM5MTJRCm40K05pcU1EVWRyd1Fab0NwMTl2MDhGOFBSTTZCWFVIRUNnQUtvanhSeVpyTzkrZDVESndXNFRHUzhJWVNleTUKR2xnUkFyQk5LMXBzb09ZMHhaZGpxTUxDNnpkTEMyT01PaVhOcXVMUGNkUFk0aHFGSDZ0cXFOY09kTllNaWVBUwpwNGM3YS83WFA1cnF0M1NiSlhYbkZmVFFmL1dablJiUWh1SDRxRmpUUUVrdEdieW9rT0tEQ09QS0lGNEVwc1czCmR5M2ZVNkhDaWR2V2JCSHEvV21JVWpRWmxlSE5UWkZack5HVC9tcmF2R0RsWjZFaHkvY2psam5DZi90NXduK1AKb1BOZUVjaWxkZDhmNWpqQTc4TW1ValNiV3Z0c01xY2VKTEQ3eUx4RUhGNEZnOXJ1c2wyZU9SZDBQSFkzYmFOZAp4SUlVcVF0M3JORmc4Q0NoSnA0N2w1T3YzTHQzWTI5SjVTRHo2dkM4c3psUkl3NE1TNVQ0OGQ0T200T0VrcVRuCkFmWGw2TjFETzVGRjJKOHZCeHA4Q2Q3NEd3SURBUUFCbzBJd1FEQU9CZ05WSFE4QkFmOEVCQU1DQWdRd0R3WUQKVlIwVEFRSC9CQVV3QXdFQi96QWRCZ05WSFE0RUZnUVVaa1FWVXdBcmZLZEpBSWZzUFBSdmFraktabll3RFFZSgpLb1pJaHZjTkFRRUxCUUFEZ2dHQkFIenEvaWxoekNMeEJuYkp2c2lsdkthTXRpbXRxSjNqTFh5NkdvYW56U3F0ClY1STZFZ01Jd3Zubml4ZVhOb0lrQnhJbDlGcUVmV1g0eHFGLytQb1JoU1dMZHkyZUk1d1I4WkJzaU9lcDA0ZTgKdW80Nk1tdU93alZkM2pOZExOdW1ZRlRqVzR3TStKUytPNUQ0SUo3YXpPVVF1K2gvd2ViQ1pXSjkxbUcxcUtnRQo4WVM3d3pKYlJiU0lQL2tMdWN6aGM4V2dnL1Z2czBldklaNFY2SHRFeFo1UmFjZ3lMcGhvKzNBQlFjakwvVDFWCjRLQ08zUFAyMmo5ZkRLUXNYdEppMHNmV3lMSmwybERwb1ZHMGJGZ2l1MFJxY05XM2dDTTNZYWs3dVRpelozOWYKU3I1NHU0SzBMQVdXbnRqc0NBR0RkajNFYmw0eFVGWWo5SG1zVURKNm5ncFdmejBHeHJQL29rUm5WV05MZjRJMwpuRlYzb0o4S0ZQRjVhUmQwYXNKZnc2a3VpZTNwZytYanlVRlp5ek51a1ZqcUFXdWFVV1lPUjg5Mnd3T0tnNFUrCnJQTUU3RzhqRFBjNWt0MVZ4dktQY0NOckkxdjgralBTSktDYnhCWFU2bXdRSFRnUGVzVWJuUDkrd1FobWpkdHkKME1JOFlpOWltdWtmTy9JMHpSRkJDdz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K" + }, + "masterAuthorizedNetworksConfig": { + "gcpPublicCidrsAccessEnabled": true + }, + "monitoringConfig": { + "advancedDatapathObservabilityConfig": { + "enableRelay": false + }, + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS" + ] + }, + "managedPrometheusConfig": {} + }, + "monitoringService": "monitoring.googleapis.com/kubernetes", + "name": "CLUSTER_NAME", + "network": "cft-gke-test-44kc", + "networkConfig": { + "defaultSnatStatus": {}, + "network": "projects/PROJECT_ID/global/networks/cft-gke-test-44kc", + "serviceExternalIpsConfig": {}, + "subnetwork": "projects/PROJECT_ID/regions/us-west1/subnetworks/cft-gke-test-44kc" + }, + "nodeConfig": { + "diskSizeGb": 100, + "diskType": "pd-balanced", + "imageType": "COS_CONTAINERD", + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "e2-medium", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "gke-sa-int-test-p1-e155@PROJECT_ID.iam.gserviceaccount.com", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-CLUSTER_NAME", + "gke-CLUSTER_NAME-default-pool" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "nodePoolAutoConfig": {}, + "nodePoolDefaults": { + "nodeConfigDefaults": { + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + } + } + }, + "nodePools": [ + { + "config": { + "diskSizeGb": 100, + "diskType": "pd-balanced", + "imageType": "COS_CONTAINERD", + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "e2-medium", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "gke-sa-int-test-p1-e155@PROJECT_ID.iam.gserviceaccount.com", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-CLUSTER_NAME", + "gke-CLUSTER_NAME-default-pool" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "3bb00743-9dbf-4e92-9512-1e64fb8f1b8a", + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-a/instanceGroupManagers/gke-simple-regional-clus-default-pool-209983a6-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-c/instanceGroupManagers/gke-simple-regional-clus-default-pool-6094c28c-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-f/instanceGroupManagers/gke-simple-regional-clus-default-pool-a9225012-grp" + ], + "locations": [ + "us-west1-a", + "us-west1-c", + "us-west1-f" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "default-pool", + "networkConfig": { + "enablePrivateNodes": false, + "podIpv4CidrBlock": "192.168.0.0/18", + "podIpv4RangeUtilization": 0.0469, + "podRange": "cft-gke-test-pods-44kc" + }, + "podIpv4CidrSize": 24, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-west1/clusters/CLUSTER_NAME/nodePools/default-pool", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.27.3-gke.100" + }, + { + "autoscaling": { + "enabled": true, + "locationPolicy": "BALANCED", + "maxNodeCount": 100, + "minNodeCount": 1 + }, + "config": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "imageType": "COS_CONTAINERD", + "labels": { + "cluster_name": "CLUSTER_NAME", + "node_pool": "default-node-pool" + }, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "e2-medium", + "metadata": { + "cluster_name": "CLUSTER_NAME", + "disable-legacy-endpoints": "true", + "node_pool": "default-node-pool" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "gke-sa-int-test-p1-e155@PROJECT_ID.iam.gserviceaccount.com", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-CLUSTER_NAME", + "gke-CLUSTER_NAME-default-node-pool" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "c1cb03bd-8b4e-4a06-9c4e-213b87aa86a3", + "initialNodeCount": 1, + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-a/instanceGroupManagers/gke-simple-regional--default-node-poo-8fb4fd0f-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-c/instanceGroupManagers/gke-simple-regional--default-node-poo-24ffed9f-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/us-west1-f/instanceGroupManagers/gke-simple-regional--default-node-poo-1d2dc357-grp" + ], + "locations": [ + "us-west1-a", + "us-west1-c", + "us-west1-f" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "default-node-pool", + "networkConfig": { + "enablePrivateNodes": false, + "podIpv4CidrBlock": "192.168.0.0/18", + "podIpv4RangeUtilization": 0.0469, + "podRange": "cft-gke-test-pods-44kc" + }, + "podIpv4CidrSize": 24, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-west1/clusters/CLUSTER_NAME/nodePools/default-node-pool", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.27.3-gke.100" + } + ], + "notificationConfig": { + "pubsub": {} + }, + "privateClusterConfig": { + "privateEndpoint": "10.0.0.2", + "publicEndpoint": "35.238.117.51" + }, + "releaseChannel": { + "channel": "REGULAR" + }, + "securityPostureConfig": { + "mode": "BASIC", + "vulnerabilityMode": "VULNERABILITY_MODE_UNSPECIFIED" + }, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/us-west1/clusters/CLUSTER_NAME", + "servicesIpv4Cidr": "192.168.64.0/18", + "shieldedNodes": { + "enabled": true + }, + "status": "RUNNING", + "subnetwork": "cft-gke-test-44kc", + "verticalPodAutoscaling": {}, + "workloadIdentityConfig": { + "workloadPool": "PROJECT_ID.svc.id.goog" + }, + "zone": "us-west1" +}