diff --git a/.kitchen.yml b/.kitchen.yml index c8a72788f3..62beeda102 100644 --- a/.kitchen.yml +++ b/.kitchen.yml @@ -78,19 +78,6 @@ suites: systems: - name: stub_domains_upstream_nameservers backend: local - - name: "workload_identity" - transport: - root_module_directory: test/fixtures/workload_identity - verifier: - systems: - - name: gcloud - backend: local - controls: - - gcloud - - name: gcp - backend: gcp - controls: - - gcp - name: "workload_metadata_config" transport: root_module_directory: test/fixtures/workload_metadata_config @@ -98,20 +85,3 @@ suites: systems: - name: workload_metadata_config backend: local - - name: "node_pool" - transport: - root_module_directory: test/fixtures/node_pool - verifier: - systems: - - name: node_pool - backend: local - controls: - - gcloud - - kubectl - - name: "safer_cluster_iap_bastion" - transport: - root_module_directory: test/fixtures/safer_cluster_iap_bastion - verifier: - systems: - - name: safer_cluster_iap_bastion - backend: local diff --git a/build/int.cloudbuild.yaml b/build/int.cloudbuild.yaml index 8a22a3dbeb..d0daea9996 100644 --- a/build/int.cloudbuild.yaml +++ b/build/int.cloudbuild.yaml @@ -309,17 +309,17 @@ steps: waitFor: - create-all name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge node-pool-local'] + args: ['/bin/bash', '-c', 'cft test run TestNodePool --stage apply --verbose'] - id: verify node-pool-local waitFor: - converge node-pool-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify node-pool-local'] + args: ['/bin/bash', '-c', 'cft test run TestNodePool --stage verify --verbose'] - id: destroy node-pool-local waitFor: - verify node-pool-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy node-pool-local'] + args: ['/bin/bash', '-c', 'cft test run TestNodePool --stage destroy --verbose'] - id: apply sandbox-enabled-local waitFor: - create-all @@ -339,32 +339,32 @@ steps: waitFor: - create-all name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge workload-identity-local'] + args: ['/bin/bash', '-c', 'cft test run TestWorkloadIdentity --stage apply --verbose'] - id: verify workload-identity-local waitFor: - converge workload-identity-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify workload-identity-local'] + args: ['/bin/bash', '-c', 'cft test run TestWorkloadIdentity --stage verify --verbose'] - id: destroy workload-identity-local waitFor: - verify workload-identity-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy workload-identity-local'] + args: ['/bin/bash', '-c', 'cft test run TestWorkloadIdentity --stage destroy --verbose'] - id: converge safer-cluster-iap-bastion-local waitFor: - create-all name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge safer-cluster-iap-bastion-local'] + args: ['/bin/bash', '-c', 'cft test run TestSaferClusterIapBastion --stage apply --verbose'] - id: verify safer-cluster-iap-bastion-local waitFor: - converge safer-cluster-iap-bastion-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify safer-cluster-iap-bastion-local'] + args: ['/bin/bash', '-c', 'cft test run TestSaferClusterIapBastion --stage verify --verbose'] - id: destroy safer-cluster-iap-bastion-local waitFor: - verify safer-cluster-iap-bastion-local name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS' - args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy safer-cluster-iap-bastion-local'] + args: ['/bin/bash', '-c', 'cft test run TestSaferClusterIapBastion --stage teardown --verbose'] - id: apply simple-zonal-with-asm-local waitFor: - create-all diff --git a/examples/node_pool/main.tf b/examples/node_pool/main.tf index 6c073919b5..ffbb8030dd 100644 --- a/examples/node_pool/main.tf +++ b/examples/node_pool/main.tf @@ -43,6 +43,7 @@ module "gke" { disable_legacy_metadata_endpoints = false cluster_autoscaling = var.cluster_autoscaling deletion_protection = false + service_account = "default" node_pools = [ { diff --git a/examples/workload_identity/main.tf b/examples/workload_identity/main.tf index 09b1acbf86..1032cb2cd3 100644 --- a/examples/workload_identity/main.tf +++ b/examples/workload_identity/main.tf @@ -1,5 +1,5 @@ /** - * Copyright 2018 Google LLC + * Copyright 2018-2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,6 +24,10 @@ provider "kubernetes" { host = "https://${module.gke.endpoint}" token = data.google_client_config.default.access_token cluster_ca_certificate = base64decode(module.gke.ca_certificate) + + ignore_annotations = [ + "^iam.gke.io\\/.*" + ] } module "gke" { diff --git a/test/fixtures/safer_cluster_iap_bastion/example.tf b/test/fixtures/safer_cluster_iap_bastion/example.tf index c89179e0e9..060141a61b 100644 --- a/test/fixtures/safer_cluster_iap_bastion/example.tf +++ b/test/fixtures/safer_cluster_iap_bastion/example.tf @@ -1,5 +1,5 @@ /** - * Copyright 2020 Google LLC + * Copyright 2020-2024 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -15,7 +15,7 @@ */ locals { - test_command = "gcloud beta compute ssh ${module.example.bastion_name} --tunnel-through-iap --verbosity=error --project ${var.project_ids[1]} --zone ${module.example.bastion_zone} --ssh-flag=\"-T\" -q -- curl -sS https://${module.example.endpoint}/version -k" + test_command = "gcloud beta compute ssh ${module.example.bastion_name} --tunnel-through-iap --verbosity=error --project ${var.project_ids[1]} --zone ${module.example.bastion_zone} -q -- curl -sS https://${module.example.endpoint}/version -k" } module "example" { diff --git a/test/integration/node_pool/node_pool_test.go b/test/integration/node_pool/node_pool_test.go new file mode 100644 index 0000000000..f0c2cecc8c --- /dev/null +++ b/test/integration/node_pool/node_pool_test.go @@ -0,0 +1,88 @@ +// Copyright 2022-2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package node_pool + +import ( + "fmt" + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" + "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" + gkeutils "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/utils" +) + +func TestNodePool(t *testing.T) { + bpt := tft.NewTFBlueprintTest(t, + tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), + ) + + bpt.DefineVerify(func(assert *assert.Assertions) { + //Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token + // bpt.DefaultVerify(assert) + gkeutils.TGKEVerify(t, bpt, assert) // Verify Resources + + projectId := bpt.GetStringOutput("project_id") + location := bpt.GetStringOutput("location") + clusterName := bpt.GetStringOutput("cluster_name") + + op := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId) + assert.Contains([]string{"RUNNING", "RECONCILING"}, op.Get("status").String(), "Cluster is Running") + + //or _, np := range op.Get("nodePools").Array() { + // npName := np.Get("name").String() + // switch npName { + + //TODO + + // case "pool-03": + // assert.JSONEq(fmt.Sprintf(`["%s-b", "%s-c"]`, location, location) , np.Get("locations").String()) + + // case "pool-05": + + // } + //} + assert.Equal("pool-03", op.Get("nodePools.#(name==\"pool-03\").name").String(), "pool-3 exists") + assert.JSONEq(fmt.Sprintf(`["%s-b", "%s-c"]`, location, location), op.Get("nodePools.#(name==\"pool-03\").locations").String(), "has nodes in correct locations") + assert.Equal("n1-standard-2", op.Get("nodePools.#(name==\"pool-03\").config.machineType").String(), "is the expected machine type") + assert.True(op.Get("nodePools.#(name==\"pool-03\").autoscaling.enabled").Bool(), "has autoscaling enabled") + assert.Equal(2, op.Get("nodePools.#(name==\"pool-03\").initialNodeCount").Float(), "is the expected machine type") + assert.True(op.Get("nodePools.#(name==\"pool-03\").management.autoRepair").Bool(), "has autorepair enabled") + assert.True(op.Get("nodePools.#(name==\"pool-03\").management.autoUpgrade").Bool(), "has automatic upgrades enabled") + assert.JSONEq(fmt.Sprintf(`{"all-pools-example": "true", "cluster_name": %s, "node_pool": "pool-03", "sandbox.gke.io/runtime": "gvisor"}`, clusterName), + op.Get("nodePools.#(name==\"pool-03\").config.labels").String(), "has the expected labels") + assert.Contains([]string{"all-node-example", + fmt.Sprintf("gke-%s", clusterName), + fmt.Sprintf("gke-%s-pool-03", clusterName)}, + op.Get("nodePools.#(name==\"pool-03\").config.tags").Array(), "has the expected network tags") + assert.Equal("172.16.0.0/18", op.Get("nodePools.#(name==\"pool-03\").networkConfig.podIpv4CidrBlock").String(), "has the expected pod range") + assert.Equal("test", op.Get("nodePools.#(name==\"pool-03\").networkConfig.podRange").String(), "has the expected pod range") + assert.Equal("COS_CONTAINERD", op.Get("nodePools.#(name==\"pool-03\").config.imageType").String(), "has the expected image") + assert.Equal("static", op.Get("nodePools.#(name==\"pool-03\").config.kubeletConfig.cpuManagerPolicy").String(), "has the expected kubelet config") + assert.True(op.Get("nodePools.#(name==\"pool-03\").management.config.kubeletConfig.cpuCfsQuota").Bool(), "has the expected kubelet config") + assert.Equal(20000, op.Get("nodePools.#(name==\"pool-03\").config.linuxNodeConfig.sysctls.net.core.netdev_max_backlog").Float(), "has the expected linux node config sysctls") + + + assert.Equal("pool-04", op.Get("nodePools.#(name==\"pool-04\").name").String(), "pool-4 exists") + assert.True(true, op.Get("nodePools.#(name==\"pool-04\").queued_provisioning.enabled").Bool(), "has queued provisioning enabled") + + assert.Equal("pool-05", op.Get("nodePools.#(name==\"pool-05\").name").String(), "pool-5 exists") + assert.True(op.Get("nodePools.#(name==\"pool-05\").advanced_machine_features.enable_nested_virtualization").Bool(), "has enable_nested_virtualization enabled") + + }) + + bpt.Test() +} diff --git a/test/integration/safer_cluster_iap_bastion/controls/e2e.rb b/test/integration/safer_cluster_iap_bastion/controls/e2e.rb deleted file mode 100644 index de185c4871..0000000000 --- a/test/integration/safer_cluster_iap_bastion/controls/e2e.rb +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -test_command = attribute('test_command') -cluster_version = attribute('cluster_version') -# pre run ssh command so that ssh-keygen can run -%x( #{test_command} ) -control "e2e" do - title "SSH into VM and verify connectivity to GKE" - describe command(test_command) do - its(:exit_status) { should eq 0 } - its(:stderr) { should eq '' } - let!(:data) do - if subject.exit_status == 0 - JSON.parse(subject.stdout) - else - {} - end - end - describe "gke version" do - it "is correct" do - expect(data['gitVersion']).to eq "v#{cluster_version}" - end - end - end -end diff --git a/test/integration/safer_cluster_iap_bastion/inspec.yml b/test/integration/safer_cluster_iap_bastion/inspec.yml deleted file mode 100644 index 1ad18a1c00..0000000000 --- a/test/integration/safer_cluster_iap_bastion/inspec.yml +++ /dev/null @@ -1,22 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: safer_cluster -attributes: - - name: test_command - required: true - type: string - - name: cluster_version - required: true - type: string diff --git a/test/integration/safer_cluster_iap_bastion/safer_cluster_iap_bastion_test.go b/test/integration/safer_cluster_iap_bastion/safer_cluster_iap_bastion_test.go new file mode 100644 index 0000000000..3ed30f73fe --- /dev/null +++ b/test/integration/safer_cluster_iap_bastion/safer_cluster_iap_bastion_test.go @@ -0,0 +1,56 @@ +// Copyright 2022-2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package safer_cluster_iap_bastion + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" + "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" + gkeutils "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/utils" +) + +func TestSaferClusterIapBastion(t *testing.T) { + bpt := tft.NewTFBlueprintTest(t, + tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), + ) + + bpt.DefineVerify(func(assert *assert.Assertions) { + //Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token + // bpt.DefaultVerify(assert) + gkeutils.TGKEVerify(t, bpt, assert) // Verify Resources + + test_command, _ := strings.CutPrefix(bpt.GetStringOutput("test_command"), "gcloud ") + + //pre run ssh command so that ssh-keygen can run + gcloud.RunCmd(t, test_command, + gcloud.WithCommonArgs([]string{}), + ) + + cluster_version := fmt.Sprintf("v%s", bpt.GetStringOutput("cluster_version")) + + op := gcloud.Run(t, test_command, + gcloud.WithCommonArgs([]string{}), + ) + + assert.Equal(cluster_version, op.Get("gitVersion").String(), "SSH into VM and verify connectivity to GKE") + }) + + bpt.Test() +} diff --git a/test/integration/workload_identity/controls/gcloud.rb b/test/integration/workload_identity/controls/gcloud.rb deleted file mode 100644 index 1c956052eb..0000000000 --- a/test/integration/workload_identity/controls/gcloud.rb +++ /dev/null @@ -1,80 +0,0 @@ -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -project_id = attribute('project_id') -location = attribute('location') -cluster_name = attribute('cluster_name') -wi_gsa_to_k8s_sa = { - attribute('default_wi_email') => attribute('default_wi_ksa_name'), - attribute('existing_ksa_email') => attribute('existing_ksa_name'), - attribute('existing_gsa_email') => attribute('existing_gsa_name') -} - -control "gcloud" do - title "Google Compute Engine GKE configuration" - describe command("gcloud beta --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json --format=\"json(nodePools[0].config.workloadMetadataConfig.nodeMetadata)\"") do - its(:exit_status) { should eq 0 } - its(:stderr) { should eq '' } - - let!(:data) do - if subject.exit_status == 0 - JSON.parse(subject.stdout) - else - {} - end - end - - describe "workload metada config" do - it "is secure" do - expect(data['nodePools'][0]["config"]["workloadMetadataConfig"]["nodeMetadata"]).to eq 'GKE_METADATA_SERVER' - end - end - end - - describe command("gcloud beta --project=#{project_id} container clusters --zone=#{location} describe #{cluster_name} --format=json --format=\"json(workloadIdentityConfig)\"") do - its(:exit_status) { should eq 0 } - its(:stderr) { should eq '' } - - let!(:data) do - if subject.exit_status == 0 - JSON.parse(subject.stdout) - else - {} - end - end - - describe "workload identity config" do - it "is has correct namespace" do - expect(data["workloadIdentityConfig"]["identityNamespace"]).to eq "#{project_id}.svc.id.goog" - end - end - end - wi_gsa_to_k8s_sa.each do |gsa_email,ksa_name| - describe command("gcloud iam service-accounts get-iam-policy #{gsa_email} --format=json") do - its(:exit_status) { should eq 0 } - its(:stderr) { should eq '' } - - let!(:iam) do - if subject.exit_status == 0 - JSON.parse(subject.stdout) - else - {} - end - end - it "has expected workload identity user roles" do - expect(iam['bindings'][0]).to include("members" => ["serviceAccount:#{project_id}.svc.id.goog[default/#{ksa_name}]"], "role" => "roles/iam.workloadIdentityUser") - end - end - end -end diff --git a/test/integration/workload_identity/inspec.yml b/test/integration/workload_identity/inspec.yml deleted file mode 100644 index 61f2f306aa..0000000000 --- a/test/integration/workload_identity/inspec.yml +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright 2021 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: workload_metadata_config -attributes: - - name: cluster_name - required: true - type: string - - name: location - required: true - type: string - - name: project_id - required: true - type: string - - name: default_wi_email - required: true - type: string - - name: default_wi_ksa_name - required: true - type: string - - name: existing_ksa_email - required: true - type: string - - name: existing_ksa_name - required: true - type: string - - name: existing_gsa_email - required: true - type: string - - name: existing_gsa_name - required: true - type: string diff --git a/test/integration/workload_identity/workload_identity_test.go b/test/integration/workload_identity/workload_identity_test.go new file mode 100644 index 0000000000..e3e68936a3 --- /dev/null +++ b/test/integration/workload_identity/workload_identity_test.go @@ -0,0 +1,49 @@ +// Copyright 2022-2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package workload_identity + +import ( + "fmt" + "testing" + "time" + + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/stretchr/testify/assert" + "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" + gkeutils "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/utils" +) + +func TestWorkloadIdentity(t *testing.T) { + bpt := tft.NewTFBlueprintTest(t, + tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute), + ) + + bpt.DefineVerify(func(assert *assert.Assertions) { + //Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token + // bpt.DefaultVerify(assert) + gkeutils.TGKEVerify(t, bpt, assert) // Verify Resources + + projectId := bpt.GetStringOutput("project_id") + location := bpt.GetStringOutput("location") + clusterName := bpt.GetStringOutput("cluster_name") + + op := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId) + assert.Contains([]string{"RUNNING", "RECONCILING"}, op.Get("status").String(), "Cluster is Running") + assert.Equal("GKE_METADATA", op.Get("nodePools.0.config.workloadMetadataConfig.mode").String(), "workload metada config is secure") + assert.Equal(fmt.Sprintf("%s.svc.id.goog", projectId), op.Get("workloadIdentityConfig.workloadPool").String(), "workload identity config has correct project") + }) + + bpt.Test() +}