diff --git a/test/fixtures/node_pool/outputs.tf b/test/fixtures/node_pool/outputs.tf index a62317bf64..74103ff0be 100644 --- a/test/fixtures/node_pool/outputs.tf +++ b/test/fixtures/node_pool/outputs.tf @@ -83,3 +83,7 @@ output "service_account" { output "registry_project_ids" { value = var.registry_project_ids } + +output "random_string" { + value = random_string.suffix.result +} diff --git a/test/fixtures/safer_cluster_iap_bastion/example.tf b/test/fixtures/safer_cluster_iap_bastion/example.tf index 060141a61b..1de8da7fae 100644 --- a/test/fixtures/safer_cluster_iap_bastion/example.tf +++ b/test/fixtures/safer_cluster_iap_bastion/example.tf @@ -15,7 +15,7 @@ */ locals { - test_command = "gcloud beta compute ssh ${module.example.bastion_name} --tunnel-through-iap --verbosity=error --project ${var.project_ids[1]} --zone ${module.example.bastion_zone} -q -- curl -sS https://${module.example.endpoint}/version -k" + test_command = "gcloud beta compute ssh ${module.example.bastion_name} --tunnel-through-iap --verbosity=error --project ${var.project_ids[1]} --zone ${module.example.bastion_zone} -q --command='curl -sS https://${module.example.endpoint}/version -k'" } module "example" { diff --git a/test/integration/node_pool/node_pool_test.go b/test/integration/node_pool/node_pool_test.go index 5ce1e5ca32..7df2e1990d 100644 --- a/test/integration/node_pool/node_pool_test.go +++ b/test/integration/node_pool/node_pool_test.go @@ -15,16 +15,20 @@ package node_pool import ( "fmt" + "slices" + "strings" "testing" "time" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/cai" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" "github.com/gruntwork-io/terratest/modules/k8s" "github.com/stretchr/testify/assert" "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" - gkeutils "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/utils" + "golang.org/x/sync/errgroup" ) func TestNodePool(t *testing.T) { @@ -35,15 +39,18 @@ func TestNodePool(t *testing.T) { bpt.DefineVerify(func(assert *assert.Assertions) { // Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token // bpt.DefaultVerify(assert) - gkeutils.TGKEVerify(t, bpt, assert) // Verify Resources + testutils.TGKEVerify(t, bpt, assert) // Verify Resources projectId := bpt.GetStringOutput("project_id") location := bpt.GetStringOutput("location") clusterName := bpt.GetStringOutput("cluster_name") + randomString := bpt.GetStringOutput("random_string") + kubernetesEndpoint := bpt.GetStringOutput("kubernetes_endpoint") + serviceAccount := bpt.GetStringOutput("service_account") // CAI clusterResourceName := fmt.Sprintf("//container.googleapis.com/projects/%s/locations/%s/clusters/%s", projectId, location, clusterName) - cluster := gkeutils.GetProjectResources(t, projectId, gkeutils.WithAssetTypes([]string{"container.googleapis.com/Cluster"})).Get("#(name=\"" + clusterResourceName + "\").resource.data") + cluster := cai.GetProjectResources(t, projectId, cai.WithAssetTypes([]string{"container.googleapis.com/Cluster"})).Get("#(name=\"" + clusterResourceName + "\").resource.data") // Equivalent gcloud describe command // cluster := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId) @@ -71,9 +78,12 @@ func TestNodePool(t *testing.T) { // Cluster (using golden image with sanitizer) g := golden.NewOrUpdate(t, cluster.String(), + golden.WithSanitizer(golden.StringSanitizer(serviceAccount, "SERVICE_ACCOUNT")), golden.WithSanitizer(golden.StringSanitizer(projectId, "PROJECT_ID")), golden.WithSanitizer(golden.StringSanitizer(location, "LOCATION")), - golden.WithSanitizer(golden.StringSanitizer(clusterName, "CLUSTER_NAME")), + //golden.WithSanitizer(golden.StringSanitizer(clusterName, "CLUSTER_NAME")), + golden.WithSanitizer(golden.StringSanitizer(randomString, "RANDOM_STRING")), + golden.WithSanitizer(golden.StringSanitizer(kubernetesEndpoint, "KUBERNETES_ENDPOINT")), ) validateJSONPaths := []string{ "autoscaling.autoprovisioningNodePoolDefaults.imageType", @@ -92,6 +102,56 @@ func TestNodePool(t *testing.T) { g.JSONEq(assert, cluster, pth) } + fmt.Println("START one path") + g.JSONPathEqs(assert, cluster, []string{"autoscaling.autoprovisioningNodePoolDefaults.imageType"}) + fmt.Println("END one path") + + fmt.Println("START multi path") + g.JSONPathEqs(assert, cluster, validateJSONPaths) + fmt.Println("END multi path") + + fmt.Println("START all paths 1") + // Test validating all paths in golden image + jsonPaths := utils.GetTerminalJSONPaths(g.GetJSON()) + + // List of paths exempt from validation + exemptJSONPathPrefixes := []string{ + "nodePools", // nodePools are unordered + } + + // Remove exempt paths by prefix + jsonPaths = slices.DeleteFunc(jsonPaths, func(s string) bool { + for _, path := range exemptJSONPathPrefixes { + if strings.HasPrefix(s, path) { + // prefix match + return true + } + } + // no prefix match + return false + }) + + syncGroup := new(errgroup.Group) + syncGroup.SetLimit(24) + t.Logf("Checking %d JSON paths with max %d goroutines", len(jsonPaths), 24) + for _, jsonPath := range jsonPaths { + jsonPath := jsonPath + syncGroup.Go(func() error { + g.JSONEq(assert, cluster, jsonPath) + return nil + }) + } + if err := syncGroup.Wait(); err != nil { + t.Fatal(err) + } + fmt.Println("END all paths 1") + + //fmt.Println("all paths 2") + // Test validating all Paths + //evalPaths := utils.GetJSONPaths(cluster) + //fmt.Println(evalPaths) + //g.JSONPathEqs(assert, cluster, evalPaths) + // Pool-01 assert.Equal("pool-01", cluster.Get("nodePools.#(name==\"pool-01\").name").String(), "pool-1 exists") assert.Equal("e2-medium", cluster.Get("nodePools.#(name==\"pool-01\").config.machineType").String(), "is the expected machine type") @@ -156,7 +216,7 @@ func TestNodePool(t *testing.T) { k8sOpts := k8s.KubectlOptions{} clusterNodesOp, err := k8s.RunKubectlAndGetOutputE(t, &k8sOpts, "get", "nodes", "-o", "json") assert.NoError(err) - clusterNodes := testutils.ParseKubectlJSONResult(t, clusterNodesOp) + clusterNodes := utils.ParseKubectlJSONResult(t, clusterNodesOp) assert.JSONEq(`[ { "effect": "PreferNoSchedule", diff --git a/test/integration/node_pool/testdata/TestNodePool.json b/test/integration/node_pool/testdata/TestNodePool.json index 8040fe2659..1cdf0c282a 100644 --- a/test/integration/node_pool/testdata/TestNodePool.json +++ b/test/integration/node_pool/testdata/TestNodePool.json @@ -57,8 +57,7 @@ "clusterIpv4Cidr": "192.168.0.0/18", "controlPlaneEndpointsConfig": { "dnsEndpointConfig": { - "allowExternalTraffic": false, - "endpoint": "gke-2cc8593b774242f79d867b885e0b37891ba4-943708007946.europe-west4.gke.goog" + "allowExternalTraffic": false }, "ipEndpointsConfig": { "authorizedNetworksConfig": { @@ -67,10 +66,9 @@ "enablePublicEndpoint": true, "enabled": true, "privateEndpoint": "10.0.0.2", - "publicEndpoint": "35.204.175.149" + "publicEndpoint": "KUBERNETES_ENDPOINT" } }, - "createTime": "2024-11-15T22:43:28+00:00", "currentMasterVersion": "1.30.5-gke.1443001", "currentNodeCount": 8, "currentNodeVersion": "1.30.5-gke.1443001", @@ -81,33 +79,21 @@ "defaultMaxPodsConstraint": { "maxPodsPerNode": "110" }, - "endpoint": "35.204.175.149", + "endpoint": "KUBERNETES_ENDPOINT", "enterpriseConfig": { "clusterTier": "STANDARD" }, - "etag": "7339f7a4-2cb5-4e8b-990f-ee01fe8e65a9", - "id": "2cc8593b774242f79d867b885e0b37891ba4c55df2d347bf82d56722ed3c2b41", "identityServiceConfig": {}, "initialClusterVersion": "1.30.5-gke.1443001", - "instanceGroupUrls": [ - "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nc-default-pool-2af7cfca-grp", - "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-02-4887009c-grp", - "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-01-868e4268-grp", - "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-04-171eda64-grp", - "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-05-7bc6f02a-grp", - "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-03-402f8758-grp", - "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-c/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-03-52ff6abd-grp", - "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluste-nap-e2-medium-1d-71102fd7-grp" - ], "ipAllocationPolicy": { "clusterIpv4Cidr": "192.168.0.0/18", "clusterIpv4CidrBlock": "192.168.0.0/18", - "clusterSecondaryRangeName": "cft-gke-test-pods-nca3", + "clusterSecondaryRangeName": "cft-gke-test-pods-RANDOM_STRING", "defaultPodIpv4RangeUtilization": 0.0624, "podCidrOverprovisionConfig": {}, "servicesIpv4Cidr": "192.168.64.0/18", "servicesIpv4CidrBlock": "192.168.64.0/18", - "servicesSecondaryRangeName": "cft-gke-test-services-nca3", + "servicesSecondaryRangeName": "cft-gke-test-services-RANDOM_STRING", "stackType": "IPV4", "useIpAliases": true }, @@ -115,7 +101,7 @@ "legacyAbac": {}, "location": "LOCATION", "locations": [ - "europe-west4-b" + "LOCATION-b" ], "loggingConfig": { "componentConfig": { @@ -136,8 +122,7 @@ } }, "masterAuth": { - "clientCertificateConfig": {}, - "clusterCaCertificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVMRENDQXBTZ0F3SUJBZ0lRZjNPYlBFSXBKK203Y0I5Uzg0Yzd0REFOQmdrcWhraUc5dzBCQVFzRkFEQXYKTVMwd0t3WURWUVFERXlSak4yRTFNR0V6TWkwNFl6WTRMVFE1TXpndE9EUTFPQzFpTkdGbU1qSm1OR1F4Wm1ZdwpJQmNOTWpReE1URTFNakUwTXpJNFdoZ1BNakExTkRFeE1EZ3lNalF6TWpoYU1DOHhMVEFyQmdOVkJBTVRKR00zCllUVXdZVE15TFRoak5qZ3RORGt6T0MwNE5EVTRMV0kwWVdZeU1tWTBaREZtWmpDQ0FhSXdEUVlKS29aSWh2Y04KQVFFQkJRQURnZ0dQQURDQ0FZb0NnZ0dCQU92dFViZ2VYdDAwYllMd0dWU0hZT0Q5eG9tOTMyOFV5YjIrbzBuRQp4bDQ1amozY2dDb1htNFprNUhJR1dtWnV2a3JDV2M1V1ZxcTJQTHJjclJFNzQvTS9YMVdpVzFBTzMwWTZjckYzCnUzNGUyV0pSMm4vWkw2dStPZmwwVWpoWm5RSXBwT2VGemVFY24zVm1CWWp1N0hzdWoyd0w2RW9OaDVlcmI3anMKVCtzeHM3a3d5VWhMTDBkc21rREFDZE0wc2h6S1lRdXNMcnhPTEZ6djhGOENRVnczcGlHSmdmYzdIazBWVjhlbAp4WGpLMlFzL0UvY2xzVnRvQU93ejZ0a2lZY042alNyY0xGeDJSb0JvVXBWc2E3cENYN3lpeiswYy9kcFFoZ2NrCllYZDgwMmRrZmx2Nk9yc2JURjZLSWlWRnNTaFdqOEhURHZwSGJteWdiSGJMOFZBMDdzc0hadzBzWjVKMG9WYXQKNzU3Z0hhUjFkS3VkbXVERlVFdHRveWZEbS84cFJUQVhYUWRpcERyTEtBdDhjNUdpUEVJNFRMNmZ5NmtKOWFzTQpSQjRtT25jU2lTemhKSE1vcUtWZDE2dGhNMEF4alczZDA2UnZHMjJyaTg2clFGZllPTFE0MUtWQjY0Nzg4S3NvClg1VTd1aVJuTEdsUWVHb2lZakRIcjAvYm5RSURBUUFCbzBJd1FEQU9CZ05WSFE4QkFmOEVCQU1DQWdRd0R3WUQKVlIwVEFRSC9CQVV3QXdFQi96QWRCZ05WSFE0RUZnUVVRVVpGSFBkR3FpQUErQ1NUcW1SNmUxa0dDUXd3RFFZSgpLb1pJaHZjTkFRRUxCUUFEZ2dHQkFPazJwczFOUTB0WTMwRithTkN2c1U4QkZzUmhPZmNvaDVuQVpoSjF2Tzh2CklSYVcraTZub3VKbWVqZnFxTjI3WG5GN2FTTHc1TlplOEVPeVg1aDBLYmRIZ1QzV05aWVEzdE1VVnZQU2M0N2EKTFpHeFh6ZW9vbHdWM0l1cW5SaDltMTkrVS9aUlBOdmdCRERNOXZqRVV0RmZvQWxqLzE2azAxdGEwV1AxZ1hpUgpON1RqRmZBcW9jOCtQR0JoR3NmVW5ablZJREF1YnpFKzB1R2RNVGdCcDdTazk1dThscnBGZjk0ZVVJbkZmWnFNClMrTVZ1RWJ0RUhOUkZhaHNCLzVDYmlTdEpIMVRwaCtEZzFOZ1hrVndjTDc4ZDR1Ny92NnZZMWs1NUJsUlg3bzIKVjVmTEZMR1BUaytmUGhyS3FmczFHM2I4ZDYyL0VHZjRCZk9DQ1VlWjNIQUNybi83eENyTzdiblFUaE1ha1NrVAp0QkJYQzRkdUdWYm9hQ29OakI3a2NUaE9UYjdJT1A4RnpIUlVkNUp0T2pqb25udEJOc05sWFhUQUF6S3QxK29aClFHcUlCUWsxTi94dTliV2dlMnd5ZVNCOGNpUW54UUwwSDhPMm00QS85VGdyUDRqTmtySEpOZFBoaVFSZ1Zva2MKRHJhaTI1UlJwK1lsRnFXbUpCT2FYdz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K" + "clientCertificateConfig": {} }, "masterAuthorizedNetworksConfig": { "gcpPublicCidrsAccessEnabled": true @@ -165,13 +150,13 @@ } }, "monitoringService": "monitoring.googleapis.com/kubernetes", - "name": "CLUSTER_NAME", - "network": "cft-gke-test-nca3", + "name": "node-pool-cluster-RANDOM_STRING", + "network": "cft-gke-test-RANDOM_STRING", "networkConfig": { "defaultSnatStatus": {}, - "network": "projects/PROJECT_ID/global/networks/cft-gke-test-nca3", + "network": "projects/PROJECT_ID/global/networks/cft-gke-test-RANDOM_STRING", "serviceExternalIpsConfig": {}, - "subnetwork": "projects/PROJECT_ID/regions/europe-west4/subnetworks/cft-gke-test-nca3" + "subnetwork": "projects/PROJECT_ID/regions/LOCATION/subnetworks/cft-gke-test-RANDOM_STRING" }, "nodeConfig": { "diskSizeGb": 100, @@ -197,8 +182,8 @@ "enableIntegrityMonitoring": true }, "tags": [ - "gke-node-pool-cluster-nca3", - "gke-node-pool-cluster-nca3-default-pool", + "gke-node-pool-cluster-RANDOM_STRING", + "gke-node-pool-cluster-RANDOM_STRING-default-pool", "all-node-example", "pool-01-example" ], @@ -240,7 +225,7 @@ "https://www.googleapis.com/auth/userinfo.email", "https://www.googleapis.com/auth/cloud-platform" ], - "serviceAccount": "SERVICE_ACCOUNT", + "serviceAccount": "default", "shieldedInstanceConfig": { "enableIntegrityMonitoring": true }, @@ -255,12 +240,8 @@ "mode": "GKE_METADATA" } }, - "etag": "e4ce2569-5f6c-49ff-ac35-60dffd51e34a", - "instanceGroupUrls": [ - "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nc-default-pool-2af7cfca-grp" - ], "locations": [ - "europe-west4-b" + "LOCATION-b" ], "management": { "autoRepair": true, @@ -273,10 +254,10 @@ "networkConfig": { "podIpv4CidrBlock": "192.168.0.0/18", "podIpv4RangeUtilization": 0.0624, - "podRange": "cft-gke-test-pods-nca3" + "podRange": "cft-gke-test-pods-RANDOM_STRING" }, "podIpv4CidrSize": 24, - "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/europe-west4/clusters/node-pool-cluster-nca3/nodePools/default-pool", + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/LOCATION/clusters/node-pool-cluster-RANDOM_STRING/nodePools/default-pool", "status": "RUNNING", "upgradeSettings": { "maxSurge": 1, @@ -329,7 +310,7 @@ "oauthScopes": [ "https://www.googleapis.com/auth/cloud-platform" ], - "serviceAccount": "SERVICE_ACCOUNT", + "serviceAccount": "default", "shieldedInstanceConfig": { "enableIntegrityMonitoring": true }, @@ -355,13 +336,9 @@ "mode": "GKE_METADATA" } }, - "etag": "31fce54e-a3f0-4206-84d7-93e3d7c0d773", "initialNodeCount": 1, - "instanceGroupUrls": [ - "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-02-4887009c-grp" - ], "locations": [ - "europe-west4-b" + "LOCATION-b" ], "management": { "autoUpgrade": true @@ -373,10 +350,10 @@ "networkConfig": { "podIpv4CidrBlock": "192.168.0.0/18", "podIpv4RangeUtilization": 0.0624, - "podRange": "cft-gke-test-pods-nca3" + "podRange": "cft-gke-test-pods-RANDOM_STRING" }, "podIpv4CidrSize": 24, - "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/europe-west4/clusters/node-pool-cluster-nca3/nodePools/pool-02", + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/LOCATION/clusters/node-pool-cluster-nca3/nodePools/pool-02", "status": "RUNNING", "upgradeSettings": { "maxSurge": 1, @@ -424,7 +401,7 @@ "oauthScopes": [ "https://www.googleapis.com/auth/cloud-platform" ], - "serviceAccount": "SERVICE_ACCOUNT", + "serviceAccount": "default", "shieldedInstanceConfig": { "enableIntegrityMonitoring": true }, @@ -451,13 +428,9 @@ "mode": "GKE_METADATA" } }, - "etag": "9d43f000-7e58-4903-ba9c-a3f9e8532d33", "initialNodeCount": 1, - "instanceGroupUrls": [ - "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-01-868e4268-grp" - ], "locations": [ - "europe-west4-b" + "LOCATION-b" ], "management": { "autoRepair": true, @@ -470,10 +443,10 @@ "networkConfig": { "podIpv4CidrBlock": "192.168.0.0/18", "podIpv4RangeUtilization": 0.0624, - "podRange": "cft-gke-test-pods-nca3" + "podRange": "cft-gke-test-pods-RANDOM_STRING" }, "podIpv4CidrSize": 24, - "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/europe-west4/clusters/node-pool-cluster-nca3/nodePools/pool-01", + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/LOCATION/clusters/node-pool-cluster-nca3/nodePools/pool-01", "status": "RUNNING", "upgradeSettings": { "maxSurge": 1, @@ -519,7 +492,7 @@ "reservationAffinity": { "consumeReservationType": "NO_RESERVATION" }, - "serviceAccount": "SERVICE_ACCOUNT", + "serviceAccount": "default", "shieldedInstanceConfig": { "enableIntegrityMonitoring": true }, @@ -545,12 +518,8 @@ "mode": "GKE_METADATA" } }, - "etag": "ad27e635-3daf-466a-91ec-863489f4ca78", - "instanceGroupUrls": [ - "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-04-171eda64-grp" - ], "locations": [ - "europe-west4-b" + "LOCATION-b" ], "management": { "autoRepair": true, @@ -563,13 +532,13 @@ "networkConfig": { "podIpv4CidrBlock": "192.168.0.0/18", "podIpv4RangeUtilization": 0.0624, - "podRange": "cft-gke-test-pods-nca3" + "podRange": "cft-gke-test-pods-RANDOM_STRING" }, "podIpv4CidrSize": 24, "queuedProvisioning": { "enabled": true }, - "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/europe-west4/clusters/node-pool-cluster-nca3/nodePools/pool-04", + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/LOCATION/clusters/node-pool-cluster-nca3/nodePools/pool-04", "status": "RUNNING", "upgradeSettings": { "maxSurge": 1, @@ -637,13 +606,9 @@ "mode": "GKE_METADATA" } }, - "etag": "2f3e3fb1-a3f9-48de-8218-27c281a1e800", "initialNodeCount": 1, - "instanceGroupUrls": [ - "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-05-7bc6f02a-grp" - ], "locations": [ - "europe-west4-b" + "LOCATION-b" ], "management": { "autoRepair": true, @@ -656,10 +621,10 @@ "networkConfig": { "podIpv4CidrBlock": "192.168.0.0/18", "podIpv4RangeUtilization": 0.0624, - "podRange": "cft-gke-test-pods-nca3" + "podRange": "cft-gke-test-pods-RANDOM_STRING" }, "podIpv4CidrSize": 24, - "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/europe-west4/clusters/node-pool-cluster-nca3/nodePools/pool-05", + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/LOCATION/clusters/node-pool-cluster-nca3/nodePools/pool-05", "status": "RUNNING", "upgradeSettings": { "maxSurge": 1, @@ -707,7 +672,7 @@ "sandboxConfig": { "type": "GVISOR" }, - "serviceAccount": "SERVICE_ACCOUNT", + "serviceAccount": "default", "shieldedInstanceConfig": { "enableIntegrityMonitoring": true }, @@ -733,15 +698,10 @@ "mode": "GKE_METADATA" } }, - "etag": "096b8f17-4c88-4e7d-ab95-999ae1c779e3", "initialNodeCount": 2, - "instanceGroupUrls": [ - "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-03-402f8758-grp", - "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-c/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-03-52ff6abd-grp" - ], "locations": [ - "europe-west4-b", - "europe-west4-c" + "LOCATION-b", + "LOCATION-c" ], "management": { "autoRepair": true, @@ -758,7 +718,7 @@ "podRange": "test" }, "podIpv4CidrSize": 24, - "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/europe-west4/clusters/node-pool-cluster-nca3/nodePools/pool-03", + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/LOCATION/clusters/node-pool-cluster-nca3/nodePools/pool-03", "status": "RUNNING", "upgradeSettings": { "maxSurge": 1, @@ -785,7 +745,7 @@ "oauthScopes": [ "https://www.googleapis.com/auth/cloud-platform" ], - "serviceAccount": "SERVICE_ACCOUNT", + "serviceAccount": "default", "shieldedInstanceConfig": { "enableIntegrityMonitoring": true }, @@ -794,12 +754,8 @@ "mode": "GKE_METADATA" } }, - "etag": "18f6eb8a-9ef5-455e-9732-ead4b2dd101b", - "instanceGroupUrls": [ - "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluste-nap-e2-medium-1d-71102fd7-grp" - ], "locations": [ - "europe-west4-b" + "LOCATION-b" ], "management": { "autoRepair": true, @@ -813,11 +769,11 @@ "networkConfig": { "podIpv4CidrBlock": "192.168.0.0/18", "podIpv4RangeUtilization": 0.0624, - "podRange": "cft-gke-test-pods-nca3" + "podRange": "cft-gke-test-pods-RANDOM_STRING" }, "placementPolicy": {}, "podIpv4CidrSize": 24, - "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/europe-west4/clusters/node-pool-cluster-nca3/nodePools/nap-e2-medium-1d469r1p", + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/LOCATION/clusters/node-pool-cluster-nca3/nodePools/nap-e2-medium-1d469r1p", "status": "RUNNING", "upgradeSettings": { "maxSurge": 1, @@ -831,7 +787,7 @@ }, "privateClusterConfig": { "privateEndpoint": "10.0.0.2", - "publicEndpoint": "35.204.175.149" + "publicEndpoint": "KUBERNETES_ENDPOINT" }, "rbacBindingConfig": { "enableInsecureBindingSystemAuthenticated": true, @@ -847,17 +803,17 @@ "mode": "DISABLED", "vulnerabilityMode": "VULNERABILITY_DISABLED" }, - "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/europe-west4/clusters/node-pool-cluster-nca3", + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/LOCATION/clusters/node-pool-cluster-nca3", "servicesIpv4Cidr": "192.168.64.0/18", "shieldedNodes": { "enabled": true }, "status": "RUNNING", - "subnetwork": "cft-gke-test-nca3", + "subnetwork": "cft-gke-test-RANDOM_STRING", "verticalPodAutoscaling": {}, "workloadIdentityConfig": { "workloadPool": "PROJECT_ID.svc.id.goog" }, - "zone": "europe-west4" + "zone": "LOCATION" } diff --git a/test/integration/safer_cluster_iap_bastion/safer_cluster_iap_bastion_test.go b/test/integration/safer_cluster_iap_bastion/safer_cluster_iap_bastion_test.go index 2e46f573f9..aa912771dd 100644 --- a/test/integration/safer_cluster_iap_bastion/safer_cluster_iap_bastion_test.go +++ b/test/integration/safer_cluster_iap_bastion/safer_cluster_iap_bastion_test.go @@ -23,7 +23,6 @@ import ( "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" "github.com/stretchr/testify/assert" "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" - gkeutils "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/utils" ) func TestSaferClusterIapBastion(t *testing.T) { @@ -34,7 +33,7 @@ func TestSaferClusterIapBastion(t *testing.T) { bpt.DefineVerify(func(assert *assert.Assertions) { // Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token // bpt.DefaultVerify(assert) - gkeutils.TGKEVerify(t, bpt, assert) // Verify Resources + testutils.TGKEVerify(t, bpt, assert) // Verify Resources test_command, _ := strings.CutPrefix(bpt.GetStringOutput("test_command"), "gcloud ") diff --git a/test/integration/testutils/cai.go b/test/integration/testutils/cai.go deleted file mode 100644 index f219f6875b..0000000000 --- a/test/integration/testutils/cai.go +++ /dev/null @@ -1,82 +0,0 @@ -/** - * Copyright 2024 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -// Package cai provides a set of helpers to interact with Cloud Asset Inventory -package utils - -import ( - "fmt" - "strings" - "testing" - "time" - - "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" - "github.com/tidwall/gjson" -) - -type CmdCfg struct { - sleep int // minutes to sleep prior to CAI retreval. default: 2 - assetTypes []string // asset types to retrieve. empty: all - args []string // arguments to pass to call -} - -type cmdOption func(*CmdCfg) - -// newCmdConfig sets defaults and options -func newCmdConfig(opts ...cmdOption) (*CmdCfg) { - caiOpts := &CmdCfg{ - sleep: 2, - assetTypes: nil, - args: nil, - } - - for _, opt := range opts { - opt(caiOpts) - } - - if caiOpts.assetTypes != nil { - caiOpts.args = []string{"--asset-types", strings.Join(caiOpts.assetTypes, ",")} - } - caiOpts.args = append(caiOpts.args, "--content-type", "resource") - - return caiOpts -} - -// Set custom sleep minutes -func WithSleep(sleep int) cmdOption { - return func(f *CmdCfg) { - f.sleep = sleep - } -} - -// Set asset types -func WithAssetTypes(assetTypes []string) cmdOption { - return func(f *CmdCfg) { - f.assetTypes = assetTypes - } -} - -// GetProjectResources returns the cloud asset inventory resources for a project as a gjson.Result -func GetProjectResources(t testing.TB, project string, opts ...cmdOption) gjson.Result { - caiOpts := newCmdConfig(opts...) - - // Cloud Asset Inventory offers best-effort data freshness. - t.Logf("Sleeping for %d minutes before retrieving Cloud Asset Inventory...", caiOpts.sleep) - time.Sleep(time.Duration(caiOpts.sleep) * time.Minute) - - cmd := fmt.Sprintf("asset list --project %s", project) - return gcloud.Runf(t, strings.Join(append([]string{cmd}, caiOpts.args...), " ")) -} diff --git a/test/integration/workload_identity/workload_identity_test.go b/test/integration/workload_identity/workload_identity_test.go index fe06e5321c..92ebc59541 100644 --- a/test/integration/workload_identity/workload_identity_test.go +++ b/test/integration/workload_identity/workload_identity_test.go @@ -22,7 +22,6 @@ import ( "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" "github.com/stretchr/testify/assert" "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils" - gkeutils "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/utils" ) func TestWorkloadIdentity(t *testing.T) { @@ -33,7 +32,7 @@ func TestWorkloadIdentity(t *testing.T) { bpt.DefineVerify(func(assert *assert.Assertions) { // Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token // bpt.DefaultVerify(assert) - gkeutils.TGKEVerify(t, bpt, assert) // Verify Resources + testutils.TGKEVerify(t, bpt, assert) // Verify Resources projectId := bpt.GetStringOutput("project_id") location := bpt.GetStringOutput("location")