validate all paths

test/fixtures/node_pool/outputs.tf

@@ -83,3 +83,7 @@ output "service_account" {
 output "registry_project_ids" {
   value = var.registry_project_ids
 }
+
+output "random_string" {
+  value = random_string.suffix.result
+}

test/fixtures/safer_cluster_iap_bastion/example.tf

@@ -15,7 +15,7 @@
  */
 
 locals {
-  test_command = "gcloud beta compute ssh ${module.example.bastion_name} --tunnel-through-iap --verbosity=error --project ${var.project_ids[1]} --zone ${module.example.bastion_zone} -q -- curl -sS https://${module.example.endpoint}/version -k"
+  test_command = "gcloud beta compute ssh ${module.example.bastion_name} --tunnel-through-iap --verbosity=error --project ${var.project_ids[1]} --zone ${module.example.bastion_zone} -q --command='curl -sS https://${module.example.endpoint}/version -k'"
 }
 
 module "example" {

test/integration/node_pool/node_pool_test.go

@@ -15,16 +15,20 @@ package node_pool
 
 import (
 	"fmt"
+	"slices"
+	"strings"
 	"testing"
 	"time"
 
+	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/cai"
 	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
 	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden"
 	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
+	"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils"
 	"github.com/gruntwork-io/terratest/modules/k8s"
 	"github.com/stretchr/testify/assert"
 	"github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils"
-	gkeutils "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/utils"
+	"golang.org/x/sync/errgroup"
 )
 
 func TestNodePool(t *testing.T) {
@@ -35,15 +39,18 @@ func TestNodePool(t *testing.T) {
 	bpt.DefineVerify(func(assert *assert.Assertions) {
 		// Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token
 		// bpt.DefaultVerify(assert)
-		gkeutils.TGKEVerify(t, bpt, assert) // Verify Resources
+		testutils.TGKEVerify(t, bpt, assert) // Verify Resources
 
 		projectId := bpt.GetStringOutput("project_id")
 		location := bpt.GetStringOutput("location")
 		clusterName := bpt.GetStringOutput("cluster_name")
+		randomString := bpt.GetStringOutput("random_string")
+		kubernetesEndpoint := bpt.GetStringOutput("kubernetes_endpoint")
+		serviceAccount := bpt.GetStringOutput("service_account")
 
 		// CAI
 		clusterResourceName := fmt.Sprintf("//container.googleapis.com/projects/%s/locations/%s/clusters/%s", projectId, location, clusterName)
-		cluster := gkeutils.GetProjectResources(t, projectId, gkeutils.WithAssetTypes([]string{"container.googleapis.com/Cluster"})).Get("#(name=\"" + clusterResourceName + "\").resource.data")
+		cluster := cai.GetProjectResources(t, projectId, cai.WithAssetTypes([]string{"container.googleapis.com/Cluster"})).Get("#(name=\"" + clusterResourceName + "\").resource.data")
 
 		// Equivalent gcloud describe command
 		// cluster := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId)
@@ -71,9 +78,12 @@ func TestNodePool(t *testing.T) {
 
 		// Cluster (using golden image with sanitizer)
 		g := golden.NewOrUpdate(t, cluster.String(),
+			golden.WithSanitizer(golden.StringSanitizer(serviceAccount, "SERVICE_ACCOUNT")),
 			golden.WithSanitizer(golden.StringSanitizer(projectId, "PROJECT_ID")),
 			golden.WithSanitizer(golden.StringSanitizer(location, "LOCATION")),
-			golden.WithSanitizer(golden.StringSanitizer(clusterName, "CLUSTER_NAME")),
+			//golden.WithSanitizer(golden.StringSanitizer(clusterName, "CLUSTER_NAME")),
+			golden.WithSanitizer(golden.StringSanitizer(randomString, "RANDOM_STRING")),
+			golden.WithSanitizer(golden.StringSanitizer(kubernetesEndpoint, "KUBERNETES_ENDPOINT")),
 		)
 		validateJSONPaths := []string{
 			"autoscaling.autoprovisioningNodePoolDefaults.imageType",
@@ -92,6 +102,56 @@ func TestNodePool(t *testing.T) {
 			g.JSONEq(assert, cluster, pth)
 		}
 
+		fmt.Println("START one path")
+		g.JSONPathEqs(assert, cluster, []string{"autoscaling.autoprovisioningNodePoolDefaults.imageType"})
+		fmt.Println("END one path")
+
+		fmt.Println("START multi path")
+		g.JSONPathEqs(assert, cluster, validateJSONPaths)
+		fmt.Println("END multi path")
+
+		fmt.Println("START all paths 1")
+		// Test validating all paths in golden image
+		jsonPaths := utils.GetTerminalJSONPaths(g.GetJSON())
+
+		// List of paths exempt from validation
+		exemptJSONPathPrefixes := []string{
+			"nodePools", // nodePools are unordered
+		}
+
+		// Remove exempt paths by prefix
+		jsonPaths = slices.DeleteFunc(jsonPaths, func(s string) bool {
+			for _, path := range exemptJSONPathPrefixes {
+				if strings.HasPrefix(s, path) {
+					// prefix match
+					return true
+				}
+			}
+			// no prefix match
+			return false
+		})
+
+		syncGroup := new(errgroup.Group)
+		syncGroup.SetLimit(24)
+		t.Logf("Checking %d JSON paths with max %d goroutines", len(jsonPaths), 24)
+		for _, jsonPath := range jsonPaths {
+			jsonPath := jsonPath
+			syncGroup.Go(func() error {
+				g.JSONEq(assert, cluster, jsonPath)
+				return nil
+			})
+		}
+		if err := syncGroup.Wait(); err != nil {
+			t.Fatal(err)
+		}
+		fmt.Println("END all paths 1")
+
+		//fmt.Println("all paths 2")
+		// Test validating all Paths
+		//evalPaths := utils.GetJSONPaths(cluster)
+		//fmt.Println(evalPaths)
+		//g.JSONPathEqs(assert, cluster, evalPaths)
+
 		// Pool-01
 		assert.Equal("pool-01", cluster.Get("nodePools.#(name==\"pool-01\").name").String(), "pool-1 exists")
 		assert.Equal("e2-medium", cluster.Get("nodePools.#(name==\"pool-01\").config.machineType").String(), "is the expected machine type")
@@ -156,7 +216,7 @@ func TestNodePool(t *testing.T) {
 		k8sOpts := k8s.KubectlOptions{}
 		clusterNodesOp, err := k8s.RunKubectlAndGetOutputE(t, &k8sOpts, "get", "nodes", "-o", "json")
 		assert.NoError(err)
-		clusterNodes := testutils.ParseKubectlJSONResult(t, clusterNodesOp)
+		clusterNodes := utils.ParseKubectlJSONResult(t, clusterNodesOp)
 		assert.JSONEq(`[
 				{
 					"effect": "PreferNoSchedule",