From e33b3ea821b12757f7072c77dd8dc80d6d19e215 Mon Sep 17 00:00:00 2001 From: Andrew Peabody Date: Fri, 15 Nov 2024 00:43:39 +0000 Subject: [PATCH] multi asset types --- test/integration/node_pool/node_pool_test.go | 43 +- .../node_pool/testdata/TestNodePool.json | 863 ++++++++++++++++++ test/integration/testutils/cai.go | 34 +- 3 files changed, 925 insertions(+), 15 deletions(-) create mode 100644 test/integration/node_pool/testdata/TestNodePool.json diff --git a/test/integration/node_pool/node_pool_test.go b/test/integration/node_pool/node_pool_test.go index 9aae6f514..5ce1e5ca3 100644 --- a/test/integration/node_pool/node_pool_test.go +++ b/test/integration/node_pool/node_pool_test.go @@ -19,6 +19,7 @@ import ( "time" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" + "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/golden" "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" "github.com/gruntwork-io/terratest/modules/k8s" "github.com/stretchr/testify/assert" @@ -40,14 +41,17 @@ func TestNodePool(t *testing.T) { location := bpt.GetStringOutput("location") clusterName := bpt.GetStringOutput("cluster_name") - //cluster := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId) + // CAI clusterResourceName := fmt.Sprintf("//container.googleapis.com/projects/%s/locations/%s/clusters/%s", projectId, location, clusterName) - cluster := gkeutils.GetProjectResources(t, projectId, gkeutils.WithAssetType("container.googleapis.com/Cluster")).Get("#(name=\"" + clusterResourceName + "\").resource.data") + cluster := gkeutils.GetProjectResources(t, projectId, gkeutils.WithAssetTypes([]string{"container.googleapis.com/Cluster"})).Get("#(name=\"" + clusterResourceName + "\").resource.data") + + // Equivalent gcloud describe command + // cluster := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId) // Cluster assert.Contains([]string{"RUNNING", "RECONCILING"}, cluster.Get("status").String(), "Cluster is Running") assert.Equal("COS_CONTAINERD", cluster.Get("autoscaling.autoprovisioningNodePoolDefaults.imageType").String(), "has the expected image type") - assert.Equal("[\n \"https://www.googleapis.com/auth/cloud-platform\"\n ]", cluster.Get("autoscaling.autoprovisioningNodePoolDefaults.oauthScopes").String(), "has the expected oauth scopes") + assert.Equal("https://www.googleapis.com/auth/cloud-platform", cluster.Get("autoscaling.autoprovisioningNodePoolDefaults.oauthScopes.0").String(), "has the expected oauth scopes") assert.Equal("default", cluster.Get("autoscaling.autoprovisioningNodePoolDefaults.serviceAccount").String(), "has the expected service account") assert.Equal("OPTIMIZE_UTILIZATION", cluster.Get("autoscaling.autoscalingProfile").String(), "has the expected autoscaling profile") assert.True(cluster.Get("autoscaling.enableNodeAutoprovisioning").Bool(), "has the expected node autoprovisioning") @@ -65,6 +69,29 @@ func TestNodePool(t *testing.T) { ]`, cluster.Get("autoscaling.resourceLimits").String(), "has the expected resource limits") + // Cluster (using golden image with sanitizer) + g := golden.NewOrUpdate(t, cluster.String(), + golden.WithSanitizer(golden.StringSanitizer(projectId, "PROJECT_ID")), + golden.WithSanitizer(golden.StringSanitizer(location, "LOCATION")), + golden.WithSanitizer(golden.StringSanitizer(clusterName, "CLUSTER_NAME")), + ) + validateJSONPaths := []string{ + "autoscaling.autoprovisioningNodePoolDefaults.imageType", + "autoscaling.autoprovisioningNodePoolDefaults.oauthScopes.0", + "autoscaling.autoprovisioningNodePoolDefaults.serviceAccount", + "autoscaling.autoscalingProfile", + "autoscaling.enableNodeAutoprovisioning", + "autoscaling.resourceLimits[0].maximum", + "autoscaling.resourceLimits[0].minimum", + "autoscaling.resourceLimits[0].resourceType", + "autoscaling.resourceLimits[1].maximum", + "autoscaling.resourceLimits[1].minimum", + "autoscaling.resourceLimits[1].resourceType", + } + for _, pth := range validateJSONPaths { + g.JSONEq(assert, cluster, pth) + } + // Pool-01 assert.Equal("pool-01", cluster.Get("nodePools.#(name==\"pool-01\").name").String(), "pool-1 exists") assert.Equal("e2-medium", cluster.Get("nodePools.#(name==\"pool-01\").config.machineType").String(), "is the expected machine type") @@ -148,6 +175,11 @@ func TestNodePool(t *testing.T) { "effect": "PreferNoSchedule", "key": "all-pools-example", "value": "true" + }, + { + "effect": "NoSchedule", + "key": "nvidia.com/gpu", + "value": "present" } ]`, clusterNodes.Get("items.#(metadata.labels.node_pool==\"pool-02\").spec.taints").String(), "has the expected all-pools-example taint") @@ -156,6 +188,11 @@ func TestNodePool(t *testing.T) { "effect": "PreferNoSchedule", "key": "all-pools-example", "value": "true" + }, + { + "effect": "NoSchedule", + "key": "sandbox.gke.io/runtime", + "value": "gvisor" } ]`, clusterNodes.Get("items.#(metadata.labels.node_pool==\"pool-03\").spec.taints").String(), "has the expected all-pools-example taint") diff --git a/test/integration/node_pool/testdata/TestNodePool.json b/test/integration/node_pool/testdata/TestNodePool.json new file mode 100644 index 000000000..8040fe265 --- /dev/null +++ b/test/integration/node_pool/testdata/TestNodePool.json @@ -0,0 +1,863 @@ +{ + "addonsConfig": { + "configConnectorConfig": {}, + "dnsCacheConfig": {}, + "gcePersistentDiskCsiDriverConfig": { + "enabled": true + }, + "gcpFilestoreCsiDriverConfig": {}, + "gkeBackupAgentConfig": {}, + "horizontalPodAutoscaling": {}, + "httpLoadBalancing": {}, + "kubernetesDashboard": { + "disabled": true + }, + "networkPolicyConfig": { + "disabled": true + } + }, + "autopilot": {}, + "autoscaling": { + "autoprovisioningNodePoolDefaults": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "imageType": "COS_CONTAINERD", + "management": { + "autoRepair": true, + "autoUpgrade": true, + "upgradeOptions": {} + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "default", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "upgradeSettings": { + "strategy": "SURGE" + } + }, + "autoscalingProfile": "OPTIMIZE_UTILIZATION", + "enableNodeAutoprovisioning": true, + "resourceLimits": [ + { + "maximum": "20", + "minimum": "5", + "resourceType": "cpu" + }, + { + "maximum": "30", + "minimum": "10", + "resourceType": "memory" + } + ] + }, + "binaryAuthorization": {}, + "clusterIpv4Cidr": "192.168.0.0/18", + "controlPlaneEndpointsConfig": { + "dnsEndpointConfig": { + "allowExternalTraffic": false, + "endpoint": "gke-2cc8593b774242f79d867b885e0b37891ba4-943708007946.europe-west4.gke.goog" + }, + "ipEndpointsConfig": { + "authorizedNetworksConfig": { + "gcpPublicCidrsAccessEnabled": true + }, + "enablePublicEndpoint": true, + "enabled": true, + "privateEndpoint": "10.0.0.2", + "publicEndpoint": "35.204.175.149" + } + }, + "createTime": "2024-11-15T22:43:28+00:00", + "currentMasterVersion": "1.30.5-gke.1443001", + "currentNodeCount": 8, + "currentNodeVersion": "1.30.5-gke.1443001", + "databaseEncryption": { + "currentState": "CURRENT_STATE_DECRYPTED", + "state": "DECRYPTED" + }, + "defaultMaxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "endpoint": "35.204.175.149", + "enterpriseConfig": { + "clusterTier": "STANDARD" + }, + "etag": "7339f7a4-2cb5-4e8b-990f-ee01fe8e65a9", + "id": "2cc8593b774242f79d867b885e0b37891ba4c55df2d347bf82d56722ed3c2b41", + "identityServiceConfig": {}, + "initialClusterVersion": "1.30.5-gke.1443001", + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nc-default-pool-2af7cfca-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-02-4887009c-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-01-868e4268-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-04-171eda64-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-05-7bc6f02a-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-03-402f8758-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-c/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-03-52ff6abd-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluste-nap-e2-medium-1d-71102fd7-grp" + ], + "ipAllocationPolicy": { + "clusterIpv4Cidr": "192.168.0.0/18", + "clusterIpv4CidrBlock": "192.168.0.0/18", + "clusterSecondaryRangeName": "cft-gke-test-pods-nca3", + "defaultPodIpv4RangeUtilization": 0.0624, + "podCidrOverprovisionConfig": {}, + "servicesIpv4Cidr": "192.168.64.0/18", + "servicesIpv4CidrBlock": "192.168.64.0/18", + "servicesSecondaryRangeName": "cft-gke-test-services-nca3", + "stackType": "IPV4", + "useIpAliases": true + }, + "labelFingerprint": "78cdf2f6", + "legacyAbac": {}, + "location": "LOCATION", + "locations": [ + "europe-west4-b" + ], + "loggingConfig": { + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS", + "WORKLOADS" + ] + } + }, + "loggingService": "logging.googleapis.com/kubernetes", + "maintenancePolicy": { + "resourceVersion": "ce912209", + "window": { + "dailyMaintenanceWindow": { + "duration": "PT4H0M0S", + "startTime": "05:00" + } + } + }, + "masterAuth": { + "clientCertificateConfig": {}, + "clusterCaCertificate": "LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUVMRENDQXBTZ0F3SUJBZ0lRZjNPYlBFSXBKK203Y0I5Uzg0Yzd0REFOQmdrcWhraUc5dzBCQVFzRkFEQXYKTVMwd0t3WURWUVFERXlSak4yRTFNR0V6TWkwNFl6WTRMVFE1TXpndE9EUTFPQzFpTkdGbU1qSm1OR1F4Wm1ZdwpJQmNOTWpReE1URTFNakUwTXpJNFdoZ1BNakExTkRFeE1EZ3lNalF6TWpoYU1DOHhMVEFyQmdOVkJBTVRKR00zCllUVXdZVE15TFRoak5qZ3RORGt6T0MwNE5EVTRMV0kwWVdZeU1tWTBaREZtWmpDQ0FhSXdEUVlKS29aSWh2Y04KQVFFQkJRQURnZ0dQQURDQ0FZb0NnZ0dCQU92dFViZ2VYdDAwYllMd0dWU0hZT0Q5eG9tOTMyOFV5YjIrbzBuRQp4bDQ1amozY2dDb1htNFprNUhJR1dtWnV2a3JDV2M1V1ZxcTJQTHJjclJFNzQvTS9YMVdpVzFBTzMwWTZjckYzCnUzNGUyV0pSMm4vWkw2dStPZmwwVWpoWm5RSXBwT2VGemVFY24zVm1CWWp1N0hzdWoyd0w2RW9OaDVlcmI3anMKVCtzeHM3a3d5VWhMTDBkc21rREFDZE0wc2h6S1lRdXNMcnhPTEZ6djhGOENRVnczcGlHSmdmYzdIazBWVjhlbAp4WGpLMlFzL0UvY2xzVnRvQU93ejZ0a2lZY042alNyY0xGeDJSb0JvVXBWc2E3cENYN3lpeiswYy9kcFFoZ2NrCllYZDgwMmRrZmx2Nk9yc2JURjZLSWlWRnNTaFdqOEhURHZwSGJteWdiSGJMOFZBMDdzc0hadzBzWjVKMG9WYXQKNzU3Z0hhUjFkS3VkbXVERlVFdHRveWZEbS84cFJUQVhYUWRpcERyTEtBdDhjNUdpUEVJNFRMNmZ5NmtKOWFzTQpSQjRtT25jU2lTemhKSE1vcUtWZDE2dGhNMEF4alczZDA2UnZHMjJyaTg2clFGZllPTFE0MUtWQjY0Nzg4S3NvClg1VTd1aVJuTEdsUWVHb2lZakRIcjAvYm5RSURBUUFCbzBJd1FEQU9CZ05WSFE4QkFmOEVCQU1DQWdRd0R3WUQKVlIwVEFRSC9CQVV3QXdFQi96QWRCZ05WSFE0RUZnUVVRVVpGSFBkR3FpQUErQ1NUcW1SNmUxa0dDUXd3RFFZSgpLb1pJaHZjTkFRRUxCUUFEZ2dHQkFPazJwczFOUTB0WTMwRithTkN2c1U4QkZzUmhPZmNvaDVuQVpoSjF2Tzh2CklSYVcraTZub3VKbWVqZnFxTjI3WG5GN2FTTHc1TlplOEVPeVg1aDBLYmRIZ1QzV05aWVEzdE1VVnZQU2M0N2EKTFpHeFh6ZW9vbHdWM0l1cW5SaDltMTkrVS9aUlBOdmdCRERNOXZqRVV0RmZvQWxqLzE2azAxdGEwV1AxZ1hpUgpON1RqRmZBcW9jOCtQR0JoR3NmVW5ablZJREF1YnpFKzB1R2RNVGdCcDdTazk1dThscnBGZjk0ZVVJbkZmWnFNClMrTVZ1RWJ0RUhOUkZhaHNCLzVDYmlTdEpIMVRwaCtEZzFOZ1hrVndjTDc4ZDR1Ny92NnZZMWs1NUJsUlg3bzIKVjVmTEZMR1BUaytmUGhyS3FmczFHM2I4ZDYyL0VHZjRCZk9DQ1VlWjNIQUNybi83eENyTzdiblFUaE1ha1NrVAp0QkJYQzRkdUdWYm9hQ29OakI3a2NUaE9UYjdJT1A4RnpIUlVkNUp0T2pqb25udEJOc05sWFhUQUF6S3QxK29aClFHcUlCUWsxTi94dTliV2dlMnd5ZVNCOGNpUW54UUwwSDhPMm00QS85VGdyUDRqTmtySEpOZFBoaVFSZ1Zva2MKRHJhaTI1UlJwK1lsRnFXbUpCT2FYdz09Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K" + }, + "masterAuthorizedNetworksConfig": { + "gcpPublicCidrsAccessEnabled": true + }, + "meshCertificates": { + "enableCertificates": false + }, + "monitoringConfig": { + "advancedDatapathObservabilityConfig": {}, + "componentConfig": { + "enableComponents": [ + "SYSTEM_COMPONENTS", + "STORAGE", + "HPA", + "POD", + "DAEMONSET", + "DEPLOYMENT", + "STATEFULSET", + "CADVISOR", + "KUBELET" + ] + }, + "managedPrometheusConfig": { + "enabled": true + } + }, + "monitoringService": "monitoring.googleapis.com/kubernetes", + "name": "CLUSTER_NAME", + "network": "cft-gke-test-nca3", + "networkConfig": { + "defaultSnatStatus": {}, + "network": "projects/PROJECT_ID/global/networks/cft-gke-test-nca3", + "serviceExternalIpsConfig": {}, + "subnetwork": "projects/PROJECT_ID/regions/europe-west4/subnetworks/cft-gke-test-nca3" + }, + "nodeConfig": { + "diskSizeGb": 100, + "diskType": "pd-balanced", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "gcfsConfig": {}, + "imageType": "COS_CONTAINERD", + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "e2-medium", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "SERVICE_ACCOUNT", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-node-pool-cluster-nca3", + "gke-node-pool-cluster-nca3-default-pool", + "all-node-example", + "pool-01-example" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "nodePoolDefaults": { + "nodeConfigDefaults": { + "gcfsConfig": {}, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "nodeKubeletConfig": {} + } + }, + "nodePools": [ + { + "autoscaling": {}, + "config": { + "diskSizeGb": 100, + "diskType": "pd-balanced", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "gcfsConfig": {}, + "imageType": "COS_CONTAINERD", + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "e2-medium", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/userinfo.email", + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "SERVICE_ACCOUNT", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-node-pool-cluster-nca3", + "gke-node-pool-cluster-nca3-default-pool", + "all-node-example", + "pool-01-example" + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "e4ce2569-5f6c-49ff-ac35-60dffd51e34a", + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nc-default-pool-2af7cfca-grp" + ], + "locations": [ + "europe-west4-b" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "default-pool", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podIpv4RangeUtilization": 0.0624, + "podRange": "cft-gke-test-pods-nca3" + }, + "podIpv4CidrSize": 24, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/europe-west4/clusters/node-pool-cluster-nca3/nodePools/default-pool", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.30.5-gke.1443001" + }, + { + "autoscaling": { + "enabled": true, + "locationPolicy": "BALANCED", + "maxNodeCount": 2, + "minNodeCount": 1 + }, + "config": { + "accelerators": [ + { + "acceleratorCount": "1", + "acceleratorType": "nvidia-tesla-p4", + "gpuDriverInstallationConfig": { + "gpuDriverVersion": "DEFAULT" + } + } + ], + "diskSizeGb": 30, + "diskType": "pd-standard", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "imageType": "COS_CONTAINERD", + "labels": { + "all-pools-example": "true", + "cluster_name": "node-pool-cluster-nca3", + "node_pool": "pool-02" + }, + "linuxNodeConfig": { + "sysctls": { + "net.core.netdev_max_backlog": "10000" + } + }, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "n1-standard-2", + "metadata": { + "cluster_name": "node-pool-cluster-nca3", + "disable-legacy-endpoints": "false", + "node_pool": "pool-02" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "SERVICE_ACCOUNT", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-node-pool-cluster-nca3", + "gke-node-pool-cluster-nca3-pool-02", + "all-node-example" + ], + "taints": [ + { + "effect": "PREFER_NO_SCHEDULE", + "key": "all-pools-example", + "value": "true" + }, + { + "effect": "NO_SCHEDULE", + "key": "nvidia.com/gpu", + "value": "present" + } + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "31fce54e-a3f0-4206-84d7-93e3d7c0d773", + "initialNodeCount": 1, + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-02-4887009c-grp" + ], + "locations": [ + "europe-west4-b" + ], + "management": { + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "pool-02", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podIpv4RangeUtilization": 0.0624, + "podRange": "cft-gke-test-pods-nca3" + }, + "podIpv4CidrSize": 24, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/europe-west4/clusters/node-pool-cluster-nca3/nodePools/pool-02", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.30.5-gke.1443001" + }, + { + "autoscaling": { + "enabled": true, + "locationPolicy": "BALANCED", + "maxNodeCount": 2, + "minNodeCount": 1 + }, + "config": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "gcfsConfig": {}, + "imageType": "COS_CONTAINERD", + "labels": { + "all-pools-example": "true", + "cluster_name": "node-pool-cluster-nca3", + "node_pool": "pool-01", + "pool-01-example": "true" + }, + "linuxNodeConfig": { + "sysctls": { + "net.core.netdev_max_backlog": "10000", + "net.core.rmem_max": "10000" + } + }, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "e2-medium", + "metadata": { + "cluster_name": "node-pool-cluster-nca3", + "disable-legacy-endpoints": "false", + "node_pool": "pool-01", + "shutdown-script": "kubectl --kubeconfig=/var/lib/kubelet/kubeconfig drain --force=true --ignore-daemonsets=true --delete-local-data \"$HOSTNAME\"" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "SERVICE_ACCOUNT", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-node-pool-cluster-nca3", + "gke-node-pool-cluster-nca3-pool-01", + "all-node-example", + "pool-01-example" + ], + "taints": [ + { + "effect": "PREFER_NO_SCHEDULE", + "key": "all-pools-example", + "value": "true" + }, + { + "effect": "PREFER_NO_SCHEDULE", + "key": "pool-01-example", + "value": "true" + } + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "9d43f000-7e58-4903-ba9c-a3f9e8532d33", + "initialNodeCount": 1, + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-01-868e4268-grp" + ], + "locations": [ + "europe-west4-b" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "pool-01", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podIpv4RangeUtilization": 0.0624, + "podRange": "cft-gke-test-pods-nca3" + }, + "podIpv4CidrSize": 24, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/europe-west4/clusters/node-pool-cluster-nca3/nodePools/pool-01", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.30.5-gke.1443001" + }, + { + "autoscaling": { + "enabled": true, + "locationPolicy": "ANY", + "maxNodeCount": 100 + }, + "config": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "imageType": "COS_CONTAINERD", + "labels": { + "all-pools-example": "true", + "cluster_name": "node-pool-cluster-nca3", + "node_pool": "pool-04" + }, + "linuxNodeConfig": { + "sysctls": { + "net.core.netdev_max_backlog": "10000" + } + }, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "e2-medium", + "metadata": { + "cluster_name": "node-pool-cluster-nca3", + "disable-legacy-endpoints": "false", + "node_pool": "pool-04" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "reservationAffinity": { + "consumeReservationType": "NO_RESERVATION" + }, + "serviceAccount": "SERVICE_ACCOUNT", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-node-pool-cluster-nca3", + "gke-node-pool-cluster-nca3-pool-04", + "all-node-example" + ], + "taints": [ + { + "effect": "PREFER_NO_SCHEDULE", + "key": "all-pools-example", + "value": "true" + }, + { + "effect": "NO_SCHEDULE", + "key": "cloud.google.com/gke-queued", + "value": "true" + } + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "ad27e635-3daf-466a-91ec-863489f4ca78", + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-04-171eda64-grp" + ], + "locations": [ + "europe-west4-b" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "pool-04", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podIpv4RangeUtilization": 0.0624, + "podRange": "cft-gke-test-pods-nca3" + }, + "podIpv4CidrSize": 24, + "queuedProvisioning": { + "enabled": true + }, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/europe-west4/clusters/node-pool-cluster-nca3/nodePools/pool-04", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.30.5-gke.1443001" + }, + { + "autoscaling": { + "enabled": true, + "locationPolicy": "BALANCED", + "maxNodeCount": 100, + "minNodeCount": 1 + }, + "config": { + "advancedMachineFeatures": { + "enableNestedVirtualization": true + }, + "diskSizeGb": 100, + "diskType": "pd-standard", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "imageType": "COS_CONTAINERD", + "labels": { + "all-pools-example": "true", + "cluster_name": "node-pool-cluster-nca3", + "node_pool": "pool-05" + }, + "linuxNodeConfig": { + "sysctls": { + "net.core.netdev_max_backlog": "10000" + } + }, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "n1-standard-2", + "metadata": { + "cluster_name": "node-pool-cluster-nca3", + "disable-legacy-endpoints": "false", + "node_pool": "pool-05" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "default", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-node-pool-cluster-nca3", + "gke-node-pool-cluster-nca3-pool-05", + "all-node-example" + ], + "taints": [ + { + "effect": "PREFER_NO_SCHEDULE", + "key": "all-pools-example", + "value": "true" + } + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "2f3e3fb1-a3f9-48de-8218-27c281a1e800", + "initialNodeCount": 1, + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-05-7bc6f02a-grp" + ], + "locations": [ + "europe-west4-b" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "pool-05", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podIpv4RangeUtilization": 0.0624, + "podRange": "cft-gke-test-pods-nca3" + }, + "podIpv4CidrSize": 24, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/europe-west4/clusters/node-pool-cluster-nca3/nodePools/pool-05", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.30.5-gke.1443001" + }, + { + "config": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "imageType": "COS_CONTAINERD", + "kubeletConfig": { + "cpuCfsQuota": true, + "cpuManagerPolicy": "static", + "insecureKubeletReadonlyPortEnabled": false, + "podPidsLimit": "4096" + }, + "labels": { + "all-pools-example": "true", + "cluster_name": "node-pool-cluster-nca3", + "node_pool": "pool-03", + "sandbox.gke.io/runtime": "gvisor" + }, + "linuxNodeConfig": { + "sysctls": { + "net.core.netdev_max_backlog": "20000" + } + }, + "loggingConfig": { + "variantConfig": { + "variant": "DEFAULT" + } + }, + "machineType": "n1-standard-2", + "metadata": { + "cluster_name": "node-pool-cluster-nca3", + "disable-legacy-endpoints": "false", + "node_pool": "pool-03" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "sandboxConfig": { + "type": "GVISOR" + }, + "serviceAccount": "SERVICE_ACCOUNT", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "tags": [ + "gke-node-pool-cluster-nca3", + "gke-node-pool-cluster-nca3-pool-03", + "all-node-example" + ], + "taints": [ + { + "effect": "PREFER_NO_SCHEDULE", + "key": "all-pools-example", + "value": "true" + }, + { + "effect": "NO_SCHEDULE", + "key": "sandbox.gke.io/runtime", + "value": "gvisor" + } + ], + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "096b8f17-4c88-4e7d-ab95-999ae1c779e3", + "initialNodeCount": 2, + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-03-402f8758-grp", + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-c/instanceGroupManagers/gke-node-pool-cluster-nca3-pool-03-52ff6abd-grp" + ], + "locations": [ + "europe-west4-b", + "europe-west4-c" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "pool-03", + "networkConfig": { + "enablePrivateNodes": false, + "podIpv4CidrBlock": "172.16.0.0/18", + "podIpv4RangeUtilization": 0.0625, + "podRange": "test" + }, + "podIpv4CidrSize": 24, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/europe-west4/clusters/node-pool-cluster-nca3/nodePools/pool-03", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.30.5-gke.1443001" + }, + { + "autoscaling": { + "autoprovisioned": true, + "enabled": true, + "locationPolicy": "BALANCED", + "maxNodeCount": 1000 + }, + "config": { + "diskSizeGb": 100, + "diskType": "pd-standard", + "effectiveCgroupMode": "EFFECTIVE_CGROUP_MODE_V2", + "imageType": "COS_CONTAINERD", + "machineType": "e2-medium", + "metadata": { + "disable-legacy-endpoints": "true" + }, + "oauthScopes": [ + "https://www.googleapis.com/auth/cloud-platform" + ], + "serviceAccount": "SERVICE_ACCOUNT", + "shieldedInstanceConfig": { + "enableIntegrityMonitoring": true + }, + "windowsNodeConfig": {}, + "workloadMetadataConfig": { + "mode": "GKE_METADATA" + } + }, + "etag": "18f6eb8a-9ef5-455e-9732-ead4b2dd101b", + "instanceGroupUrls": [ + "https://www.googleapis.com/compute/v1/projects/PROJECT_ID/zones/europe-west4-b/instanceGroupManagers/gke-node-pool-cluste-nap-e2-medium-1d-71102fd7-grp" + ], + "locations": [ + "europe-west4-b" + ], + "management": { + "autoRepair": true, + "autoUpgrade": true, + "upgradeOptions": {} + }, + "maxPodsConstraint": { + "maxPodsPerNode": "110" + }, + "name": "nap-e2-medium-1d469r1p", + "networkConfig": { + "podIpv4CidrBlock": "192.168.0.0/18", + "podIpv4RangeUtilization": 0.0624, + "podRange": "cft-gke-test-pods-nca3" + }, + "placementPolicy": {}, + "podIpv4CidrSize": 24, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/europe-west4/clusters/node-pool-cluster-nca3/nodePools/nap-e2-medium-1d469r1p", + "status": "RUNNING", + "upgradeSettings": { + "maxSurge": 1, + "strategy": "SURGE" + }, + "version": "1.30.5-gke.1443001" + } + ], + "notificationConfig": { + "pubsub": {} + }, + "privateClusterConfig": { + "privateEndpoint": "10.0.0.2", + "publicEndpoint": "35.204.175.149" + }, + "rbacBindingConfig": { + "enableInsecureBindingSystemAuthenticated": true, + "enableInsecureBindingSystemUnauthenticated": true + }, + "releaseChannel": { + "channel": "REGULAR" + }, + "resourceLabels": { + "goog-terraform-provisioned": "true" + }, + "securityPostureConfig": { + "mode": "DISABLED", + "vulnerabilityMode": "VULNERABILITY_DISABLED" + }, + "selfLink": "https://container.googleapis.com/v1/projects/PROJECT_ID/locations/europe-west4/clusters/node-pool-cluster-nca3", + "servicesIpv4Cidr": "192.168.64.0/18", + "shieldedNodes": { + "enabled": true + }, + "status": "RUNNING", + "subnetwork": "cft-gke-test-nca3", + "verticalPodAutoscaling": {}, + "workloadIdentityConfig": { + "workloadPool": "PROJECT_ID.svc.id.goog" + }, + "zone": "europe-west4" +} + diff --git a/test/integration/testutils/cai.go b/test/integration/testutils/cai.go index 69f819d67..f219f6875 100644 --- a/test/integration/testutils/cai.go +++ b/test/integration/testutils/cai.go @@ -18,6 +18,8 @@ package utils import ( + "fmt" + "strings" "testing" "time" @@ -26,8 +28,9 @@ import ( ) type CmdCfg struct { - sleep int // minutes to sleep prior to CAI retreval. default: 2 - assetType string // asset type to retrieve. default: all + sleep int // minutes to sleep prior to CAI retreval. default: 2 + assetTypes []string // asset types to retrieve. empty: all + args []string // arguments to pass to call } type cmdOption func(*CmdCfg) @@ -35,14 +38,20 @@ type cmdOption func(*CmdCfg) // newCmdConfig sets defaults and options func newCmdConfig(opts ...cmdOption) (*CmdCfg) { caiOpts := &CmdCfg{ - sleep: 2, - assetType: "", + sleep: 2, + assetTypes: nil, + args: nil, } for _, opt := range opts { opt(caiOpts) } + if caiOpts.assetTypes != nil { + caiOpts.args = []string{"--asset-types", strings.Join(caiOpts.assetTypes, ",")} + } + caiOpts.args = append(caiOpts.args, "--content-type", "resource") + return caiOpts } @@ -53,20 +62,21 @@ func WithSleep(sleep int) cmdOption { } } -// Set asset type -func WithAssetType(assetType string) cmdOption { +// Set asset types +func WithAssetTypes(assetTypes []string) cmdOption { return func(f *CmdCfg) { - f.assetType = assetType + f.assetTypes = assetTypes } } // GetProjectResources returns the cloud asset inventory resources for a project as a gjson.Result func GetProjectResources(t testing.TB, project string, opts ...cmdOption) gjson.Result { caiOpts := newCmdConfig(opts...) + + // Cloud Asset Inventory offers best-effort data freshness. + t.Logf("Sleeping for %d minutes before retrieving Cloud Asset Inventory...", caiOpts.sleep) time.Sleep(time.Duration(caiOpts.sleep) * time.Minute) - if caiOpts.assetType != "" { - return gcloud.Runf(t, "asset list --project=%s --asset-types=%s --content-type=resource", project, caiOpts.assetType) - } else { - return gcloud.Runf(t, "asset list --project=%s --content-type=resource", project) - } + + cmd := fmt.Sprintf("asset list --project %s", project) + return gcloud.Runf(t, strings.Join(append([]string{cmd}, caiOpts.args...), " ")) }