Skip to content

Commit

Permalink
chore(CI): migrate tests to CFT
Browse files Browse the repository at this point in the history
  • Loading branch information
apeabody committed Nov 6, 2024
1 parent b1e42c1 commit 6236300
Show file tree
Hide file tree
Showing 12 changed files with 210 additions and 224 deletions.
30 changes: 0 additions & 30 deletions .kitchen.yml
Original file line number Diff line number Diff line change
Expand Up @@ -78,40 +78,10 @@ suites:
systems:
- name: stub_domains_upstream_nameservers
backend: local
- name: "workload_identity"
transport:
root_module_directory: test/fixtures/workload_identity
verifier:
systems:
- name: gcloud
backend: local
controls:
- gcloud
- name: gcp
backend: gcp
controls:
- gcp
- name: "workload_metadata_config"
transport:
root_module_directory: test/fixtures/workload_metadata_config
verifier:
systems:
- name: workload_metadata_config
backend: local
- name: "node_pool"
transport:
root_module_directory: test/fixtures/node_pool
verifier:
systems:
- name: node_pool
backend: local
controls:
- gcloud
- kubectl
- name: "safer_cluster_iap_bastion"
transport:
root_module_directory: test/fixtures/safer_cluster_iap_bastion
verifier:
systems:
- name: safer_cluster_iap_bastion
backend: local
18 changes: 9 additions & 9 deletions build/int.cloudbuild.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -309,17 +309,17 @@ steps:
waitFor:
- create-all
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge node-pool-local']
args: ['/bin/bash', '-c', 'cft test run TestNodePool --stage apply --verbose']
- id: verify node-pool-local
waitFor:
- converge node-pool-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify node-pool-local']
args: ['/bin/bash', '-c', 'cft test run TestNodePool --stage verify --verbose']
- id: destroy node-pool-local
waitFor:
- verify node-pool-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy node-pool-local']
args: ['/bin/bash', '-c', 'cft test run TestNodePool --stage destroy --verbose']
- id: apply sandbox-enabled-local
waitFor:
- create-all
Expand All @@ -339,32 +339,32 @@ steps:
waitFor:
- create-all
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge workload-identity-local']
args: ['/bin/bash', '-c', 'cft test run TestWorkloadIdentity --stage apply --verbose']
- id: verify workload-identity-local
waitFor:
- converge workload-identity-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify workload-identity-local']
args: ['/bin/bash', '-c', 'cft test run TestWorkloadIdentity --stage verify --verbose']
- id: destroy workload-identity-local
waitFor:
- verify workload-identity-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy workload-identity-local']
args: ['/bin/bash', '-c', 'cft test run TestWorkloadIdentity --stage destroy --verbose']
- id: converge safer-cluster-iap-bastion-local
waitFor:
- create-all
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do converge safer-cluster-iap-bastion-local']
args: ['/bin/bash', '-c', 'cft test run TestSaferClusterIapBastion --stage apply --verbose']
- id: verify safer-cluster-iap-bastion-local
waitFor:
- converge safer-cluster-iap-bastion-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do verify safer-cluster-iap-bastion-local']
args: ['/bin/bash', '-c', 'cft test run TestSaferClusterIapBastion --stage verify --verbose']
- id: destroy safer-cluster-iap-bastion-local
waitFor:
- verify safer-cluster-iap-bastion-local
name: 'gcr.io/cloud-foundation-cicd/$_DOCKER_IMAGE_DEVELOPER_TOOLS:$_DOCKER_TAG_VERSION_DEVELOPER_TOOLS'
args: ['/bin/bash', '-c', 'source /usr/local/bin/task_helper_functions.sh && kitchen_do destroy safer-cluster-iap-bastion-local']
args: ['/bin/bash', '-c', 'cft test run TestSaferClusterIapBastion --stage teardown --verbose']
- id: apply simple-zonal-with-asm-local
waitFor:
- create-all
Expand Down
1 change: 1 addition & 0 deletions examples/node_pool/main.tf
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ module "gke" {
disable_legacy_metadata_endpoints = false
cluster_autoscaling = var.cluster_autoscaling
deletion_protection = false
service_account = "default"

node_pools = [
{
Expand Down
6 changes: 5 additions & 1 deletion examples/workload_identity/main.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright 2018 Google LLC
* Copyright 2018-2024 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -24,6 +24,10 @@ provider "kubernetes" {
host = "https://${module.gke.endpoint}"
token = data.google_client_config.default.access_token
cluster_ca_certificate = base64decode(module.gke.ca_certificate)

ignore_annotations = [
"^iam.gke.io\\/.*"
]
}

module "gke" {
Expand Down
4 changes: 2 additions & 2 deletions test/fixtures/safer_cluster_iap_bastion/example.tf
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/**
* Copyright 2020 Google LLC
* Copyright 2020-2024 Google LLC
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -15,7 +15,7 @@
*/

locals {
test_command = "gcloud beta compute ssh ${module.example.bastion_name} --tunnel-through-iap --verbosity=error --project ${var.project_ids[1]} --zone ${module.example.bastion_zone} --ssh-flag=\"-T\" -q -- curl -sS https://${module.example.endpoint}/version -k"
test_command = "gcloud beta compute ssh ${module.example.bastion_name} --tunnel-through-iap --verbosity=error --project ${var.project_ids[1]} --zone ${module.example.bastion_zone} -q -- curl -sS https://${module.example.endpoint}/version -k"
}

module "example" {
Expand Down
88 changes: 88 additions & 0 deletions test/integration/node_pool/node_pool_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
// Copyright 2022-2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package node_pool

import (
"fmt"
"testing"
"time"

"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
"github.com/stretchr/testify/assert"
"github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils"
gkeutils "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/utils"
)

func TestNodePool(t *testing.T) {
bpt := tft.NewTFBlueprintTest(t,
tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute),
)

bpt.DefineVerify(func(assert *assert.Assertions) {
//Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token
// bpt.DefaultVerify(assert)
gkeutils.TGKEVerify(t, bpt, assert) // Verify Resources

projectId := bpt.GetStringOutput("project_id")
location := bpt.GetStringOutput("location")
clusterName := bpt.GetStringOutput("cluster_name")

op := gcloud.Runf(t, "container clusters describe %s --zone %s --project %s", clusterName, location, projectId)
assert.Contains([]string{"RUNNING", "RECONCILING"}, op.Get("status").String(), "Cluster is Running")

//or _, np := range op.Get("nodePools").Array() {
// npName := np.Get("name").String()
// switch npName {

//TODO

// case "pool-03":
// assert.JSONEq(fmt.Sprintf(`["%s-b", "%s-c"]`, location, location) , np.Get("locations").String())

// case "pool-05":

// }
//}
assert.Equal("pool-03", op.Get("nodePools.#(name==\"pool-03\").name").String(), "pool-3 exists")
assert.JSONEq(fmt.Sprintf(`["%s-b", "%s-c"]`, location, location), op.Get("nodePools.#(name==\"pool-03\").locations").String(), "has nodes in correct locations")
assert.Equal("n1-standard-2", op.Get("nodePools.#(name==\"pool-03\").config.machineType").String(), "is the expected machine type")
assert.True(op.Get("nodePools.#(name==\"pool-03\").autoscaling.enabled").Bool(), "has autoscaling enabled")
assert.Equal(2, op.Get("nodePools.#(name==\"pool-03\").initialNodeCount").Float(), "is the expected machine type")
assert.True(op.Get("nodePools.#(name==\"pool-03\").management.autoRepair").Bool(), "has autorepair enabled")
assert.True(op.Get("nodePools.#(name==\"pool-03\").management.autoUpgrade").Bool(), "has automatic upgrades enabled")
assert.JSONEq(fmt.Sprintf(`{"all-pools-example": "true", "cluster_name": %s, "node_pool": "pool-03", "sandbox.gke.io/runtime": "gvisor"}`, clusterName),
op.Get("nodePools.#(name==\"pool-03\").config.labels").String(), "has the expected labels")
assert.Contains([]string{"all-node-example",
fmt.Sprintf("gke-%s", clusterName),
fmt.Sprintf("gke-%s-pool-03", clusterName)},
op.Get("nodePools.#(name==\"pool-03\").config.tags").Array(), "has the expected network tags")
assert.Equal("172.16.0.0/18", op.Get("nodePools.#(name==\"pool-03\").networkConfig.podIpv4CidrBlock").String(), "has the expected pod range")
assert.Equal("test", op.Get("nodePools.#(name==\"pool-03\").networkConfig.podRange").String(), "has the expected pod range")
assert.Equal("COS_CONTAINERD", op.Get("nodePools.#(name==\"pool-03\").config.imageType").String(), "has the expected image")
assert.Equal("static", op.Get("nodePools.#(name==\"pool-03\").config.kubeletConfig.cpuManagerPolicy").String(), "has the expected kubelet config")
assert.True(op.Get("nodePools.#(name==\"pool-03\").management.config.kubeletConfig.cpuCfsQuota").Bool(), "has the expected kubelet config")
assert.Equal(20000, op.Get("nodePools.#(name==\"pool-03\").config.linuxNodeConfig.sysctls.net.core.netdev_max_backlog").Float(), "has the expected linux node config sysctls")


assert.Equal("pool-04", op.Get("nodePools.#(name==\"pool-04\").name").String(), "pool-4 exists")
assert.True(true, op.Get("nodePools.#(name==\"pool-04\").queued_provisioning.enabled").Bool(), "has queued provisioning enabled")

assert.Equal("pool-05", op.Get("nodePools.#(name==\"pool-05\").name").String(), "pool-5 exists")
assert.True(op.Get("nodePools.#(name==\"pool-05\").advanced_machine_features.enable_nested_virtualization").Bool(), "has enable_nested_virtualization enabled")

})

bpt.Test()
}
37 changes: 0 additions & 37 deletions test/integration/safer_cluster_iap_bastion/controls/e2e.rb

This file was deleted.

22 changes: 0 additions & 22 deletions test/integration/safer_cluster_iap_bastion/inspec.yml

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
// Copyright 2022-2024 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package safer_cluster_iap_bastion

import (
"fmt"
"strings"
"testing"
"time"

"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud"
"github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft"
"github.com/stretchr/testify/assert"
"github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/testutils"
gkeutils "github.com/terraform-google-modules/terraform-google-kubernetes-engine/test/integration/utils"
)

func TestSaferClusterIapBastion(t *testing.T) {
bpt := tft.NewTFBlueprintTest(t,
tft.WithRetryableTerraformErrors(testutils.RetryableTransientErrors, 3, 2*time.Minute),
)

bpt.DefineVerify(func(assert *assert.Assertions) {
//Skipping Default Verify as the Verify Stage fails due to change in Client Cert Token
// bpt.DefaultVerify(assert)
gkeutils.TGKEVerify(t, bpt, assert) // Verify Resources

test_command, _ := strings.CutPrefix(bpt.GetStringOutput("test_command"), "gcloud ")

//pre run ssh command so that ssh-keygen can run
gcloud.RunCmd(t, test_command,
gcloud.WithCommonArgs([]string{}),
)

cluster_version := fmt.Sprintf("v%s", bpt.GetStringOutput("cluster_version"))

op := gcloud.Run(t, test_command,
gcloud.WithCommonArgs([]string{}),
)

assert.Equal(cluster_version, op.Get("gitVersion").String(), "SSH into VM and verify connectivity to GKE")
})

bpt.Test()
}
Loading

0 comments on commit 6236300

Please sign in to comment.