diff --git a/tests/acceptance/.gitignore b/tests/acceptance/.gitignore new file mode 100644 index 0000000000..574916ee83 --- /dev/null +++ b/tests/acceptance/.gitignore @@ -0,0 +1,8 @@ +.ssh/ +config/ +/tests/.vscode +/.idea/ +/modules/.terraform/ +*.terraform* +*.tfstate* +*plan.yaml \ No newline at end of file diff --git a/tests/terraform/.golangci.yaml b/tests/acceptance/.golangci.yaml similarity index 95% rename from tests/terraform/.golangci.yaml rename to tests/acceptance/.golangci.yaml index 273f792582..1c1aa90aba 100644 --- a/tests/terraform/.golangci.yaml +++ b/tests/acceptance/.golangci.yaml @@ -28,7 +28,7 @@ linters-settings: ignore-generated-header: true rules: - name: line-length-limit - arguments: [100] + arguments: [110] - name: cognitive-complexity arguments: [10] - name: empty-lines @@ -37,7 +37,6 @@ linters-settings: - name: blank-imports - name: confusing-naming - name: confusing-results - - name: context-as-argument - name: duplicated-imports - name: early-return - name: empty-block @@ -46,7 +45,6 @@ linters-settings: - name: error-strings - name: errorf - name: exported - - name: flag-parameter - name: get-return - name: if-return - name: increment-decrement diff --git a/tests/acceptance/Makefile b/tests/acceptance/Makefile new file mode 100644 index 0000000000..07d454553c --- /dev/null +++ b/tests/acceptance/Makefile @@ -0,0 +1,90 @@ +include ./config.mk + +TAGNAME ?= default +test-env-up: + @cd ../.. && docker build . -q -f ./tests/acceptance/scripts/Dockerfile.build -t rke2-automated-${TAGNAME} + +.PHONY: test-run +test-run: + @docker run --name rke2-automated-test-${IMGNAME} -t \ + -e AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \ + -e AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \ + -v ${ACCESS_KEY_LOCAL}:/go/src/github.com/rancher/rke2/tests/acceptance/modules/config/.ssh/aws_key.pem \ + rke2-automated-${TAGNAME} sh -c 'cd ./tests/acceptance/entrypoint; \ + if [ -n "${TESTDIR}" ]; then \ + if [ "${TESTDIR}" = "upgradecluster" ]; then \ + if [ "${TESTTAG}" = "upgradesuc" ]; then \ + go test -timeout=45m -v -tags=upgradesuc ./upgradecluster/... -upgradeVersion "${UPGRADEVERSION}"; \ + elif [ "${TESTTAG}" = "upgrademanual" ]; then \ + go test -timeout=45m -v -tags=upgrademanual ./upgradecluster/... -installVersionOrCommit "${INSTALLTYPE}"; \ + fi; \ + elif [ "${TESTDIR}" = "versionbump" ]; then \ + go test -timeout=45m -v -tags=versionbump ./versionbump/... -cmd "${CMD}" -expectedValue "${VALUE}" \ + -expectedValueUpgrade "${VALUEUPGRADE}" -installVersionOrCommit "${INSTALLTYPE}" -channel "${CHANNEL}" -testCase "${TESTCASE}" \ + -deployWorkload "${DEPLOYWORKLOAD}" -workloadName "${WORKLOADNAME}" -description "${DESCRIPTION}"; \ + fi; \ + elif [ -z "${TESTDIR}" ]; then \ + go test -timeout=45m -v ./createcluster/...; \ + fi;' + + +.PHONY: test-logs +test-logs: + @docker logs -f rke2-automated-test-${IMGNAME} + + +.PHONY: test-env-down +test-env-down: + @echo "Removing containers and images" + @docker stop $$(docker ps -a -q --filter="name=rke2-automated*") + @docker rm $$(docker ps -a -q --filter="name=rke2-automated*") + @docker rmi $$(docker images -q --filter="reference=rke2-automated*") + + +.PHONY: test-env-clean +test-env-clean: + @./scripts/delete_resources.sh + + +.PHONY: test-complete +test-complete: test-env-clean test-env-down remove-tf-state test-env-up test-run + + +.PHONY: remove-tf-state +remove-tf-state: + @rm -rf ./modules/.terraform + @rm -rf ./modules/.terraform.lock.hcl ./modules/terraform.tfstate ./modules/terraform.tfstate.backup + + +#======================= Run acceptance tests locally =========================# + +.PHONY: test-create +test-create: + @go test -timeout=45m -v ./entrypoint/createcluster/... + + +.PHONY: test-upgrade-suc +test-upgrade-suc: + @go test -timeout=45m -v -tags=upgradesuc ./entrypoint/upgradecluster/... -upgradeVersion ${UPGRADEVERSION} + + +.PHONY: test-upgrade-manual +test-upgrade-manual: + @go test -timeout=45m -v -tags=upgrademanual ./entrypoint/upgradecluster/... -installVersionOrCommit ${INSTALLTYPE} + + +.PHONY: test-version-bump +test-version-bump: + go test -timeout=45m -v -tags=versionbump ./entrypoint/versionbump/... \ + -cmd "${CMD}" \ + -expectedValue ${VALUE} \ + -expectedValueUpgrade ${VALUEUPGRADED} \ + -installVersionOrCommit ${INSTALLTYPE} -channel ${CHANNEL} \ + -testCase "${TESTCASE}" -deployWorkload ${DEPLOYWORKLOAD} -workloadName ${WORKLOADNAME} -description "${DESCRIPTION}" + + +#========================= TestCode Static Quality Check =========================# +.PHONY: vet-lint ## Run locally only inside Tests package +vet-lint: + @echo "Running go vet and lint" + @go vet ./${TESTDIR} && golangci-lint run --tests \ No newline at end of file diff --git a/tests/acceptance/README.md b/tests/acceptance/README.md new file mode 100644 index 0000000000..7691a1ff48 --- /dev/null +++ b/tests/acceptance/README.md @@ -0,0 +1,313 @@ +## Acceptance Test Framework + +The acceptance tests are a customizable way to create clusters and perform validations on them such that the requirements of specific features and functions can be validated. + +- It relies on [Terraform](https://www.terraform.io/) to provide the underlying cluster configuration. +- It uses [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) as assertion framework. + + +## Architecture +- For better maintenance, readability and productivity we encourage max of separation of concerns and loose coupling between packages so inner packages should not depend on outer packages + +### Packages: +```bash +./acceptance +│ +├── core +│ └───── Place where resides the logic and services for it +| +├── entrypoint +│ └───── Where is located the entrypoint for tests execution, separated by test runs and test suites +│ +├── fixtures +│ └───── Place where resides the fixtures for tests +│ +├── modules +│ └───── Terraform modules and configurations +│ +├── shared +│ └───── shared and reusable functions, workloads, constants, and scripts + +``` + + + +### Explanation: + +- `Core` +``` + Service: + +Act: Acts as a provider for customizations across framework +Responsibility: Should not depend on any outer layer only in the core itself, the idea is to provide not rely on. + + + Testcase: + +Act: Acts as a innermost layer where the main logic (test implementations) is handled. +Responsibility: Encapsulate test logic and should not depend on any outer layer + +``` + +- `Entrypoint` +```` +Act: Acts as the one of the outer layer to receive the input to start test execution +Responsibility: Should not need to implement any logic and only focus on orchestrating +```` + +- `Fixtures` +``` +Act: It acts as a provider for test fixtures +Responsibility: Totally independent of any other layer and should only provide +``` + +- `Modules` +``` +Act: It acts as the infra to provide the terraform modules and configurations +Responsibility: Only provides indirectly for all, should not need the knowledge of any test logic or have dependencies from internal layers. +``` + +- `Shared` +``` +Act: It acts as an intermediate module providing shared and reusable functions, constants, and scripts +Responsibility: Should not need the knowledge or "external" dependency at all, provides for all. +``` + +#### PS: "External" and "Outer" dependency here in this context is considered any other package within the acceptance framework. + +------------------- + +### `Template Bump Version Model ` + +- We have a template model interface for testing bump versions, the idea is to provide a simple and direct way to test bump of version using go test tool. + + +```You can test that like:``` + +- Adding one version or commit and ran some commands on it and check it against respective expected values then upgrade and repeat the same commands and check the respective new (or not) expected values. + + +```How can I do that?``` + +- Step 1: Add your desired first version or commit that you want to use on `local.tfvars` file on the vars `k3s_version` and `install_mode` +- Step 2: Have the commands you need to run and the expected output from them +- Step 3: Have a version or commit that you want to upgrade to. +- Step 4: On the TestConfig field you can add another test case that we already have or a newly created one. +- Step 5: You can add a standalone workload deploy if you need +- Step 6: Just fill the go test or make command with your required values +- Step 7: Run the command and wait for results. +- Step 8: (WIP) Export your customizable report. + + + +Available arguments to create your command with examples: +```` +- $ -cmd "kubectl describe pod rke2-canal- -n kube-system : | grep -i Image" +- $ -expectedValue "v0.0.21" +- $ -expectedValueUpgrade "v0.0.24" +- $ -installVersionOrCommit INSTALL_K3S_COMMIT=257fa2c54cda332e42b8aae248c152f4d1898218 +- $ -deployWorkload true +- $ -testCase "TestLocalPathProvisionerStorage" +- $ -workloadName "bandwidth-annotations.yaml" +- $ -description "Description of your test" + +* All non-boolean arguments is comma separated in case you need to send more than 1. + +* If you need to separate another command to run as a single here , separate those with " : " as this example: +`-cmd "kubectl describe pod rke2-canal- -n kube-system : | grep -i Image"` +``` + +Example of an execution with multiple values: +```bash + go test -v -timeout=45m -tags=versionbump ./entrypoint/versionbump/... \ + -cmd "(find /var/lib/rancher/rke2/data/ -type f -name runc -exec {} --version \\;), rke2 -v" \ + -expectedValue "v1.9.3, v1.25.9+rke2r1" \ + -expectedValueUpgrade "v1.10.1, v1.26.4-rc1+rke2r1" \ + -installVersionOrCommit INSTALL_RKE2_VERSION=v1.25.9+rke2r1 \ + -testCase "TestServiceClusterIp, TestIngress" \ + -deployWorkload true \ + -workloadName "ingress.yaml" \ + -description "Testing ingress and service cluster ip" + + ``` + +Example of an execution with less args: +````bash +go test -timeout=45m -v -tags=versionbump ./entrypoint/versionbump/... \ +-cmd "(find /var/lib/rancher/rke2/data/ -type f -name runc -exec {} --version \\;)" \ +-expectedValue "1.1.7" \ +-expectedValueUpgrade "1.10.1" \ +-installVersionOrCommit INSTALL_RKE2_VERSION=v1.27.2+rke2r1 \ +```` + + +#### We also have this on the `makefile` to make things easier to run just adding the values, please see below on the makefile section + + +----- +#### Testcase naming convention: +- All tests should be placed under `tests/acceptance/testcase/`. +- All test functions should be named: `Test`. + + + +## Running + +- Before running the tests, you should creat local.tfvars file in `./tests/acceptance/modules/config/local.tfvars`. There is some information there to get you started, but the empty variables should be filled in appropriately per your AWS environment. + +- Please make sure to export your correct AWS credentials before running the tests. e.g: +```bash +export AWS_ACCESS_KEY_ID= +export AWS_SECRET_ACCESS_KEY= +``` + +- The local.tfvars split roles section should be strictly followed to not cause any false positives or negatives on tests + +- Please also when creating tf var resource_name, make sure that you do not have any instances from other automations with the same name to avoid deleting wrong resources + + +*** + +Tests can be run individually per package or per test tags from acceptance package: +```bash +go test -timeout=45m -v ./entrypoint/$PACKAGE_NAME/... + +go test -timeout=45m -v -tags=upgrademanual ./entrypoint/upgradecluster/... -installVersionOrCommit INSTALL_RKE2_VERSION=v1.25.8+rke2r1 + +go test -timeout=45m -v -tags=upgradesuc ./entrypoint/upgradecluster/... -upgradeVersion v1.25.8+rke2r1 + +``` +Test flags: +``` + ${upgradeVersion} version to upgrade to as SUC + -upgradeVersion v1.26.2+rke2r1 + + ${installType} type of installation (version or commit) + desired value + -installVersionOrCommit Version or commit +``` +Test tags: +``` + -tags=upgradesuc + -tags=upgrademanual + -tags=versionbump +``` +### Run with `Makefile` through acceptance package: +```bash +- On the first run with make and docker please delete your .terraform folder, terraform.tfstate and terraform.hcl.lock file + +Args: +*Most of args are optional so you can fit to your use case. + +- ${IMGNAME} append any string to the end of image name +- ${TAGNAME} append any string to the end of tag name +- ${ARGNAME} name of the arg to pass to the test +- ${ARGVALUE} value of the arg to pass to the test +- ${TESTDIR} path to the test directory +- ${TESTFILE} path to the test file +- ${TAGTEST} name of the tag function from suite ( -tags=upgradesuc or -tags=upgrademanual ) +- ${TESTCASE} name of the testcase to run +- ${DEPLOYWORKLOAD} true or false to deploy workload +- ${CMD} command to run +- ${VALUE} value to check +- ${VALUEUPGRADE} value to check after upgrade +- ${INSTALLTYPE} type of installation (version or commit) + desired value +- &{WORKLOADNAME} name of the workload to deploy +- &{DESCRIPTION} description of the test +- &{CHANNEL} channel to use for upgrade + + + +Commands: +$ make test-env-up # create the image from Dockerfile.build +$ make test-run # runs create and upgrade cluster by passing the argname and argvalue +$ make test-env-down # removes the image and container by prefix +$ make test-env-clean # removes instances and resources created by testcase +$ make test-logs # prints logs from container the testcase +$ make test-create # runs create cluster test locally +$ make test-upgrade-suc # runs upgrade via SUC +$ make test-upgrade-manual # runs upgrade manually +$ make test-version-bump # runs version bump test locally +$ make test-run # runs create and upgrade cluster by passing the argname and argvalue +$ make remove-tf-state # removes acceptance state dir and files +$ make test-suite # runs all testcase locally in sequence not using the same state +$ make vet-lint # runs go vet and go lint +``` + +### Examples with docker: +``` +- Create an image tagged +$ make test-env-up TAGNAME=ubuntu + +- Run upgrade cluster test with `${IMGNAME}` and `${TAGNAME}` +$ make test-run IMGNAME=2 TAGNAME=ubuntu + + +- Run create and upgrade cluster just adding `INSTALLTYPE` flag to upgrade +$ make test-run + + +- Run version bump test upgrading with commit id +$ make test-run IMGNAME=x \ + TAGNAME=y \ + TESTDIR=versionbump \ + CMD="rke2 --version, kubectl get image..." \ + VALUE="v1.26.2+rke2r1, v0.0.21" \ + VALUEUPGRADE="v1.27.2+rke2r1, v0.0.24" + INSTALLTYPE=INSTALL_RKE2_COMMIT=257fa2c54cda332e42b8aae248c152f4d1898218 \ + TESTCASE=TestCaseName \ + DEPLOYWORKLOAD=true \ + WORKLOADNAME="someWorkload.yaml" \ + DESCRIPTION="some description" \ + CHANNEL="stable" + ```` + +### Examples to run locally: +```` +- Run create cluster test: +$ make test-create + +- Run upgrade cluster test SUC: +$ make test-upgrade-suc UPGRADEVERSION=v1.26.4+rke2r1 + +- Run upgrade cluster test: +$ make test-upgrade-manual INSTALLTYPE=INSTALL_RKE2_VERSION=v1.26.4+rke2r1 + + +- Logs from test +$ make tf-logs IMGNAME=1 + +- Run lint for a specific directory +$ make vet-lint TESTDIR=upgradecluster + +```` + + +### Running tests in parallel: + +- You can play around and have a lot of different test combinations like: +``` +- Build docker image with different TAGNAME="OS`s" + with different configurations( resource_name, node_os, versions, install type, nodes and etc) and have unique "IMGNAMES" + +- And in the meanwhile run also locally with different configuration while your dockers TAGNAME and IMGNAMES are running +``` + + +### In between tests: +```` +- If you want to run with same cluster do not delete ./tests/acceptance/modules/terraform.tfstate + .terraform.lock.hcl file after each test. + +- If you want to use new resources then make sure to delete the ./tests/acceptance/modules/terraform.tfstate + .terraform.lock.hcl file if you want to create a new cluster. +```` + +### Common Issues: +``` +- Issues related to terraform plugin please also delete the modules/.terraform folder +- Issues related to terraform failed to find local token , please also delete modules/.terraform folder +- In mac m1 maybe you need also to go to rke2/tests/terraform/modules and run `terraform init` to download the plugins +``` + +### Debugging +```` +To focus individual runs on specific test clauses, you can prefix with `F`. For example, in the [create cluster test](../tests/acceptance/entrypoint/createcluster_test.go), you can update the initial creation to be: `FIt("Starts up with no issues", func() {` in order to focus the run on only that clause. +Or use break points in your IDE. +```` diff --git a/tests/terraform/config.mk b/tests/acceptance/config.mk similarity index 82% rename from tests/terraform/config.mk rename to tests/acceptance/config.mk index 89a789f028..5d6aa26674 100644 --- a/tests/terraform/config.mk +++ b/tests/acceptance/config.mk @@ -1,6 +1,6 @@ SHELL := /bin/bash -TFVARS_PATH := terraform/modules/config/local.tfvars +TFVARS_PATH := acceptance/modules/config/local.tfvars ifeq ($(wildcard ${TFVARS_PATH}),) RESOURCE_NAME := diff --git a/tests/acceptance/core/service/assert/host.go b/tests/acceptance/core/service/assert/host.go new file mode 100644 index 0000000000..ec316cba83 --- /dev/null +++ b/tests/acceptance/core/service/assert/host.go @@ -0,0 +1,34 @@ +package assert + +import ( + "fmt" + "strings" + + "github.com/rancher/rke2/tests/acceptance/shared" + + . "github.com/onsi/gomega" +) + +// CheckComponentCmdHost runs a command on the host and asserts that the value received contains the specified substring +// +// you can send multiple asserts from a cmd but all of them must be true +// +// need to send sKubeconfigFile +func CheckComponentCmdHost(cmd string, asserts ...string) { + Eventually(func() error { + fmt.Println("Executing cmd: ", cmd) + res, err := shared.RunCommandHost(cmd) + if err != nil { + return fmt.Errorf("error on RunCommandHost: %v", err) + } + + for _, assert := range asserts { + if !strings.Contains(res, assert) { + return fmt.Errorf("expected substring %q not found in result %q", assert, res) + } + fmt.Println("Result:", res+"\nMatched with assert:\n", assert) + } + + return nil + }, "420s", "5s").Should(Succeed()) +} diff --git a/tests/acceptance/core/service/assert/node.go b/tests/acceptance/core/service/assert/node.go new file mode 100644 index 0000000000..d21040090d --- /dev/null +++ b/tests/acceptance/core/service/assert/node.go @@ -0,0 +1,92 @@ +package assert + +import ( + "fmt" + "strings" + + "github.com/rancher/rke2/tests/acceptance/core/service/customflag" + "github.com/rancher/rke2/tests/acceptance/shared" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type NodeAssertFunc func(g Gomega, node shared.Node) + +// NodeAssertVersionTypeUpgrade custom assertion func that asserts that node version is as expected +func NodeAssertVersionTypeUpgrade(installType customflag.FlagConfig) NodeAssertFunc { + if installType.InstallUpgrade != nil { + if strings.HasPrefix(customflag.ServiceFlag.InstallUpgrade.String(), "v") { + return assertVersion(installType) + } + return assertCommit(installType) + } + + return func(g Gomega, node shared.Node) { + GinkgoT().Errorf("no version or commit specified for upgrade assertion") + } +} + +// assertVersion returns the NodeAssertFunc for asserting version +func assertVersion(installType customflag.FlagConfig) NodeAssertFunc { + if installType.InstallUpgrade != nil { + if strings.HasPrefix(customflag.ServiceFlag.InstallUpgrade.String(), "v") { + fmt.Printf("Asserting Version: %s\n", installType.InstallUpgrade.String()) + return func(g Gomega, node shared.Node) { + g.Expect(node.Version).Should(ContainSubstring(installType.InstallUpgrade.String()), + "Nodes should all be upgraded to the specified version", node.Name) + } + } + } + + return func(g Gomega, node shared.Node) { + GinkgoT().Errorf("no version specified for upgrade assertion") + } +} + +// assertCommit returns the NodeAssertFunc for asserting commit +func assertCommit(installType customflag.FlagConfig) NodeAssertFunc { + if installType.InstallUpgrade != nil { + upgradedVersion := shared.GetRke2Version() + fmt.Printf("Asserting Commit: %s\n Version: %s", + installType.InstallUpgrade.String(), upgradedVersion) + return func(g Gomega, node shared.Node) { + g.Expect(upgradedVersion).Should(ContainSubstring(installType.InstallUpgrade.String()), + "Nodes should all be upgraded to the specified commit", node.Name) + } + } + + return func(g Gomega, node shared.Node) { + GinkgoT().Errorf("no commit specified for upgrade validation") + } +} + +// NodeAssertVersionUpgraded custom assertion func that asserts that node version is as expected +func NodeAssertVersionUpgraded() NodeAssertFunc { + return func(g Gomega, node shared.Node) { + g.Expect(&customflag.ServiceFlag.UpgradeVersionSUC).Should(ContainSubstring(node.Version), + "Nodes should all be upgraded to the specified version", node.Name) + } +} + +// NodeAssertReadyStatus custom assertion func that asserts that the node is in Ready state. +func NodeAssertReadyStatus() NodeAssertFunc { + return func(g Gomega, node shared.Node) { + g.Expect(node.Status).Should(Equal("Ready"), + "Nodes should all be in Ready state") + } +} + +// CheckComponentCmdNode runs a command on a node and asserts that the value received +// contains the specified substring. +func CheckComponentCmdNode(cmd, assert, ip string) { + Eventually(func(g Gomega) { + fmt.Println("Executing cmd: ", cmd) + res, err := shared.RunCommandOnNode(cmd, ip) + if err != nil { + return + } + g.Expect(res).Should(ContainSubstring(assert)) + fmt.Println("Result:", res+"\nMatched with assert:\n", assert) + }, "420s", "3s").Should(Succeed()) +} diff --git a/tests/acceptance/core/service/assert/pod.go b/tests/acceptance/core/service/assert/pod.go new file mode 100644 index 0000000000..f9ddc28632 --- /dev/null +++ b/tests/acceptance/core/service/assert/pod.go @@ -0,0 +1,84 @@ +package assert + +import ( + "fmt" + "strings" + + "github.com/rancher/rke2/tests/acceptance/shared" + + . "github.com/onsi/gomega" + "github.com/onsi/gomega/types" +) + +type PodAssertFunc func(g Gomega, pod shared.Pod) + +var completedAssert = "Completed" + +// PodAssertRestart custom assertion func that asserts that pods are not restarting with no reason +// controller, scheduler, helm-install pods can be restarted occasionally when cluster started if only once +func PodAssertRestart() PodAssertFunc { + return func(g Gomega, pod shared.Pod) { + if strings.Contains(pod.NameSpace, "kube-system") && + strings.Contains(pod.Name, "controller") && + strings.Contains(pod.Name, "scheduler") { + g.Expect(pod.Restarts).Should(SatisfyAny(Equal("0"), + Equal("1")), + "could be restarted occasionally when cluster started", pod.Name) + } + } +} + +// PodAssertReady custom assertion func that asserts that the pod is +// with correct numbers of ready containers. +func PodAssertReady() PodAssertFunc { + return func(g Gomega, pod shared.Pod) { + g.ExpectWithOffset(1, pod.Ready).To(checkReadyFields(), + "should have equal values in n/n format") + } +} + +// checkReadyFields is a custom matcher that checks +// if the input string is in N/N format and the same quantity. +func checkReadyFields() types.GomegaMatcher { + return WithTransform(func(s string) (bool, error) { + var a, b int + + n, err := fmt.Sscanf(s, "%d/%d", &a, &b) + if err != nil || n != 2 { + return false, fmt.Errorf("failed to parse format: %v", err) + } + + return a == b, nil + }, BeTrue()) +} + +// PodAssertStatus custom assertion that asserts that pod status is completed or in some cases +// apply pods can have an error status +func PodAssertStatus() PodAssertFunc { + return func(g Gomega, pod shared.Pod) { + if strings.Contains(pod.Name, "helm-install") { + g.Expect(pod.Status).Should(Equal(completedAssert), pod.Name) + } else if strings.Contains(pod.Name, "apply") && + strings.Contains(pod.NameSpace, "system-upgrade") { + g.Expect(pod.Status).Should(SatisfyAny( + ContainSubstring("Error"), + Equal(completedAssert), + ), pod.Name) + } else { + g.Expect(pod.Status).Should(Equal("Running"), pod.Name) + } + } +} + +// CheckPodStatusRunning asserts that the pod is running with the specified label = app name. +func CheckPodStatusRunning(name, namespace, assert string) { + cmd := "kubectl get pods -n " + namespace + " -o=name -l k8s-app=" + name + + " --field-selector=status.phase=Running --kubeconfig=" + shared.KubeConfigFile + Eventually(func(g Gomega) { + res, err := shared.RunCommandHost(cmd) + if err != nil { + return + } + g.Expect(res).Should(ContainSubstring(assert)) + }, "180s", "5s").Should(Succeed()) +} diff --git a/tests/acceptance/core/service/assert/validate.go b/tests/acceptance/core/service/assert/validate.go new file mode 100644 index 0000000000..2eea36ad57 --- /dev/null +++ b/tests/acceptance/core/service/assert/validate.go @@ -0,0 +1,98 @@ +package assert + +import ( + "fmt" + "strings" + "time" + + "github.com/rancher/rke2/tests/acceptance/shared" +) + +// validate calls runAssertion for each cmd/assert pair +// +// the first caller - process tests will spawn a go routine per ip the cluster +// +// need to send KubeconfigFile +func validate(exec func(string) (string, error), args ...string) error { + if len(args) < 2 || len(args)%2 != 0 { + return fmt.Errorf("must receive an even number of arguments as cmd/assert pairs") + } + + errorsChan := make(chan error, len(args)/2) + timeout := time.After(220 * time.Second) + ticker := time.NewTicker(3 * time.Second) + + for i := 0; i < len(args); i++ { + cmd := args[i] + if i+1 < len(args) { + assert := args[i+1] + i++ + + err := runAssertion(cmd, assert, exec, ticker.C, timeout, errorsChan) + if err != nil { + fmt.Printf("error from runAssertion:\n %s\n", err) + close(errorsChan) + return err + } + } + } + close(errorsChan) + + return nil +} + +// runAssertion runs a command and asserts that the value received against his respective command +func runAssertion( + cmd, assert string, + exec func(string) (string, error), + ticker <-chan time.Time, + timeout <-chan time.Time, + errorsChan chan<- error, +) error { + for { + select { + case <-timeout: + timeoutErr := fmt.Errorf("timeout reached for command:\n%s\n "+"Trying to assert with:\n %s", + cmd, assert) + errorsChan <- timeoutErr + return timeoutErr + + case <-ticker: + res, err := exec(cmd) + if err != nil { + errorsChan <- err + return fmt.Errorf("error from runCmd:\n %s\n %s", res, err) + } + fmt.Printf("\n---------------------\nCommand:\n"+ + "%s\n"+ + "\n---------------------\nResult:\n"+ + "%s\n"+ + "\n---------------------\nAssertion:\n"+ + "%s\n", cmd, res, assert) + if strings.Contains(res, assert) { + fmt.Printf("Matched with: \n%s\n", res) + errorsChan <- nil + return nil + } + } + } +} + +// ValidateOnHost runs an exec function on RunCommandHost and assert given is fulfilled. +// The last argument should be the assertion. +// Need to send kubeconfig file. +func ValidateOnHost(args ...string) error { + exec := func(cmd string) (string, error) { + return shared.RunCommandHost(cmd) + } + return validate(exec, args...) +} + +// ValidateOnNode runs an exec function on RunCommandHost and assert given is fulfilled. +// The last argument should be the assertion. +func ValidateOnNode(ip string, args ...string) error { + exec := func(cmd string) (string, error) { + return shared.RunCommandOnNode(cmd, ip) + } + return validate(exec, args...) +} diff --git a/tests/acceptance/core/service/customflag/model.go b/tests/acceptance/core/service/customflag/model.go new file mode 100644 index 0000000000..4f6ae51216 --- /dev/null +++ b/tests/acceptance/core/service/customflag/model.go @@ -0,0 +1,147 @@ +package customflag + +import ( + "fmt" + "strconv" + "strings" +) + +var ServiceFlag FlagConfig +var TestCaseNameFlag StringSlice + +type FlagConfig struct { + InstallType InstallTypeValueFlag + InstallUpgrade MultiValueFlag + TestConfig TestConfigFlag + ClusterConfig ClusterConfigFlag + UpgradeVersionSUC UpgradeVersionFlag +} + +// UpgradeVersionFlag is a custom type to use upgradeVersionSUC flag +type UpgradeVersionFlag struct { + Version string +} + +// InstallTypeValueFlag is a customflag type that can be used to parse the installation type +type InstallTypeValueFlag struct { + Version []string + Commit []string + Channel string +} + +// TestConfigFlag is a customflag type that can be used to parse the test case argument +type TestConfigFlag struct { + TestFuncNames []string + TestFuncs []TestCaseFlag + DeployWorkload bool + WorkloadName string + Description string +} + +// TestCaseFlag is a customflag type that can be used to parse the test case argument +type TestCaseFlag func(deployWorkload bool) + +// MultiValueFlag is a customflag type that can be used to parse multiple values +type MultiValueFlag []string + +// DestroyFlag is a customflag type that can be used to parse the destroy flag +type DestroyFlag bool + +// ClusterConfigFlag is a customFlag type that can be used to change some cluster config +type ClusterConfigFlag struct { + Destroy DestroyFlag +} + +// StringSlice defines a custom flag type for string slice +type StringSlice []string + +// String returns the string representation of the StringSlice +func (s *StringSlice) String() string { + return strings.Join(*s, ",") +} + +// Set parses the input string and sets the StringSlice using Set customflag interface +func (s *StringSlice) Set(value string) error { + *s = strings.Split(value, ",") + return nil +} + +// String returns the string representation of the MultiValueFlag +func (m *MultiValueFlag) String() string { + return strings.Join(*m, ",") +} + +// Set func sets multiValueFlag appending the value +func (m *MultiValueFlag) Set(value string) error { + *m = append(*m, value) + return nil +} + +// String returns the string representation of the TestConfigFlag +func (t *TestConfigFlag) String() string { + return fmt.Sprintf("TestFuncName: %s", t.TestFuncNames) +} + +// Set parses the customFlag value for TestConfigFlag +func (t *TestConfigFlag) Set(value string) error { + t.TestFuncNames = strings.Split(value, ",") + return nil +} + +// String returns the string representation of the InstallTypeValue +func (i *InstallTypeValueFlag) String() string { + return fmt.Sprintf("Version: %s, Commit: %s", i.Version, i.Commit) +} + +// Set parses the input string and sets the Version or Commit field using Set customflag interface +func (i *InstallTypeValueFlag) Set(value string) error { + parts := strings.Split(value, "=") + + for _, part := range parts { + subParts := strings.Split(part, "=") + if len(subParts) != 2 { + return fmt.Errorf("invalid input format") + } + switch parts[0] { + case "INSTALL_RKE2_VERSION": + i.Version = append(i.Version, subParts[1]) + case "INSTALL_RKE2_COMMIT": + i.Commit = append(i.Commit, subParts[1]) + default: + return fmt.Errorf("invalid install type: %s", parts[0]) + } + } + + return nil +} + +// String returns the string representation of the UpgradeVersion for SUC upgrade +func (t *UpgradeVersionFlag) String() string { + return t.Version +} + +// Set parses the input string and sets the Version field for SUC upgrades +func (t *UpgradeVersionFlag) Set(value string) error { + if !strings.HasPrefix(value, "v") && !strings.HasSuffix(value, "rke2r1") { + return fmt.Errorf("invalid install format: %s", value) + } + + t.Version = value + return nil +} + +// String returns the string representation of the DestroyFlag +func (d *DestroyFlag) String() string { + return fmt.Sprintf("%v", *d) +} + +// Set parses the customFlag value for DestroyFlag +func (d *DestroyFlag) Set(value string) error { + v, err := strconv.ParseBool(value) + if err != nil { + return err + } + *d = DestroyFlag(v) + + return nil +} diff --git a/tests/acceptance/core/service/factory/cluster.go b/tests/acceptance/core/service/factory/cluster.go new file mode 100644 index 0000000000..dbb3626462 --- /dev/null +++ b/tests/acceptance/core/service/factory/cluster.go @@ -0,0 +1,137 @@ +package factory + +import ( + "fmt" + "path/filepath" + "strconv" + "sync" + + "github.com/gruntwork-io/terratest/modules/terraform" + "github.com/rancher/rke2/tests/acceptance/shared" + + . "github.com/onsi/ginkgo/v2" +) + +type Cluster struct { + Status string + ServerIPs string + AgentIPs string + NumServers int + NumAgents int +} + +var ( + once sync.Once + cluster *Cluster +) + +// NewCluster creates a new cluster and returns his values from terraform config and vars +func NewCluster(g GinkgoTInterface) (*Cluster, error) { + tfDir, err := filepath.Abs(shared.BasePath() + "/acceptance/modules") + if err != nil { + return nil, err + } + + varDir, err := filepath.Abs(shared.BasePath() + "/acceptance/modules/config/local.tfvars") + if err != nil { + return nil, err + } + + terraformOptions := &terraform.Options{ + TerraformDir: tfDir, + VarFiles: []string{varDir}, + } + + NumServers, err := strconv.Atoi(terraform.GetVariableAsStringFromVarFile(g, varDir, "no_of_server_nodes")) + if err != nil { + return nil, err + } + + NumAgents, err := strconv.Atoi(terraform.GetVariableAsStringFromVarFile(g, varDir, "no_of_worker_nodes")) + if err != nil { + return nil, err + } + + splitRoles := terraform.GetVariableAsStringFromVarFile(g, varDir, "split_roles") + if splitRoles == "true" { + etcdNodes, err := strconv.Atoi(terraform.GetVariableAsStringFromVarFile(g, varDir, + "etcd_only_nodes")) + if err != nil { + return nil, err + } + etcdCpNodes, err := strconv.Atoi(terraform.GetVariableAsStringFromVarFile(g, varDir, + "etcd_cp_nodes")) + if err != nil { + return nil, err + } + etcdWorkerNodes, err := strconv.Atoi(terraform.GetVariableAsStringFromVarFile(g, varDir, + "etcd_worker_nodes")) + if err != nil { + return nil, err + } + cpNodes, err := strconv.Atoi(terraform.GetVariableAsStringFromVarFile(g, varDir, + "cp_only_nodes")) + if err != nil { + return nil, err + } + cpWorkerNodes, err := strconv.Atoi(terraform.GetVariableAsStringFromVarFile(g, varDir, + "cp_worker_nodes")) + if err != nil { + return nil, err + } + NumServers = NumServers + etcdNodes + etcdCpNodes + etcdWorkerNodes + + +cpNodes + cpWorkerNodes + } + + fmt.Println("Creating Cluster") + + terraform.InitAndApply(g, terraformOptions) + + ServerIPs := terraform.Output(g, terraformOptions, "master_ips") + AgentIPs := terraform.Output(g, terraformOptions, "worker_ips") + + shared.AwsUser = terraform.GetVariableAsStringFromVarFile(g, varDir, "aws_user") + shared.AccessKey = terraform.GetVariableAsStringFromVarFile(g, varDir, "access_key") + shared.KubeConfigFile = terraform.Output(g, terraformOptions, "kubeconfig") + return &Cluster{ + Status: "cluster created", + ServerIPs: ServerIPs, + AgentIPs: AgentIPs, + NumServers: NumServers, + NumAgents: NumAgents, + }, nil +} + +// GetCluster returns a singleton cluster +func GetCluster(g GinkgoTInterface) *Cluster { + var err error + + once.Do(func() { + cluster, err = NewCluster(g) + if err != nil { + g.Errorf("error getting cluster: %v", err) + } + }) + return cluster +} + +// DestroyCluster destroys the cluster and returns a message +func DestroyCluster(g GinkgoTInterface) (string, error) { + basepath := shared.BasePath() + tfDir, err := filepath.Abs(basepath + "/modules") + if err != nil { + return "", err + } + varDir, err := filepath.Abs(basepath + "/modules/config/local.tfvars") + if err != nil { + return "", err + } + + terraformOptions := terraform.Options{ + TerraformDir: tfDir, + VarFiles: []string{varDir}, + } + terraform.Destroy(g, &terraformOptions) + + return "cluster destroyed", nil +} diff --git a/tests/acceptance/core/service/template/helper.go b/tests/acceptance/core/service/template/helper.go new file mode 100644 index 0000000000..3d16dd74f3 --- /dev/null +++ b/tests/acceptance/core/service/template/helper.go @@ -0,0 +1,89 @@ +package template + +import ( + "fmt" + "strings" + "sync" + + "github.com/rancher/rke2/tests/acceptance/core/testcase" + "github.com/rancher/rke2/tests/acceptance/shared" +) + +// upgradeVersion upgrades the version of RKE2 and updates the expected values +func upgradeVersion(template VersionTestTemplate, version string) error { + err := testcase.TestUpgradeClusterManually(version) + if err != nil { + return err + } + + for i := range template.TestCombination.Run { + template.TestCombination.Run[i].ExpectedValue = + template.TestCombination.Run[i].ExpectedValueUpgrade + } + + return nil +} + +// checkVersion checks the version of RKE2 by calling processTestCombination +func checkVersion(v VersionTestTemplate) error { + ips, err := getIPs() + if err != nil { + return fmt.Errorf("failed to get IPs: %v", err) + } + + var wg sync.WaitGroup + errorChanList := make( + chan error, + len(ips)*(len(v.TestCombination.Run)), + ) + + processTestCombination(errorChanList, &wg, ips, *v.TestCombination) + + wg.Wait() + close(errorChanList) + + for chanErr := range errorChanList { + if chanErr != nil { + return chanErr + } + } + + if v.TestConfig != nil { + TestCaseWrapper(v) + } + + return nil +} + +// getIPs gets the IPs of the nodes +func getIPs() (ips []string, err error) { + ips = shared.FetchNodeExternalIP() + return ips, nil +} + +// AddTestCases returns the test case based on the name to be used as customflag. +func AddTestCases(names []string) ([]TestCase, error) { + var testCases []TestCase + + testCase := map[string]TestCase{ + "TestDaemonset": testcase.TestDaemonset, + "TestIngress": testcase.TestIngress, + "TestDnsAccess": testcase.TestDnsAccess, + "TestServiceClusterIp": testcase.TestServiceClusterIp, + "TestServiceNodePort": testcase.TestServiceNodePort, + "TestCoredns": testcase.TestCoredns, + } + + for _, name := range names { + name = strings.TrimSpace(name) + if name == "" { + testCases = append(testCases, func(deployWorkload bool) {}) + } else if test, ok := testCase[name]; ok { + testCases = append(testCases, test) + } else { + return nil, fmt.Errorf("invalid test case name") + } + } + + return testCases, nil +} diff --git a/tests/acceptance/core/service/template/model.go b/tests/acceptance/core/service/template/model.go new file mode 100644 index 0000000000..72d41912cf --- /dev/null +++ b/tests/acceptance/core/service/template/model.go @@ -0,0 +1,52 @@ +package template + +import "github.com/rancher/rke2/tests/acceptance/core/service/customflag" + +var TestMapFlag TestMap + +// VersionTestTemplate represents a version test scenario with test configurations and commands. +type VersionTestTemplate struct { + TestCombination *RunCmd + InstallUpgrade []string + TestConfig *TestConfig + Description string +} + +// RunCmd represents the command sets to run on host and node. +type RunCmd struct { + Run []TestMap +} + +// TestMap represents a single test command with key:value pairs. +type TestMap struct { + Cmd string + ExpectedValue string + ExpectedValueUpgrade string +} + +// TestConfig represents the testcase function configuration +type TestConfig struct { + TestFunc []TestCase + DeployWorkload bool + WorkloadName string +} + +// TestCase is a custom type representing the test function. +type TestCase func(deployWorkload bool) + +// TestCaseWrapper wraps a test function and calls it with the given VersionTestTemplate. +func TestCaseWrapper(v VersionTestTemplate) { + for _, testFunc := range v.TestConfig.TestFunc { + testFunc(v.TestConfig.DeployWorkload) + } +} + +// ConvertToTestCase converts the TestCaseFlag to TestCase +func ConvertToTestCase(testCaseFlags []customflag.TestCaseFlag) []TestCase { + var testCases []TestCase + for _, tcf := range testCaseFlags { + tc := TestCase(tcf) + testCases = append(testCases, tc) + } + return testCases +} diff --git a/tests/acceptance/core/service/template/processor.go b/tests/acceptance/core/service/template/processor.go new file mode 100644 index 0000000000..5d94465cc4 --- /dev/null +++ b/tests/acceptance/core/service/template/processor.go @@ -0,0 +1,118 @@ +package template + +import ( + "fmt" + "strings" + "sync" + + "github.com/rancher/rke2/tests/acceptance/core/service/assert" + "github.com/rancher/rke2/tests/acceptance/shared" + + . "github.com/onsi/ginkgo/v2" +) + +// processCmds runs the tests per ips using processOnNode and processOnHost validation. +// +// it will spawn a go routine per testCombination and ip. +func processCmds(resultChan chan error, wg *sync.WaitGroup, ip string, cmds []string, expectedValues []string) { + if len(cmds) != len(expectedValues) { + resultChan <- fmt.Errorf("mismatched length commands x expected values:"+ + " %s x %s", cmds, expectedValues) + return + } + + for i := range cmds { + cmd := cmds[i] + expectedValue := expectedValues[i] + wg.Add(1) + go func(ip string, cmd, expectedValue string) { + defer wg.Done() + defer GinkgoRecover() + + if strings.Contains(cmd, "kubectl") || strings.HasPrefix(cmd, "helm") { + processOnHost(resultChan, ip, cmd, expectedValue) + } else { + processOnNode(resultChan, ip, cmd, expectedValue) + } + }(ip, cmd, expectedValue) + } +} + +func processTestCombination(resultChan chan error, wg *sync.WaitGroup, ips []string, testCombination RunCmd) { + if testCombination.Run != nil { + for _, ip := range ips { + for _, testMap := range testCombination.Run { + cmds := strings.Split(testMap.Cmd, ",") + expectedValues := strings.Split(testMap.ExpectedValue, ",") + processCmds(resultChan, wg, ip, cmds, expectedValues) + } + } + } +} + +// processOnNode runs the test on the node calling ValidateOnNode. +func processOnNode(resultChan chan error, ip, cmd, expectedValue string) { + if expectedValue == "" { + err := fmt.Errorf("\nexpected value should be sent to node") + fmt.Println("error:", err) + resultChan <- err + close(resultChan) + return + } + + version := shared.GetRke2Version() + fmt.Printf("\n---------------------\n"+ + "Version Checked: %s\n"+ + "IP Address: %s\n"+ + "Command Executed: %s\n"+ + "Execution Location: Node\n"+ + "Expected Value: %s\n---------------------\n", + version, ip, cmd, expectedValue) + + cmds := strings.Split(cmd, ",") + for _, c := range cmds { + err := assert.ValidateOnNode( + ip, + c, + expectedValue, + ) + if err != nil { + resultChan <- err + close(resultChan) + return + } + } +} + +// processOnHost runs the test on the host calling ValidateOnHost. +func processOnHost(resultChan chan error, ip, cmd, expectedValue string) { + if expectedValue == "" { + err := fmt.Errorf("\nexpected value should be sent to host") + fmt.Println("error:", err) + resultChan <- err + close(resultChan) + return + } + + kubeconfigFlag := " --kubeconfig=" + shared.KubeConfigFile + fullCmd := shared.JoinCommands(cmd, kubeconfigFlag) + + version := shared.GetRke2Version() + fmt.Printf("\n---------------------\n"+ + "Version Checked: %s\n"+ + "IP Address: %s\n"+ + "Command Executed: %s\n"+ + "Execution Location: Host\n"+ + "Expected Value: %s\n---------------------\n", + version, ip, cmd, expectedValue) + + err := assert.ValidateOnHost( + fullCmd, + expectedValue, + ) + if err != nil { + resultChan <- err + close(resultChan) + return + } +} diff --git a/tests/acceptance/core/service/template/versiontemplate.go b/tests/acceptance/core/service/template/versiontemplate.go new file mode 100644 index 0000000000..6ed00faac0 --- /dev/null +++ b/tests/acceptance/core/service/template/versiontemplate.go @@ -0,0 +1,57 @@ +package template + +import ( + "fmt" + "strings" + + "github.com/rancher/rke2/tests/acceptance/core/service/customflag" + "github.com/rancher/rke2/tests/acceptance/shared" + + . "github.com/onsi/ginkgo/v2" +) + +// VersionTemplate is a template for testing RKE2 versions + test cases and upgrading cluster if needed +func VersionTemplate(test VersionTestTemplate) { + if customflag.ServiceFlag.TestConfig.WorkloadName != "" && + strings.HasSuffix(customflag.ServiceFlag.TestConfig.WorkloadName, ".yaml") { + _, err := shared.ManageWorkload( + "create", + customflag.ServiceFlag.TestConfig.WorkloadName, + ) + if err != nil { + GinkgoT().Errorf(err.Error()) + return + } + } + + err := checkVersion(test) + if err != nil { + GinkgoT().Errorf(err.Error()) + return + } + + if test.InstallUpgrade != nil { + for _, version := range test.InstallUpgrade { + if GinkgoT().Failed() { + fmt.Println("checkVersion failed, upgrade not performed") + return + } + + err = upgradeVersion(test, version) + if err != nil { + GinkgoT().Errorf("error upgrading: %v\n", err) + return + } + + err = checkVersion(test) + if err != nil { + GinkgoT().Errorf(err.Error()) + return + } + + if test.TestConfig != nil { + TestCaseWrapper(test) + } + } + } +} diff --git a/tests/acceptance/core/testcase/cluster.go b/tests/acceptance/core/testcase/cluster.go new file mode 100644 index 0000000000..2282840c89 --- /dev/null +++ b/tests/acceptance/core/testcase/cluster.go @@ -0,0 +1,31 @@ +package testcase + +import ( + "fmt" + + "github.com/rancher/rke2/tests/acceptance/core/service/factory" + "github.com/rancher/rke2/tests/acceptance/shared" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// TestBuildCluster test the creation of a cluster using terraform +func TestBuildCluster(g GinkgoTInterface, destroy bool) { + cluster := factory.GetCluster(g) + + Expect(cluster.Status).To(Equal("cluster created")) + + shared.PrintFileContents(shared.KubeConfigFile) + Expect(shared.KubeConfigFile).ShouldNot(BeEmpty()) + Expect(cluster.ServerIPs).ShouldNot(BeEmpty()) + + fmt.Println("Server Node IPS:", cluster.ServerIPs) + fmt.Println("Agent Node IPS:", cluster.AgentIPs) + + if cluster.NumAgents > 0 { + Expect(cluster.AgentIPs).ShouldNot(BeEmpty()) + } else { + Expect(cluster.AgentIPs).Should(BeEmpty()) + } +} diff --git a/tests/acceptance/core/testcase/coredns.go b/tests/acceptance/core/testcase/coredns.go new file mode 100644 index 0000000000..043a95832f --- /dev/null +++ b/tests/acceptance/core/testcase/coredns.go @@ -0,0 +1,36 @@ +package testcase + +import ( + "log" + + "github.com/rancher/rke2/tests/acceptance/core/service/assert" + "github.com/rancher/rke2/tests/acceptance/shared" + + . "github.com/onsi/gomega" +) + +func TestCoredns(deployWorkload bool) { + if deployWorkload { + _, err := shared.ManageWorkload("create", "dnsutils.yaml") + Expect(err).NotTo(HaveOccurred(), + "dnsutils manifest not deployed", err) + } + + _, err := shared.AddHelmRepo("traefik", "https://helm.traefik.io/traefik") + if err != nil { + log.Fatalf("failed to add Helm repo: %v", err) + } + + kubeconfigFlag := " --kubeconfig=" + shared.KubeConfigFile + fullCmd := shared.JoinCommands("helm list --all-namespaces ", kubeconfigFlag) + assert.CheckComponentCmdHost( + fullCmd, + "rke2-coredns-1.19.402", + ) + + err = assert.ValidateOnHost(ExecDnsUtils+shared.KubeConfigFile+ + " -- nslookup kubernetes.default", Nslookup) + if err != nil { + return + } +} diff --git a/tests/acceptance/core/testcase/daemonset.go b/tests/acceptance/core/testcase/daemonset.go new file mode 100644 index 0000000000..09ee7aa5e5 --- /dev/null +++ b/tests/acceptance/core/testcase/daemonset.go @@ -0,0 +1,24 @@ +package testcase + +import ( + "github.com/rancher/rke2/tests/acceptance/shared" + + . "github.com/onsi/gomega" +) + +func TestDaemonset(deployWorkload bool) { + if deployWorkload { + _, err := shared.ManageWorkload("create", "daemonset.yaml") + Expect(err).NotTo(HaveOccurred(), + "Daemonset manifest not deployed") + + nodes, _ := shared.ParseNodes(false) + pods, _ := shared.ParsePods(false) + + Eventually(func(g Gomega) { + count := shared.CountOfStringInSlice("test-daemonset", pods) + g.Expect(count).Should(Equal(len(nodes)), + "Daemonset pod count does not match node count") + }, "420s", "5s").Should(Succeed()) + } +} diff --git a/tests/acceptance/core/testcase/ingressdns.go b/tests/acceptance/core/testcase/ingressdns.go new file mode 100644 index 0000000000..0030234252 --- /dev/null +++ b/tests/acceptance/core/testcase/ingressdns.go @@ -0,0 +1,68 @@ +package testcase + +import ( + "github.com/rancher/rke2/tests/acceptance/core/service/assert" + "github.com/rancher/rke2/tests/acceptance/shared" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var Running = "Running" +var ExecDnsUtils = "kubectl exec -n auto-dns -t dnsutils --kubeconfig=" +var Nslookup = "kubernetes.default.svc.cluster.local" + +func TestIngress(deployWorkload bool) { + var ingressIps []string + if deployWorkload { + _, err := shared.ManageWorkload("create", "ingress.yaml") + Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed") + } + + getIngressRunning := "kubectl get pods -n auto-ingress -l k8s-app=nginx-app-ingress " + + "--field-selector=status.phase=Running --kubeconfig=" + err := assert.ValidateOnHost(getIngressRunning+shared.KubeConfigFile, Running) + if err != nil { + GinkgoT().Errorf("Error: %v", err) + } + + nodes, err := shared.ParseNodes(false) + Expect(err).NotTo(HaveOccurred()) + + Eventually(func(Gomega) bool { + ingressIps, err = shared.FetchIngressIP("auto-ingress") + if err != nil { + return false + } + if len(ingressIps) != len(nodes) { + return false + } + return true + }, "400s", "3s").Should(BeTrue()) + + for _, ip := range ingressIps { + if assert.CheckComponentCmdHost("curl -s --header host:foo1.bar.com"+" "+ + "http://"+ip+"/name.html", "test-ingress"); err != nil { + return + } + } +} + +func TestDnsAccess(deployWorkload bool) { + if deployWorkload { + _, err := shared.ManageWorkload("create", "dnsutils.yaml") + Expect(err).NotTo(HaveOccurred(), + "dnsutils manifest not deployed", err) + } + + getDnsUtils := "kubectl get pods -n auto-dns dnsutils --kubeconfig=" + err := assert.ValidateOnHost(getDnsUtils+shared.KubeConfigFile, Running) + if err != nil { + GinkgoT().Errorf("Error: %v", err) + } + + assert.CheckComponentCmdHost( + ExecDnsUtils+shared.KubeConfigFile+" -- nslookup kubernetes.default", + Nslookup, + ) +} diff --git a/tests/acceptance/core/testcase/node.go b/tests/acceptance/core/testcase/node.go new file mode 100644 index 0000000000..88f250f8b6 --- /dev/null +++ b/tests/acceptance/core/testcase/node.go @@ -0,0 +1,42 @@ +package testcase + +import ( + "fmt" + + "github.com/rancher/rke2/tests/acceptance/core/service/assert" + "github.com/rancher/rke2/tests/acceptance/core/service/factory" + "github.com/rancher/rke2/tests/acceptance/shared" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// TestNodeStatus test the status of the nodes in the cluster using 2 custom assert functions +func TestNodeStatus( + nodeAssertReadyStatus assert.NodeAssertFunc, + nodeAssertVersion assert.NodeAssertFunc, +) { + cluster := factory.GetCluster(GinkgoT()) + fmt.Printf("\nFetching node status\n") + + expectedNodeCount := cluster.NumServers + cluster.NumAgents + Eventually(func(g Gomega) { + nodes, err := shared.ParseNodes(false) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(len(nodes)).To(Equal(expectedNodeCount), + "Number of nodes should match the spec") + + for _, node := range nodes { + if nodeAssertReadyStatus != nil { + nodeAssertReadyStatus(g, node) + } + if nodeAssertVersion != nil { + nodeAssertVersion(g, node) + + } + } + }, "800s", "5s").Should(Succeed()) + + _, err := shared.ParseNodes(true) + Expect(err).NotTo(HaveOccurred()) +} diff --git a/tests/acceptance/core/testcase/pod.go b/tests/acceptance/core/testcase/pod.go new file mode 100644 index 0000000000..a014943c23 --- /dev/null +++ b/tests/acceptance/core/testcase/pod.go @@ -0,0 +1,84 @@ +package testcase + +import ( + "fmt" + "strings" + + "github.com/rancher/rke2/tests/acceptance/core/service/assert" + "github.com/rancher/rke2/tests/acceptance/shared" + + . "github.com/onsi/gomega" +) + +// TestPodStatus test the status of the pods in the cluster using 2 custom assert functions +func TestPodStatus( + podAssertRestarts assert.PodAssertFunc, + podAssertReady assert.PodAssertFunc, + podAssertStatus assert.PodAssertFunc, +) { + fmt.Printf("\nFetching pod status\n") + + Eventually(func(g Gomega) { + pods, err := shared.ParsePods(false) + g.Expect(err).NotTo(HaveOccurred()) + + for _, pod := range pods { + if strings.Contains(pod.Name, "helm-install") { + g.Expect(pod.Status).Should(Equal("Completed"), pod.Name) + } else if strings.Contains(pod.Name, "apply") && + strings.Contains(pod.NameSpace, "system-upgrade") { + g.Expect(pod.Status).Should(SatisfyAny( + ContainSubstring("Unknown"), + ContainSubstring("Init:Error"), + Equal("Completed"), + ), pod.Name) + } else { + g.Expect(pod.Status).Should(Equal(Running), pod.Name) + if podAssertRestarts != nil { + podAssertRestarts(g, pod) + } + if podAssertReady != nil { + podAssertReady(g, pod) + } + if podAssertStatus != nil { + podAssertStatus(g, pod) + } + } + } + }, "900s", "5s").Should(Succeed()) +} + +func testCrossNodeServiceRequest(services, ports, expected []string) error { + var err error + if len(services) != len(ports) && len(ports) != len(expected) { + return fmt.Errorf("array parameters must have equal length") + } + + if len(services) < 2 || len(ports) < 2 || len(expected) < 2 { + return fmt.Errorf("array parameters must not be less than or equal to 2") + } + + // Iterating services first to last + for i := 0; i < len(services); i++ { + for j := i + 1; j < len(services); j++ { + cmd := fmt.Sprintf("kubectl exec svc/%s --kubeconfig=%s -- curl -m7 %s:%s", + services[i], shared.KubeConfigFile, services[j], ports[j]) + Eventually(func() (string, error) { + return shared.RunCommandHost(cmd) + }, "120s", "5s").Should(ContainSubstring(expected[j])) + } + } + + // Iterating services last to first + for i := len(services) - 1; i > 0; i-- { + for j := 1; j <= i; j++ { + cmd := fmt.Sprintf("kubectl exec svc/%s --kubeconfig=%s -- curl -m7 %s:%s", + services[i], shared.KubeConfigFile, services[i-j], ports[i-j]) + Eventually(func() (string, error) { + return shared.RunCommandHost(cmd) + }, "120s", "5s").Should(ContainSubstring(expected[i-j])) + } + } + + return err +} diff --git a/tests/acceptance/core/testcase/service.go b/tests/acceptance/core/testcase/service.go new file mode 100644 index 0000000000..485721d196 --- /dev/null +++ b/tests/acceptance/core/testcase/service.go @@ -0,0 +1,60 @@ +package testcase + +import ( + "github.com/rancher/rke2/tests/acceptance/core/service/assert" + "github.com/rancher/rke2/tests/acceptance/shared" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestServiceClusterIp(deployWorkload bool) { + if deployWorkload { + _, err := shared.ManageWorkload("create", "clusterip.yaml") + Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed") + } + + getClusterIp := "kubectl get pods -n auto-clusterip -l k8s-app=nginx-app-clusterip" + + " --field-selector=status.phase=Running --kubeconfig=" + err := assert.ValidateOnHost(getClusterIp+shared.KubeConfigFile, Running) + if err != nil { + GinkgoT().Errorf("Error: %v", err) + } + + clusterip, port, _ := shared.FetchClusterIP("auto-clusterip", + "nginx-clusterip-svc") + nodeExternalIP := shared.FetchNodeExternalIP() + for _, ip := range nodeExternalIP { + err = assert.ValidateOnNode(ip, "curl -sL --insecure http://"+clusterip+ + ":"+port+"/name.html", "test-clusterip") + if err != nil { + GinkgoT().Errorf("Error: %v", err) + } + } +} + +func TestServiceNodePort(deployWorkload bool) { + if deployWorkload { + _, err := shared.ManageWorkload("create", "nodeport.yaml") + Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed") + } + + nodeExternalIP := shared.FetchNodeExternalIP() + getNodePortSVC := "kubectl get service -n auto-nodeport nginx-nodeport-svc" + + " --output jsonpath={.spec.ports[0].nodePort} --kubeconfig=" + nodeport, err := shared.RunCommandHost(getNodePortSVC + shared.KubeConfigFile) + if err != nil { + GinkgoT().Errorf("Error: %v", err) + } + + for _, ip := range nodeExternalIP { + assert.CheckPodStatusRunning("nginx-app-nodeport", + "auto-nodeport", "test-nodeport") + + assert.CheckComponentCmdNode("curl -sL --insecure http://"+ip+":"+nodeport+"/name.html", + "test-nodeport", ip) + if err != nil { + GinkgoT().Errorf("Error: %v", err) + } + } +} diff --git a/tests/acceptance/core/testcase/upgradecluster.go b/tests/acceptance/core/testcase/upgradecluster.go new file mode 100644 index 0000000000..20862f45b3 --- /dev/null +++ b/tests/acceptance/core/testcase/upgradecluster.go @@ -0,0 +1,139 @@ +package testcase + +import ( + "fmt" + "os" + "strings" + "sync" + "time" + + "github.com/rancher/rke2/tests/acceptance/core/service/assert" + "github.com/rancher/rke2/tests/acceptance/core/service/customflag" + "github.com/rancher/rke2/tests/acceptance/core/service/factory" + "github.com/rancher/rke2/tests/acceptance/shared" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// TestUpgradeClusterSUC upgrades cluster using the system-upgrade-controller. +func TestUpgradeClusterSUC(version string) error { + fmt.Printf("\nUpgrading cluster to version: %s\n", version) + + _, err := shared.ManageWorkload("create", "suc.yaml") + Expect(err).NotTo(HaveOccurred(), + "system-upgrade-controller manifest did not deploy successfully") + + getPodsSystemUpgrade := "kubectl get pods -n system-upgrade --kubeconfig=" + assert.CheckComponentCmdHost( + getPodsSystemUpgrade+shared.KubeConfigFile, + "system-upgrade-controller", + Running, + ) + Expect(err).NotTo(HaveOccurred()) + + originalFilePath := shared.BasePath() + "/acceptance/workloads" + "/upgrade-plan.yaml" + newFilePath := shared.BasePath() + "/acceptance/workloads" + "/plan.yaml" + + content, err := os.ReadFile(originalFilePath) + if err != nil { + return fmt.Errorf("failed to read file: %s", err) + } + + newContent := strings.ReplaceAll(string(content), "$UPGRADEVERSION", version) + err = os.WriteFile(newFilePath, []byte(newContent), 0644) + if err != nil { + return fmt.Errorf("failed to write file: %s", err) + } + + _, err = shared.ManageWorkload("create", "plan.yaml") + Expect(err).NotTo(HaveOccurred(), "failed to upgrade cluster.") + + return nil +} + +// TestUpgradeClusterManually upgrades cluster "manually" +func TestUpgradeClusterManually(version string) error { + if version == "" { + return fmt.Errorf("please provide a non-empty rke2 version to upgrade to") + } + cluster := factory.GetCluster(GinkgoT()) + + serverIPs := strings.Split(cluster.ServerIPs, ",") + agentIPs := strings.Split(cluster.AgentIPs, ",") + + if cluster.NumServers == 0 && cluster.NumAgents == 0 { + return fmt.Errorf("no nodes found to upgrade") + } + + if cluster.NumServers > 0 { + if err := upgradeServer(version, serverIPs); err != nil { + return err + } + } + + if cluster.NumAgents > 0 { + if err := upgradeAgent(version, agentIPs); err != nil { + return err + } + } + + return nil +} + +// upgradeNode upgrades a node server or agent type to the specified version +func upgradeNode(nodeType string, installType string, ips []string) error { + var wg sync.WaitGroup + var installFlag string + errCh := make(chan error, len(ips)) + + if strings.HasPrefix(installType, "v") { + installFlag = fmt.Sprintf("INSTALL_RKE2_VERSION=%s", installType) + } else { + installFlag = fmt.Sprintf("INSTALL_RKE2_COMMIT=%s", installType) + } + + channel := fmt.Sprintf("INSTALL_RKE2_CHANNEL=%s", "stable") + if customflag.ServiceFlag.InstallType.Channel != "" { + channel = fmt.Sprintf("INSTALL_RKE2_CHANNEL=%s", customflag.ServiceFlag.InstallType.Channel) + } + + installRke2Command := "sudo curl -sfL https://get.rke2.io | sudo %s %s INSTALL_RKE2_TYPE=" + nodeType + " sh - " + for _, ip := range ips { + upgradeCommand := fmt.Sprintf(installRke2Command, installFlag, channel) + wg.Add(1) + go func(ip, installFlag string) { + defer wg.Done() + defer GinkgoRecover() + + fmt.Println("Upgrading " + nodeType + " to: " + upgradeCommand) + if _, err := shared.RunCommandOnNode(upgradeCommand, ip); err != nil { + fmt.Printf("\nError upgrading %s %s: %v\n\n", nodeType, ip, err) + errCh <- err + close(errCh) + return + } + + fmt.Println("Restarting " + nodeType + ": " + ip) + if _, err := shared.RestartCluster(ip); err != nil { + fmt.Printf("\nError restarting %s %s: %v\n\n", nodeType, ip, err) + errCh <- err + close(errCh) + return + } + time.Sleep(20 * time.Second) + }(ip, installType) + } + wg.Wait() + close(errCh) + + return nil +} + +func upgradeServer(installType string, serverIPs []string) error { + return upgradeNode("server", installType, serverIPs) +} + +func upgradeAgent(installType string, agentIPs []string) error { + return upgradeNode("agent", installType, agentIPs) +} diff --git a/tests/acceptance/entrypoint/createcluster/createcluster_suite_test.go b/tests/acceptance/entrypoint/createcluster/createcluster_suite_test.go new file mode 100644 index 0000000000..7c6ecfcfe8 --- /dev/null +++ b/tests/acceptance/entrypoint/createcluster/createcluster_suite_test.go @@ -0,0 +1,34 @@ +package createcluster + +import ( + "flag" + "os" + "testing" + + "github.com/rancher/rke2/tests/acceptance/core/service/customflag" + "github.com/rancher/rke2/tests/acceptance/core/service/factory" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMain(m *testing.M) { + flag.Var(&customflag.ServiceFlag.ClusterConfig.Destroy, "destroy", "Destroy cluster after test") + flag.Parse() + os.Exit(m.Run()) +} + +func TestClusterCreateSuite(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Create Cluster Test Suite") +} + +var _ = AfterSuite(func() { + g := GinkgoT() + if customflag.ServiceFlag.ClusterConfig.Destroy { + status, err := factory.DestroyCluster(g) + Expect(err).NotTo(HaveOccurred()) + Expect(status).To(Equal("cluster destroyed")) + } +}) diff --git a/tests/acceptance/entrypoint/createcluster/createcluster_test.go b/tests/acceptance/entrypoint/createcluster/createcluster_test.go new file mode 100644 index 0000000000..9e539ae5bc --- /dev/null +++ b/tests/acceptance/entrypoint/createcluster/createcluster_test.go @@ -0,0 +1,66 @@ +package createcluster + +import ( + "fmt" + + "github.com/rancher/rke2/tests/acceptance/core/service/assert" + "github.com/rancher/rke2/tests/acceptance/core/testcase" + "github.com/rancher/rke2/tests/acceptance/shared" + + . "github.com/onsi/ginkgo/v2" +) + +var _ = Describe("Test:", func() { + + It("Start Up with no issues", func() { + testcase.TestBuildCluster(GinkgoT(), false) + }) + + It("Validate Nodes", func() { + testcase.TestNodeStatus( + assert.NodeAssertReadyStatus(), + nil, + ) + }) + + It("Validate Pods", func() { + testcase.TestPodStatus( + assert.PodAssertRestart(), + assert.PodAssertReady(), + assert.PodAssertStatus(), + ) + }) + + It("Verifies ClusterIP Service", func() { + testcase.TestServiceClusterIp(true) + defer shared.ManageWorkload("delete", "clusterip.yaml") + }) + + It("Verifies NodePort Service", func() { + testcase.TestServiceNodePort(true) + defer shared.ManageWorkload("delete", "nodeport.yaml") + }) + + It("Verifies Ingress", func() { + testcase.TestIngress(true) + defer shared.ManageWorkload("delete", "ingress.yaml") + }) + + It("Verifies Daemonset", func() { + testcase.TestDaemonset(true) + defer shared.ManageWorkload("delete", "daemonset.yaml") + }) + + It("Verifies dns access", func() { + testcase.TestDnsAccess(true) + defer shared.ManageWorkload("delete", "dnsutils.yaml") + }) +}) + +var _ = AfterEach(func() { + if CurrentSpecReport().Failed() { + fmt.Printf("\nFAILED! %s\n", CurrentSpecReport().FullText()) + } else { + fmt.Printf("\nPASSED! %s\n", CurrentSpecReport().FullText()) + } +}) diff --git a/tests/acceptance/entrypoint/upgradecluster/upgradecluster_suite_test.go b/tests/acceptance/entrypoint/upgradecluster/upgradecluster_suite_test.go new file mode 100644 index 0000000000..fdee672515 --- /dev/null +++ b/tests/acceptance/entrypoint/upgradecluster/upgradecluster_suite_test.go @@ -0,0 +1,37 @@ +package upgradecluster + +import ( + "flag" + "os" + "testing" + + "github.com/rancher/rke2/tests/acceptance/core/service/customflag" + "github.com/rancher/rke2/tests/acceptance/core/service/factory" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMain(m *testing.M) { + flag.Var(&customflag.ServiceFlag.InstallUpgrade, "installVersionOrCommit", "Install upgrade customflag for version bump") + flag.Var(&customflag.ServiceFlag.ClusterConfig.Destroy, "destroy", "Destroy cluster after test") + flag.Var(&customflag.ServiceFlag.UpgradeVersionSUC, "upgradeVersion", "Upgrade SUC model") + flag.Parse() + + os.Exit(m.Run()) +} + +func TestClusterUpgradeSuite(t *testing.T) { + RegisterFailHandler(Fail) + + RunSpecs(t, "Upgrade Cluster Test Suite") +} + +var _ = AfterSuite(func() { + g := GinkgoT() + if customflag.ServiceFlag.ClusterConfig.Destroy { + status, err := factory.DestroyCluster(g) + Expect(err).NotTo(HaveOccurred()) + Expect(status).To(Equal("cluster destroyed")) + } +}) diff --git a/tests/acceptance/entrypoint/upgradecluster/upgrademanual_test.go b/tests/acceptance/entrypoint/upgradecluster/upgrademanual_test.go new file mode 100644 index 0000000000..fb4d81823e --- /dev/null +++ b/tests/acceptance/entrypoint/upgradecluster/upgrademanual_test.go @@ -0,0 +1,110 @@ +//go:build upgrademanual + +package upgradecluster + +import ( + "fmt" + + "github.com/rancher/rke2/tests/acceptance/core/service/assert" + "github.com/rancher/rke2/tests/acceptance/core/service/customflag" + "github.com/rancher/rke2/tests/acceptance/core/testcase" + "github.com/rancher/rke2/tests/acceptance/shared" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Test:", func() { + + It("Starts up with no issues", func() { + testcase.TestBuildCluster(GinkgoT(), false) + }) + + It("Validate Nodes", func() { + testcase.TestNodeStatus( + assert.NodeAssertReadyStatus(), + nil, + ) + }) + + It("Validate Pods", func() { + testcase.TestPodStatus( + assert.PodAssertRestart(), + assert.PodAssertReady(), + assert.PodAssertStatus(), + ) + }) + + It("Verifies ClusterIP Service Pre upgrade", func() { + testcase.TestServiceClusterIp(true) + }) + + It("Verifies NodePort Service Pre upgrade", func() { + testcase.TestServiceNodePort(true) + }) + + It("Verifies Ingress Pre upgrade", func() { + testcase.TestIngress(true) + }) + + It("Verifies Daemonset Pre upgrade", func() { + testcase.TestDaemonset(true) + }) + + It("Verifies DNS Access Pre upgrade", func() { + testcase.TestDnsAccess(true) + }) + + It("Upgrade manual", func() { + err := testcase.TestUpgradeClusterManually(customflag.ServiceFlag.InstallUpgrade.String()) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Checks Node Status pos upgrade", func() { + testcase.TestNodeStatus( + assert.NodeAssertReadyStatus(), + assert.NodeAssertVersionTypeUpgrade(customflag.ServiceFlag), + ) + }) + + It("Checks Pod Status pos upgrade", func() { + testcase.TestPodStatus( + nil, + assert.PodAssertReady(), + assert.PodAssertStatus(), + ) + }) + + It("Verifies ClusterIP Service Post upgrade", func() { + testcase.TestServiceClusterIp(false) + defer shared.ManageWorkload("delete", "clusterip.yaml") + }) + + It("Verifies NodePort Service Post upgrade", func() { + testcase.TestServiceNodePort(false) + defer shared.ManageWorkload("delete", "nodeport.yaml") + }) + + It("Verifies Ingress Post upgrade", func() { + testcase.TestIngress(false) + defer shared.ManageWorkload("delete", "ingress.yaml") + }) + + It("Verifies Daemonset Post upgrade", func() { + testcase.TestDaemonset(false) + defer shared.ManageWorkload("delete", "daemonset.yaml") + }) + + It("Verifies DNS Access Post upgrade", func() { + testcase.TestDnsAccess(true) + defer shared.ManageWorkload("delete", "dns.yaml") + }) +}) + +var _ = AfterEach(func() { + if CurrentSpecReport().Failed() { + fmt.Printf("\nFAILED! %s\n", CurrentSpecReport().FullText()) + } else { + fmt.Printf("\nPASSED! %s\n", CurrentSpecReport().FullText()) + } +}) diff --git a/tests/acceptance/entrypoint/upgradecluster/upgradesuc_test.go b/tests/acceptance/entrypoint/upgradecluster/upgradesuc_test.go new file mode 100644 index 0000000000..963f36af84 --- /dev/null +++ b/tests/acceptance/entrypoint/upgradecluster/upgradesuc_test.go @@ -0,0 +1,110 @@ +//go:build upgradesuc + +package upgradecluster + +import ( + "fmt" + + "github.com/rancher/rke2/tests/acceptance/core/service/assert" + "github.com/rancher/rke2/tests/acceptance/core/service/customflag" + "github.com/rancher/rke2/tests/acceptance/core/testcase" + "github.com/rancher/rke2/tests/acceptance/shared" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("SUC Upgrade Tests:", func() { + + It("Starts up with no issues", func() { + testcase.TestBuildCluster(GinkgoT(), false) + }) + + It("Validate Nodes", func() { + testcase.TestNodeStatus( + assert.NodeAssertReadyStatus(), + nil, + ) + }) + + It("Validate Pods", func() { + testcase.TestPodStatus( + assert.PodAssertRestart(), + assert.PodAssertReady(), + assert.PodAssertStatus(), + ) + }) + + It("Verifies ClusterIP Service pre upgrade", func() { + testcase.TestServiceClusterIp(true) + }) + + It("Verifies NodePort Service pre upgrade", func() { + testcase.TestServiceNodePort(true) + }) + + It("Verifies Ingress pre upgrade", func() { + testcase.TestIngress(true) + }) + + It("Verifies Daemonset pre upgrade", func() { + testcase.TestDaemonset(true) + }) + + It("Verifies DNS Access pre upgrade", func() { + testcase.TestDnsAccess(true) + }) + + It("\nUpgrade via SUC", func() { + err := testcase.TestUpgradeClusterSUC(customflag.ServiceFlag.UpgradeVersionSUC.String()) + Expect(err).NotTo(HaveOccurred()) + }) + + It("Checks Node Status pos upgrade suc", func() { + testcase.TestNodeStatus( + assert.NodeAssertReadyStatus(), + assert.NodeAssertVersionUpgraded(), + ) + }) + + It("Checks Pod Status pos upgrade suc", func() { + testcase.TestPodStatus( + nil, + assert.PodAssertReady(), + assert.PodAssertStatus(), + ) + }) + + It("Verifies ClusterIP Service pos upgrade", func() { + testcase.TestServiceClusterIp(false) + defer shared.ManageWorkload("delete", "clusterip.yaml") + }) + + It("Verifies NodePort Service pos upgrade", func() { + testcase.TestServiceNodePort(false) + defer shared.ManageWorkload("delete", "nodeport.yaml") + }) + + It("Verifies Ingress pos upgrade", func() { + testcase.TestIngress(false) + defer shared.ManageWorkload("delete", "ingress.yaml") + }) + + It("Verifies Daemonset pos upgrade", func() { + testcase.TestDaemonset(false) + defer shared.ManageWorkload("delete", "daemonset.yaml") + }) + + It("Verifies DNS Access pos upgrade", func() { + testcase.TestDnsAccess(true) + defer shared.ManageWorkload("delete", "dns.yaml") + }) +}) + +var _ = AfterEach(func() { + if CurrentSpecReport().Failed() { + fmt.Printf("\nFAILED! %s\n", CurrentSpecReport().FullText()) + } else { + fmt.Printf("\nPASSED! %s\n", CurrentSpecReport().FullText()) + } +}) diff --git a/tests/acceptance/entrypoint/versionbump/version_suite_test.go b/tests/acceptance/entrypoint/versionbump/version_suite_test.go new file mode 100644 index 0000000000..5861b3eb96 --- /dev/null +++ b/tests/acceptance/entrypoint/versionbump/version_suite_test.go @@ -0,0 +1,69 @@ +package versionbump + +import ( + "flag" + "fmt" + "os" + "strings" + "testing" + + "github.com/rancher/rke2/tests/acceptance/core/service/customflag" + "github.com/rancher/rke2/tests/acceptance/core/service/factory" + "github.com/rancher/rke2/tests/acceptance/core/service/template" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestMain(m *testing.M) { + flag.StringVar(&template.TestMapFlag.Cmd, "cmd", "", "Comma separated list of commands to execute") + flag.StringVar(&template.TestMapFlag.ExpectedValue, "expectedValue", "", "Comma separated list of expected values for commands") + flag.StringVar(&template.TestMapFlag.ExpectedValueUpgrade, "expectedValueUpgrade", "", "Expected value of the command ran on Node after upgrading") + flag.Var(&customflag.ServiceFlag.InstallUpgrade, "installVersionOrCommit", "Install upgrade customflag") + flag.StringVar(&customflag.ServiceFlag.InstallType.Channel, "channel", "", "channel to use on install or upgrade") + flag.Var(&customflag.TestCaseNameFlag, "testCase", "Comma separated list of test case names to run") + flag.BoolVar(&customflag.ServiceFlag.TestConfig.DeployWorkload, "deployWorkload", false, "Deploy workload customflag") + flag.StringVar(&customflag.ServiceFlag.TestConfig.WorkloadName, "workloadName", "", "Name of the workload to a standalone deploy") + flag.Var(&customflag.ServiceFlag.ClusterConfig.Destroy, "destroy", "Destroy cluster after test") + flag.StringVar(&customflag.ServiceFlag.TestConfig.Description, "description", "", "Description of the test") + flag.Parse() + + if flag.Parsed() { + installVersionOrCommit := strings.Split(customflag.ServiceFlag.InstallUpgrade.String(), ",") + if len(installVersionOrCommit) == 1 && installVersionOrCommit[0] == "" { + customflag.ServiceFlag.InstallUpgrade = nil + } else { + customflag.ServiceFlag.InstallUpgrade = installVersionOrCommit + } + } + + customflag.ServiceFlag.TestConfig.TestFuncNames = customflag.TestCaseNameFlag + testFuncs, err := template.AddTestCases(customflag.ServiceFlag.TestConfig.TestFuncNames) + if err != nil { + fmt.Printf("error: %v\n", err) + return + } + if len(testFuncs) > 0 { + testCaseFlags := make([]customflag.TestCaseFlag, len(testFuncs)) + for i, j := range testFuncs { + testCaseFlags[i] = customflag.TestCaseFlag(j) + } + customflag.ServiceFlag.TestConfig.TestFuncs = testCaseFlags + } + + os.Exit(m.Run()) +} + +func TestVersionTestSuite(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Version Test Suite") +} + +var _ = AfterSuite(func() { + g := GinkgoT() + if customflag.ServiceFlag.ClusterConfig.Destroy { + status, err := factory.DestroyCluster(g) + Expect(err).NotTo(HaveOccurred()) + Expect(status).To(Equal("cluster destroyed")) + } +}) diff --git a/tests/acceptance/entrypoint/versionbump/versionbump_test.go b/tests/acceptance/entrypoint/versionbump/versionbump_test.go new file mode 100644 index 0000000000..00f46a93b1 --- /dev/null +++ b/tests/acceptance/entrypoint/versionbump/versionbump_test.go @@ -0,0 +1,65 @@ +//go:build versionbump + +package versionbump + +import ( + "fmt" + + "github.com/rancher/rke2/tests/acceptance/core/service/assert" + "github.com/rancher/rke2/tests/acceptance/core/service/customflag" + "github.com/rancher/rke2/tests/acceptance/core/service/template" + "github.com/rancher/rke2/tests/acceptance/core/testcase" + + . "github.com/onsi/ginkgo/v2" +) + +var _ = Describe("VersionTemplate Upgrade:", func() { + + It("Start Up with no issues", func() { + testcase.TestBuildCluster(GinkgoT(), false) + }) + + It("Validate Nodes", func() { + testcase.TestNodeStatus( + assert.NodeAssertReadyStatus(), + nil, + ) + }) + + It("Validate Pods", func() { + testcase.TestPodStatus( + assert.PodAssertRestart(), + assert.PodAssertReady(), + assert.PodAssertStatus(), + ) + }) + + It("Test Bump version", func() { + template.VersionTemplate(template.VersionTestTemplate{ + TestCombination: &template.RunCmd{ + Run: []template.TestMap{ + { + Cmd: template.TestMapFlag.Cmd, + ExpectedValue: template.TestMapFlag.ExpectedValue, + ExpectedValueUpgrade: template.TestMapFlag.ExpectedValueUpgrade, + }, + }, + }, + InstallUpgrade: customflag.ServiceFlag.InstallUpgrade, + TestConfig: &template.TestConfig{ + TestFunc: template.ConvertToTestCase(customflag.ServiceFlag.TestConfig.TestFuncs), + DeployWorkload: customflag.ServiceFlag.TestConfig.DeployWorkload, + WorkloadName: customflag.ServiceFlag.TestConfig.WorkloadName, + }, + Description: customflag.ServiceFlag.TestConfig.Description, + }) + }) +}) + +var _ = AfterEach(func() { + if CurrentSpecReport().Failed() { + fmt.Printf("\nFAILED! %s\n", CurrentSpecReport().FullText()) + } else { + fmt.Printf("\nPASSED! %s\n", CurrentSpecReport().FullText()) + } +}) diff --git a/tests/terraform/go.mod b/tests/acceptance/go.mod similarity index 56% rename from tests/terraform/go.mod rename to tests/acceptance/go.mod index 6859cdc3b5..310c9401a8 100644 --- a/tests/terraform/go.mod +++ b/tests/acceptance/go.mod @@ -1,37 +1,38 @@ -module github.com/rancher/rke2/tests/terraform +module github.com/rancher/rke2/tests/acceptance -go 1.19 +go 1.20 require ( - github.com/gruntwork-io/terratest v0.41.15 - github.com/onsi/ginkgo/v2 v2.9.1 - github.com/onsi/gomega v1.27.3 - golang.org/x/crypto v0.7.0 + github.com/gruntwork-io/terratest v0.43.6 + github.com/onsi/ginkgo/v2 v2.11.0 + github.com/onsi/gomega v1.27.8 + golang.org/x/crypto v0.11.0 ) require ( - cloud.google.com/go v0.105.0 // indirect - cloud.google.com/go/compute v1.12.1 // indirect - cloud.google.com/go/compute/metadata v0.2.1 // indirect - cloud.google.com/go/iam v0.7.0 // indirect - cloud.google.com/go/storage v1.27.0 // indirect + cloud.google.com/go v0.110.2 // indirect + cloud.google.com/go/compute v1.19.3 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/iam v1.1.1 // indirect + cloud.google.com/go/storage v1.29.0 // indirect github.com/agext/levenshtein v1.2.3 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/aws/aws-sdk-go v1.44.122 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect - github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect + github.com/go-logr/logr v1.2.4 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 // indirect + github.com/google/s2a-go v0.1.4 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect - github.com/googleapis/gax-go/v2 v2.7.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect + github.com/googleapis/gax-go/v2 v2.11.0 // indirect github.com/hashicorp/errwrap v1.0.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-getter v1.7.0 // indirect + github.com/hashicorp/go-getter v1.7.1 // indirect github.com/hashicorp/go-multierror v1.1.0 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect @@ -47,19 +48,21 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/testify v1.8.1 // indirect github.com/tmccombs/hcl2json v0.3.3 // indirect - github.com/ulikunitz/xz v0.5.10 // indirect + github.com/ulikunitz/xz v0.5.11 // indirect github.com/zclconf/go-cty v1.9.1 // indirect go.opencensus.io v0.24.0 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/oauth2 v0.1.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect - golang.org/x/tools v0.7.0 // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/oauth2 v0.9.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect + golang.org/x/tools v0.9.3 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/api v0.103.0 // indirect + google.golang.org/api v0.130.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c // indirect - google.golang.org/grpc v1.51.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 // indirect + google.golang.org/grpc v1.56.1 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/tests/terraform/go.sum b/tests/acceptance/go.sum similarity index 94% rename from tests/terraform/go.sum rename to tests/acceptance/go.sum index 20a94b6d33..1aa3ab260f 100644 --- a/tests/terraform/go.sum +++ b/tests/acceptance/go.sum @@ -30,8 +30,8 @@ cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w9 cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= -cloud.google.com/go v0.105.0 h1:DNtEKRBAAzeS4KyIory52wWHuClNaXJ5x1F7xa4q+5Y= -cloud.google.com/go v0.105.0/go.mod h1:PrLgOJNe5nfE9UMxKxgXj4mD3voiP+YQ6gdt6KMFOKM= +cloud.google.com/go v0.110.2 h1:sdFPBr6xG9/wkBbfhmUz/JmZC7X6LavQgcrVINrKiVA= +cloud.google.com/go v0.110.2/go.mod h1:k04UEeEtb6ZBRTv3dZz4CeJC3jKGxyhl0sAiVVquxiw= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= @@ -68,10 +68,10 @@ cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= -cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0= -cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU= -cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48= -cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM= +cloud.google.com/go/compute v1.19.3 h1:DcTwsFgGev/wV5+q8o2fzgcHOaac+DKGC91ZlvpsQds= +cloud.google.com/go/compute v1.19.3/go.mod h1:qxvISKp/gYnXkSAD1ppcSOveRAmzxicEv/JlizULFrI= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= cloud.google.com/go/containeranalysis v0.6.0/go.mod h1:HEJoiEIu+lEXM+k7+qLCci0h33lX3ZqoYFdmPcoO7s4= cloud.google.com/go/datacatalog v1.3.0/go.mod h1:g9svFY6tuR+j+hrTw3J2dNcmI0dzmSiyOzm8kpLq0a0= @@ -109,13 +109,12 @@ cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y97 cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= -cloud.google.com/go/iam v0.7.0 h1:k4MuwOsS7zGJJ+QfZ5vBK8SgHBAvYN/23BWsiihJ1vs= -cloud.google.com/go/iam v0.7.0/go.mod h1:H5Br8wRaDGNc8XP3keLc4unfUUZeyH3Sfl9XpQEYOeg= +cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= +cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= -cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= @@ -171,8 +170,9 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= -cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= +cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= @@ -236,10 +236,10 @@ github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeME github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/go-test/deep v1.0.7 h1:/VSMRlnY/JSyqxQUzQLKVMAskpY/NZKFA5j2P+0pP2M= github.com/go-test/deep v1.0.7/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8= @@ -301,8 +301,8 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -319,13 +319,16 @@ github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.5 h1:UR4rDjcgpgEnqpIEvkiqTYKBCKLNmlge2eVjoZfySzM= +github.com/googleapis/enterprise-certificate-proxy v0.2.5/go.mod h1:RxW0N9901Cko1VOCW3SXCpWP+mlIEkk2tP7jnHy9a3w= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= @@ -335,18 +338,18 @@ github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99 github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= -github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= -github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= +github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/gruntwork-io/terratest v0.41.15 h1:od3neRQRBuf+Zwz5SSByrf3F90iyqFanV9nFF/86uuM= -github.com/gruntwork-io/terratest v0.41.15/go.mod h1:a9QNOPa3/nbsLy1ufNGT33X9sMRknun7qsf4bIVEbPo= +github.com/gruntwork-io/terratest v0.43.6 h1:4WaimQtrUt1hbXEeHdI4JAKkHvFyudCkRGwOXJERASU= +github.com/gruntwork-io/terratest v0.43.6/go.mod h1:Tw+6/fcJFiBPpsx9NNSkLG5oHKIeaqiJHVLpQ+ORIfQ= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= -github.com/hashicorp/go-getter v1.7.0 h1:bzrYP+qu/gMrL1au7/aDvkoOVGUJpeKBgbqRHACAFDY= -github.com/hashicorp/go-getter v1.7.0/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= +github.com/hashicorp/go-getter v1.7.1 h1:SWiSWN/42qdpR0MdhaOc/bLR48PLuP1ZQtYLRlM69uY= +github.com/hashicorp/go-getter v1.7.1/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= @@ -397,10 +400,10 @@ github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZX github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk= -github.com/onsi/ginkgo/v2 v2.9.1/go.mod h1:FEcmzVcCHl+4o9bQZVab+4dC9+j+91t2FHSzmGAPfuo= -github.com/onsi/gomega v1.27.3 h1:5VwIwnBY3vbBDOJrNtA4rVdiTZCsq9B5F12pvy1Drmk= -github.com/onsi/gomega v1.27.3/go.mod h1:5vG284IBtfDAmDyrK+eGyZmUgUlmi+Wngqo557cZ6Gw= +github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= +github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -426,8 +429,9 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/tmccombs/hcl2json v0.3.3 h1:+DLNYqpWE0CsOQiEZu+OZm5ZBImake3wtITYxQ8uLFQ= github.com/tmccombs/hcl2json v0.3.3/go.mod h1:Y2chtz2x9bAeRTvSibVRVgbLJhLJXKlUeIvjeVdnm4w= -github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= +github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4= github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI= @@ -460,8 +464,9 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -498,6 +503,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -534,6 +540,7 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= @@ -546,8 +553,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -572,8 +579,9 @@ golang.org/x/oauth2 v0.0.0-20220622183110-fd043fe589d2/go.mod h1:jaDAt6Dkxork7Lm golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= -golang.org/x/oauth2 v0.1.0 h1:isLCZuhj4v+tYv7eskaN4v/TM+A1begWWgyVJDdl1+Y= golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= +golang.org/x/oauth2 v0.9.0 h1:BPpt2kU7oMRq3kCHAA1tbSEshXRw1LpG2ztgDwrzuAs= +golang.org/x/oauth2 v0.9.0/go.mod h1:qYgFZaFiu6Wg24azG8bdV52QJXJGbZzIIsRCdVKzbLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -588,6 +596,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -652,12 +661,12 @@ golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= +golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -667,9 +676,10 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -725,8 +735,8 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/tools v0.9.3/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -784,8 +794,8 @@ google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= -google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= -google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.130.0 h1:A50ujooa1h9iizvfzA4rrJr2B7uRmWexwbekQ2+5FPQ= +google.golang.org/api v0.130.0/go.mod h1:J/LCJMYSDFvAVREGCbrESb53n4++NMBDetSHGL5I5RY= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -895,8 +905,12 @@ google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqw google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c h1:S34D59DS2GWOEwWNt4fYmTcFrtlOgukG2k9WsomZ7tg= -google.golang.org/genproto v0.0.0-20221201164419-0e50fba7f41c/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc h1:8DyZCyvI8mE1IdLy/60bS+52xfymkE72wv1asokgtao= +google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529 h1:DEH99RbiLZhMxrpEJCZ0A+wdTe0EOgou/poSLx9vWf4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230629202037-9506855d4529/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -932,8 +946,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.51.0 h1:E1eGv1FTqoLIdnBCZufiSHgKjlqG6fKFf6pPWtMTh8U= -google.golang.org/grpc v1.51.0/go.mod h1:wgNDFcnuBGmxLKI/qn4T+m5BtEBYXJPvibbUPsAIPww= +google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= +google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -949,8 +963,9 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/tests/terraform/modules/define_node_role.sh b/tests/acceptance/modules/install/define_node_role.sh similarity index 100% rename from tests/terraform/modules/define_node_role.sh rename to tests/acceptance/modules/install/define_node_role.sh diff --git a/tests/terraform/modules/join_rke2_agent.sh b/tests/acceptance/modules/install/join_rke2_agent.sh old mode 100755 new mode 100644 similarity index 98% rename from tests/terraform/modules/join_rke2_agent.sh rename to tests/acceptance/modules/install/join_rke2_agent.sh index 4e3ee7092a..98f08dcf8d --- a/tests/terraform/modules/join_rke2_agent.sh +++ b/tests/acceptance/modules/install/join_rke2_agent.sh @@ -79,4 +79,4 @@ then systemctl restart systemd-sysctl fi sudo systemctl enable rke2-agent -sudo systemctl start rke2-agent +sudo systemctl start rke2-agent \ No newline at end of file diff --git a/tests/terraform/modules/join_rke2_master.sh b/tests/acceptance/modules/install/join_rke2_master.sh old mode 100755 new mode 100644 similarity index 98% rename from tests/terraform/modules/join_rke2_master.sh rename to tests/acceptance/modules/install/join_rke2_master.sh index 1c1af0ec4c..67a68dea28 --- a/tests/terraform/modules/join_rke2_master.sh +++ b/tests/acceptance/modules/install/join_rke2_master.sh @@ -85,4 +85,4 @@ then fi sudo systemctl enable rke2-server -sudo systemctl start --no-block rke2-server +sudo systemctl start --no-block rke2-server \ No newline at end of file diff --git a/tests/terraform/modules/optional_write_files.sh b/tests/acceptance/modules/install/optional_write_files.sh similarity index 99% rename from tests/terraform/modules/optional_write_files.sh rename to tests/acceptance/modules/install/optional_write_files.sh index 809b4542d2..0a3bcd7d6e 100644 --- a/tests/terraform/modules/optional_write_files.sh +++ b/tests/acceptance/modules/install/optional_write_files.sh @@ -16,4 +16,4 @@ then curl -s "$raw_data" -o "$file_location" done -fi +fi \ No newline at end of file diff --git a/tests/terraform/modules/install_rke2_master.sh b/tests/acceptance/modules/install/rke2_master.sh old mode 100755 new mode 100644 similarity index 100% rename from tests/terraform/modules/install_rke2_master.sh rename to tests/acceptance/modules/install/rke2_master.sh diff --git a/tests/terraform/modules/main.tf b/tests/acceptance/modules/main.tf similarity index 99% rename from tests/terraform/modules/main.tf rename to tests/acceptance/modules/main.tf index d4defe860c..a700bbca07 100644 --- a/tests/terraform/modules/main.tf +++ b/tests/acceptance/modules/main.tf @@ -74,4 +74,4 @@ module "worker" { install_method = var.install_method rke2_channel = var.rke2_channel worker_flags = var.worker_flags -} +} \ No newline at end of file diff --git a/tests/terraform/modules/master/instances_server.tf b/tests/acceptance/modules/master/instances_server.tf similarity index 94% rename from tests/terraform/modules/master/instances_server.tf rename to tests/acceptance/modules/master/instances_server.tf index 2ba6d7f4e8..088d38767a 100644 --- a/tests/terraform/modules/master/instances_server.tf +++ b/tests/acceptance/modules/master/instances_server.tf @@ -21,7 +21,7 @@ resource "aws_instance" "master" { "kubernetes.io/cluster/clusterid" = "owned" } provisioner "file" { - source = "optional_write_files.sh" + source = "install/optional_write_files.sh" destination = "/tmp/optional_write_files.sh" } provisioner "remote-exec" { @@ -31,7 +31,7 @@ resource "aws_instance" "master" { ] } provisioner "file" { - source = "define_node_role.sh" + source = "install/define_node_role.sh" destination = "/tmp/define_node_role.sh" } provisioner "remote-exec" { @@ -41,13 +41,13 @@ resource "aws_instance" "master" { ] } provisioner "file" { - source = "install_rke2_master.sh" - destination = "/tmp/install_rke2_master.sh" + source = "install/rke2_master.sh" + destination = "/tmp/rke2_master.sh" } provisioner "remote-exec" { inline = [ - "chmod +x /tmp/install_rke2_master.sh", - "sudo /tmp/install_rke2_master.sh ${var.node_os} ${var.create_lb ? aws_route53_record.aws_route53[0].fqdn : "fake.fqdn.value"} ${var.rke2_version} ${self.public_ip} ${var.rke2_channel} \"${var.server_flags}\" ${var.install_mode} ${var.username} ${var.password} \"${var.install_method}\"", + "chmod +x /tmp/rke2_master.sh", + "sudo /tmp/rke2_master.sh ${var.node_os} ${var.create_lb ? aws_route53_record.aws_route53[0].fqdn : "fake.fqdn.value"} ${var.rke2_version} ${self.public_ip} ${var.rke2_channel} \"${var.server_flags}\" ${var.install_mode} ${var.username} ${var.password} \"${var.install_method}\"", ] } provisioner "local-exec" { @@ -89,7 +89,7 @@ resource "aws_instance" "master2" { } depends_on = [aws_instance.master] provisioner "file" { - source = "optional_write_files.sh" + source = "install/optional_write_files.sh" destination = "/tmp/optional_write_files.sh" } provisioner "remote-exec" { @@ -99,7 +99,7 @@ resource "aws_instance" "master2" { ] } provisioner "file" { - source = "define_node_role.sh" + source = "install/define_node_role.sh" destination = "/tmp/define_node_role.sh" } provisioner "remote-exec" { @@ -109,7 +109,7 @@ resource "aws_instance" "master2" { ] } provisioner "file" { - source = "join_rke2_master.sh" + source = "install/join_rke2_master.sh" destination = "/tmp/join_rke2_master.sh" } provisioner "remote-exec" { diff --git a/tests/terraform/modules/master/outputs.tf b/tests/acceptance/modules/master/outputs.tf similarity index 100% rename from tests/terraform/modules/master/outputs.tf rename to tests/acceptance/modules/master/outputs.tf diff --git a/tests/terraform/modules/master/variables.tf b/tests/acceptance/modules/master/variables.tf similarity index 97% rename from tests/terraform/modules/master/variables.tf rename to tests/acceptance/modules/master/variables.tf index 9405da1765..d46deecaaa 100644 --- a/tests/terraform/modules/master/variables.tf +++ b/tests/acceptance/modules/master/variables.tf @@ -43,4 +43,4 @@ variable "etcd_cp_nodes" {} variable "etcd_worker_nodes" {} variable "cp_only_nodes" {} variable "cp_worker_nodes" {} -variable "optional_files" {} +variable "optional_files" {} \ No newline at end of file diff --git a/tests/terraform/modules/outputs.tf b/tests/acceptance/modules/outputs.tf similarity index 99% rename from tests/terraform/modules/outputs.tf rename to tests/acceptance/modules/outputs.tf index 6c5d8c9792..9b42ae9edf 100644 --- a/tests/terraform/modules/outputs.tf +++ b/tests/acceptance/modules/outputs.tf @@ -11,4 +11,4 @@ output "worker_ips" { output "kubeconfig" { value = module.master.kubeconfig description = "kubeconfig of the cluster created" -} +} \ No newline at end of file diff --git a/tests/acceptance/modules/providers.tf b/tests/acceptance/modules/providers.tf new file mode 100644 index 0000000000..d9c349e24c --- /dev/null +++ b/tests/acceptance/modules/providers.tf @@ -0,0 +1,3 @@ +provider "aws" { + region = "${var.region}" +} \ No newline at end of file diff --git a/tests/terraform/modules/variables.tf b/tests/acceptance/modules/variables.tf similarity index 99% rename from tests/terraform/modules/variables.tf rename to tests/acceptance/modules/variables.tf index 8aecf32adf..e7232ede80 100644 --- a/tests/terraform/modules/variables.tf +++ b/tests/acceptance/modules/variables.tf @@ -70,4 +70,4 @@ variable "cp_worker_nodes" { } variable "optional_files" { description = "File location and raw data url separate by commas, with a space for other pairs. E.g. file1,url1 file2,url2" -} +} \ No newline at end of file diff --git a/tests/terraform/modules/worker/instances_worker.tf b/tests/acceptance/modules/worker/instances_worker.tf similarity index 97% rename from tests/terraform/modules/worker/instances_worker.tf rename to tests/acceptance/modules/worker/instances_worker.tf index 545396f49c..6e6b03a268 100644 --- a/tests/terraform/modules/worker/instances_worker.tf +++ b/tests/acceptance/modules/worker/instances_worker.tf @@ -27,7 +27,7 @@ resource "aws_instance" "worker" { "kubernetes.io/cluster/clusterid" = "owned" } provisioner "file" { - source = "join_rke2_agent.sh" + source = "install/join_rke2_agent.sh" destination = "/tmp/join_rke2_agent.sh" } provisioner "remote-exec" { diff --git a/tests/terraform/modules/worker/outputs.tf b/tests/acceptance/modules/worker/outputs.tf similarity index 97% rename from tests/terraform/modules/worker/outputs.tf rename to tests/acceptance/modules/worker/outputs.tf index 87238eef9a..473c4c8375 100644 --- a/tests/terraform/modules/worker/outputs.tf +++ b/tests/acceptance/modules/worker/outputs.tf @@ -1,3 +1,3 @@ output "worker_ips" { value = join(",", aws_instance.worker.*.public_ip) -} +} \ No newline at end of file diff --git a/tests/terraform/modules/worker/variables.tf b/tests/acceptance/modules/worker/variables.tf similarity index 95% rename from tests/terraform/modules/worker/variables.tf rename to tests/acceptance/modules/worker/variables.tf index 4d509beb59..6fca0ca6f6 100644 --- a/tests/terraform/modules/worker/variables.tf +++ b/tests/acceptance/modules/worker/variables.tf @@ -27,4 +27,4 @@ variable "username" { default = "username" } variable "vpc_id" {} -variable "worker_flags" {} +variable "worker_flags" {} \ No newline at end of file diff --git a/tests/terraform/scripts/Dockerfile.build b/tests/acceptance/scripts/Dockerfile.build similarity index 100% rename from tests/terraform/scripts/Dockerfile.build rename to tests/acceptance/scripts/Dockerfile.build diff --git a/tests/terraform/scripts/Jenkinsfile b/tests/acceptance/scripts/Jenkinsfile similarity index 84% rename from tests/terraform/scripts/Jenkinsfile rename to tests/acceptance/scripts/Jenkinsfile index de955cf4dc..6e0e4ee0a4 100644 --- a/tests/terraform/scripts/Jenkinsfile +++ b/tests/acceptance/scripts/Jenkinsfile @@ -6,7 +6,7 @@ node { job_name = job_names[job_names.size() - 1] } def testContainer = "${job_name}${env.BUILD_NUMBER}_test" - def imageName = "rke2-tf-${job_name}${env.BUILD_NUMBER}" + def imageName = "rke2-test-${job_name}${env.BUILD_NUMBER}" def envFile = ".env" def branch = "master" if ("${env.BRANCH}" != "null" && "${env.BRANCH}" != "") { @@ -50,32 +50,32 @@ node { try { stage('Configure and Build') { if (env.AWS_SSH_PEM_KEY && env.AWS_SSH_KEY_NAME) { - dir("./tests/terraform/modules/config/.ssh") { + dir("./tests/acceptance/modules/config/.ssh") { def decoded = new String(AWS_SSH_PEM_KEY.decodeBase64()) writeFile file: "aws_key.pem", text: decoded } } - dir("./tests/terraform/modules/config") { + dir("./tests/acceptance/modules/config") { def filename = "local.tfvars" def configContents = env.TFVARS writeFile file: filename, text: configContents + "\npassword = \"" + RKE2_RHEL_PASSWORD + "\"" + "\nssh_key = \"" + AWS_SSH_KEY_NAME + "\"" + - "\naccess_key = \"/go/src/github.com/rancher/rke2/tests/terraform/modules/config/.ssh/aws_key.pem\"" + + "\naccess_key = \"/go/src/github.com/rancher/rke2/tests/acceptance/modules/config/.ssh/aws_key.pem\"" + "\nresource_name = \"" + RKE2_HOSTNAME_PREFIX + "\"" + "\nrke2_version = \"" + RKE2_VERSION + "\"" + "\nrke2_channel = \"" + RKE2_CHANNEL + "\"" } - sh "./tests/terraform/scripts/configure.sh" - sh "./tests/terraform/scripts/build.sh" + sh "./tests/acceptance/scripts/configure.sh" + sh "./tests/acceptance/scripts/build.sh" } - stage('Run Tests') { + stage('Run TestCombination') { sh "docker run --name ${testContainer} -t --env-file ${envFile} " + - "${imageName} sh -c \"chmod 400 /go/src/github.com/rancher/rke2/tests/terraform/modules/config/.ssh/aws_key.pem && " + - "cd ./tests/terraform && go test -timeout=${timeout} -v ./${RKE2_TESTCASE}/... ${RKE2_TEST_ARGS}\"" + "${imageName} sh -c \"chmod 400 /go/src/github.com/rancher/rke2/tests/acceptance/modules/config/.ssh/aws_key.pem && " + + "cd ./tests/acceptance/entrypoint && go test -timeout=${timeout} -v ./${RKE2_TESTCASE}/... ${RKE2_TEST_ARGS}\"" } } finally { stage('Cleanup') { diff --git a/tests/terraform/scripts/build.sh b/tests/acceptance/scripts/build.sh similarity index 74% rename from tests/terraform/scripts/build.sh rename to tests/acceptance/scripts/build.sh index b4b4b258db..607ac07de4 100755 --- a/tests/terraform/scripts/build.sh +++ b/tests/acceptance/scripts/build.sh @@ -14,7 +14,7 @@ fi count=0 while [[ 3 -gt $count ]]; do - docker build . -f tests/terraform/scripts/Dockerfile.build -t rke2-tf-"${TRIM_JOB_NAME}""${BUILD_NUMBER}" + docker build . -f tests/acceptance/scripts/Dockerfile.build -t rke2-test-"${TRIM_JOB_NAME}""${BUILD_NUMBER}" if [[ $? -eq 0 ]]; then break; fi count=$(($count + 1)) diff --git a/tests/terraform/scripts/configure.sh b/tests/acceptance/scripts/configure.sh similarity index 100% rename from tests/terraform/scripts/configure.sh rename to tests/acceptance/scripts/configure.sh diff --git a/tests/acceptance/scripts/delete_resources.sh b/tests/acceptance/scripts/delete_resources.sh new file mode 100755 index 0000000000..020e51647d --- /dev/null +++ b/tests/acceptance/scripts/delete_resources.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +#Get resource name from tfvarslocal && change name to make more sense in this context +RESOURCE_NAME=$(grep resource_name /dev/null 2>&1 + + +#Get the list of load balancer ARNs +lb_arn_list=$(aws elbv2 describe-load-balancers \ + --query "LoadBalancers[?starts_with(LoadBalancerName, '${name_prefix}') && Type=='network'].LoadBalancerArn" \ + --output text) + + +#Loop through the load balancer ARNs and delete the load balancers +for lb_arn in $lb_arn_list; do + echo "Deleting load balancer $lb_arn" + aws elbv2 delete-load-balancer --load-balancer-arn "$lb_arn" +done + + +#Get the list of target group ARNs +tg_arn_list=$(aws elbv2 describe-target-groups \ + --query "TargetGroups[?starts_with(TargetGroupName, '${name_prefix}') && Protocol=='TCP'].TargetGroupArn" \ + --output text) + + +#Loop through the target group ARNs and delete the target groups +for tg_arn in $tg_arn_list; do + echo "Deleting target group $tg_arn" + aws elbv2 delete-target-group --target-group-arn "$tg_arn" +done + + +#Get the ID and recordName with lower case of the hosted zone that contains the Route 53 record sets +name_prefix_lower=$(echo "$name_prefix" | tr '[:upper:]' '[:lower:]') +r53_zone_id=$(aws route53 list-hosted-zones-by-name --dns-name "${name_prefix}." \ + --query "HostedZones[0].Id" --output text) +r53_record=$(aws route53 list-resource-record-sets \ + --hosted-zone-id "${r53_zone_id}" \ + --query "ResourceRecordSets[?starts_with(Name, '${name_prefix_lower}.') && Type == 'CNAME'].Name" \ + --output text) + + +#Get ResourceRecord Value +record_value=$(aws route53 list-resource-record-sets \ + --hosted-zone-id "${r53_zone_id}" \ + --query "ResourceRecordSets[?starts_with(Name, '${name_prefix_lower}.') \ + && Type == 'CNAME'].ResourceRecords[0].Value" --output text) + + +#Delete Route53 record +if [[ "$r53_record" == "${name_prefix_lower}."* ]]; then + echo "Deleting Route53 record ${r53_record}" + change_status=$(aws route53 change-resource-record-sets --hosted-zone-id "${r53_zone_id}" \ + --change-batch '{"Changes": [ + { + "Action": "DELETE", + "ResourceRecordSet": { + "Name": "'"${r53_record}"'", + "Type": "CNAME", + "TTL": 300, + "ResourceRecords": [ + { + "Value": "'"${record_value}"'" + } + ] + } + } + ] + }') + status_id=$(echo "$change_status" | jq -r '.ChangeInfo.Id') + #Get status from the change + aws route53 wait resource-record-sets-changed --id "$status_id" + echo "Successfully deleted Route53 record ${r53_record}: status: ${status_id}" +else + echo "No Route53 record found" +fi \ No newline at end of file diff --git a/tests/acceptance/shared/aux.go b/tests/acceptance/shared/aux.go new file mode 100644 index 0000000000..4fb66aca8a --- /dev/null +++ b/tests/acceptance/shared/aux.go @@ -0,0 +1,203 @@ +package shared + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "golang.org/x/crypto/ssh" +) + +// RunCommandHost executes a command on the host +func RunCommandHost(cmds ...string) (string, error) { + if cmds == nil { + return "", fmt.Errorf("cmd should not be empty") + } + + var output, errOut bytes.Buffer + for _, cmd := range cmds { + c := exec.Command("bash", "-c", cmd) + c.Stdout = &output + c.Stderr = &errOut + if errOut.Len() > 0 { + fmt.Println("returning Stderr if not null, this might not be an error", + errOut.String()) + } + + err := c.Run() + if err != nil { + return output.String(), fmt.Errorf("executing command: %s: %w", cmd, err) + } + } + + return output.String(), nil +} + +// RunCommandOnNode executes a command on the node SSH +func RunCommandOnNode(cmd string, ServerIP string) (string, error) { + if cmd == "" { + return "", fmt.Errorf("cmd should not be empty") + } + + host := ServerIP + ":22" + conn, err := configureSSH(host) + if err != nil { + return fmt.Errorf("failed to configure SSH: %v", err).Error(), err + } + + stdout, stderr, err := runsshCommand(cmd, conn) + if err != nil { + return fmt.Errorf("\ncommand: %s \n failed with error: %v", cmd, err).Error(), err + } + + stdout = strings.TrimSpace(stdout) + stderr = strings.TrimSpace(stderr) + + if stderr != "" && (!strings.Contains(stderr, "error") || + !strings.Contains(stderr, "exit status 1") || + !strings.Contains(stderr, "exit status 2")) { + return stderr, nil + } else if stderr != "" { + return fmt.Errorf("\ncommand: %s \n failed with error: %v", cmd, stderr).Error(), err + } + + return stdout, err +} + +// BasePath returns the base path of the project. +func BasePath() string { + _, b, _, _ := runtime.Caller(0) + return filepath.Join(filepath.Dir(b), "../..") +} + +// PrintFileContents prints the contents of the file as [] string. +func PrintFileContents(f ...string) error { + for _, file := range f { + content, err := os.ReadFile(file) + if err != nil { + return err + } + fmt.Println(string(content) + "\n") + } + + return nil +} + +// CountOfStringInSlice Used to count the pods using prefix passed in the list of pods. +func CountOfStringInSlice(str string, pods []Pod) int { + var count int + for _, p := range pods { + if strings.Contains(p.Name, str) { + count++ + } + } + return count +} + +// GetRke2Version returns the rke2 version with commit hash +func GetRke2Version() string { + ips := FetchNodeExternalIP() + for _, ip := range ips { + res, err := RunCommandOnNode("rke2 --version", ip) + if err != nil { + return err.Error() + } + return res + } + + return "" +} + +// AddHelmRepo adds a helm repo to the cluster. +func AddHelmRepo(name, url string) (string, error) { + InstallHelm := "curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash" + addRepo := fmt.Sprintf("helm repo add %s %s", name, url) + update := "helm repo update" + installRepo := fmt.Sprintf("helm install %s %s/%s -n kube-system --kubeconfig=%s", + name, name, name, KubeConfigFile) + + nodeExternalIP := FetchNodeExternalIP() + for _, ip := range nodeExternalIP { + _, err := RunCommandOnNode(InstallHelm, ip) + if err != nil { + return "", err + } + } + + return RunCommandHost(addRepo, update, installRepo) +} + +func publicKey(path string) (ssh.AuthMethod, error) { + key, err := os.ReadFile(path) + if err != nil { + return nil, err + } + signer, err := ssh.ParsePrivateKey(key) + if err != nil { + return nil, err + } + + return ssh.PublicKeys(signer), nil +} + +func configureSSH(host string) (*ssh.Client, error) { + var config *ssh.ClientConfig + + authMethod, err := publicKey(AccessKey) + if err != nil { + return nil, err + } + config = &ssh.ClientConfig{ + User: AwsUser, + Auth: []ssh.AuthMethod{ + authMethod, + }, + HostKeyCallback: ssh.InsecureIgnoreHostKey(), + } + conn, err := ssh.Dial("tcp", host, config) + if err != nil { + return nil, err + } + + return conn, nil +} + +func runsshCommand(cmd string, conn *ssh.Client) (string, string, error) { + session, err := conn.NewSession() + if err != nil { + return "", "", err + } + defer session.Close() + + var stdoutBuf bytes.Buffer + var stderrBuf bytes.Buffer + session.Stdout = &stdoutBuf + session.Stderr = &stderrBuf + + errssh := session.Run(cmd) + stdoutStr := stdoutBuf.String() + stderrStr := stderrBuf.String() + + if errssh != nil { + return stdoutStr, stderrStr, fmt.Errorf("error on command execution: %v", errssh) + } + + return stdoutStr, stderrStr, nil +} + +// JoinCommands joins the first command with some arg +func JoinCommands(cmd, kubeconfigFlag string) string { + cmds := strings.Split(cmd, ":") + joinedCmd := cmds[0] + kubeconfigFlag + + if len(cmds) > 1 { + secondCmd := strings.Join(cmds[1:], ",") + joinedCmd += " " + secondCmd + } + + return joinedCmd +} diff --git a/tests/acceptance/shared/cluster.go b/tests/acceptance/shared/cluster.go new file mode 100644 index 0000000000..26cddb46f2 --- /dev/null +++ b/tests/acceptance/shared/cluster.go @@ -0,0 +1,279 @@ +package shared + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" +) + +var ( + KubeConfigFile string + AwsUser string + AccessKey string +) + +type Node struct { + Name string + Status string + Roles string + Version string + InternalIP string + ExternalIP string +} + +type Pod struct { + NameSpace string + Name string + Ready string + Status string + Restarts string + NodeIP string + Node string +} + +// ManageWorkload creates or deletes a workload based on the action: create or delete. +func ManageWorkload(action, workload string) (string, error) { + if action != "create" && action != "delete" { + return "", fmt.Errorf("invalid action: %s. Must be 'create' or 'delete'", action) + } + var res string + var err error + + resourceDir := BasePath() + "/acceptance/workloads/" + + files, err := os.ReadDir(resourceDir) + if err != nil { + err = fmt.Errorf("%s : Unable to read resource manifest file for %s", err, workload) + return "", err + } + + for _, f := range files { + filename := filepath.Join(resourceDir, f.Name()) + if strings.TrimSpace(f.Name()) == workload { + if action == "create" { + res, err = createWorkload(workload, filename) + if err != nil { + return "", fmt.Errorf("failed to create workload %s: %s", workload, err) + } + } else { + err = deleteWorkload(workload, filename) + if err != nil { + return "", fmt.Errorf("failed to delete workload %s: %s", workload, err) + } + } + return res, err + } + } + + return "", fmt.Errorf("workload %s not found", workload) +} + +// createWorkload creates a workload. +func createWorkload(workload, filename string) (string, error) { + fmt.Println("\nDeploying", workload) + return RunCommandHost("kubectl apply -f " + filename + " --kubeconfig=" + KubeConfigFile) +} + +// deleteWorkload deletes a workload and asserts that the workload is deleted. +func deleteWorkload(workload, filename string) error { + fmt.Println("\nRemoving", workload) + cmd := "kubectl delete -f " + filename + " --kubeconfig=" + KubeConfigFile + + _, err := RunCommandHost(cmd) + if err != nil { + return fmt.Errorf("failed to run kubectl delete: %v", err) + } + + timeout := time.After(60 * time.Second) + tick := time.Tick(5 * time.Second) + + for { + select { + case <-timeout: + return errors.New("workload deletion timed out") + case <-tick: + res, err := RunCommandHost("kubectl get all -A --kubeconfig=" + KubeConfigFile) + if err != nil { + return err + } + isDeleted := !strings.Contains(res, workload) + if isDeleted { + return nil + } + } + } +} + +// KubectlCommand return results from various commands, it receives an "action" , source and args. +// it already has KubeConfigFile +// +// destination = host or node +// +// action = get,describe... +// +// source = pods, node , exec, service ... +// +// args = the rest of your command arguments. +func KubectlCommand(destination, action, source string, args ...string) (string, error) { + kubeconfigFlag := " --kubeconfig=" + KubeConfigFile + shortCmd := map[string]string{ + "get": "kubectl get", + "describe": "kubectl describe", + "exec": "kubectl exec", + "delete": "kubectl delete", + "apply": "kubectl apply", + } + + cmdPrefix, ok := shortCmd[action] + if !ok { + cmdPrefix = action + } + + cmd := cmdPrefix + " " + source + " " + strings.Join(args, " ") + kubeconfigFlag + + var res string + var err error + if destination == "host" { + res, err = RunCommandHost(cmd) + if err != nil { + return "", err + } + } else if destination == "node" { + ips := FetchNodeExternalIP() + for _, ip := range ips { + res, err = RunCommandOnNode(cmd, ip) + if err != nil { + return "", err + } + } + } else { + return "", fmt.Errorf("invalid destination: %s", destination) + } + + return res, nil +} + +// FetchClusterIP returns the cluster IP and port of the service. +func FetchClusterIP( + namespace string, + serviceName string) (string, string, error) { + ip, err := RunCommandHost("kubectl get svc " + serviceName + " -n " + namespace + + " -o jsonpath='{.spec.clusterIP}' --kubeconfig=" + KubeConfigFile) + if err != nil { + return "", "", err + } + + port, err := RunCommandHost("kubectl get svc " + serviceName + " -n " + namespace + + " -o jsonpath='{.spec.ports[0].port}' --kubeconfig=" + KubeConfigFile) + if err != nil { + return "", "", err + } + + return ip, port, err +} + +// FetchNodeExternalIP returns the external IP of the nodes. +func FetchNodeExternalIP() []string { + res, _ := RunCommandHost("kubectl get nodes " + + "--output=jsonpath='{.items[*].status.addresses[?(@.type==\"ExternalIP\")].address}' " + + "--kubeconfig=" + KubeConfigFile) + nodeExternalIP := strings.Trim(res, " ") + nodeExternalIPs := strings.Split(nodeExternalIP, " ") + + return nodeExternalIPs +} + +// RestartCluster restarts the rke2 service on each node given by external IP. +func RestartCluster(ip string) (string, error) { + return RunCommandOnNode("sudo systemctl restart rke2-*", ip) +} + +// FetchIngressIP returns the ingress IP of the given namespace +func FetchIngressIP(namespace string) ([]string, error) { + res, err := RunCommandHost( + "kubectl get ingress -n " + + namespace + + " -o jsonpath='{.items[0].status.loadBalancer.ingress[*].ip}' --kubeconfig=" + + KubeConfigFile, + ) + if err != nil { + return nil, err + } + + ingressIP := strings.Trim(res, " ") + if ingressIP == "" { + return nil, nil + } + ingressIPs := strings.Split(ingressIP, " ") + + return ingressIPs, nil +} + +// ParseNodes returns nodes parsed from kubectl get nodes. +func ParseNodes(print bool) ([]Node, error) { + nodes := make([]Node, 0, 10) + + cmd := "kubectl get nodes --no-headers -o wide --kubeconfig=" + KubeConfigFile + res, err := RunCommandHost(cmd) + if err != nil { + return nil, err + } + + nodelist := strings.TrimSpace(res) + split := strings.Split(nodelist, "\n") + for _, rec := range split { + if strings.TrimSpace(rec) != "" { + fields := strings.Fields(rec) + n := Node{ + Name: fields[0], + Status: fields[1], + Roles: fields[2], + Version: fields[4], + InternalIP: fields[5], + ExternalIP: fields[6], + } + nodes = append(nodes, n) + } + } + if print { + fmt.Println(nodelist) + } + + return nodes, nil +} + +// ParsePods returns pods parsed from kubectl get pods. +func ParsePods(print bool) ([]Pod, error) { + pods := make([]Pod, 0, 10) + + cmd := "kubectl get pods -o wide --no-headers -A --kubeconfig=" + KubeConfigFile + res, err := RunCommandHost(cmd) + if err != nil { + return nil, err + } + + podList := strings.TrimSpace(res) + split := strings.Split(podList, "\n") + for _, rec := range split { + fields := strings.Fields(rec) + p := Pod{ + NameSpace: fields[0], + Name: fields[1], + Ready: fields[2], + Status: fields[3], + Restarts: fields[4], + NodeIP: fields[6], + Node: fields[7], + } + pods = append(pods, p) + } + + if print { + fmt.Println(podList) + } + + return pods, nil +} diff --git a/tests/terraform/resource_files/clusterip.yaml b/tests/acceptance/workloads/clusterip.yaml similarity index 100% rename from tests/terraform/resource_files/clusterip.yaml rename to tests/acceptance/workloads/clusterip.yaml diff --git a/tests/terraform/resource_files/daemonset.yaml b/tests/acceptance/workloads/daemonset.yaml similarity index 100% rename from tests/terraform/resource_files/daemonset.yaml rename to tests/acceptance/workloads/daemonset.yaml diff --git a/tests/terraform/resource_files/dnsutils.yaml b/tests/acceptance/workloads/dnsutils.yaml similarity index 100% rename from tests/terraform/resource_files/dnsutils.yaml rename to tests/acceptance/workloads/dnsutils.yaml diff --git a/tests/terraform/resource_files/ingress.yaml b/tests/acceptance/workloads/ingress.yaml similarity index 100% rename from tests/terraform/resource_files/ingress.yaml rename to tests/acceptance/workloads/ingress.yaml diff --git a/tests/terraform/resource_files/nodeport.yaml b/tests/acceptance/workloads/nodeport.yaml similarity index 100% rename from tests/terraform/resource_files/nodeport.yaml rename to tests/acceptance/workloads/nodeport.yaml diff --git a/tests/terraform/resource_files/suc.yaml b/tests/acceptance/workloads/suc.yaml similarity index 100% rename from tests/terraform/resource_files/suc.yaml rename to tests/acceptance/workloads/suc.yaml diff --git a/tests/acceptance/workloads/traefik-logs.yaml b/tests/acceptance/workloads/traefik-logs.yaml new file mode 100644 index 0000000000..bdefe87d12 --- /dev/null +++ b/tests/acceptance/workloads/traefik-logs.yaml @@ -0,0 +1,13 @@ +apiVersion: helm.cattle.io/v1 +kind: HelmChartConfig +metadata: + name: traefik + namespace: kube-system +spec: + valuesContent: |- + service: + spec: + externalTrafficPolicy: Local + logs: + access: + enabled: true \ No newline at end of file diff --git a/tests/terraform/.gitignore b/tests/terraform/.gitignore deleted file mode 100644 index 493038c7c0..0000000000 --- a/tests/terraform/.gitignore +++ /dev/null @@ -1,4 +0,0 @@ -.ssh/ -*.tfstate* -.terraform* -config/ diff --git a/tests/terraform/Makefile b/tests/terraform/Makefile deleted file mode 100644 index d258311980..0000000000 --- a/tests/terraform/Makefile +++ /dev/null @@ -1,75 +0,0 @@ -##========================= Terraform Tests =========================# -include ./config.mk - - -TAGNAME ?= default -tf-up: - @cd ../.. && docker build . -q -f ./tests/terraform/scripts/Dockerfile.build -t rke2-tf-${TAGNAME} - -.PHONY: tf-run -tf-run: - @docker run -d --name rke2-tf-test-${IMGNAME} -t \ - -e AWS_ACCESS_KEY_ID="${AWS_ACCESS_KEY_ID}" \ - -e AWS_SECRET_ACCESS_KEY="${AWS_SECRET_ACCESS_KEY}" \ - -v ${ACCESS_KEY_LOCAL}:/go/src/github.com/rancher/rke2/tests/terraform/modules/config/.ssh/aws_key.pem \ - rke2-tf-${TAGNAME} sh -c 'cd ./tests/terraform ; \ - if [ -n "${ARGNAME}" ]; then \ - go test -v -timeout=45m \ - ./${TESTDIR}/... \ - -"${ARGNAME}"="${ARGVALUE}"; \ - elif [ -z "${TESTDIR}" ]; then \ - go test -v -timeout=40m \ - ./createcluster/...; \ - else \ - go test -v -timeout=45m \ - ./${TESTDIR}/...; \ - fi' - -.PHONY: tf-logs -tf-logs: - @docker logs -f rke2-tf-test-${IMGNAME} - -.PHONY: tf-down -tf-down: - @echo "Removing containers and images" - @docker stop $$(docker ps -a -q --filter="name=rke2-tf*") - @docker rm $$(docker ps -a -q --filter="name=rke2-tf*") - @docker rmi $$(docker images -q --filter="reference=rke2-tf*") - -.PHONY: tf-clean -tf-clean: - @./scripts/delete_resources.sh - -.PHONY: tf-complete -tf-tests-complete: tf-clean tf-down tf-remove-state tf-up tf-run - - -#========================= Run terraform tests locally =========================# - -.PHONY: tf-create -tf-create: - @go test -timeout=45m -v ./createcluster/... - -.PHONY: tf-upgrade -tf-upgrade: - @go test -timeout=45m -v ./upgradecluster/... -${ARGNAME}=${ARGVALUE} - -.PHONY: tf-remove-state -tf-remove-state: - @rm -rf ./modules/.terraform - @rm -rf ./modules/.terraform.lock.hcl ./modules/terraform.tfstate ./modules/terraform.tfstate.backup - -.PHONY: tf-test-suite -tf-test-suite: - @make tf-remove-state && make tf-create ; sleep 5 && \ - make tf-remove-state && make tf-upgrade ${ARGNAME}=${ARGVALUE} - -.PHONY: tf-test-suite-same-cluster -tf-test-suite-same-cluster: - @make tf-create ; sleep 5 && make tf-upgrade ${ARGNAME}=${ARGVALUE} - -#========================= TestCode Static Quality Check =========================# -.PHONY: vet-lint ## Run locally only inside Tests package -vet-lint: - @echo "Running go vet and lint" - @go vet ./${TESTDIR} && golangci-lint run --tests \ No newline at end of file diff --git a/tests/terraform/README.md b/tests/terraform/README.md deleted file mode 100644 index ad62079bcc..0000000000 --- a/tests/terraform/README.md +++ /dev/null @@ -1,107 +0,0 @@ -# Terraform (TF) Tests - -Terraform (TF) tests are an additional form of End-to-End (E2E) tests that cover multi-node RKE2 configuration and administration: install, update, teardown, etc. across a wide range of operating systems. Terraform tests are used as part of RKE2 quality assurance (QA) to bring up clusters with different configurations on demand, perform specific functionality tests, and keep them up and running to perform some exploratory tests in real-world scenarios. - -## Framework -TF tests utilize [Ginkgo](https://onsi.github.io/ginkgo/) and [Gomega](https://onsi.github.io/gomega/) like the e2e tests. They rely on [Terraform](https://www.terraform.io/) to provide the underlying cluster configuration. - -## Format - -- All TF tests should be placed under `tests/terraform/`. -- All TF test functions should be named: `Test_TF`. - -See the [create cluster test](../tests/terraform/createcluster_test.go) as an example. - -## Running - -- Before running the tests, it's required to create a tfvars file in `./tests/terraform/modules/config/local.tfvars`. This should be filled in to match the desired variables, including those relevant for your AWS environment. All variables that are necessary can be seen in [main.tf](../tests/terraform/modules/main.tf). -It is also required to have standard AWS environment variables present: `AWS_ACCESS_KEY_ID` , `AWS_SECRET_ACCESS_KEY` and `ACCESS_KEY_LOCAL` - - -- The local.tfvars split roles section should be strictly followed to not cause any false positives or negatives on tests - - -- Please also when creating tf var resource_name, make sure that you do not have any instances from other automations with the same name to avoid deleting wrong resources - - -- If you want to run tests locally totally in parallel, please make sure that you have different resource_name for each test - -*** - -Tests can be run per package with: -```bash -go test -timeout=30m -v ./tests/terraform/$PACKAGE_NAME/... -``` -Additionally, you can use docker to run the tests, which may be beneficial when wanting to run multiple tests in parallel. Just be sure to change the resource name in the tfvars file to ensure there won't be overwrites! Provided example below is for running two separate packages using docker: -```bash -$ docker build . -f ./tests/terraform/scripts/Dockerfile.build -t rke2-tf -# These next commands assume you have the following environment variable in your config/local.tfvars: 'access_key = "/tmp/aws_key.pem"' -$ docker run --name rke2-tf-creation-test -t -e AWS_ACCESS_KEY_ID= -e AWS_SECRET_ACCESS_KEY= -v /path/to/aws/key.pem:/tmp/aws_key.pem rke2-tf sh -c "go test -timeout=30m -v ./tests/terraform/createcluster/..." -$ docker run --name rke2-tf-upgrade-test -t -e AWS_ACCESS_KEY_ID= -e AWS_SECRET_ACCESS_KEY= -v /path/to/aws/key.pem:/tmp/aws_key.pem rke2-tf sh -c "go test -timeout=45m -v ./tests/terraform/upgradecluster/... -upgradeVersion=v1.24.8+rke2r1" -``` -Test Flags: -``` -- ${upgradeVersion} version to upgrade to -``` -We can also run tests through the Makefile through ./test/terraform directory: - -- On the first run with make and docker please delete your .terraform folder, terraform.tfstate and terraform.hcl.lock file -```bash -Args: -*All args are optional and can be used with: - -`$make tf-run` `$make tf-logs`, -`$make vet-lint` `$make tf-complete`, -`$make tf-upgrade` `$make tf-test-suite-same-cluster`, -`$make tf-test-suite` - -- ${IMGNAME} append any string to the end of image name -- ${TAGNAME} append any string to the end of tag name -- ${ARGNAME} name of the arg to pass to the test -- ${ARGVALUE} value of the arg to pass to the test -- ${TESTDIR} path to the test directory - -Commands: -$ make tf-up # create the image from Dockerfile.build -$ make tf-run # runs all tests if no flags or args provided -$ make tf-down # removes the image -$ make tf-clean # removes instances and resources created by tests -$ make tf-logs # prints logs from container the tests -$ make tf--complete # clean resources + remove images + run tests -$ make tf-create # runs create cluster test locally -$ make tf-upgrade # runs upgrade cluster test locally -$ make tf-test-suite-same-cluster # runs all tests locally in sequence using the same state -$ make tf-remove-state # removes terraform state dir and files -$ make tf-test-suite # runs all tests locally in sequence not using the same state -$ make vet-lint # runs go vet and go lint - - -Examples: -$ make tf-up TAGNAME=ubuntu -$ make tf-run IMGNAME=2 TAGNAME=ubuntu TESTDIR=upgradecluster ARGNAME=upgradeVersion ARGVALUE=v1.26.2+rke2r1 -$ make tf-run TESTDIR=createcluster -$ make tf-logs IMGNAME=1 -$ make vet-lint TESTDIR=upgradecluster -``` - -# Running tests in parallel: -- You can play around and have a lot of different test combinations like: -``` -- Build docker image with different TAGNAME="OS`s" + with different configurations( resource_name, node_os, versions, install type, nodes and etc) and have unique "IMGNAMES" -- And in the meanwhile run also locally with different configuration while your dockers TAGNAME and IMGNAMES are running -``` - -# In between tests: -- If you want to run with same cluster do not delete ./tests/terraform/modules/terraform.tfstate + .terraform.lock.hcl file after each test. - -- If you want to use new resources then make sure to delete the ./tests/terraform/modules/terraform.tfstate + .terraform.lock.hcl file if you want to create a new cluster. - - -# Common Issues: - -- Issues related to terraform plugin please also delete the modules/.terraform folder -- In mac m1 maybe you need also to go to rke2/tests/terraform/modules and run `terraform init` to download the plugins - - -# Debugging -To focus individual runs on specific test clauses, you can prefix with `F`. For example, in the [create cluster test](../tests/terraform/createcluster_test.go), you can update the initial creation to be: `FIt("Starts up with no issues", func() {` in order to focus the run on only that clause. diff --git a/tests/terraform/createcluster/createcluster.go b/tests/terraform/createcluster/createcluster.go deleted file mode 100644 index 7eb4e30352..0000000000 --- a/tests/terraform/createcluster/createcluster.go +++ /dev/null @@ -1,92 +0,0 @@ -package createcluster - -import ( - "fmt" - - "path/filepath" - "strconv" - "testing" - - "github.com/gruntwork-io/terratest/modules/terraform" - tf "github.com/rancher/rke2/tests/terraform" -) - -var ( - KubeConfigFile string - MasterIPs string - WorkerIPs string - NumServers int - NumWorkers int - AwsUser string - AccessKey string - modulesPath = "/tests/terraform/modules" - tfVarsPath = "/tests/terraform/modules/config/local.tfvars" -) - -func BuildCluster(t *testing.T, destroy bool) (string, error) { - tfDir, err := filepath.Abs(tf.Basepath() + modulesPath) - if err != nil { - return "", err - } - - varDir, err := filepath.Abs(tf.Basepath() + tfVarsPath) - if err != nil { - return "", err - } - terraformOptions := terraform.Options{ - TerraformDir: tfDir, - VarFiles: []string{varDir}, - } - - NumServers, err = strconv.Atoi(terraform.GetVariableAsStringFromVarFile(t, varDir, "no_of_server_nodes")) - if err != nil { - return "", err - } - NumWorkers, err = strconv.Atoi(terraform.GetVariableAsStringFromVarFile(t, varDir, "no_of_worker_nodes")) - if err != nil { - return "", err - } - - splitRoles := terraform.GetVariableAsStringFromVarFile(t, varDir, "split_roles") - if splitRoles == "true" { - etcdNodes, err := strconv.Atoi(terraform.GetVariableAsStringFromVarFile(t, varDir, "etcd_only_nodes")) - if err != nil { - return "", err - } - etcdCpNodes, err := strconv.Atoi(terraform.GetVariableAsStringFromVarFile(t, varDir, "etcd_cp_nodes")) - if err != nil { - return "", err - } - etcdWorkerNodes, err := strconv.Atoi(terraform.GetVariableAsStringFromVarFile(t, varDir, "etcd_worker_nodes")) - if err != nil { - return "", err - } - cpNodes, err := strconv.Atoi(terraform.GetVariableAsStringFromVarFile(t, varDir, "cp_only_nodes")) - if err != nil { - return "", err - } - cpWorkerNodes, err := strconv.Atoi(terraform.GetVariableAsStringFromVarFile(t, varDir, "cp_worker_nodes")) - if err != nil { - return "", err - } - NumServers = NumServers + etcdNodes + etcdCpNodes + etcdWorkerNodes + cpNodes + cpWorkerNodes - } - - AwsUser = terraform.GetVariableAsStringFromVarFile(t, varDir, "aws_user") - AccessKey = terraform.GetVariableAsStringFromVarFile(t, varDir, "access_key") - - if destroy { - fmt.Printf("Cluster is being deleted") - terraform.Destroy(t, &terraformOptions) - return "cluster destroyed", nil - } - - fmt.Printf("Creating Cluster") - - terraform.InitAndApply(t, &terraformOptions) - KubeConfigFile = terraform.Output(t, &terraformOptions, "kubeconfig") - MasterIPs = terraform.Output(t, &terraformOptions, "master_ips") - WorkerIPs = terraform.Output(t, &terraformOptions, "worker_ips") - - return "cluster created", nil -} diff --git a/tests/terraform/createcluster/createcluster_test.go b/tests/terraform/createcluster/createcluster_test.go deleted file mode 100644 index 62fcfb982c..0000000000 --- a/tests/terraform/createcluster/createcluster_test.go +++ /dev/null @@ -1,259 +0,0 @@ -package createcluster - -import ( - "flag" - "fmt" - "regexp" - "strings" - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/rancher/rke2/tests/terraform" -) - -var destroy = flag.Bool("destroy", false, "a bool") - -func Test_TFClusterCreateValidation(t *testing.T) { - RegisterFailHandler(Fail) - flag.Parse() - - RunSpecs(t, "Create Cluster Test Suite") -} - -var _ = Describe("Test:", func() { - Context("Build Cluster:", func() { - It("Starts up with no issues", func() { - status, err := BuildCluster(&testing.T{}, false) - Expect(err).NotTo(HaveOccurred()) - Expect(status).To(Equal("cluster created")) - - defer GinkgoRecover() - - fmt.Println("Server Node IPS:", MasterIPs) - fmt.Println("Agent Node IPS:", WorkerIPs) - terraform.PrintFileContents(KubeConfigFile) - - Expect(KubeConfigFile).ShouldNot(BeEmpty()) - Expect(MasterIPs).ShouldNot(BeEmpty()) - - if NumWorkers > 0 { - Expect(WorkerIPs).ShouldNot(BeEmpty()) - } else { - Expect(WorkerIPs).Should(BeEmpty()) - } - Expect(KubeConfigFile).ShouldNot(BeEmpty()) - }) - - fmt.Printf("\nFetching node status\n") - It("Checks Node and Pod Status", func() { - defer func() { - _, err := terraform.Nodes(KubeConfigFile, true) - if err != nil { - fmt.Println("Error retrieving nodes: ", err) - } - _, err = terraform.Pods(KubeConfigFile, true) - if err != nil { - fmt.Println("Error retrieving pods: ", err) - } - }() - - expectedNodeCount := NumServers + NumWorkers - Eventually(func(g Gomega) { - nodes, err := terraform.Nodes(KubeConfigFile, false) - - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(len(nodes)).To(Equal(expectedNodeCount), - "Number of nodes should match the spec") - - for _, node := range nodes { - g.Expect(node.Status).Should(Equal("Ready"), - "Nodes should all be in Ready state") - } - }, "420s", "5s").Should(Succeed()) - - fmt.Printf("\nFetching pod status\n") - Eventually(func(g Gomega) { - pods, err := terraform.Pods(KubeConfigFile, false) - - g.Expect(err).NotTo(HaveOccurred()) - - re := regexp.MustCompile("[0-9]+") - for _, pod := range pods { - if strings.Contains(pod.Name, "helm-install") { - g.Expect(pod.Status).Should(Equal("Completed"), pod.Name) - } else { - g.Expect(pod.Status).Should(Equal("Running"), pod.Name) - g.Expect(pod.Restarts).Should(Equal("0"), pod.Name) - numRunning := re.FindAllString(pod.Ready, 2) - g.Expect(numRunning[0]).Should(Equal(numRunning[1]), pod.Name, - "should have all containers running") - } - } - }, "600s", "5s").Should(Succeed()) - }) - - It("Verifies ClusterIP Service", func() { - namespace := "auto-clusterip" - _, err := terraform.DeployWorkload("clusterip.yaml", KubeConfigFile) - Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed") - - defer terraform.RemoveWorkload("clusterip.yaml", KubeConfigFile) - - Eventually(func(g Gomega) { - cmd := "kubectl get pods -n " + namespace + " -o=name -l k8s-app=nginx-app-clusterip " + - "--field-selector=status.phase=Running --kubeconfig=" + KubeConfigFile - res, err := terraform.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-clusterip")) - }, "420s", "5s").Should(Succeed()) - - clusterip, port, _ := terraform.FetchClusterIP(KubeConfigFile, namespace, - "nginx-clusterip-svc") - - cmd := "curl -sL --insecure http://" + clusterip + ":" + port + "/name.html" - nodeExternalIP := terraform.FetchNodeExternalIP(KubeConfigFile) - - for _, ip := range nodeExternalIP { - Eventually(func(g Gomega) { - res, err := terraform.RunCommandOnNode(cmd, ip, AwsUser, AccessKey) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-clusterip")) - }, "420s", "10s").Should(Succeed()) - } - }) - - It("Verifies NodePort Service", func() { - namespace := "auto-nodeport" - _, err := terraform.DeployWorkload("nodeport.yaml", KubeConfigFile) - Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed") - - defer terraform.RemoveWorkload("nodeport.yaml", KubeConfigFile) - - nodeExternalIP := terraform.FetchNodeExternalIP(KubeConfigFile) - cmd := "kubectl get service -n " + namespace + " nginx-nodeport-svc --kubeconfig=" + - KubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" - - nodeport, err := terraform.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) - - for _, ip := range nodeExternalIP { - Eventually(func(g Gomega) { - cmd := "kubectl get pods -n " + namespace + " -o=name -l k8s-app=nginx-app-nodeport " + - "--field-selector=status.phase=Running --kubeconfig=" + KubeConfigFile - res, err := terraform.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-nodeport")) - }, "240s", "5s").Should(Succeed()) - - cmd = "curl -sL --insecure http://" + ip + ":" + nodeport + "/name.html" - Eventually(func(g Gomega) { - res, err := terraform.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-nodeport")) - }, "240s", "5s").Should(Succeed()) - } - }) - - It("Verifies Ingress", func() { - namespace := "auto-ingress" - _, err := terraform.DeployWorkload("ingress.yaml", KubeConfigFile) - Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed") - - defer terraform.RemoveWorkload("ingress.yaml", KubeConfigFile) - - Eventually(func(g Gomega) { - cmd := "kubectl get pods -n " + namespace + " -o=name -l k8s-app=nginx-app-ingress" + - " --field-selector=status.phase=Running --kubeconfig=" + KubeConfigFile - res, err := terraform.RunCommand(cmd) - - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-ingress")) - }, "240s", "5s").Should(Succeed()) - - var ingressIps []string - nodes, err := terraform.WorkerNodes(KubeConfigFile, false) - if err != nil { - fmt.Println("Error retrieving nodes: ", err) - } - - Eventually(func(g Gomega) { - ingressIps, err = terraform.FetchIngressIP(namespace, KubeConfigFile) - - g.Expect(err).NotTo(HaveOccurred(), "Ingress ip is not returned") - g.Expect(len(ingressIps)).To(Equal(len(nodes)), - "Number of ingress IPs should match the number of nodes") - }, "240s", "5s").Should(Succeed()) - - for _, ip := range ingressIps { - cmd := "curl -s --header host:foo1.bar.com" + " http://" + ip + "/name.html" - Eventually(func(g Gomega) { - res, err := terraform.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-ingress")) - }, "240s", "5s").Should(Succeed()) - } - }) - - It("Verifies Daemonset", func() { - _, err := terraform.DeployWorkload("daemonset.yaml", KubeConfigFile) - Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed") - - defer terraform.RemoveWorkload("daemonset.yaml", KubeConfigFile) - - nodes, _ := terraform.WorkerNodes(KubeConfigFile, false) - pods, _ := terraform.Pods(KubeConfigFile, false) - - Eventually(func(g Gomega) { - count := terraform.CountOfStringInSlice("test-daemonset", pods) - g.Expect(count).Should(Equal(len(nodes)), - "Daemonset pod count does not match node count") - }, "420s", "10s").Should(Succeed()) - }) - - It("Verifies dns access", func() { - namespace := "auto-dns" - _, err := terraform.DeployWorkload("dnsutils.yaml", KubeConfigFile) - Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed") - - defer terraform.RemoveWorkload("dnsutils.yaml", KubeConfigFile) - - Eventually(func(g Gomega) { - cmd := "kubectl get pods dnsutils " + "-n " + namespace + " --kubeconfig=" + KubeConfigFile - res, _ := terraform.RunCommand(cmd) - g.Expect(res).Should(ContainSubstring("dnsutils")) - g.Expect(res).Should(ContainSubstring("Running")) - }, "420s", "2s").Should(Succeed()) - - Eventually(func(g Gomega) { - cmd := "kubectl -n " + namespace + " --kubeconfig=" + - KubeConfigFile + " exec -t dnsutils -- nslookup kubernetes.default" - res, _ := terraform.RunCommand(cmd) - g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local")) - }, "420s", "2s").Should(Succeed()) - }) - }) -}) - -var _ = BeforeEach(func() { - if *destroy { - Skip("Cluster is being Deleted") - } -}) - -var _ = AfterEach(func() { - if CurrentSpecReport().Failed() { - fmt.Printf("\nFAILED! %s\n", CurrentSpecReport().FullText()) - } else { - fmt.Printf("\nPASSED! %s\n", CurrentSpecReport().FullText()) - } -}) - -var _ = AfterSuite(func() { - if *destroy { - status, err := BuildCluster(&testing.T{}, *destroy) - Expect(err).NotTo(HaveOccurred()) - Expect(status).To(Equal("cluster destroyed")) - } -}) diff --git a/tests/terraform/manual_upgrade/upgradecluster_test.go b/tests/terraform/manual_upgrade/upgradecluster_test.go deleted file mode 100644 index 3c15819fc7..0000000000 --- a/tests/terraform/manual_upgrade/upgradecluster_test.go +++ /dev/null @@ -1,387 +0,0 @@ -package manual_upgrade - -import ( - "flag" - "fmt" - "regexp" - "strings" - "testing" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - - "github.com/rancher/rke2/tests/terraform" - "github.com/rancher/rke2/tests/terraform/createcluster" -) - -var tfVars = flag.String("tfvars", "/tests/terraform/modules/config/local.tfvars", "custom .tfvars file from base project path") -var destroy = flag.Bool("destroy", false, "a bool") -var upgradeVersion = flag.String("upgradeVersion", "", "Version to upgrade the cluster to") -var channel = flag.String("channel", "", "Channel to use inside the installing command") -var failed bool - -func Test_TFUpgradeClusterValidation(t *testing.T) { - RegisterFailHandler(Fail) - flag.Parse() - - RunSpecs(t, "Upgrade Cluster Test Suite") -} - -var _ = Describe("Upgrade Tests:", func() { - Context("Build Cluster:", func() { - It("Starts up with no issues", func() { - status, err := createcluster.BuildCluster(&testing.T{}, false) - Expect(err).NotTo(HaveOccurred()) - Expect(status).To(Equal("cluster created")) - defer GinkgoRecover() - fmt.Println("Server Node IPS:", createcluster.MasterIPs) - fmt.Println("Agent Node IPS:", createcluster.WorkerIPs) - terraform.PrintFileContents(createcluster.KubeConfigFile) - Expect(createcluster.MasterIPs).ShouldNot(BeEmpty()) - if createcluster.NumWorkers > 0 { - Expect(createcluster.WorkerIPs).ShouldNot(BeEmpty()) - } else { - Expect(createcluster.WorkerIPs).Should(BeEmpty()) - } - Expect(createcluster.KubeConfigFile).ShouldNot(BeEmpty()) - }) - - It("Checks Node and Pod Status", func() { - defer func() { - _, err := terraform.Nodes(createcluster.KubeConfigFile, true) - if err != nil { - fmt.Println("Error retrieving nodes preupgrade: ", err) - } - _, err = terraform.Pods(createcluster.KubeConfigFile, true) - if err != nil { - fmt.Println("Error retrieving pods preupgrade: ", err) - } - }() - - fmt.Printf("\nFetching node status preupgrade\n") - expectedNodeCount := createcluster.NumServers + createcluster.NumWorkers - Eventually(func(g Gomega) { - nodes, err := terraform.Nodes(createcluster.KubeConfigFile, false) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(len(nodes)).To(Equal(expectedNodeCount), "Number of nodes should match the spec") - for _, node := range nodes { - g.Expect(node.Status).Should(Equal("Ready"), "Nodes should all be in Ready state") - } - }, "420s", "5s").Should(Succeed()) - - re := regexp.MustCompile("[0-9]+") - fmt.Printf("\nFetching pod status preupgrade\n") - Eventually(func(g Gomega) { - pods, err := terraform.Pods(createcluster.KubeConfigFile, false) - g.Expect(err).NotTo(HaveOccurred()) - for _, pod := range pods { - if strings.Contains(pod.Name, "helm-install") { - g.Expect(pod.Status).Should(Equal("Completed"), pod.Name) - } else { - g.Expect(pod.Status).Should(Equal("Running"), pod.Name) - g.Expect(pod.Restarts).Should(Equal("0"), pod.Name) - numRunning := re.FindAllString(pod.Ready, 2) - g.Expect(numRunning[0]).Should(Equal(numRunning[1]), pod.Name, "should have all containers running") - } - } - }, "600s", "5s").Should(Succeed()) - }) - }) - - Context("Preupgrade Validations:", func() { - It("Verifies ClusterIP Service Preupgrade", func() { - namespace := "auto-clusterip" - _, err := terraform.DeployWorkload("clusterip.yaml", createcluster.KubeConfigFile) - Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed") - - Eventually(func(g Gomega) { - res, err := terraform.IsAppRunning(namespace, "nginx-app-clusterip", createcluster.KubeConfigFile) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should((ContainSubstring("test-clusterip"))) - }, "420s", "5s").Should(Succeed()) - - clusterip, port, _ := terraform.FetchClusterIP(createcluster.KubeConfigFile, namespace, "nginx-clusterip-svc") - cmd := "curl -sL --insecure http://" + clusterip + ":" + port + "/name.html" - nodeExternalIP := terraform.FetchNodeExternalIP(createcluster.KubeConfigFile) - for _, ip := range nodeExternalIP { - Eventually(func(g Gomega) { - res, err := terraform.RunCommandOnNode(cmd, ip, createcluster.AwsUser, createcluster.AccessKey) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-clusterip")) - }, "420s", "10s").Should(Succeed()) - } - }) - - It("Verifies NodePort Service Preupgrade", func() { - namespace := "auto-nodeport" - _, err := terraform.DeployWorkload("nodeport.yaml", createcluster.KubeConfigFile) - Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed") - - nodeExternalIP := terraform.FetchNodeExternalIP(createcluster.KubeConfigFile) - cmd := "kubectl get service -n " + namespace + " nginx-nodeport-svc --kubeconfig=" + createcluster.KubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" - nodeport, err := terraform.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) - - for _, ip := range nodeExternalIP { - Eventually(func(g Gomega) { - res, err := terraform.IsAppRunning(namespace, "nginx-app-nodeport", createcluster.KubeConfigFile) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-nodeport")) - }, "240s", "5s").Should(Succeed()) - - cmd = "curl -sL --insecure http://" + ip + ":" + nodeport + "/name.html" - Eventually(func(g Gomega) { - res, err := terraform.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-nodeport")) - }, "240s", "5s").Should(Succeed()) - } - }) - - It("Verifies Ingress Preupgrade", func() { - namespace := "auto-ingress" - _, err := terraform.DeployWorkload("ingress.yaml", createcluster.KubeConfigFile) - Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed") - - Eventually(func(g Gomega) { - res, err := terraform.IsAppRunning(namespace, "nginx-app-ingress", createcluster.KubeConfigFile) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-ingress")) - }, "240s", "5s").Should(Succeed()) - - var ingressIps []string - nodes, err := terraform.WorkerNodes(createcluster.KubeConfigFile, false) - if err != nil { - fmt.Println("Error retrieving nodes: ", err) - } - Eventually(func(g Gomega) { - ingressIps, err = terraform.FetchIngressIP(namespace, createcluster.KubeConfigFile) - g.Expect(err).NotTo(HaveOccurred(), "Ingress ip is not returned") - g.Expect(len(ingressIps)).To(Equal(len(nodes)), "Number of ingress IPs should match the number of nodes") - }, "240s", "5s").Should(Succeed()) - - for _, ip := range ingressIps { - cmd := "curl -s --header host:foo1.bar.com" + " http://" + ip + "/name.html" - Eventually(func(g Gomega) { - res, err := terraform.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-ingress")) - }, "240s", "5s").Should(Succeed()) - } - }) - - It("Verifies Daemonset Preupgrade", func() { - _, err := terraform.DeployWorkload("daemonset.yaml", createcluster.KubeConfigFile) - Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed") - - nodes, _ := terraform.WorkerNodes(createcluster.KubeConfigFile, false) - pods, _ := terraform.Pods(createcluster.KubeConfigFile, false) - - Eventually(func(g Gomega) { - count := terraform.CountOfStringInSlice("test-daemonset", pods) - g.Expect(count).Should((Equal(len(nodes))), "Daemonset pod count does not match node count") - }, "420s", "10s").Should(Succeed()) - }) - - It("Verifies DNS Access Preupgrade", func() { - namespace := "auto-dns" - _, err := terraform.DeployWorkload("dnsutils.yaml", createcluster.KubeConfigFile) - Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed") - - Eventually(func(g Gomega) { - cmd := "kubectl get pods dnsutils " + "-n " + namespace + " --kubeconfig=" + createcluster.KubeConfigFile - res, _ := terraform.RunCommand(cmd) - g.Expect(res).Should(ContainSubstring("dnsutils")) - g.Expect(res).Should(ContainSubstring("Running")) - }, "420s", "2s").Should(Succeed()) - - Eventually(func(g Gomega) { - cmd := "kubectl -n " + namespace + " --kubeconfig=" + createcluster.KubeConfigFile + " exec -t dnsutils -- nslookup kubernetes.default" - res, _ := terraform.RunCommand(cmd) - g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local")) - }, "420s", "2s").Should(Succeed()) - }) - }) - - Context("Upgrade nodes via manual:", func() { - It("Upgrade nodes", func() { - var cmdUpgradeVersion string - if *channel != "" { - // If a channel is specified, upgrade using that channel - cmdUpgradeVersion = "sudo curl -sfL https://get.rke2.io | sudo INSTALL_RKE2_VERSION=" + *upgradeVersion + " INSTALL_RKE2_CHANNEL=" + *channel + " sh -" - } else { - // If no channel is specified, upgrade using the version specified - cmdUpgradeVersion = "sudo curl -sfL https://get.rke2.io | sudo INSTALL_RKE2_VERSION=" + *upgradeVersion + " sh -" - } - - versionRegex := regexp.MustCompile("-rc[0-9]+") - k8sVersion := versionRegex.ReplaceAllString(*upgradeVersion, "") - re := regexp.MustCompile("[0-9]+") - - nodes, err := terraform.Nodes(createcluster.KubeConfigFile, false) - if err != nil { - fmt.Println("Error retrieving nodes: ", err) - } - - for _, node := range nodes { - // Every node will be upgraded and restarted - - Eventually(func(g Gomega) { - fmt.Println("\nUpgrading ", node.Roles, node.ExternalIP) - terraform.RunCommandOnNode(cmdUpgradeVersion, node.ExternalIP, createcluster.AwsUser, createcluster.AccessKey) - }, "120s", "2s").Should(Succeed()) - - Eventually(func(g Gomega) { - fmt.Println("Restarting node: ", node.Roles, node.ExternalIP) - terraform.RunCommandOnNode("sudo systemctl restart rke2-*", node.ExternalIP, createcluster.AwsUser, createcluster.AccessKey) - }, "240s", "2s").Should(Succeed()) - } - - //Fetch node status & version PostUpgrade - Eventually(func(g Gomega) { - expectedNodeCount := createcluster.NumServers + createcluster.NumWorkers - nodes, err := terraform.Nodes(createcluster.KubeConfigFile, false) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(len(nodes)).To(Equal(expectedNodeCount), "Number of nodes should match the spec") - for _, node := range nodes { - g.Expect(node.Status).Should(Equal("Ready"), "Nodes should all be in Ready state after upgrading") - g.Expect(node.Version).Should(Equal(k8sVersion), "Nodes should all be upgraded to the specified version") - } - }, "900s", "30s").Should(Succeed()) - - //Fetch pods status PostUpgrade - Eventually(func(g Gomega) { - pods, err := terraform.Pods(createcluster.KubeConfigFile, false) - g.Expect(err).NotTo(HaveOccurred()) - for _, pod := range pods { - if strings.Contains(pod.Name, "helm-install") { - g.Expect(pod.Status).Should(Equal("Completed"), pod.Name) - } else { - g.Expect(pod.Status).Should(Equal("Running"), pod.Name) - numRunning := re.FindAllString(pod.Ready, 2) - g.Expect(numRunning[0]).Should(Equal(numRunning[1]), pod.Name, "should have all containers running") - } - } - }, "600s", "5s").Should(Succeed()) - }) - }) - - Context("Postupgrade Validations:", func() { - It("Verifies ClusterIP Service Postupgrade", func() { - namespace := "auto-clusterip" - defer terraform.RemoveWorkload("clusterip.yaml", createcluster.KubeConfigFile) - - res, err := terraform.IsAppRunning(namespace, "nginx-app-clusterip", createcluster.KubeConfigFile) - Expect(err).NotTo(HaveOccurred()) - Expect(res).Should((ContainSubstring("test-clusterip"))) - - clusterip, port, _ := terraform.FetchClusterIP(createcluster.KubeConfigFile, namespace, "nginx-clusterip-svc") - cmd := "curl -sL --insecure http://" + clusterip + ":" + port + "/name.html" - nodeExternalIP := terraform.FetchNodeExternalIP(createcluster.KubeConfigFile) - for _, ip := range nodeExternalIP { - Eventually(func(g Gomega) { - res, err := terraform.RunCommandOnNode(cmd, ip, createcluster.AwsUser, createcluster.AccessKey) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-clusterip")) - }, "120s", "10s").Should(Succeed()) - } - }) - - It("Verifies NodePort Service Postupgrade", func() { - namespace := "auto-nodeport" - defer terraform.RemoveWorkload("nodeport.yaml", createcluster.KubeConfigFile) - - nodeExternalIP := terraform.FetchNodeExternalIP(createcluster.KubeConfigFile) - cmd := "kubectl get service -n " + namespace + " nginx-nodeport-svc --kubeconfig=" + createcluster.KubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" - nodeport, err := terraform.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) - - for _, ip := range nodeExternalIP { - Eventually(func(g Gomega) { - res, err := terraform.IsAppRunning(namespace, "nginx-app-nodeport", createcluster.KubeConfigFile) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-nodeport")) - }, "120s", "5s").Should(Succeed()) - - cmd = "curl -sL --insecure http://" + ip + ":" + nodeport + "/name.html" - Eventually(func(g Gomega) { - res, err := terraform.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-nodeport")) - }, "120s", "5s").Should(Succeed()) - } - }) - - It("Verifies Ingress Postupgrade", func() { - namespace := "auto-ingress" - defer terraform.RemoveWorkload("ingress.yaml", createcluster.KubeConfigFile) - - res, err := terraform.IsAppRunning(namespace, "nginx-app-ingress", createcluster.KubeConfigFile) - Expect(err).NotTo(HaveOccurred()) - Expect(res).Should(ContainSubstring("test-ingress")) - - var ingressIps []string - nodes, err := terraform.WorkerNodes(createcluster.KubeConfigFile, false) - if err != nil { - fmt.Println("Error retrieving nodes: ", err) - } - Eventually(func(g Gomega) { - ingressIps, err = terraform.FetchIngressIP(namespace, createcluster.KubeConfigFile) - g.Expect(err).NotTo(HaveOccurred(), "Ingress ip is not returned") - g.Expect(len(ingressIps)).To(Equal(len(nodes)), "Number of ingress IPs should match the number of nodes") - }, "120s", "5s").Should(Succeed()) - - for _, ip := range ingressIps { - cmd := "curl -s --header host:foo1.bar.com" + " http://" + ip + "/name.html" - Eventually(func(g Gomega) { - res, err := terraform.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-ingress")) - }, "120s", "5s").Should(Succeed()) - } - }) - - It("Verifies Daemonset Postupgrade", func() { - defer terraform.RemoveWorkload("daemonset.yaml", createcluster.KubeConfigFile) - nodes, _ := terraform.WorkerNodes(createcluster.KubeConfigFile, false) - pods, _ := terraform.Pods(createcluster.KubeConfigFile, false) - count := terraform.CountOfStringInSlice("test-daemonset", pods) - Expect(count).Should((Equal(len(nodes))), "Daemonset pod count does not match node count") - }) - - It("Verifies DNS Access Postupgrade", func() { - namespace := "auto-dns" - _, err := terraform.DeployWorkload("dnsutils.yaml", createcluster.KubeConfigFile) - Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed") - defer terraform.RemoveWorkload("dnsutils.yaml", createcluster.KubeConfigFile) - - Eventually(func(g Gomega) { - cmd := "kubectl get pods dnsutils " + "-n " + namespace + " --kubeconfig=" + createcluster.KubeConfigFile - res, _ := terraform.RunCommand(cmd) - g.Expect(res).Should(ContainSubstring("dnsutils")) - g.Expect(res).Should(ContainSubstring("Running")) - }, "120s", "2s").Should(Succeed()) - - Eventually(func(g Gomega) { - cmd := "kubectl -n " + namespace + " --kubeconfig=" + createcluster.KubeConfigFile + " exec -t dnsutils -- nslookup kubernetes.default" - res, _ := terraform.RunCommand(cmd) - g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local")) - }, "120s", "2s").Should(Succeed()) - }) - }) - -}) - -var _ = BeforeEach(func() { - if *destroy { - Skip("Cluster is being Deleted") - } -}) - -var _ = AfterEach(func() { - if CurrentSpecReport().Failed() { - fmt.Printf("\nFAILED! %s\n", CurrentSpecReport().FullText()) - } else { - fmt.Printf("\nPASSED! %s\n", CurrentSpecReport().FullText()) - } -}) diff --git a/tests/terraform/modules/providers.tf b/tests/terraform/modules/providers.tf deleted file mode 100644 index 0ced9d7521..0000000000 --- a/tests/terraform/modules/providers.tf +++ /dev/null @@ -1,3 +0,0 @@ -provider "aws" { - region = "${var.region}" -} diff --git a/tests/terraform/resource_files/upgrade-plan.yaml b/tests/terraform/resource_files/upgrade-plan.yaml deleted file mode 100644 index 9a574e6f7c..0000000000 --- a/tests/terraform/resource_files/upgrade-plan.yaml +++ /dev/null @@ -1,66 +0,0 @@ -apiVersion: upgrade.cattle.io/v1 -kind: Plan -metadata: - name: rke2-server-cp - namespace: system-upgrade - labels: - rke2-upgrade: server -spec: - concurrency: 1 - version: $UPGRADEVERSION - nodeSelector: - matchExpressions: - - {key: node-role.kubernetes.io/control-plane, operator: In, values: ["true"]} - tolerations: - - operator: Exists - serviceAccountName: system-upgrade - cordon: true - upgrade: - image: rancher/rke2-upgrade ---- -apiVersion: upgrade.cattle.io/v1 -kind: Plan -metadata: - name: rke2-server-etcd - namespace: system-upgrade - labels: - rke2-upgrade: server -spec: - concurrency: 1 - version: $UPGRADEVERSION - nodeSelector: - matchExpressions: - - {key: node-role.kubernetes.io/etcd, operator: In, values: ["true"]} - - {key: node-role.kubernetes.io/control-plane, operator: NotIn, values: ["true"]} - tolerations: - - operator: Exists - serviceAccountName: system-upgrade - prepare: - image: rancher/rke2-upgrade - args: ["prepare", "rke2-server-cp"] - cordon: true - upgrade: - image: rancher/rke2-upgrade ---- -apiVersion: upgrade.cattle.io/v1 -kind: Plan -metadata: - name: rke2-agent - namespace: system-upgrade - labels: - rke2-upgrade: agent -spec: - concurrency: 2 - version: $UPGRADEVERSION - nodeSelector: - matchExpressions: - - {key: node-role.kubernetes.io/etcd, operator: NotIn, values: ["true"]} - - {key: node-role.kubernetes.io/control-plane, operator: NotIn, values: ["true"]} - serviceAccountName: system-upgrade - prepare: - image: rancher/rke2-upgrade - args: ["prepare", "rke2-server-etcd"] - drain: - force: true - upgrade: - image: rancher/rke2-upgrade \ No newline at end of file diff --git a/tests/terraform/scripts/delete_resources.sh b/tests/terraform/scripts/delete_resources.sh deleted file mode 100755 index af24ea329a..0000000000 --- a/tests/terraform/scripts/delete_resources.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/bin/bash - -#Get resource name from tfvarslocal && change name to make more sense in this context -RESOURCE_NAME=$(grep resource_name /dev/null 2>&1 - - -#Get the list of load balancer ARNs -LB_ARN_LIST=$(aws elbv2 describe-load-balancers \ - --query "LoadBalancers[?starts_with(LoadBalancerName, '${NAME_PREFIX}') && Type=='network'].LoadBalancerArn" \ - --output text) - - -#Loop through the load balancer ARNs and delete the load balancers -for LB_ARN in $LB_ARN_LIST; do - echo "Deleting load balancer $LB_ARN" - aws elbv2 delete-load-balancer --load-balancer-arn "$LB_ARN" -done - - -#Get the list of target group ARNs -TG_ARN_LIST=$(aws elbv2 describe-target-groups \ - --query "TargetGroups[?starts_with(TargetGroupName, '${NAME_PREFIX}') && Protocol=='TCP'].TargetGroupArn" \ - --output text) - - -#Loop through the target group ARNs and delete the target groups -for TG_ARN in $TG_ARN_LIST; do - echo "Deleting target group $TG_ARN" - aws elbv2 delete-target-group --target-group-arn "$TG_ARN" -done - - -#Get the ID and recordName with lower case of the hosted zone that contains the Route 53 record sets -NAME_PREFIX_LOWER=$(echo "$NAME_PREFIX" | tr '[:upper:]' '[:lower:]') -R53_ZONE_ID=$(aws route53 list-hosted-zones-by-name --dns-name "${NAME_PREFIX}." \ - --query "HostedZones[0].Id" --output text) -R53_RECORD=$(aws route53 list-resource-record-sets \ - --hosted-zone-id "${R53_ZONE_ID}" \ - --query "ResourceRecordSets[?starts_with(Name, '${NAME_PREFIX_LOWER}.') && Type == 'CNAME'].Name" \ - --output text) - - -#Get ResourceRecord Value -RECORD_VALUE=$(aws route53 list-resource-record-sets \ - --hosted-zone-id "${R53_ZONE_ID}" \ - --query "ResourceRecordSets[?starts_with(Name, '${NAME_PREFIX_LOWER}.') \ - && Type == 'CNAME'].ResourceRecords[0].Value" --output text) - - -#Delete Route53 record -if [[ "$R53_RECORD" == "${NAME_PREFIX_LOWER}."* ]]; then - echo "Deleting Route53 record ${R53_RECORD}" - CHANGE_STATUS=$(aws route53 change-resource-record-sets --hosted-zone-id "${R53_ZONE_ID}" \ - --change-batch '{"Changes": [ - { - "Action": "DELETE", - "ResourceRecordSet": { - "Name": "'"${R53_RECORD}"'", - "Type": "CNAME", - "TTL": 300, - "ResourceRecords": [ - { - "Value": "'"${RECORD_VALUE}"'" - } - ] - } - } - ] - }') - STATUS_ID=$(echo "$CHANGE_STATUS" | jq -r '.ChangeInfo.Id') - #Get status from the change - aws route53 wait resource-record-sets-changed --id "$STATUS_ID" - echo "Successfully deleted Route53 record ${R53_RECORD}: status: ${STATUS_ID}" -else - echo "No Route53 record found" -fi \ No newline at end of file diff --git a/tests/terraform/suc_upgrade/upgradecluster.go b/tests/terraform/suc_upgrade/upgradecluster.go deleted file mode 100644 index fee95468b2..0000000000 --- a/tests/terraform/suc_upgrade/upgradecluster.go +++ /dev/null @@ -1,28 +0,0 @@ -package suc_upgrade - -import ( - "fmt" - "os" - "regexp" - "strings" - - "github.com/rancher/rke2/tests/terraform" -) - -func upgradeCluster(version string, kubeconfig string) error { - if strings.TrimSpace(version) == "" { - return fmt.Errorf("please provide a non-empty rke2 version to upgrade to") - } - regex := regexp.MustCompile(`\+`) - sucVersion := regex.ReplaceAllString(version, "-") - originalFilePath := terraform.Basepath() + "/tests/terraform/resource_files" + "/upgrade-plan.yaml" - newFilePath := terraform.Basepath() + "/tests/terraform/resource_files" + "/plan.yaml" - content, err := os.ReadFile(originalFilePath) - if err != nil { - return err - } - newContent := strings.ReplaceAll(string(content), "$UPGRADEVERSION", sucVersion) - os.WriteFile(newFilePath, []byte(newContent), 0777) - _, err = terraform.DeployWorkload("plan.yaml", kubeconfig) - return err -} diff --git a/tests/terraform/suc_upgrade/upgradecluster_test.go b/tests/terraform/suc_upgrade/upgradecluster_test.go deleted file mode 100644 index 80dd8da3cc..0000000000 --- a/tests/terraform/suc_upgrade/upgradecluster_test.go +++ /dev/null @@ -1,453 +0,0 @@ -package suc_upgrade - -import ( - "flag" - "fmt" - "regexp" - "strings" - "testing" - - "github.com/rancher/rke2/tests/terraform" - "github.com/rancher/rke2/tests/terraform/createcluster" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var ( - destroy = flag.Bool("destroy", false, "a bool") - upgradeVersion = flag.String("upgradeVersion", "", "Version to upgrade the cluster to") -) - -func Test_TFUpgradeClusterValidation(t *testing.T) { - RegisterFailHandler(Fail) - flag.Parse() - - RunSpecs(t, "Upgrade Cluster Test Suite") -} - -var _ = Describe("Upgrade Tests:", func() { - Context("Build Cluster:", func() { - It("Starts up with no issues", func() { - status, err := createcluster.BuildCluster(&testing.T{}, false) - Expect(err).NotTo(HaveOccurred()) - Expect(status).To(Equal("cluster created")) - - defer GinkgoRecover() - - fmt.Println("Server Node IPS:", createcluster.MasterIPs) - fmt.Println("Agent Node IPS:", createcluster.WorkerIPs) - terraform.PrintFileContents(createcluster.KubeConfigFile) - - Expect(createcluster.KubeConfigFile).ShouldNot(BeEmpty()) - Expect(createcluster.MasterIPs).ShouldNot(BeEmpty()) - - if createcluster.NumWorkers > 0 { - Expect(createcluster.WorkerIPs).ShouldNot(BeEmpty()) - } else { - Expect(createcluster.WorkerIPs).Should(BeEmpty()) - } - Expect(createcluster.KubeConfigFile).ShouldNot(BeEmpty()) - }) - - It("Checks Node and Pod Status", func() { - defer func() { - fmt.Printf("\nFetching node status preupgrade\n") - - _, err := terraform.Nodes(createcluster.KubeConfigFile, true) - if err != nil { - fmt.Println("Error retrieving nodes preupgrade: ", err) - } - _, err = terraform.Pods(createcluster.KubeConfigFile, true) - if err != nil { - fmt.Println("Error retrieving pods preupgrade: ", err) - } - }() - - expectedNodeCount := createcluster.NumServers + createcluster.NumWorkers - Eventually(func(g Gomega) { - nodes, err := terraform.Nodes(createcluster.KubeConfigFile, false) - - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(len(nodes)).To(Equal(expectedNodeCount), - "Number of nodes should match the spec") - - for _, node := range nodes { - g.Expect(node.Status).Should(Equal("Ready"), - "Nodes should all be in Ready state", node.Name) - } - }, "420s", "5s").Should(Succeed()) - - fmt.Printf("\nFetching pod status preupgrade\n") - Eventually(func(g Gomega) { - pods, err := terraform.Pods(createcluster.KubeConfigFile, false) - g.Expect(err).NotTo(HaveOccurred()) - - re := regexp.MustCompile("[0-9]+") - for _, pod := range pods { - if strings.Contains(pod.Name, "helm-install") { - g.Expect(pod.Status).Should(Equal("Completed"), pod.Name) - } else { - g.Expect(pod.Status).Should(Equal("Running"), pod.Name) - g.Expect(pod.Restarts).Should(Equal("0"), pod.Name) - numRunning := re.FindAllString(pod.Ready, 2) - g.Expect(numRunning[0]).Should(Equal(numRunning[1]), pod.Name, - "should have all containers running") - } - } - }, "600s", "5s").Should(Succeed()) - }) - }) - - Context("Preupgrade Validations:", func() { - It("Verifies ClusterIP Service Preupgrade", func() { - namespace := "auto-clusterip" - _, err := terraform.DeployWorkload("clusterip.yaml", createcluster.KubeConfigFile) - - Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed") - - Eventually(func(g Gomega) { - cmd := "kubectl get pods -n " + namespace + " -o=name " + - "-l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running " + - "--kubeconfig=" + createcluster.KubeConfigFile - res, err := terraform.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-clusterip")) - }, "240s", "5s").Should(Succeed()) - - clusterip, port, _ := terraform.FetchClusterIP(createcluster.KubeConfigFile, - namespace, "nginx-clusterip-svc") - - cmd := "curl -sL --insecure http://" + clusterip + ":" + port + "/name.html" - nodeExternalIP := terraform.FetchNodeExternalIP(createcluster.KubeConfigFile) - - for _, ip := range nodeExternalIP { - Eventually(func(g Gomega) { - res, err := terraform.RunCommandOnNode(cmd, ip, createcluster.AwsUser, createcluster.AccessKey) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-clusterip")) - }, "240s", "10s").Should(Succeed()) - } - }) - - It("Verifies NodePort Service Preupgrade", func() { - namespace := "auto-nodeport" - _, err := terraform.DeployWorkload("nodeport.yaml", createcluster.KubeConfigFile) - - Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed") - - nodeExternalIP := terraform.FetchNodeExternalIP(createcluster.KubeConfigFile) - cmd := "kubectl get service -n " + namespace + " nginx-nodeport-svc --kubeconfig=" + - createcluster.KubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" - - nodeport, err := terraform.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) - - for _, ip := range nodeExternalIP { - Eventually(func(g Gomega) { - cmd := "kubectl get pods -n " + namespace + " -o=name -l k8s-app=nginx-app-nodeport " + - "--field-selector=status.phase=Running --kubeconfig=" + createcluster.KubeConfigFile - res, err := terraform.RunCommand(cmd) - - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-nodeport")) - }, "240s", "5s").Should(Succeed()) - - cmd = "curl -sL --insecure http://" + ip + ":" + nodeport + "/name.html" - Eventually(func(g Gomega) { - res, err := terraform.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-nodeport")) - }, "240s", "5s").Should(Succeed()) - } - }) - - It("Verifies Ingress Preupgrade", func() { - namespace := "auto-ingress" - _, err := terraform.DeployWorkload("ingress.yaml", createcluster.KubeConfigFile) - - Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed") - - Eventually(func(g Gomega) { - cmd := "kubectl get pods -n " + namespace + " -o=name -l k8s-app=nginx-app-ingress " + - "--field-selector=status.phase=Running --kubeconfig=" + createcluster.KubeConfigFile - res, err := terraform.RunCommand(cmd) - - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-ingress")) - }, "240s", "5s").Should(Succeed()) - - var ingressIps []string - nodes, err := terraform.WorkerNodes(createcluster.KubeConfigFile, false) - if err != nil { - fmt.Println("Error retrieving nodes: ", err) - } - - Eventually(func(g Gomega) { - ingressIps, err = terraform.FetchIngressIP(namespace, createcluster.KubeConfigFile) - - g.Expect(err).NotTo(HaveOccurred(), "Ingress ip is not returned") - g.Expect(len(ingressIps)).To(Equal(len(nodes)), - "Number of ingress IPs should match the number of nodes") - }, "240s", "5s").Should(Succeed()) - - for _, ip := range ingressIps { - cmd := "curl -s --header host:foo1.bar.com" + " http://" + ip + "/name.html" - Eventually(func(g Gomega) { - res, err := terraform.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-ingress")) - }, "240s", "5s").Should(Succeed()) - } - }) - - It("Verifies Daemonset Preupgrade", func() { - _, err := terraform.DeployWorkload("daemonset.yaml", createcluster.KubeConfigFile) - - Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed") - - nodes, _ := terraform.WorkerNodes(createcluster.KubeConfigFile, false) - pods, _ := terraform.Pods(createcluster.KubeConfigFile, false) - Eventually(func(g Gomega) { - count := terraform.CountOfStringInSlice("test-daemonset", pods) - g.Expect(count).Should(Equal(len(nodes)), - "Daemonset pod count does not match node count") - }, "240s", "10s").Should(Succeed()) - }) - - It("Verifies DNS Access Preupgrade", func() { - namespace := "auto-dns" - _, err := terraform.DeployWorkload("dnsutils.yaml", createcluster.KubeConfigFile) - - Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed") - - Eventually(func(g Gomega) { - cmd := "kubectl get pods dnsutils " + "-n " + namespace + - " --kubeconfig=" + createcluster.KubeConfigFile - res, _ := terraform.RunCommand(cmd) - - g.Expect(res).Should(ContainSubstring("dnsutils")) - g.Expect(res).Should(ContainSubstring("Running")) - }, "240s", "2s").Should(Succeed()) - - Eventually(func(g Gomega) { - cmd := "kubectl -n " + namespace + " --kubeconfig=" + - createcluster.KubeConfigFile + " exec -t dnsutils -- nslookup kubernetes.default" - res, _ := terraform.RunCommand(cmd) - - g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local")) - }, "240s", "2s").Should(Succeed()) - }) - }) - - Context("Upgrade via SUC:", func() { - It("Verifies Upgrade", func() { - namespace := "system-upgrade" - _, err := terraform.DeployWorkload("suc.yaml", createcluster.KubeConfigFile) - Expect(err).NotTo(HaveOccurred(), - "system-upgrade-controller manifest did not deploy successfully") - - Eventually(func(g Gomega) { - cmd := "kubectl get pods " + "-n " + namespace + " --kubeconfig=" + createcluster.KubeConfigFile - res, _ := terraform.RunCommand(cmd) - - g.Expect(res).Should(ContainSubstring("system-upgrade-controller")) - g.Expect(res).Should(ContainSubstring("Running")) - }, "120s", "2s").Should(Succeed()) - - err = upgradeCluster(*upgradeVersion, createcluster.KubeConfigFile) - Expect(err).NotTo(HaveOccurred(), "failed to upgrade cluster.") - - defer func() { - _, err := terraform.Nodes(createcluster.KubeConfigFile, true) - if err != nil { - fmt.Println("Error retrieving nodes postupgrade: ", err) - } - _, err = terraform.Pods(createcluster.KubeConfigFile, true) - if err != nil { - fmt.Println("Error retrieving pods postupgrade: ", err) - } - }() - - versionRegex := regexp.MustCompile("-rc[0-9]+") - k8sVersion := versionRegex.ReplaceAllString(*upgradeVersion, "") - - fmt.Printf("\nFetching node status postupgrade\n") - - expectedNodeCount := createcluster.NumServers + createcluster.NumWorkers - Eventually(func(g Gomega) { - nodes, err := terraform.Nodes(createcluster.KubeConfigFile, false) - - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(len(nodes)).To(Equal(expectedNodeCount), - "Number of nodes should match the spec") - - for _, node := range nodes { - g.Expect(node.Status).Should(Equal("Ready"), - "Nodes should all be in Ready state after upgrading", node.Name) - g.Expect(node.Version).Should(Equal(k8sVersion), - "Nodes should all be upgraded to the specified version", node.Name) - } - }, "900s", "30s").Should(Succeed()) - - fmt.Printf("\n Fetching pod status postupgrade\n") - Eventually(func(g Gomega) { - pods, err := terraform.Pods(createcluster.KubeConfigFile, false) - g.Expect(err).NotTo(HaveOccurred()) - for _, pod := range pods { - if strings.Contains(pod.Name, "helm-install") { - g.Expect(pod.Status).Should(Equal("Completed"), pod.Name) - } else { - g.Expect(pod.Status).Should(Equal("Running"), pod.Name) - numRunning := versionRegex.FindAllString(pod.Ready, 2) - g.Expect(numRunning[0]).Should(Equal(numRunning[1]), pod.Name, "should have all containers running") - } - } - }, "600s", "5s").Should(Succeed()) - }) - }) - - Context("Postupgrade Validations:", func() { - It("Verifies ClusterIP Service Postupgrade", func() { - namespace := "auto-clusterip" - defer terraform.RemoveWorkload("clusterip.yaml", createcluster.KubeConfigFile) - - cmd := "kubectl get pods -n " + namespace + " -o=name -l " + - "k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + - createcluster.KubeConfigFile - res, err := terraform.RunCommand(cmd) - - Expect(err).NotTo(HaveOccurred()) - Expect(res).Should(ContainSubstring("test-clusterip")) - - clusterip, port, _ := terraform.FetchClusterIP(createcluster.KubeConfigFile, - namespace, "nginx-clusterip-svc") - cmd = "curl -sL --insecure http://" + clusterip + ":" + port + "/name.html" - nodeExternalIP := terraform.FetchNodeExternalIP(createcluster.KubeConfigFile) - - for _, ip := range nodeExternalIP { - Eventually(func(g Gomega) { - res, err := terraform.RunCommandOnNode(cmd, ip, createcluster.AwsUser, createcluster.AccessKey) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-clusterip")) - }, "120s", "10s").Should(Succeed()) - } - }) - - It("Verifies NodePort Service Postupgrade", func() { - namespace := "auto-nodeport" - defer terraform.RemoveWorkload("nodeport.yaml", createcluster.KubeConfigFile) - - nodeExternalIP := terraform.FetchNodeExternalIP(createcluster.KubeConfigFile) - cmd := "kubectl get service -n " + namespace + " nginx-nodeport-svc --kubeconfig=" + - createcluster.KubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" - nodeport, err := terraform.RunCommand(cmd) - - Expect(err).NotTo(HaveOccurred()) - - for _, ip := range nodeExternalIP { - Eventually(func(g Gomega) { - cmd := "kubectl get pods -n " + namespace + " -o=name -l " + - "k8s-app=nginx-app-nodeport --field-selector=status.phase=Running --kubeconfig=" + - createcluster.KubeConfigFile - res, err := terraform.RunCommand(cmd) - - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-nodeport")) - }, "120s", "5s").Should(Succeed()) - - cmd = "curl -sL --insecure http://" + ip + ":" + nodeport + "/name.html" - Eventually(func(g Gomega) { - res, err := terraform.RunCommand(cmd) - - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-nodeport")) - }, "120s", "5s").Should(Succeed()) - } - }) - - It("Verifies Ingress Postupgrade", func() { - namespace := "auto-ingress" - - defer terraform.RemoveWorkload("ingress.yaml", createcluster.KubeConfigFile) - - cmd := "kubectl get pods -n " + namespace + " -o=name -l k8s-app=nginx-app-ingress " + - "--field-selector=status.phase=Running --kubeconfig=" + createcluster.KubeConfigFile - res, err := terraform.RunCommand(cmd) - - Expect(err).NotTo(HaveOccurred()) - Expect(res).Should(ContainSubstring("test-ingress")) - - var ingressIps []string - nodes, err := terraform.WorkerNodes(createcluster.KubeConfigFile, false) - if err != nil { - fmt.Println("Error retrieving nodes: ", err) - } - - Eventually(func(g Gomega) { - ingressIps, err = terraform.FetchIngressIP(namespace, createcluster.KubeConfigFile) - g.Expect(err).NotTo(HaveOccurred(), "Ingress ip is not returned") - g.Expect(len(ingressIps)).To(Equal(len(nodes)), - "Number of ingress IPs should match the number of nodes") - }, "120s", "5s").Should(Succeed()) - - for _, ip := range ingressIps { - cmd := "curl -s --header host:foo1.bar.com" + " http://" + ip + "/name.html" - Eventually(func(g Gomega) { - res, err := terraform.RunCommand(cmd) - - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-ingress")) - }, "120s", "5s").Should(Succeed()) - } - }) - - It("Verifies Daemonset Postupgrade", func() { - nodes, _ := terraform.WorkerNodes(createcluster.KubeConfigFile, false) - pods, _ := terraform.Pods(createcluster.KubeConfigFile, false) - count := terraform.CountOfStringInSlice("test-daemonset", pods) - - defer terraform.RemoveWorkload("daemonset.yaml", createcluster.KubeConfigFile) - Expect(count).Should((Equal(len(nodes))), - "Daemonset pod count does not match node count") - }) - - It("Verifies DNS Access Postupgrade", func() { - namespace := "auto-dns" - _, err := terraform.DeployWorkload("dnsutils.yaml", createcluster.KubeConfigFile) - - Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed") - - defer terraform.RemoveWorkload("dnsutils.yaml", createcluster.KubeConfigFile) - Eventually(func(g Gomega) { - cmd := "kubectl get pods dnsutils " + "-n " + namespace + - " --kubeconfig=" + createcluster.KubeConfigFile - res, _ := terraform.RunCommand(cmd) - - g.Expect(res).Should(ContainSubstring("dnsutils")) - g.Expect(res).Should(ContainSubstring("Running")) - }, "120s", "2s").Should(Succeed()) - - Eventually(func(g Gomega) { - cmd := "kubectl -n " + namespace + " --kubeconfig=" + createcluster.KubeConfigFile + " exec -t dnsutils -- nslookup kubernetes.default" - res, _ := terraform.RunCommand(cmd) - - g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local")) - }, "120s", "2s").Should(Succeed()) - }) - }) -}) - -var _ = BeforeEach(func() { - if *destroy { - Skip("Cluster is being Deleted") - } -}) - -var _ = AfterEach(func() { - if CurrentSpecReport().Failed() { - fmt.Printf("\nFAILED! %s\n", CurrentSpecReport().FullText()) - } else { - fmt.Printf("\nPASSED! %s\n", CurrentSpecReport().FullText()) - } -}) diff --git a/tests/terraform/testutils.go b/tests/terraform/testutils.go deleted file mode 100644 index f4aa5c514d..0000000000 --- a/tests/terraform/testutils.go +++ /dev/null @@ -1,292 +0,0 @@ -package terraform - -import ( - "bytes" - "fmt" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" - "time" - - "golang.org/x/crypto/ssh" -) - -type KubectlCommand string - -var config *ssh.ClientConfig - -type Node struct { - Name string - Status string - Roles string - Version string - InternalIP string - ExternalIP string -} - -type Pod struct { - NameSpace string - Name string - Ready string - Status string - Restarts string - NodeIP string - Node string -} - -func publicKey(path string) (ssh.AuthMethod, error) { - key, err := os.ReadFile(path) - if err != nil { - return nil, err - } - signer, err := ssh.ParsePrivateKey(key) - if err != nil { - return nil, err - } - return ssh.PublicKeys(signer), nil -} - -func configureSSH(host string, sshUser string, sshKey string) (*ssh.Client, error) { - authMethod, err := publicKey(sshKey) - if err != nil { - return nil, err - } - config = &ssh.ClientConfig{ - User: sshUser, - Auth: []ssh.AuthMethod{ - authMethod, - }, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - } - conn, err := ssh.Dial("tcp", host, config) - if err != nil { - return nil, err - } - return conn, nil -} - -func runsshCommand(cmd string, conn *ssh.Client) (string, error) { - session, err := conn.NewSession() - if err != nil { - return "", err - } - defer session.Close() - var stdoutBuf bytes.Buffer - var stderrBuf bytes.Buffer - session.Stdout = &stdoutBuf - session.Stderr = &stderrBuf - if err := session.Run(cmd); err != nil { - return "", err - } - return fmt.Sprintf("%s", stdoutBuf.String()), err -} - -func parseNodes(kubeConfig string, print bool, cmd string) ([]Node, error) { - nodes := make([]Node, 0, 10) - res, err := RunCommand(cmd) - if err != nil { - return nil, err - } - rawNodes := strings.TrimSpace(res) - split := strings.Split(rawNodes, "\n") - for _, rec := range split { - if strings.TrimSpace(rec) != "" { - fields := strings.Fields(rec) - n := Node{ - Name: fields[0], - Status: fields[1], - Roles: fields[2], - Version: fields[4], - InternalIP: fields[5], - ExternalIP: fields[6], - } - nodes = append(nodes, n) - } - } - if print { - fmt.Println(rawNodes) - } - - return nodes, nil -} - -func parsePods(kubeconfig string, print bool, cmd string) ([]Pod, error) { - pods := make([]Pod, 0, 10) - res, _ := RunCommand(cmd) - rawPods := strings.TrimSpace(res) - - split := strings.Split(rawPods, "\n") - for _, rec := range split { - fields := strings.Fields(string(rec)) - p := Pod{ - NameSpace: fields[0], - Name: fields[1], - Ready: fields[2], - Status: fields[3], - Restarts: fields[4], - NodeIP: fields[6], - Node: fields[7], - } - pods = append(pods, p) - } - if print { - fmt.Println(rawPods) - } - - return pods, nil -} - -func Basepath() string { - _, b, _, _ := runtime.Caller(0) - return filepath.Join(filepath.Dir(b), "../..") -} - -func PrintFileContents(f ...string) error { - for _, file := range f { - content, err := os.ReadFile(file) - if err != nil { - return err - } - fmt.Println(string(content) + "\n") - } - - return nil -} - -// RunCommandOnNode executes a command from within the given node -func RunCommandOnNode(cmd string, ServerIP string, sshUser string, sshKey string) (string, error) { - Server := ServerIP + ":22" - conn, err := configureSSH(Server, sshUser, sshKey) - if err != nil { - return "", err - } - res, err := runsshCommand(cmd, conn) - res = strings.TrimSpace(res) - - return res, err -} - -// RunCommand executes a command on the host -func RunCommand(cmd string) (string, error) { - c := exec.Command("bash", "-c", cmd) - out, err := c.CombinedOutput() - - return string(out), err -} - -// CountOfStringInSlice Used to count the pods using prefix passed in the list of pods -func CountOfStringInSlice(str string, pods []Pod) int { - var count int - for _, p := range pods { - if strings.Contains(p.Name, str) { - count++ - } - } - - return count -} - -func DeployWorkload(workload, kubeconfig string) (string, error) { - resourceDir := Basepath() + "/tests/terraform/resource_files" - files, err := os.ReadDir(resourceDir) - if err != nil { - return "", fmt.Errorf("%s : Unable to read resource manifest file for %s", err, workload) - } - for _, f := range files { - filename := filepath.Join(resourceDir, f.Name()) - if strings.TrimSpace(f.Name()) == workload { - cmd := "kubectl apply -f " + filename + " --kubeconfig=" + kubeconfig - return RunCommand(cmd) - } - } - - return "", nil -} - -func RemoveWorkload(workload, kubeconfig string) (string, error) { - resourceDir := Basepath() + "/tests/terraform/resource_files" - files, err := os.ReadDir(resourceDir) - if err != nil { - return "", fmt.Errorf("%s : Unable to read resource manifest file for %s", err, workload) - } - for _, f := range files { - filename := filepath.Join(resourceDir, f.Name()) - if strings.TrimSpace(f.Name()) == workload { - cmd := "kubectl delete -f " + filename + " --kubeconfig=" + kubeconfig - return RunCommand(cmd) - } - } - - return "", nil -} - -func FetchClusterIP(kubeconfig string, namespace string, servicename string) (string, string, error) { - ipCmd := "kubectl get svc " + servicename + " -n " + namespace + - " -o jsonpath='{.spec.clusterIP}' --kubeconfig=" + kubeconfig - ip, err := RunCommand(ipCmd) - if err != nil { - return "", "", err - } - portCmd := "kubectl get svc " + servicename + " -n " + namespace + - " -o jsonpath='{.spec.ports[0].port}' --kubeconfig=" + kubeconfig - port, err := RunCommand(portCmd) - if err != nil { - return "", "", err - } - - return ip, port, err -} - -func FetchNodeExternalIP(kubeconfig string) []string { - cmd := "kubectl get node --output=jsonpath='{range .items[*]} " + - "{ .status.addresses[?(@.type==\"ExternalIP\")].address}' --kubeconfig=" + kubeconfig - time.Sleep(10 * time.Second) - res, _ := RunCommand(cmd) - nodeExternalIP := strings.Trim(res, " ") - nodeExternalIPs := strings.Split(nodeExternalIP, " ") - - return nodeExternalIPs -} - -func FetchIngressIP(namespace string, kubeconfig string) ([]string, error) { - cmd := "kubectl get ingress -n " + namespace + - " -o jsonpath='{.items[0].status.loadBalancer.ingress[*].ip}' --kubeconfig=" + kubeconfig - res, err := RunCommand(cmd) - if err != nil { - return nil, err - } - ingressIP := strings.Trim(res, " ") - if ingressIP != "" { - ingressIPs := strings.Split(ingressIP, " ") - return ingressIPs, nil - } - - return nil, nil -} - -func Nodes(kubeConfig string, print bool) ([]Node, error) { - cmd := "kubectl get nodes --no-headers -o wide --kubeconfig=" + kubeConfig - return parseNodes(kubeConfig, print, cmd) -} - -func WorkerNodes(kubeConfig string, print bool) ([]Node, error) { - cmd := "kubectl get node -o jsonpath='{range .items[*]}{@.metadata.name} " + - "{@.status.conditions[-1].type} {@.status.nodeInfo.kubeletVersion} " + - "{@.status.addresses[?(@.type==\"InternalIP\")].address} " + - "{@.status.addresses[?(@.type==\"ExternalIP\")].address} {@.spec.taints[*].effect}{\"\\n\"}{end}' " + - "--kubeconfig=" + kubeConfig + " | grep -v NoSchedule | grep -v NoExecute" - - return parseNodes(kubeConfig, print, cmd) -} - -func Pods(kubeconfig string, print bool) ([]Pod, error) { - cmd := "kubectl get pods -o wide --no-headers -A --kubeconfig=" + kubeconfig - return parsePods(kubeconfig, print, cmd) -} - -func IsAppRunning(namespace, appName string, kubeconfig string) (string, error) { - cmd := "kubectl get pods -n" + namespace + " -o=name -l k8s-app=" + appName + " --field-selector=status.phase=Running --kubeconfig=" + kubeconfig - return RunCommand(cmd) -} diff --git a/tests/terraform/upgradecluster/upgradecluster_test.go b/tests/terraform/upgradecluster/upgradecluster_test.go deleted file mode 100644 index ec282def38..0000000000 --- a/tests/terraform/upgradecluster/upgradecluster_test.go +++ /dev/null @@ -1,453 +0,0 @@ -package upgradecluster - -import ( - "flag" - "fmt" - "regexp" - "strings" - "testing" - - "github.com/rancher/rke2/tests/terraform" - "github.com/rancher/rke2/tests/terraform/createcluster" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var ( - destroy = flag.Bool("destroy", false, "a bool") - upgradeVersion = flag.String("upgradeVersion", "", "Version to upgrade the cluster to") -) - -func Test_TFUpgradeClusterValidation(t *testing.T) { - RegisterFailHandler(Fail) - flag.Parse() - - RunSpecs(t, "Upgrade Cluster Test Suite") -} - -var _ = Describe("Upgrade Tests:", func() { - Context("Build Cluster:", func() { - It("Starts up with no issues", func() { - status, err := createcluster.BuildCluster(&testing.T{}, false) - Expect(err).NotTo(HaveOccurred()) - Expect(status).To(Equal("cluster created")) - - defer GinkgoRecover() - - fmt.Println("Server Node IPS:", createcluster.MasterIPs) - fmt.Println("Agent Node IPS:", createcluster.WorkerIPs) - terraform.PrintFileContents(createcluster.KubeConfigFile) - - Expect(createcluster.KubeConfigFile).ShouldNot(BeEmpty()) - Expect(createcluster.MasterIPs).ShouldNot(BeEmpty()) - - if createcluster.NumWorkers > 0 { - Expect(createcluster.WorkerIPs).ShouldNot(BeEmpty()) - } else { - Expect(createcluster.WorkerIPs).Should(BeEmpty()) - } - Expect(createcluster.KubeConfigFile).ShouldNot(BeEmpty()) - }) - - It("Checks Node and Pod Status", func() { - defer func() { - fmt.Printf("\nFetching node status preupgrade\n") - - _, err := terraform.Nodes(createcluster.KubeConfigFile, true) - if err != nil { - fmt.Println("Error retrieving nodes preupgrade: ", err) - } - _, err = terraform.Pods(createcluster.KubeConfigFile, true) - if err != nil { - fmt.Println("Error retrieving pods preupgrade: ", err) - } - }() - - expectedNodeCount := createcluster.NumServers + createcluster.NumWorkers - Eventually(func(g Gomega) { - nodes, err := terraform.Nodes(createcluster.KubeConfigFile, false) - - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(len(nodes)).To(Equal(expectedNodeCount), - "Number of nodes should match the spec") - - for _, node := range nodes { - g.Expect(node.Status).Should(Equal("Ready"), - "Nodes should all be in Ready state", node.Name) - } - }, "420s", "5s").Should(Succeed()) - - fmt.Printf("\nFetching pod status preupgrade\n") - Eventually(func(g Gomega) { - pods, err := terraform.Pods(createcluster.KubeConfigFile, false) - g.Expect(err).NotTo(HaveOccurred()) - - re := regexp.MustCompile("[0-9]+") - for _, pod := range pods { - if strings.Contains(pod.Name, "helm-install") { - g.Expect(pod.Status).Should(Equal("Completed"), pod.Name) - } else { - g.Expect(pod.Status).Should(Equal("Running"), pod.Name) - g.Expect(pod.Restarts).Should(Equal("0"), pod.Name) - numRunning := re.FindAllString(pod.Ready, 2) - g.Expect(numRunning[0]).Should(Equal(numRunning[1]), pod.Name, - "should have all containers running") - } - } - }, "600s", "5s").Should(Succeed()) - }) - }) - - Context("Preupgrade Validations:", func() { - It("Verifies ClusterIP Service Preupgrade", func() { - namespace := "auto-clusterip" - _, err := terraform.DeployWorkload("clusterip.yaml", createcluster.KubeConfigFile) - - Expect(err).NotTo(HaveOccurred(), "Cluster IP manifest not deployed") - - Eventually(func(g Gomega) { - cmd := "kubectl get pods -n " + namespace + " -o=name " + - "-l k8s-app=nginx-app-clusterip --field-selector=status.phase=Running " + - "--kubeconfig=" + createcluster.KubeConfigFile - res, err := terraform.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-clusterip")) - }, "240s", "5s").Should(Succeed()) - - clusterip, port, _ := terraform.FetchClusterIP(createcluster.KubeConfigFile, - namespace, "nginx-clusterip-svc") - - cmd := "curl -sL --insecure http://" + clusterip + ":" + port + "/name.html" - nodeExternalIP := terraform.FetchNodeExternalIP(createcluster.KubeConfigFile) - - for _, ip := range nodeExternalIP { - Eventually(func(g Gomega) { - res, err := terraform.RunCommandOnNode(cmd, ip, createcluster.AwsUser, createcluster.AccessKey) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-clusterip")) - }, "240s", "10s").Should(Succeed()) - } - }) - - It("Verifies NodePort Service Preupgrade", func() { - namespace := "auto-nodeport" - _, err := terraform.DeployWorkload("nodeport.yaml", createcluster.KubeConfigFile) - - Expect(err).NotTo(HaveOccurred(), "NodePort manifest not deployed") - - nodeExternalIP := terraform.FetchNodeExternalIP(createcluster.KubeConfigFile) - cmd := "kubectl get service -n " + namespace + " nginx-nodeport-svc --kubeconfig=" + - createcluster.KubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" - - nodeport, err := terraform.RunCommand(cmd) - Expect(err).NotTo(HaveOccurred()) - - for _, ip := range nodeExternalIP { - Eventually(func(g Gomega) { - cmd := "kubectl get pods -n " + namespace + " -o=name -l k8s-app=nginx-app-nodeport " + - "--field-selector=status.phase=Running --kubeconfig=" + createcluster.KubeConfigFile - res, err := terraform.RunCommand(cmd) - - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-nodeport")) - }, "240s", "5s").Should(Succeed()) - - cmd = "curl -sL --insecure http://" + ip + ":" + nodeport + "/name.html" - Eventually(func(g Gomega) { - res, err := terraform.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-nodeport")) - }, "240s", "5s").Should(Succeed()) - } - }) - - It("Verifies Ingress Preupgrade", func() { - namespace := "auto-ingress" - _, err := terraform.DeployWorkload("ingress.yaml", createcluster.KubeConfigFile) - - Expect(err).NotTo(HaveOccurred(), "Ingress manifest not deployed") - - Eventually(func(g Gomega) { - cmd := "kubectl get pods -n " + namespace + " -o=name -l k8s-app=nginx-app-ingress " + - "--field-selector=status.phase=Running --kubeconfig=" + createcluster.KubeConfigFile - res, err := terraform.RunCommand(cmd) - - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-ingress")) - }, "240s", "5s").Should(Succeed()) - - var ingressIps []string - nodes, err := terraform.WorkerNodes(createcluster.KubeConfigFile, false) - if err != nil { - fmt.Println("Error retrieving nodes: ", err) - } - - Eventually(func(g Gomega) { - ingressIps, err = terraform.FetchIngressIP(namespace, createcluster.KubeConfigFile) - - g.Expect(err).NotTo(HaveOccurred(), "Ingress ip is not returned") - g.Expect(len(ingressIps)).To(Equal(len(nodes)), - "Number of ingress IPs should match the number of nodes") - }, "240s", "5s").Should(Succeed()) - - for _, ip := range ingressIps { - cmd := "curl -s --header host:foo1.bar.com" + " http://" + ip + "/name.html" - Eventually(func(g Gomega) { - res, err := terraform.RunCommand(cmd) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-ingress")) - }, "240s", "5s").Should(Succeed()) - } - }) - - It("Verifies Daemonset Preupgrade", func() { - _, err := terraform.DeployWorkload("daemonset.yaml", createcluster.KubeConfigFile) - - Expect(err).NotTo(HaveOccurred(), "Daemonset manifest not deployed") - - nodes, _ := terraform.WorkerNodes(createcluster.KubeConfigFile, false) - pods, _ := terraform.Pods(createcluster.KubeConfigFile, false) - Eventually(func(g Gomega) { - count := terraform.CountOfStringInSlice("test-daemonset", pods) - g.Expect(count).Should(Equal(len(nodes)), - "Daemonset pod count does not match node count") - }, "240s", "10s").Should(Succeed()) - }) - - It("Verifies DNS Access Preupgrade", func() { - namespace := "auto-dns" - _, err := terraform.DeployWorkload("dnsutils.yaml", createcluster.KubeConfigFile) - - Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed") - - Eventually(func(g Gomega) { - cmd := "kubectl get pods dnsutils " + "-n " + namespace + - " --kubeconfig=" + createcluster.KubeConfigFile - res, _ := terraform.RunCommand(cmd) - - g.Expect(res).Should(ContainSubstring("dnsutils")) - g.Expect(res).Should(ContainSubstring("Running")) - }, "240s", "2s").Should(Succeed()) - - Eventually(func(g Gomega) { - cmd := "kubectl -n " + namespace + " --kubeconfig=" + - createcluster.KubeConfigFile + " exec -t dnsutils -- nslookup kubernetes.default" - res, _ := terraform.RunCommand(cmd) - - g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local")) - }, "240s", "2s").Should(Succeed()) - }) - }) - - Context("Upgrade via SUC:", func() { - It("Verifies Upgrade", func() { - namespace := "system-upgrade" - _, err := terraform.DeployWorkload("suc.yaml", createcluster.KubeConfigFile) - Expect(err).NotTo(HaveOccurred(), - "system-upgrade-controller manifest did not deploy successfully") - - Eventually(func(g Gomega) { - cmd := "kubectl get pods " + "-n " + namespace + " --kubeconfig=" + createcluster.KubeConfigFile - res, _ := terraform.RunCommand(cmd) - - g.Expect(res).Should(ContainSubstring("system-upgrade-controller")) - g.Expect(res).Should(ContainSubstring("Running")) - }, "120s", "2s").Should(Succeed()) - - err = upgradeCluster(*upgradeVersion, createcluster.KubeConfigFile) - Expect(err).NotTo(HaveOccurred(), "failed to upgrade cluster.") - - defer func() { - _, err := terraform.Nodes(createcluster.KubeConfigFile, true) - if err != nil { - fmt.Println("Error retrieving nodes postupgrade: ", err) - } - _, err = terraform.Pods(createcluster.KubeConfigFile, true) - if err != nil { - fmt.Println("Error retrieving pods postupgrade: ", err) - } - }() - - versionRegex := regexp.MustCompile("-rc[0-9]+") - k8sVersion := versionRegex.ReplaceAllString(*upgradeVersion, "") - - fmt.Printf("\nFetching node status postupgrade\n") - - expectedNodeCount := createcluster.NumServers + createcluster.NumWorkers - Eventually(func(g Gomega) { - nodes, err := terraform.Nodes(createcluster.KubeConfigFile, false) - - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(len(nodes)).To(Equal(expectedNodeCount), - "Number of nodes should match the spec") - - for _, node := range nodes { - g.Expect(node.Status).Should(Equal("Ready"), - "Nodes should all be in Ready state after upgrading", node.Name) - g.Expect(node.Version).Should(Equal(k8sVersion), - "Nodes should all be upgraded to the specified version", node.Name) - } - }, "900s", "30s").Should(Succeed()) - - fmt.Printf("\n Fetching pod status postupgrade\n") - Eventually(func(g Gomega) { - pods, err := terraform.Pods(createcluster.KubeConfigFile, false) - g.Expect(err).NotTo(HaveOccurred()) - for _, pod := range pods { - if strings.Contains(pod.Name, "helm-install") { - g.Expect(pod.Status).Should(Equal("Completed"), pod.Name) - } else { - g.Expect(pod.Status).Should(Equal("Running"), pod.Name) - numRunning := versionRegex.FindAllString(pod.Ready, 2) - g.Expect(numRunning[0]).Should(Equal(numRunning[1]), pod.Name, "should have all containers running") - } - } - }, "600s", "5s").Should(Succeed()) - }) - }) - - Context("Postupgrade Validations:", func() { - It("Verifies ClusterIP Service Postupgrade", func() { - namespace := "auto-clusterip" - defer terraform.RemoveWorkload("clusterip.yaml", createcluster.KubeConfigFile) - - cmd := "kubectl get pods -n " + namespace + " -o=name -l " + - "k8s-app=nginx-app-clusterip --field-selector=status.phase=Running --kubeconfig=" + - createcluster.KubeConfigFile - res, err := terraform.RunCommand(cmd) - - Expect(err).NotTo(HaveOccurred()) - Expect(res).Should(ContainSubstring("test-clusterip")) - - clusterip, port, _ := terraform.FetchClusterIP(createcluster.KubeConfigFile, - namespace, "nginx-clusterip-svc") - cmd = "curl -sL --insecure http://" + clusterip + ":" + port + "/name.html" - nodeExternalIP := terraform.FetchNodeExternalIP(createcluster.KubeConfigFile) - - for _, ip := range nodeExternalIP { - Eventually(func(g Gomega) { - res, err := terraform.RunCommandOnNode(cmd, ip, createcluster.AwsUser, createcluster.AccessKey) - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-clusterip")) - }, "120s", "10s").Should(Succeed()) - } - }) - - It("Verifies NodePort Service Postupgrade", func() { - namespace := "auto-nodeport" - defer terraform.RemoveWorkload("nodeport.yaml", createcluster.KubeConfigFile) - - nodeExternalIP := terraform.FetchNodeExternalIP(createcluster.KubeConfigFile) - cmd := "kubectl get service -n " + namespace + " nginx-nodeport-svc --kubeconfig=" + - createcluster.KubeConfigFile + " --output jsonpath=\"{.spec.ports[0].nodePort}\"" - nodeport, err := terraform.RunCommand(cmd) - - Expect(err).NotTo(HaveOccurred()) - - for _, ip := range nodeExternalIP { - Eventually(func(g Gomega) { - cmd := "kubectl get pods -n " + namespace + " -o=name -l " + - "k8s-app=nginx-app-nodeport --field-selector=status.phase=Running --kubeconfig=" + - createcluster.KubeConfigFile - res, err := terraform.RunCommand(cmd) - - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-nodeport")) - }, "120s", "5s").Should(Succeed()) - - cmd = "curl -sL --insecure http://" + ip + ":" + nodeport + "/name.html" - Eventually(func(g Gomega) { - res, err := terraform.RunCommand(cmd) - - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-nodeport")) - }, "120s", "5s").Should(Succeed()) - } - }) - - It("Verifies Ingress Postupgrade", func() { - namespace := "auto-ingress" - - defer terraform.RemoveWorkload("ingress.yaml", createcluster.KubeConfigFile) - - cmd := "kubectl get pods -n " + namespace + " -o=name -l k8s-app=nginx-app-ingress " + - "--field-selector=status.phase=Running --kubeconfig=" + createcluster.KubeConfigFile - res, err := terraform.RunCommand(cmd) - - Expect(err).NotTo(HaveOccurred()) - Expect(res).Should(ContainSubstring("test-ingress")) - - var ingressIps []string - nodes, err := terraform.WorkerNodes(createcluster.KubeConfigFile, false) - if err != nil { - fmt.Println("Error retrieving nodes: ", err) - } - - Eventually(func(g Gomega) { - ingressIps, err = terraform.FetchIngressIP(namespace, createcluster.KubeConfigFile) - g.Expect(err).NotTo(HaveOccurred(), "Ingress ip is not returned") - g.Expect(len(ingressIps)).To(Equal(len(nodes)), - "Number of ingress IPs should match the number of nodes") - }, "120s", "5s").Should(Succeed()) - - for _, ip := range ingressIps { - cmd := "curl -s --header host:foo1.bar.com" + " http://" + ip + "/name.html" - Eventually(func(g Gomega) { - res, err := terraform.RunCommand(cmd) - - g.Expect(err).NotTo(HaveOccurred()) - g.Expect(res).Should(ContainSubstring("test-ingress")) - }, "120s", "5s").Should(Succeed()) - } - }) - - It("Verifies Daemonset Postupgrade", func() { - nodes, _ := terraform.WorkerNodes(createcluster.KubeConfigFile, false) - pods, _ := terraform.Pods(createcluster.KubeConfigFile, false) - count := terraform.CountOfStringInSlice("test-daemonset", pods) - - defer terraform.RemoveWorkload("daemonset.yaml", createcluster.KubeConfigFile) - Expect(count).Should((Equal(len(nodes))), - "Daemonset pod count does not match node count") - }) - - It("Verifies DNS Access Postupgrade", func() { - namespace := "auto-dns" - _, err := terraform.DeployWorkload("dnsutils.yaml", createcluster.KubeConfigFile) - - Expect(err).NotTo(HaveOccurred(), "dnsutils manifest not deployed") - - defer terraform.RemoveWorkload("dnsutils.yaml", createcluster.KubeConfigFile) - Eventually(func(g Gomega) { - cmd := "kubectl get pods dnsutils " + "-n " + namespace + - " --kubeconfig=" + createcluster.KubeConfigFile - res, _ := terraform.RunCommand(cmd) - - g.Expect(res).Should(ContainSubstring("dnsutils")) - g.Expect(res).Should(ContainSubstring("Running")) - }, "120s", "2s").Should(Succeed()) - - Eventually(func(g Gomega) { - cmd := "kubectl -n " + namespace + " --kubeconfig=" + createcluster.KubeConfigFile + " exec -t dnsutils -- nslookup kubernetes.default" - res, _ := terraform.RunCommand(cmd) - - g.Expect(res).Should(ContainSubstring("kubernetes.default.svc.cluster.local")) - }, "120s", "2s").Should(Succeed()) - }) - }) -}) - -var _ = BeforeEach(func() { - if *destroy { - Skip("Cluster is being Deleted") - } -}) - -var _ = AfterEach(func() { - if CurrentSpecReport().Failed() { - fmt.Printf("\nFAILED! %s\n", CurrentSpecReport().FullText()) - } else { - fmt.Printf("\nPASSED! %s\n", CurrentSpecReport().FullText()) - } -})