From a9651b3429f28b0d9bca13f52cf77f2302a94509 Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Fri, 13 Sep 2024 14:32:08 +0800 Subject: [PATCH] Skip uninstall and resource cleanup when fail-fast is enabled. Signed-off-by: Xun Jiang --- test/Makefile | 5 +- test/e2e/README.md | 8 +- test/e2e/backup/backup.go | 2 +- test/e2e/backups/deletion.go | 4 +- test/e2e/backups/sync_backups.go | 6 +- test/e2e/backups/ttl.go | 2 +- .../api-group/enable_api_group_extentions.go | 2 +- .../api-group/enable_api_group_versions.go | 2 +- test/e2e/basic/namespace-mapping.go | 2 +- test/e2e/basic/pvc-selected-node-changing.go | 2 +- test/e2e/basic/resources-check/rbac.go | 2 +- test/e2e/basic/storage-class-changing.go | 2 +- test/e2e/bsl-mgmt/deletion.go | 2 +- test/e2e/e2e_suite_test.go | 6 +- test/e2e/migration/migration.go | 2 +- .../resourcemodifiers/resource_modifiers.go | 2 +- .../e2e/resourcepolicies/resource_policies.go | 2 +- test/e2e/schedule/ordered_resources.go | 2 +- test/e2e/schedule/schedule-backup-creation.go | 84 +++++++++---------- test/e2e/schedule/schedule.go | 2 +- test/e2e/test/test.go | 2 +- test/e2e/upgrade/upgrade.go | 2 +- test/perf/e2e_suite_test.go | 10 ++- test/perf/test/test.go | 2 +- test/types.go | 2 +- test/util/kibishii/kibishii_utils.go | 2 +- 26 files changed, 85 insertions(+), 76 deletions(-) diff --git a/test/Makefile b/test/Makefile index 2cec6307d7..4ea01186cf 100644 --- a/test/Makefile +++ b/test/Makefile @@ -97,7 +97,6 @@ ADDITIONAL_BSL_PREFIX ?= ADDITIONAL_BSL_CONFIG ?= FEATURES ?= -DEBUG_E2E_TEST ?= false DEBUG_VELERO_POD_RESTART ?= false VELERO_SERVER_DEBUG_MODE ?= false @@ -148,10 +147,10 @@ COMMON_ARGS := --velerocli=$(VELERO_CLI) \ --features=$(FEATURES) \ --install-velero=$(INSTALL_VELERO) \ --registry-credential-file=$(REGISTRY_CREDENTIAL_FILE) \ - --debug-e2e-test=$(DEBUG_E2E_TEST) \ --velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \ --uploader-type=$(UPLOADER_TYPE) \ - --debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART) + --debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART) \ + --fail-fast=$(FAIL_FAST) # Make sure ginkgo is in $GOBIN .PHONY:ginkgo diff --git a/test/e2e/README.md b/test/e2e/README.md index 1afb113fb2..735e336819 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -116,7 +116,6 @@ Below is a mapping between `make` variables to E2E configuration flags. 1. `FEATURES`: `-features`. Optional. 1. `REGISTRY_CREDENTIAL_FILE`: `-registry-credential-file`. Optional. 1. `KIBISHII_DIRECTORY`: `-kibishii-directory`. Optional. -1. `DEBUG_E2E_TEST`: `-debug-e2e-test`. Optional. 1. `VELERO_SERVER_DEBUG_MODE`: `-velero-server-debug-mode`. Optional. 1. `DEFAULT_CLUSTER`: `-default-cluster-context`. Optional. 1. `STANDBY_CLUSTER`: `-standby-cluster-context`. Optional. @@ -128,6 +127,7 @@ Below is a mapping between `make` variables to E2E configuration flags. 1. `STANDBY_CLUSTER_OBJECT_STORE_PROVIDER`: `-standby-cluster-object-store-provider`. Optional. 1. `INSTALL_VELERO `: `-install-velero`. Optional. 1. `DEBUG_VELERO_POD_RESTART`: `-debug-velero-pod-restart`. Optional. +1. `FAIL_FAST`: `--fail-fast`. Optional. @@ -310,7 +310,7 @@ For programming conventions, Take ResourcePoliciesCase case for example: - It only cleans up resources in currently test namespaces, if you wish to clean up all resources including resources created which are not in currently test namespaces, it's better to override base Destroy function ##### Clean - Clean function only clean resources in namespaces which has the prefix CaseBaseName. So the the names of test namespaces should start with prefix of CaseBaseName. -- If wantting to clean up all resources including resources created which is not in currently test namespaces, it's better to override base Clean function +- It's better to override base Clean function, if cleaning up all resources including resources created which is not in currently test namespaces is needed. #### Velero Installation - Velero is installed with global velero config before the E2E test start, so if the case (such as upgrade/migration, etc.) does not want to use the global velero config, it is NEED TO UNINSTALL velero, or the global velero config may affect the current test. ### TestFunc @@ -323,3 +323,7 @@ Look for the ⛵ emoji printed at the end of each install and uninstall log. The #### Troubleshooting If velero log shows `level=error msg="Failed to get bucket region, bucket: xbucket, error: operation error S3: HeadBucket, failed to resolve service endpoint, endpoint rule error, A region must be set when sending requests to S3." backup-storage-location=velero/default cmd=/plugins/velero-plugin-for-aws controller=backup-storage-location logSource="/go/src/velero-plugin-for-aws/velero-plugin-for-aws/object_store.go:136" pluginName=velero-plugin-for-aws`, it means you need to set `BSL_CONFIG` to include `region=`. + +If need to debug the failed test case, please set the `FAIL_FAST=true` for the `make test-e2e` CLI. +If `FAIL_FAST` is enabled, the pipeline with failed test case will not delete the test bed, +and the resources(including Velero instance, CRs, and other k8s resources) created during the test are kept. diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index 1c8c434d8d..db6643bd60 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -97,7 +97,7 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) { }) AfterEach(func() { - if !veleroCfg.Debug { + if !veleroCfg.FailFast { By("Clean backups after test", func() { DeleteAllBackups(context.Background(), &veleroCfg) if backupRestoreTestConfig.isRetainPVTest { diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index 23e24a7f62..d1c0147328 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -62,7 +62,7 @@ func backup_deletion_test(useVolumeSnapshots bool) { }) AfterEach(func() { - if !veleroCfg.Debug { + if !veleroCfg.FailFast { By("Clean backups after test", func() { DeleteAllBackups(context.Background(), &veleroCfg) }) @@ -104,7 +104,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc if err := CreateNamespace(oneHourTimeout, client, ns); err != nil { return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", ns) } - if !veleroCfg.Debug { + if !veleroCfg.FailFast { defer func() { if err := DeleteNamespace(context.Background(), client, ns, true); err != nil { fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", ns)) diff --git a/test/e2e/backups/sync_backups.go b/test/e2e/backups/sync_backups.go index de5c1a5f03..6778a5a84a 100644 --- a/test/e2e/backups/sync_backups.go +++ b/test/e2e/backups/sync_backups.go @@ -63,7 +63,7 @@ func BackupsSyncTest() { }) AfterEach(func() { - if !veleroCfg.Debug { + if !veleroCfg.FailFast { By("Clean backups after test", func() { DeleteAllBackups(context.Background(), &veleroCfg) }) @@ -82,7 +82,7 @@ func BackupsSyncTest() { By(fmt.Sprintf("Prepare workload as target to backup by creating namespace %s namespace", test.testNS)) Expect(CreateNamespace(ctx, *veleroCfg.ClientToInstallVelero, test.testNS)).To(Succeed(), fmt.Sprintf("Failed to create %s namespace", test.testNS)) - if !veleroCfg.Debug { + if !veleroCfg.FailFast { defer func() { Expect(DeleteNamespace(ctx, *veleroCfg.ClientToInstallVelero, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) }() @@ -125,7 +125,7 @@ func BackupsSyncTest() { Expect(CreateNamespace(ctx, *veleroCfg.ClientToInstallVelero, test.testNS)).To(Succeed(), fmt.Sprintf("Failed to create %s namespace", test.testNS)) }) - if !veleroCfg.Debug { + if !veleroCfg.FailFast { defer func() { Expect(DeleteNamespace(ctx, *veleroCfg.ClientToInstallVelero, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 578f6161d8..3d85b8d794 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -74,7 +74,7 @@ func TTLTest() { AfterEach(func() { veleroCfg.GCFrequency = "" - if !veleroCfg.Debug { + if !veleroCfg.FailFast { By("Clean backups after test", func() { DeleteAllBackups(context.Background(), &veleroCfg) }) diff --git a/test/e2e/basic/api-group/enable_api_group_extentions.go b/test/e2e/basic/api-group/enable_api_group_extentions.go index f78db34175..da14f25380 100644 --- a/test/e2e/basic/api-group/enable_api_group_extentions.go +++ b/test/e2e/basic/api-group/enable_api_group_extentions.go @@ -67,7 +67,7 @@ func APIExtensionsVersionsTest() { }) }) AfterEach(func() { - if !veleroCfg.Debug { + if !veleroCfg.FailFast { By("Clean backups after test", func() { DeleteAllBackups(context.Background(), &veleroCfg) }) diff --git a/test/e2e/basic/api-group/enable_api_group_versions.go b/test/e2e/basic/api-group/enable_api_group_versions.go index 7f67dbf6e1..3ed8eb1e0c 100644 --- a/test/e2e/basic/api-group/enable_api_group_versions.go +++ b/test/e2e/basic/api-group/enable_api_group_versions.go @@ -79,7 +79,7 @@ func APIGroupVersionsTest() { }) AfterEach(func() { - if !veleroCfg.Debug { + if !veleroCfg.FailFast { for i := 0; i < testCaseNum; i++ { curResource := fmt.Sprintf("rockband%ds", i) curGroup := fmt.Sprintf("%s.%d", group, i) diff --git a/test/e2e/basic/namespace-mapping.go b/test/e2e/basic/namespace-mapping.go index 479c37a226..0d70227101 100644 --- a/test/e2e/basic/namespace-mapping.go +++ b/test/e2e/basic/namespace-mapping.go @@ -116,7 +116,7 @@ func (n *NamespaceMapping) Verify() error { } func (n *NamespaceMapping) Clean() error { - if !n.VeleroCfg.Debug { + if !n.VeleroCfg.FailFast { if err := DeleteStorageClass(context.Background(), n.Client, KibishiiStorageClassName); err != nil { return err } diff --git a/test/e2e/basic/pvc-selected-node-changing.go b/test/e2e/basic/pvc-selected-node-changing.go index d3e22b8922..8fef2dbe4c 100644 --- a/test/e2e/basic/pvc-selected-node-changing.go +++ b/test/e2e/basic/pvc-selected-node-changing.go @@ -149,7 +149,7 @@ func (p *PVCSelectedNodeChanging) Verify() error { } func (p *PVCSelectedNodeChanging) Clean() error { - if !p.VeleroCfg.Debug { + if !p.VeleroCfg.FailFast { p.TestCase.Clean() By(fmt.Sprintf("Clean namespace with prefix %s after test", p.mappedNS), func() { CleanupNamespaces(p.Ctx, p.Client, p.mappedNS) diff --git a/test/e2e/basic/resources-check/rbac.go b/test/e2e/basic/resources-check/rbac.go index 7afedd13c6..c6b7141a99 100644 --- a/test/e2e/basic/resources-check/rbac.go +++ b/test/e2e/basic/resources-check/rbac.go @@ -176,7 +176,7 @@ func (r *RBACCase) Destroy() error { } func (r *RBACCase) Clean() error { - if !r.VeleroCfg.Debug { + if !r.VeleroCfg.FailFast { return r.Destroy() } return nil diff --git a/test/e2e/basic/storage-class-changing.go b/test/e2e/basic/storage-class-changing.go index d61e0b4353..64db1a73f1 100644 --- a/test/e2e/basic/storage-class-changing.go +++ b/test/e2e/basic/storage-class-changing.go @@ -142,7 +142,7 @@ func (s *StorageClasssChanging) Verify() error { } func (s *StorageClasssChanging) Clean() error { - if !s.VeleroCfg.Debug { + if !s.VeleroCfg.FailFast { By(fmt.Sprintf("Start to destroy namespace %s......", s.CaseBaseName), func() { Expect(CleanupNamespacesWithPoll(s.Ctx, s.Client, s.CaseBaseName)).To(Succeed(), fmt.Sprintf("Failed to delete namespace %s", s.CaseBaseName)) diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 35037efa37..d181ad9463 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -74,7 +74,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) AfterEach(func() { - if !veleroCfg.Debug { + if !veleroCfg.FailFast { By("Clean backups after test", func() { veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient DeleteAllBackups(context.Background(), &veleroCfg) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 1031958084..39a5475b48 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -85,7 +85,6 @@ func init() { flag.StringVar(&VeleroCfg.AdditionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2") flag.StringVar(&VeleroCfg.AdditionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.") flag.StringVar(&VeleroCfg.Features, "features", "", "comma-separated list of features to enable for this Velero process.") - flag.BoolVar(&VeleroCfg.Debug, "debug-e2e-test", false, "A Switch for enable or disable test data cleaning action.") flag.StringVar(&VeleroCfg.GCFrequency, "garbage-collection-frequency", "", "frequency of garbage collection.") flag.StringVar(&VeleroCfg.DefaultClusterContext, "default-cluster-context", "", "default cluster's kube config context, it's for migration test.") flag.StringVar(&VeleroCfg.StandbyClusterContext, "standby-cluster-context", "", "standby cluster's kube config context, it's for migration test.") @@ -103,6 +102,7 @@ func init() { flag.StringVar(&VeleroCfg.EKSPolicyARN, "eks-policy-arn", "", "EKS plicy ARN for creating AWS IAM service account.") flag.StringVar(&VeleroCfg.DefaultCLSServiceAccountName, "default-cls-service-account-name", "", "default cluster service account name.") flag.StringVar(&VeleroCfg.StandbyCLSServiceAccountName, "standby-cls-service-account-name", "", "standby cluster service account name.") + flag.BoolVar(&VeleroCfg.FailFast, "fail-fast", true, "a switch for failing fast on meeting error.") } // Add label [SkipVanillaZfs]: @@ -302,7 +302,9 @@ var _ = BeforeSuite(func() { }) var _ = AfterSuite(func() { - if InstallVelero && !VeleroCfg.Debug { + // If the Velero is installed during test, and the FailFast is not enabled, + // uninstall Velero. If not, either Velero is not installed, or kept it for debug. + if InstallVelero && !VeleroCfg.FailFast { By("release test resources after testing") ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 0cf96f1bfd..68a95d2668 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -81,7 +81,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) } }) AfterEach(func() { - if !veleroCfg.Debug { + if !veleroCfg.FailFast { By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.DefaultClusterContext), func() { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() diff --git a/test/e2e/resourcemodifiers/resource_modifiers.go b/test/e2e/resourcemodifiers/resource_modifiers.go index 745963c48b..6bbef7b677 100644 --- a/test/e2e/resourcemodifiers/resource_modifiers.go +++ b/test/e2e/resourcemodifiers/resource_modifiers.go @@ -131,7 +131,7 @@ func (r *ResourceModifiersCase) Verify() error { func (r *ResourceModifiersCase) Clean() error { // If created some resources which is not in current test namespace, we NEED to override the base Clean function - if !r.VeleroCfg.Debug { + if !r.VeleroCfg.FailFast { if err := DeleteConfigmap(r.Client.ClientGo, r.VeleroCfg.VeleroNamespace, r.cmName); err != nil { return err } diff --git a/test/e2e/resourcepolicies/resource_policies.go b/test/e2e/resourcepolicies/resource_policies.go index 62d495abb3..49a2270206 100644 --- a/test/e2e/resourcepolicies/resource_policies.go +++ b/test/e2e/resourcepolicies/resource_policies.go @@ -178,7 +178,7 @@ func (r *ResourcePoliciesCase) Verify() error { func (r *ResourcePoliciesCase) Clean() error { // If created some resources which is not in current test namespace, we NEED to override the base Clean function - if !r.VeleroCfg.Debug { + if !r.VeleroCfg.FailFast { if err := r.deleteTestStorageClassList([]string{StorageClassName, StorageClassName2}); err != nil { return err } diff --git a/test/e2e/schedule/ordered_resources.go b/test/e2e/schedule/ordered_resources.go index b6a6a6e428..7cfac1c4af 100644 --- a/test/e2e/schedule/ordered_resources.go +++ b/test/e2e/schedule/ordered_resources.go @@ -153,7 +153,7 @@ func (o *OrderedResources) Verify() error { } func (o *OrderedResources) Clean() error { - if !o.VeleroCfg.Debug { + if !o.VeleroCfg.FailFast { Expect(VeleroScheduleDelete(o.Ctx, o.VeleroCfg.VeleroCLI, o.VeleroCfg.VeleroNamespace, o.ScheduleName)).To(Succeed()) Expect(o.TestCase.Clean()).To(Succeed()) } diff --git a/test/e2e/schedule/schedule-backup-creation.go b/test/e2e/schedule/schedule-backup-creation.go index 8ece31292d..2a08fe16a4 100644 --- a/test/e2e/schedule/schedule-backup-creation.go +++ b/test/e2e/schedule/schedule-backup-creation.go @@ -32,63 +32,63 @@ type ScheduleBackupCreation struct { var ScheduleBackupCreationTest func() = TestFunc(&ScheduleBackupCreation{}) -func (n *ScheduleBackupCreation) Init() error { - n.TestCase.Init() - n.CaseBaseName = "schedule-backup-creation-test" + n.UUIDgen - n.ScheduleName = "schedule-" + n.CaseBaseName - n.namespace = n.GetTestCase().CaseBaseName - n.Period = 3 // Unit is minute - n.verifyTimes = 5 // More larger verify times more confidence we have +func (s *ScheduleBackupCreation) Init() error { + s.TestCase.Init() + s.CaseBaseName = "schedule-backup-creation-test" + s.UUIDgen + s.ScheduleName = "schedule-" + s.CaseBaseName + s.namespace = s.GetTestCase().CaseBaseName + s.Period = 3 // Unit is minute + s.verifyTimes = 5 // More larger verify times more confidence we have podSleepDurationStr := "300s" - n.podSleepDuration, _ = time.ParseDuration(podSleepDurationStr) - n.TestMsg = &TestMSG{ + s.podSleepDuration, _ = time.ParseDuration(podSleepDurationStr) + s.TestMsg = &TestMSG{ Desc: "Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", FailedMSG: "Failed to verify schedule back creation behavior", Text: "Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", } - n.podAnn = map[string]string{ - "pre.hook.backup.velero.io/container": n.podName, + s.podAnn = map[string]string{ + "pre.hook.backup.velero.io/container": s.podName, "pre.hook.backup.velero.io/command": "[\"sleep\", \"" + podSleepDurationStr + "\"]", "pre.hook.backup.velero.io/timeout": "600s", } - n.volume = "volume-1" - n.podName = "pod-1" - n.pvcName = "pvc-1" - n.ScheduleArgs = []string{ - "--include-namespaces", n.namespace, - "--schedule=*/" + fmt.Sprintf("%v", n.Period) + " * * * *", + s.volume = "volume-1" + s.podName = "pod-1" + s.pvcName = "pvc-1" + s.ScheduleArgs = []string{ + "--include-namespaces", s.namespace, + "--schedule=*/" + fmt.Sprintf("%v", s.Period) + " * * * *", } - Expect(n.Period).To(BeNumerically("<", 30)) + Expect(s.Period).To(BeNumerically("<", 30)) return nil } -func (p *ScheduleBackupCreation) CreateResources() error { - By(fmt.Sprintf("Create namespace %s", p.namespace), func() { - Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(), - fmt.Sprintf("Failed to create namespace %s", p.namespace)) +func (s *ScheduleBackupCreation) CreateResources() error { + By(fmt.Sprintf("Create namespace %s", s.namespace), func() { + Expect(CreateNamespace(s.Ctx, s.Client, s.namespace)).To(Succeed(), + fmt.Sprintf("Failed to create namespace %s", s.namespace)) }) - By(fmt.Sprintf("Create pod %s in namespace %s", p.podName, p.namespace), func() { - _, err := CreatePod(p.Client, p.namespace, p.podName, "default", p.pvcName, []string{p.volume}, nil, p.podAnn) + By(fmt.Sprintf("Create pod %s in namespace %s", s.podName, s.namespace), func() { + _, err := CreatePod(s.Client, s.namespace, s.podName, "default", s.pvcName, []string{s.volume}, nil, s.podAnn) Expect(err).To(Succeed()) - err = WaitForPods(p.Ctx, p.Client, p.namespace, []string{p.podName}) + err = WaitForPods(s.Ctx, s.Client, s.namespace, []string{s.podName}) Expect(err).To(Succeed()) }) return nil } -func (n *ScheduleBackupCreation) Backup() error { +func (s *ScheduleBackupCreation) Backup() error { // Wait until the beginning of the given period to create schedule, it will give us // a predictable period to wait for the first scheduled backup, and verify no immediate // scheduled backup was created between schedule creation and first scheduled backup. - By(fmt.Sprintf("Creating schedule %s ......\n", n.ScheduleName), func() { - for i := 0; i < n.Period*60/30; i++ { + By(fmt.Sprintf("Creating schedule %s ......\n", s.ScheduleName), func() { + for i := 0; i < s.Period*60/30; i++ { time.Sleep(30 * time.Second) now := time.Now().Minute() - triggerNow := now % n.Period + triggerNow := now % s.Period if triggerNow == 0 { - Expect(VeleroScheduleCreate(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName, n.ScheduleArgs)).To(Succeed(), func() string { - RunDebug(context.Background(), n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, "", "") + Expect(VeleroScheduleCreate(s.Ctx, s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, s.ScheduleName, s.ScheduleArgs)).To(Succeed(), func() string { + RunDebug(context.Background(), s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, "", "") return "Fail to create schedule" }) break @@ -100,13 +100,13 @@ func (n *ScheduleBackupCreation) Backup() error { time.Sleep(1 * time.Minute) }) - By(fmt.Sprintf("Get backups every %d minute, and backups count should increase 1 more step in the same pace\n", n.Period), func() { - for i := 1; i <= n.verifyTimes; i++ { - fmt.Printf("Start to sleep %d minute #%d time...\n", n.podSleepDuration, i) + By(fmt.Sprintf("Get backups every %d minute, and backups count should increase 1 more step in the same pace\n", s.Period), func() { + for i := 1; i <= s.verifyTimes; i++ { + fmt.Printf("Start to sleep %d minute #%d time...\n", s.podSleepDuration, i) mi, _ := time.ParseDuration("60s") - time.Sleep(n.podSleepDuration + mi) + time.Sleep(s.podSleepDuration + mi) bMap := make(map[string]string) - backupsInfo, err := GetScheduledBackupsCreationTime(n.Ctx, n.VeleroCfg.VeleroCLI, "default", n.ScheduleName) + backupsInfo, err := GetScheduledBackupsCreationTime(s.Ctx, s.VeleroCfg.VeleroCLI, "default", s.ScheduleName) Expect(err).To(Succeed()) Expect(backupsInfo).To(HaveLen(i)) for index, bi := range backupsInfo { @@ -116,19 +116,19 @@ func (n *ScheduleBackupCreation) Backup() error { _, err := time.Parse("2006-01-02 15:04:05 -0700 MST", bList[1]) Expect(err).To(Succeed()) } - if i == n.verifyTimes-1 { + if i == s.verifyTimes-1 { backupInfo := backupsInfo[rand.Intn(len(backupsInfo))] - n.randBackupName = strings.Split(backupInfo, ",")[0] + s.randBackupName = strings.Split(backupInfo, ",")[0] } } }) return nil } -func (n *ScheduleBackupCreation) Clean() error { - if !n.VeleroCfg.Debug { - Expect(VeleroScheduleDelete(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed()) - Expect(n.TestCase.Clean()).To(Succeed()) +func (s *ScheduleBackupCreation) Clean() error { + if !s.VeleroCfg.FailFast { + Expect(VeleroScheduleDelete(s.Ctx, s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, s.ScheduleName)).To(Succeed()) + Expect(s.TestCase.Clean()).To(Succeed()) } return nil } diff --git a/test/e2e/schedule/schedule.go b/test/e2e/schedule/schedule.go index b728623aa7..c2943ec432 100644 --- a/test/e2e/schedule/schedule.go +++ b/test/e2e/schedule/schedule.go @@ -203,7 +203,7 @@ func (n *ScheduleBackup) Verify() error { } func (n *ScheduleBackup) Clean() error { - if !n.VeleroCfg.Debug { + if !n.VeleroCfg.FailFast { Expect(VeleroScheduleDelete(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed()) Expect(n.TestCase.Clean()).To(Succeed()) } diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index f46a147609..2c0a8537b4 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -182,7 +182,7 @@ func (t *TestCase) Start() error { func (t *TestCase) Clean() error { veleroCfg := t.GetTestCase().VeleroCfg - if !veleroCfg.Debug { + if !veleroCfg.FailFast { By(fmt.Sprintf("Clean namespace with prefix %s after test", t.CaseBaseName), func() { if err := CleanupNamespaces(t.Ctx, t.Client, t.CaseBaseName); err != nil { fmt.Println("Fail to cleanup namespaces: ", err) diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 0f33d2eff0..a59ecca32f 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -84,7 +84,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC } }) AfterEach(func() { - if !veleroCfg.Debug { + if !veleroCfg.FailFast { By("Clean backups after test", func() { DeleteAllBackups(context.Background(), &veleroCfg) }) diff --git a/test/perf/e2e_suite_test.go b/test/perf/e2e_suite_test.go index e047749c22..59da106119 100644 --- a/test/perf/e2e_suite_test.go +++ b/test/perf/e2e_suite_test.go @@ -69,7 +69,6 @@ func init() { //vmware-tanzu-experiments flag.StringVar(&VeleroCfg.Features, "features", "", "Comma-separated list of features to enable for this Velero process.") flag.StringVar(&VeleroCfg.DefaultClusterContext, "default-cluster-context", "", "Default cluster context for migration test.") - flag.BoolVar(&VeleroCfg.Debug, "debug-e2e-test", true, "Switch to control namespace cleaning.") flag.StringVar(&VeleroCfg.UploaderType, "uploader-type", "kopia", "Identify persistent volume backup uploader.") flag.BoolVar(&VeleroCfg.VeleroServerDebugMode, "velero-server-debug-mode", false, "Identify persistent volume backup uploader.") flag.StringVar(&VeleroCfg.NFSServerPath, "nfs-server-path", "", "the path of nfs server") @@ -77,6 +76,7 @@ func init() { flag.StringVar(&VeleroCfg.BackupForRestore, "backup-for-restore", "", "the name of backup for restore") flag.BoolVar(&VeleroCfg.DeleteClusterResource, "delete-cluster-resource", false, "delete cluster resource after test") flag.BoolVar(&VeleroCfg.DebugVeleroPodRestart, "debug-velero-pod-restart", false, "Switch for debugging velero pod restart.") + flag.BoolVar(&VeleroCfg.FailFast, "fail-fast", true, "a switch for failing fast on meeting error.") } func initConfig() error { @@ -128,8 +128,12 @@ var _ = BeforeSuite(func() { var _ = AfterSuite(func() { Expect(report.GenerateYamlReport()).To(Succeed()) - if InstallVelero && !VeleroCfg.Debug { + // If the Velero is installed during test, and the FailFast is not enabled, + // uninstall Velero. If not, either Velero is not installed, or kept it for debug. + if InstallVelero && !VeleroCfg.FailFast { By("release test resources after testing") - Expect(VeleroUninstall(context.Background(), VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) + ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) + defer ctxCancel() + Expect(VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) } }) diff --git a/test/perf/test/test.go b/test/perf/test/test.go index 36f46bbf54..529cd6dc06 100644 --- a/test/perf/test/test.go +++ b/test/perf/test/test.go @@ -161,7 +161,7 @@ func (t *TestCase) Verify() error { } func (t *TestCase) Clean() error { - if !VeleroCfg.Debug || VeleroCfg.DeleteClusterResource { + if !VeleroCfg.FailFast || VeleroCfg.DeleteClusterResource { By("Clean backups and restore after test", func() { if len(t.BackupArgs) != 0 { if err := VeleroBackupDelete(t.Ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace, t.BackupName); err != nil { diff --git a/test/types.go b/test/types.go index dc1b100d92..083fcdbbad 100644 --- a/test/types.go +++ b/test/types.go @@ -84,7 +84,6 @@ type VeleroConfig struct { Plugins string AddBSLPlugins string KibishiiDirectory string - Debug bool GCFrequency string DefaultClusterContext string StandbyClusterContext string @@ -110,6 +109,7 @@ type VeleroConfig struct { StandbyCLSServiceAccountName string ServiceAccountNameToInstall string EKSPolicyARN string + FailFast bool } type VeleroCfgInPerf struct { diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index 2fe223ea96..c72c4bca8b 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -86,7 +86,7 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", kibishiiNamespace) } defer func() { - if !veleroCfg.Debug { + if !veleroCfg.FailFast { if err := DeleteNamespace(context.Background(), client, kibishiiNamespace, true); err != nil { fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", kibishiiNamespace)) }