Skip uninstall and resource cleanup when fail-fast is enabled.

Signed-off-by: Xun Jiang <[email protected]>

test/Makefile

@@ -97,7 +97,6 @@ ADDITIONAL_BSL_PREFIX ?=
 ADDITIONAL_BSL_CONFIG ?=
 
 FEATURES ?=
-DEBUG_E2E_TEST ?= false
 DEBUG_VELERO_POD_RESTART ?= false
 VELERO_SERVER_DEBUG_MODE ?= false
 
@@ -148,10 +147,10 @@ COMMON_ARGS := --velerocli=$(VELERO_CLI) \
 	--features=$(FEATURES) \
 	--install-velero=$(INSTALL_VELERO) \
 	--registry-credential-file=$(REGISTRY_CREDENTIAL_FILE) \
-	--debug-e2e-test=$(DEBUG_E2E_TEST) \
 	--velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \
 	--uploader-type=$(UPLOADER_TYPE) \
-	--debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART)
+	--debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART) \
+	--fail-fast=$(FAIL_FAST)
 
 # Make sure ginkgo is in $GOBIN
 .PHONY:ginkgo

test/e2e/README.md

@@ -116,7 +116,6 @@ Below is a mapping between `make` variables to E2E configuration flags.
 1. `FEATURES`: `-features`. Optional.
 1. `REGISTRY_CREDENTIAL_FILE`: `-registry-credential-file`. Optional.
 1. `KIBISHII_DIRECTORY`: `-kibishii-directory`. Optional.
-1. `DEBUG_E2E_TEST`: `-debug-e2e-test`. Optional.
 1. `VELERO_SERVER_DEBUG_MODE`: `-velero-server-debug-mode`. Optional.
 1. `DEFAULT_CLUSTER`: `-default-cluster-context`. Optional.
 1. `STANDBY_CLUSTER`: `-standby-cluster-context`. Optional.
@@ -128,6 +127,7 @@ Below is a mapping between `make` variables to E2E configuration flags.
 1. `STANDBY_CLUSTER_OBJECT_STORE_PROVIDER`: `-standby-cluster-object-store-provider`. Optional.
 1. `INSTALL_VELERO `: `-install-velero`. Optional.
 1. `DEBUG_VELERO_POD_RESTART`: `-debug-velero-pod-restart`. Optional.
+1. `FAIL_FAST`: `--fail-fast`. Optional.
 
 
 
@@ -310,7 +310,7 @@ For programming conventions, Take ResourcePoliciesCase case for example:
 - It only cleans up resources in currently test namespaces, if you wish to clean up all resources including resources created which are not in currently test namespaces, it's better to override base Destroy function
 ##### Clean
 - Clean function only clean resources in namespaces which has the prefix CaseBaseName. So the the names of test namespaces should start with prefix of CaseBaseName.
-- If wantting to clean up all resources including resources created which is not in currently test namespaces, it's better to override base Clean function
+- It's better to override base Clean function, if cleaning up all resources including resources created which is not in currently test namespaces is needed.
 #### Velero Installation
 - Velero is installed with global velero config before the E2E test start, so if the case (such as upgrade/migration, etc.) does not want to use the global velero config, it is NEED TO UNINSTALL velero, or the global velero config may affect the current test.
 ### TestFunc 
@@ -323,3 +323,7 @@ Look for the ⛵ emoji printed at the end of each install and uninstall log. The
 #### Troubleshooting
 
 If velero log shows `level=error msg="Failed to get bucket region, bucket: xbucket, error: operation error S3: HeadBucket, failed to resolve service endpoint, endpoint rule error, A region must be set when sending requests to S3." backup-storage-location=velero/default cmd=/plugins/velero-plugin-for-aws controller=backup-storage-location logSource="/go/src/velero-plugin-for-aws/velero-plugin-for-aws/object_store.go:136" pluginName=velero-plugin-for-aws`, it means you need to set `BSL_CONFIG` to include `region=<region>`.
+
+If need to debug the failed test case, please set the `FAIL_FAST=true` for the `make test-e2e` CLI.
+If `FAIL_FAST` is enabled, the pipeline with failed test case will not delete the test bed,
+and the resources(including Velero instance, CRs, and other k8s resources) created during the test are kept.

test/e2e/backup/backup.go

@@ -97,7 +97,7 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) {
 	})
 
 	AfterEach(func() {
-		if !veleroCfg.Debug {
+		if !veleroCfg.FailFast {
 			By("Clean backups after test", func() {
 				DeleteAllBackups(context.Background(), &veleroCfg)
 				if backupRestoreTestConfig.isRetainPVTest {

test/e2e/backups/deletion.go

@@ -62,7 +62,7 @@ func backup_deletion_test(useVolumeSnapshots bool) {
 	})
 
 	AfterEach(func() {
-		if !veleroCfg.Debug {
+		if !veleroCfg.FailFast {
 			By("Clean backups after test", func() {
 				DeleteAllBackups(context.Background(), &veleroCfg)
 			})
@@ -104,7 +104,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc
 		if err := CreateNamespace(oneHourTimeout, client, ns); err != nil {
 			return errors.Wrapf(err, "Failed to create namespace %s to install Kibishii workload", ns)
 		}
-		if !veleroCfg.Debug {
+		if !veleroCfg.FailFast {
 			defer func() {
 				if err := DeleteNamespace(context.Background(), client, ns, true); err != nil {
 					fmt.Println(errors.Wrapf(err, "failed to delete the namespace %q", ns))

test/e2e/backups/sync_backups.go

@@ -63,7 +63,7 @@ func BackupsSyncTest() {
 	})
 
 	AfterEach(func() {
-		if !veleroCfg.Debug {
+		if !veleroCfg.FailFast {
 			By("Clean backups after test", func() {
 				DeleteAllBackups(context.Background(), &veleroCfg)
 			})
@@ -82,7 +82,7 @@ func BackupsSyncTest() {
 		By(fmt.Sprintf("Prepare workload as target to backup by creating namespace %s namespace", test.testNS))
 		Expect(CreateNamespace(ctx, *veleroCfg.ClientToInstallVelero, test.testNS)).To(Succeed(),
 			fmt.Sprintf("Failed to create %s namespace", test.testNS))
-		if !veleroCfg.Debug {
+		if !veleroCfg.FailFast {
 			defer func() {
 				Expect(DeleteNamespace(ctx, *veleroCfg.ClientToInstallVelero, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS))
 			}()
@@ -125,7 +125,7 @@ func BackupsSyncTest() {
 			Expect(CreateNamespace(ctx, *veleroCfg.ClientToInstallVelero, test.testNS)).To(Succeed(),
 				fmt.Sprintf("Failed to create %s namespace", test.testNS))
 		})
-		if !veleroCfg.Debug {
+		if !veleroCfg.FailFast {
 			defer func() {
 				Expect(DeleteNamespace(ctx, *veleroCfg.ClientToInstallVelero, test.testNS, false)).To(Succeed(),
 					fmt.Sprintf("Failed to delete the namespace %s", test.testNS))

test/e2e/backups/ttl.go

@@ -74,7 +74,7 @@ func TTLTest() {
 
 	AfterEach(func() {
 		veleroCfg.GCFrequency = ""
-		if !veleroCfg.Debug {
+		if !veleroCfg.FailFast {
 			By("Clean backups after test", func() {
 				DeleteAllBackups(context.Background(), &veleroCfg)
 			})

test/e2e/basic/api-group/enable_api_group_extentions.go

@@ -67,7 +67,7 @@ func APIExtensionsVersionsTest() {
 		})
 	})
 	AfterEach(func() {
-		if !veleroCfg.Debug {
+		if !veleroCfg.FailFast {
 			By("Clean backups after test", func() {
 				DeleteAllBackups(context.Background(), &veleroCfg)
 			})

test/e2e/basic/api-group/enable_api_group_versions.go

@@ -79,7 +79,7 @@ func APIGroupVersionsTest() {
 	})
 
 	AfterEach(func() {
-		if !veleroCfg.Debug {
+		if !veleroCfg.FailFast {
 			for i := 0; i < testCaseNum; i++ {
 				curResource := fmt.Sprintf("rockband%ds", i)
 				curGroup := fmt.Sprintf("%s.%d", group, i)

test/e2e/basic/namespace-mapping.go

@@ -116,7 +116,7 @@ func (n *NamespaceMapping) Verify() error {
 }
 
 func (n *NamespaceMapping) Clean() error {
-	if !n.VeleroCfg.Debug {
+	if !n.VeleroCfg.FailFast {
 		if err := DeleteStorageClass(context.Background(), n.Client, KibishiiStorageClassName); err != nil {
 			return err
 		}

test/e2e/basic/pvc-selected-node-changing.go

@@ -149,7 +149,7 @@ func (p *PVCSelectedNodeChanging) Verify() error {
 }
 
 func (p *PVCSelectedNodeChanging) Clean() error {
-	if !p.VeleroCfg.Debug {
+	if !p.VeleroCfg.FailFast {
 		p.TestCase.Clean()
 		By(fmt.Sprintf("Clean namespace with prefix %s after test", p.mappedNS), func() {
 			CleanupNamespaces(p.Ctx, p.Client, p.mappedNS)

test/e2e/basic/resources-check/rbac.go

@@ -176,7 +176,7 @@ func (r *RBACCase) Destroy() error {
 }
 
 func (r *RBACCase) Clean() error {
-	if !r.VeleroCfg.Debug {
+	if !r.VeleroCfg.FailFast {
 		return r.Destroy()
 	}
 	return nil

test/e2e/basic/storage-class-changing.go

@@ -142,7 +142,7 @@ func (s *StorageClasssChanging) Verify() error {
 }
 
 func (s *StorageClasssChanging) Clean() error {
-	if !s.VeleroCfg.Debug {
+	if !s.VeleroCfg.FailFast {
 		By(fmt.Sprintf("Start to destroy namespace %s......", s.CaseBaseName), func() {
 			Expect(CleanupNamespacesWithPoll(s.Ctx, s.Client, s.CaseBaseName)).To(Succeed(),
 				fmt.Sprintf("Failed to delete namespace %s", s.CaseBaseName))

test/e2e/bsl-mgmt/deletion.go

@@ -74,7 +74,7 @@ func BslDeletionTest(useVolumeSnapshots bool) {
 	})
 
 	AfterEach(func() {
-		if !veleroCfg.Debug {
+		if !veleroCfg.FailFast {
 			By("Clean backups after test", func() {
 				veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient
 				DeleteAllBackups(context.Background(), &veleroCfg)

test/e2e/e2e_suite_test.go

@@ -85,7 +85,6 @@ func init() {
 	flag.StringVar(&VeleroCfg.AdditionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2")
 	flag.StringVar(&VeleroCfg.AdditionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.")
 	flag.StringVar(&VeleroCfg.Features, "features", "", "comma-separated list of features to enable for this Velero process.")
-	flag.BoolVar(&VeleroCfg.Debug, "debug-e2e-test", false, "A Switch for enable or disable test data cleaning action.")
 	flag.StringVar(&VeleroCfg.GCFrequency, "garbage-collection-frequency", "", "frequency of garbage collection.")
 	flag.StringVar(&VeleroCfg.DefaultClusterContext, "default-cluster-context", "", "default cluster's kube config context, it's for migration test.")
 	flag.StringVar(&VeleroCfg.StandbyClusterContext, "standby-cluster-context", "", "standby cluster's kube config context, it's for migration test.")
@@ -103,6 +102,7 @@ func init() {
 	flag.StringVar(&VeleroCfg.EKSPolicyARN, "eks-policy-arn", "", "EKS plicy ARN for creating AWS IAM service account.")
 	flag.StringVar(&VeleroCfg.DefaultCLSServiceAccountName, "default-cls-service-account-name", "", "default cluster service account name.")
 	flag.StringVar(&VeleroCfg.StandbyCLSServiceAccountName, "standby-cls-service-account-name", "", "standby cluster service account name.")
+	flag.BoolVar(&VeleroCfg.FailFast, "fail-fast", true, "a switch for failing fast on meeting error.")
 }
 
 // Add label [SkipVanillaZfs]:
@@ -302,7 +302,9 @@ var _ = BeforeSuite(func() {
 })
 
 var _ = AfterSuite(func() {
-	if InstallVelero && !VeleroCfg.Debug {
+	// If the Velero is installed during test, and the FailFast is not enabled,
+	// uninstall Velero. If not, either Velero is not installed, or kept it for debug.
+	if InstallVelero && !VeleroCfg.FailFast {
 		By("release test resources after testing")
 		ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
 		defer ctxCancel()

test/e2e/migration/migration.go

@@ -81,7 +81,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 		}
 	})
 	AfterEach(func() {
-		if !veleroCfg.Debug {
+		if !veleroCfg.FailFast {
 			By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.DefaultClusterContext), func() {
 				ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5)
 				defer ctxCancel()

test/e2e/resourcemodifiers/resource_modifiers.go

@@ -131,7 +131,7 @@ func (r *ResourceModifiersCase) Verify() error {
 
 func (r *ResourceModifiersCase) Clean() error {
 	// If created some resources which is not in current test namespace, we NEED to override the base Clean function
-	if !r.VeleroCfg.Debug {
+	if !r.VeleroCfg.FailFast {
 		if err := DeleteConfigmap(r.Client.ClientGo, r.VeleroCfg.VeleroNamespace, r.cmName); err != nil {
 			return err
 		}

test/e2e/resourcepolicies/resource_policies.go

@@ -178,7 +178,7 @@ func (r *ResourcePoliciesCase) Verify() error {
 
 func (r *ResourcePoliciesCase) Clean() error {
 	// If created some resources which is not in current test namespace, we NEED to override the base Clean function
-	if !r.VeleroCfg.Debug {
+	if !r.VeleroCfg.FailFast {
 		if err := r.deleteTestStorageClassList([]string{StorageClassName, StorageClassName2}); err != nil {
 			return err
 		}

test/e2e/schedule/ordered_resources.go

@@ -153,7 +153,7 @@ func (o *OrderedResources) Verify() error {
 }
 
 func (o *OrderedResources) Clean() error {
-	if !o.VeleroCfg.Debug {
+	if !o.VeleroCfg.FailFast {
 		Expect(VeleroScheduleDelete(o.Ctx, o.VeleroCfg.VeleroCLI, o.VeleroCfg.VeleroNamespace, o.ScheduleName)).To(Succeed())
 		Expect(o.TestCase.Clean()).To(Succeed())
 	}

test/e2e/schedule/schedule-backup-creation.go

@@ -32,63 +32,63 @@ type ScheduleBackupCreation struct {
 
 var ScheduleBackupCreationTest func() = TestFunc(&ScheduleBackupCreation{})
 
-func (n *ScheduleBackupCreation) Init() error {
-	n.TestCase.Init()
-	n.CaseBaseName = "schedule-backup-creation-test" + n.UUIDgen
-	n.ScheduleName = "schedule-" + n.CaseBaseName
-	n.namespace = n.GetTestCase().CaseBaseName
-	n.Period = 3      // Unit is minute
-	n.verifyTimes = 5 // More larger verify times more confidence we have
+func (s *ScheduleBackupCreation) Init() error {
+	s.TestCase.Init()
+	s.CaseBaseName = "schedule-backup-creation-test" + s.UUIDgen
+	s.ScheduleName = "schedule-" + s.CaseBaseName
+	s.namespace = s.GetTestCase().CaseBaseName
+	s.Period = 3      // Unit is minute
+	s.verifyTimes = 5 // More larger verify times more confidence we have
 	podSleepDurationStr := "300s"
-	n.podSleepDuration, _ = time.ParseDuration(podSleepDurationStr)
-	n.TestMsg = &TestMSG{
+	s.podSleepDuration, _ = time.ParseDuration(podSleepDurationStr)
+	s.TestMsg = &TestMSG{
 		Desc:      "Schedule controller wouldn't create a new backup when it still has pending or InProgress backup",
 		FailedMSG: "Failed to verify schedule back creation behavior",
 		Text:      "Schedule controller wouldn't create a new backup when it still has pending or InProgress backup",
 	}
-	n.podAnn = map[string]string{
-		"pre.hook.backup.velero.io/container": n.podName,
+	s.podAnn = map[string]string{
+		"pre.hook.backup.velero.io/container": s.podName,
 		"pre.hook.backup.velero.io/command":   "[\"sleep\", \"" + podSleepDurationStr + "\"]",
 		"pre.hook.backup.velero.io/timeout":   "600s",
 	}
-	n.volume = "volume-1"
-	n.podName = "pod-1"
-	n.pvcName = "pvc-1"
-	n.ScheduleArgs = []string{
-		"--include-namespaces", n.namespace,
-		"--schedule=*/" + fmt.Sprintf("%v", n.Period) + " * * * *",
+	s.volume = "volume-1"
+	s.podName = "pod-1"
+	s.pvcName = "pvc-1"
+	s.ScheduleArgs = []string{
+		"--include-namespaces", s.namespace,
+		"--schedule=*/" + fmt.Sprintf("%v", s.Period) + " * * * *",
 	}
-	Expect(n.Period).To(BeNumerically("<", 30))
+	Expect(s.Period).To(BeNumerically("<", 30))
 	return nil
 }
 
-func (p *ScheduleBackupCreation) CreateResources() error {
-	By(fmt.Sprintf("Create namespace %s", p.namespace), func() {
-		Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(),
-			fmt.Sprintf("Failed to create namespace %s", p.namespace))
+func (s *ScheduleBackupCreation) CreateResources() error {
+	By(fmt.Sprintf("Create namespace %s", s.namespace), func() {
+		Expect(CreateNamespace(s.Ctx, s.Client, s.namespace)).To(Succeed(),
+			fmt.Sprintf("Failed to create namespace %s", s.namespace))
 	})
 
-	By(fmt.Sprintf("Create pod %s in namespace %s", p.podName, p.namespace), func() {
-		_, err := CreatePod(p.Client, p.namespace, p.podName, "default", p.pvcName, []string{p.volume}, nil, p.podAnn)
+	By(fmt.Sprintf("Create pod %s in namespace %s", s.podName, s.namespace), func() {
+		_, err := CreatePod(s.Client, s.namespace, s.podName, "default", s.pvcName, []string{s.volume}, nil, s.podAnn)
 		Expect(err).To(Succeed())
-		err = WaitForPods(p.Ctx, p.Client, p.namespace, []string{p.podName})
+		err = WaitForPods(s.Ctx, s.Client, s.namespace, []string{s.podName})
 		Expect(err).To(Succeed())
 	})
 	return nil
 }
 
-func (n *ScheduleBackupCreation) Backup() error {
+func (s *ScheduleBackupCreation) Backup() error {
 	// Wait until the beginning of the given period to create schedule, it will give us
 	//   a predictable period to wait for the first scheduled backup, and verify no immediate
 	//   scheduled backup was created between schedule creation and first scheduled backup.
-	By(fmt.Sprintf("Creating schedule %s ......\n", n.ScheduleName), func() {
-		for i := 0; i < n.Period*60/30; i++ {
+	By(fmt.Sprintf("Creating schedule %s ......\n", s.ScheduleName), func() {
+		for i := 0; i < s.Period*60/30; i++ {
 			time.Sleep(30 * time.Second)
 			now := time.Now().Minute()
-			triggerNow := now % n.Period
+			triggerNow := now % s.Period
 			if triggerNow == 0 {
-				Expect(VeleroScheduleCreate(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName, n.ScheduleArgs)).To(Succeed(), func() string {
-					RunDebug(context.Background(), n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, "", "")
+				Expect(VeleroScheduleCreate(s.Ctx, s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, s.ScheduleName, s.ScheduleArgs)).To(Succeed(), func() string {
+					RunDebug(context.Background(), s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, "", "")
 					return "Fail to create schedule"
 				})
 				break
@@ -100,13 +100,13 @@ func (n *ScheduleBackupCreation) Backup() error {
 		time.Sleep(1 * time.Minute)
 	})
 
-	By(fmt.Sprintf("Get backups every %d minute, and backups count should increase 1 more step in the same pace\n", n.Period), func() {
-		for i := 1; i <= n.verifyTimes; i++ {
-			fmt.Printf("Start to sleep %d minute #%d time...\n", n.podSleepDuration, i)
+	By(fmt.Sprintf("Get backups every %d minute, and backups count should increase 1 more step in the same pace\n", s.Period), func() {
+		for i := 1; i <= s.verifyTimes; i++ {
+			fmt.Printf("Start to sleep %d minute #%d time...\n", s.podSleepDuration, i)
 			mi, _ := time.ParseDuration("60s")
-			time.Sleep(n.podSleepDuration + mi)
+			time.Sleep(s.podSleepDuration + mi)
 			bMap := make(map[string]string)
-			backupsInfo, err := GetScheduledBackupsCreationTime(n.Ctx, n.VeleroCfg.VeleroCLI, "default", n.ScheduleName)
+			backupsInfo, err := GetScheduledBackupsCreationTime(s.Ctx, s.VeleroCfg.VeleroCLI, "default", s.ScheduleName)
 			Expect(err).To(Succeed())
 			Expect(backupsInfo).To(HaveLen(i))
 			for index, bi := range backupsInfo {
@@ -116,19 +116,19 @@ func (n *ScheduleBackupCreation) Backup() error {
 				_, err := time.Parse("2006-01-02 15:04:05 -0700 MST", bList[1])
 				Expect(err).To(Succeed())
 			}
-			if i == n.verifyTimes-1 {
+			if i == s.verifyTimes-1 {
 				backupInfo := backupsInfo[rand.Intn(len(backupsInfo))]
-				n.randBackupName = strings.Split(backupInfo, ",")[0]
+				s.randBackupName = strings.Split(backupInfo, ",")[0]
 			}
 		}
 	})
 	return nil
 }
 
-func (n *ScheduleBackupCreation) Clean() error {
-	if !n.VeleroCfg.Debug {
-		Expect(VeleroScheduleDelete(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed())
-		Expect(n.TestCase.Clean()).To(Succeed())
+func (s *ScheduleBackupCreation) Clean() error {
+	if !s.VeleroCfg.FailFast {
+		Expect(VeleroScheduleDelete(s.Ctx, s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, s.ScheduleName)).To(Succeed())
+		Expect(s.TestCase.Clean()).To(Succeed())
 	}
 	return nil
 }

test/e2e/schedule/schedule.go

@@ -203,7 +203,7 @@ func (n *ScheduleBackup) Verify() error {
 }
 
 func (n *ScheduleBackup) Clean() error {
-	if !n.VeleroCfg.Debug {
+	if !n.VeleroCfg.FailFast {
 		Expect(VeleroScheduleDelete(n.Ctx, n.VeleroCfg.VeleroCLI, n.VeleroCfg.VeleroNamespace, n.ScheduleName)).To(Succeed())
 		Expect(n.TestCase.Clean()).To(Succeed())
 	}