From 82fc557bd112ae492506798cf58693a7c6998cc3 Mon Sep 17 00:00:00 2001 From: danfengl Date: Fri, 29 Mar 2024 02:29:27 +0000 Subject: [PATCH] Add checkpoint of VSC for data movement migration test 1. In data movement scenario, volumesnapshotcontent by Velero backup will be deleted instead of retained in CSI scenaito, so add a checkpoint for data movement scenario to verify no volumesnapshotcontent left after Velero backup; 2. Fix global context varaible issue, context varaible is not effective due to it's initialized right after the very beginning of all tests instead of beginning of each test, so if someone script a new E2E test and did not overwrite it in the test body, then it will fail the test if it was triggerd one hour later; 3. Due to CSI plugin is deprecated, it breaked down migration tests, because v1.13 still needs to install CSI plugin for the test. Signed-off-by: danfengl --- test/e2e/backups/ttl.go | 2 +- test/e2e/basic/backup-volume-info/base.go | 42 ++++--- .../backup-volume-info/native_snapshot.go | 5 +- test/e2e/basic/namespace-mapping.go | 2 - test/e2e/basic/nodeport.go | 4 +- test/e2e/basic/pvc-selected-node-changing.go | 2 - test/e2e/basic/resources-check/namespaces.go | 1 - .../resources-check/namespaces_annotation.go | 3 - test/e2e/basic/resources-check/rbac.go | 3 - test/e2e/basic/storage-class-changing.go | 3 +- test/e2e/migration/migration.go | 103 ++++++++++-------- test/e2e/pv-backup/pv-backup-filter.go | 2 - test/e2e/resource-filtering/base.go | 2 - test/e2e/resource-filtering/exclude_label.go | 3 - .../resource-filtering/exclude_namespaces.go | 3 - .../resource-filtering/include_namespaces.go | 3 - test/e2e/resource-filtering/label_selector.go | 3 - .../resourcemodifiers/resource_modifiers.go | 5 - .../e2e/resourcepolicies/resource_policies.go | 5 - test/e2e/schedule/ordered_resources.go | 3 - test/e2e/schedule/schedule-backup-creation.go | 1 - test/e2e/schedule/schedule.go | 2 - test/e2e/test/test.go | 91 +++++++--------- test/e2e/upgrade/upgrade.go | 9 +- test/types.go | 4 +- test/util/csi/common.go | 94 +++++++--------- test/util/k8s/common.go | 3 +- test/util/k8s/namespace.go | 7 +- test/util/k8s/rbac.go | 2 +- test/util/k8s/serviceaccount.go | 11 +- test/util/kibishii/kibishii_utils.go | 16 ++- test/util/providers/common.go | 2 +- test/util/velero/install.go | 5 +- test/util/velero/velero_utils.go | 27 ++--- 34 files changed, 203 insertions(+), 270 deletions(-) diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 409773264f..0e5a4d653d 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -139,7 +139,7 @@ func TTLTest() { fmt.Sprintf("Failed to delete namespace %s", BackupCfg.BackupName)) }) - if veleroCfg.CloudProvider == Aws && useVolumeSnapshots { + if veleroCfg.CloudProvider == AWS && useVolumeSnapshots { fmt.Println("Waiting 7 minutes to make sure the snapshots are ready...") time.Sleep(7 * time.Minute) } diff --git a/test/e2e/basic/backup-volume-info/base.go b/test/e2e/basic/backup-volume-info/base.go index b23fe15671..e6ba12ce1a 100644 --- a/test/e2e/basic/backup-volume-info/base.go +++ b/test/e2e/basic/backup-volume-info/base.go @@ -17,7 +17,6 @@ limitations under the License. package basic import ( - "context" "fmt" "strconv" "strings" @@ -45,26 +44,6 @@ type BackupVolumeInfo struct { func (v *BackupVolumeInfo) Init() error { v.TestCase.Init() - - BeforeEach(func() { - if v.VeleroCfg.CloudProvider == Vsphere && (!strings.Contains(v.CaseBaseName, "fs-upload") && !strings.Contains(v.CaseBaseName, "skipped")) { - fmt.Printf("Skip snapshot case %s for vsphere environment.\n", v.CaseBaseName) - Skip("Skip snapshot case due to vsphere environment doesn't cover the CSI test, and it doesn't have a Velero native snapshot plugin.") - } - - if strings.Contains(v.VeleroCfg.Features, FeatureCSI) { - if strings.Contains(v.CaseBaseName, "native-snapshot") { - fmt.Printf("Skip native snapshot case %s when the CSI feature is enabled.\n", v.CaseBaseName) - Skip("Skip native snapshot case due to CSI feature is enabled.") - } - } else { - if strings.Contains(v.CaseBaseName, "csi") { - fmt.Printf("Skip CSI related case %s when the CSI feature is not enabled.\n", v.CaseBaseName) - Skip("Skip CSI cases due to CSI feature is not enabled.") - } - } - }) - v.CaseBaseName = v.CaseBaseName + v.UUIDgen v.BackupName = "backup-" + v.CaseBaseName v.RestoreName = "restore-" + v.CaseBaseName @@ -98,8 +77,27 @@ func (v *BackupVolumeInfo) Init() error { return nil } +func (v *BackupVolumeInfo) Start() error { + if v.VeleroCfg.CloudProvider == Vsphere && (!strings.Contains(v.CaseBaseName, "fs-upload") && !strings.Contains(v.CaseBaseName, "skipped")) { + fmt.Printf("Skip snapshot case %s for vsphere environment.\n", v.CaseBaseName) + Skip("Skip snapshot case due to vsphere environment doesn't cover the CSI test, and it doesn't have a Velero native snapshot plugin.") + } + + if strings.Contains(v.VeleroCfg.Features, FeatureCSI) { + if strings.Contains(v.CaseBaseName, "native-snapshot") { + fmt.Printf("Skip native snapshot case %s when the CSI feature is enabled.\n", v.CaseBaseName) + Skip("Skip native snapshot case due to CSI feature is enabled.") + } + } else { + if strings.Contains(v.CaseBaseName, "csi") { + fmt.Printf("Skip CSI related case %s when the CSI feature is not enabled.\n", v.CaseBaseName) + Skip("Skip CSI cases due to CSI feature is not enabled.") + } + } + v.TestCase.Start() + return nil +} func (v *BackupVolumeInfo) CreateResources() error { - v.Ctx, v.CtxCancel = context.WithTimeout(context.Background(), v.TimeoutDuration) labels := map[string]string{ "volume-info": "true", } diff --git a/test/e2e/basic/backup-volume-info/native_snapshot.go b/test/e2e/basic/backup-volume-info/native_snapshot.go index fb3a6f3d8c..c8ec0be1b5 100644 --- a/test/e2e/basic/backup-volume-info/native_snapshot.go +++ b/test/e2e/basic/backup-volume-info/native_snapshot.go @@ -30,7 +30,8 @@ var NativeSnapshotVolumeInfoTest func() = TestFunc(&NativeSnapshotVolumeInfo{ BackupVolumeInfo{ SnapshotVolumes: true, TestCase: TestCase{ - CaseBaseName: "native-snapshot-volumeinfo", + UseVolumeSnapshots: true, + CaseBaseName: "native-snapshot-volumeinfo", TestMsg: &TestMSG{ Desc: "Test backup's VolumeInfo metadata content for native snapshot case.", Text: "The VolumeInfo should be generated, and the NativeSnapshotInfo structure should not be nil.", @@ -54,7 +55,7 @@ func (n *NativeSnapshotVolumeInfo) Verify() error { BackupObjectsPrefix+"/"+n.BackupName, ) - Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Fail to get VolumeInfo metadata in the Backup Repository.")) + Expect(err).ShouldNot(HaveOccurred(), "Fail to get VolumeInfo metadata in the Backup Repository.") fmt.Printf("The VolumeInfo metadata content: %+v\n", *volumeInfo[0]) Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true)) diff --git a/test/e2e/basic/namespace-mapping.go b/test/e2e/basic/namespace-mapping.go index c85fe247e0..48aa0cdd5c 100644 --- a/test/e2e/basic/namespace-mapping.go +++ b/test/e2e/basic/namespace-mapping.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "strings" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -86,7 +85,6 @@ func (n *NamespaceMapping) Init() error { } func (n *NamespaceMapping) CreateResources() error { - n.Ctx, n.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) for index, ns := range *n.NSIncluded { n.kibishiiData.Levels = len(*n.NSIncluded) + index By(fmt.Sprintf("Creating namespaces ...%s\n", ns), func() { diff --git a/test/e2e/basic/nodeport.go b/test/e2e/basic/nodeport.go index 846edb9a2a..a127d0d391 100644 --- a/test/e2e/basic/nodeport.go +++ b/test/e2e/basic/nodeport.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "strings" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -61,9 +60,8 @@ func (n *NodePort) Init() error { } return nil } -func (n *NodePort) CreateResources() error { - n.Ctx, n.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) +func (n *NodePort) CreateResources() error { for _, ns := range *n.NSIncluded { By(fmt.Sprintf("Creating service %s in namespaces %s ......\n", n.serviceName, ns), func() { Expect(CreateNamespace(n.Ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) diff --git a/test/e2e/basic/pvc-selected-node-changing.go b/test/e2e/basic/pvc-selected-node-changing.go index e073946158..073c874df6 100644 --- a/test/e2e/basic/pvc-selected-node-changing.go +++ b/test/e2e/basic/pvc-selected-node-changing.go @@ -3,7 +3,6 @@ package basic import ( "context" "fmt" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -64,7 +63,6 @@ func (p *PVCSelectedNodeChanging) Init() error { } func (p *PVCSelectedNodeChanging) CreateResources() error { - p.Ctx, p.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) By(fmt.Sprintf("Create namespace %s", p.namespace), func() { Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", p.namespace)) diff --git a/test/e2e/basic/resources-check/namespaces.go b/test/e2e/basic/resources-check/namespaces.go index e4a88d4a77..b99154a563 100644 --- a/test/e2e/basic/resources-check/namespaces.go +++ b/test/e2e/basic/resources-check/namespaces.go @@ -84,7 +84,6 @@ func (m *MultiNSBackup) Init() error { } func (m *MultiNSBackup) CreateResources() error { - m.Ctx, m.CtxCancel = context.WithTimeout(context.Background(), m.TimeoutDuration) fmt.Printf("Creating namespaces ...\n") labels := map[string]string{ "ns-test": "true", diff --git a/test/e2e/basic/resources-check/namespaces_annotation.go b/test/e2e/basic/resources-check/namespaces_annotation.go index c8642c22c7..e698d48185 100644 --- a/test/e2e/basic/resources-check/namespaces_annotation.go +++ b/test/e2e/basic/resources-check/namespaces_annotation.go @@ -17,10 +17,8 @@ limitations under the License. package basic import ( - "context" "fmt" "strings" - "time" "github.com/pkg/errors" @@ -63,7 +61,6 @@ func (n *NSAnnotationCase) Init() error { } func (n *NSAnnotationCase) CreateResources() error { - n.Ctx, n.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) for nsNum := 0; nsNum < n.NamespacesTotal; nsNum++ { createNSName := fmt.Sprintf("%s-%00000d", n.CaseBaseName, nsNum) createAnnotationName := fmt.Sprintf("annotation-%s-%00000d", n.CaseBaseName, nsNum) diff --git a/test/e2e/basic/resources-check/rbac.go b/test/e2e/basic/resources-check/rbac.go index 37e2ce5fbc..7afedd13c6 100644 --- a/test/e2e/basic/resources-check/rbac.go +++ b/test/e2e/basic/resources-check/rbac.go @@ -33,10 +33,8 @@ limitations under the License. package basic import ( - "context" "fmt" "strings" - "time" "github.com/pkg/errors" @@ -79,7 +77,6 @@ func (r *RBACCase) Init() error { } func (r *RBACCase) CreateResources() error { - r.Ctx, r.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ { createNSName := fmt.Sprintf("%s-%00000d", r.CaseBaseName, nsNum) fmt.Printf("Creating namespaces ...%s\n", createNSName) diff --git a/test/e2e/basic/storage-class-changing.go b/test/e2e/basic/storage-class-changing.go index fae10c9f89..8bd344700c 100644 --- a/test/e2e/basic/storage-class-changing.go +++ b/test/e2e/basic/storage-class-changing.go @@ -3,7 +3,6 @@ package basic import ( "context" "fmt" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -67,11 +66,11 @@ func (s *StorageClasssChanging) Init() error { } return nil } + func (s *StorageClasssChanging) CreateResources() error { label := map[string]string{ "app": "test", } - s.Ctx, s.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) By(("Installing storage class..."), func() { Expect(InstallTestStorageClasses(fmt.Sprintf("../testdata/storage-class/%s.yaml", s.VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install storage class") diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index f84a3f7a54..c57b04b469 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -27,6 +27,7 @@ import ( . "github.com/onsi/gomega" . "github.com/vmware-tanzu/velero/test" + util "github.com/vmware-tanzu/velero/test/util/csi" . "github.com/vmware-tanzu/velero/test/util/k8s" . "github.com/vmware-tanzu/velero/test/util/kibishii" . "github.com/vmware-tanzu/velero/test/util/providers" @@ -54,9 +55,11 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) var ( backupName, restoreName string backupScName, restoreScName string + kibishiiWorkerCount int err error ) BeforeEach(func() { + kibishiiWorkerCount = 3 veleroCfg = VeleroCfg UUIDgen, err = uuid.NewRandom() migrationNamespace = "migration-" + UUIDgen.String() @@ -116,10 +119,6 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) if !useVolumeSnapshots { Skip("FSB migration test is not needed in data mover scenario") } - // TODO: remove this block once Velero version in cluster A is great than V1.11 for all migration path. - if veleroCLI2Version.VeleroVersion != "self" { - Skip(fmt.Sprintf("Only V1.12 support data mover scenario instead of %s", veleroCLI2Version.VeleroVersion)) - } } oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60) defer ctxCancel() @@ -128,20 +127,46 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) Expect(err).To(Succeed()) supportUploaderType, err := IsSupportUploaderType(veleroCLI2Version.VeleroVersion) Expect(err).To(Succeed()) + + OriginVeleroCfg := veleroCfg if veleroCLI2Version.VeleroCLI == "" { //Assume tag of velero server image is identical to velero CLI version //Download velero CLI if it's empty according to velero CLI version By(fmt.Sprintf("Install the expected version Velero CLI (%s) for installing Velero", veleroCLI2Version.VeleroVersion), func() { + //"self" represents 1.14.x and future versions if veleroCLI2Version.VeleroVersion == "self" { veleroCLI2Version.VeleroCLI = veleroCfg.VeleroCLI } else { + fmt.Printf("Using default images address of Velero CLI %s\n", veleroCLI2Version.VeleroVersion) + OriginVeleroCfg.VeleroImage = "" + OriginVeleroCfg.RestoreHelperImage = "" + OriginVeleroCfg.Plugins = "" + + // It is for v1.13.x migration scenario only, because since v1.14, nightly CI won't + // pass velero-plugin-for-csi to E2E test anymore, and velero installation will not + // fetch velero-plugin-for-csi automatically, so add it as hardcode below. + // TODO: remove this section from future version like v1.15, e.g. + if OriginVeleroCfg.CloudProvider == Azure { + OriginVeleroCfg.Plugins = "velero/velero-plugin-for-microsoft-azure:v1.9.0" + } + if OriginVeleroCfg.CloudProvider == AWS { + OriginVeleroCfg.Plugins = "velero/velero-plugin-for-aws:v1.9.0" + } + if strings.Contains(OriginVeleroCfg.Features, FeatureCSI) { + OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + ",velero/velero-plugin-for-csi:v0.7.0" + } + if OriginVeleroCfg.SnapshotMoveData { + if OriginVeleroCfg.CloudProvider == Azure { + OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + ",velero/velero-plugin-for-aws:v1.9.0" + } + } veleroCLI2Version.VeleroCLI, err = InstallVeleroCLI(veleroCLI2Version.VeleroVersion) Expect(err).To(Succeed()) } }) } - OriginVeleroCfg := veleroCfg + By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultClusterContext), func() { Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) OriginVeleroCfg.MigrateFromVeleroVersion = veleroCLI2Version.VeleroVersion @@ -156,21 +181,10 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) Expect(err).To(Succeed(), "Fail to get Velero version") OriginVeleroCfg.VeleroVersion = version - // self represents v1.12 - if veleroCLI2Version.VeleroVersion == "self" { - if OriginVeleroCfg.SnapshotMoveData { - OriginVeleroCfg.UseNodeAgent = true - } - } else { - Expect(err).To(Succeed()) - fmt.Printf("Using default images address of Velero CLI %s\n", veleroCLI2Version.VeleroVersion) - OriginVeleroCfg.VeleroImage = "" - OriginVeleroCfg.RestoreHelperImage = "" - OriginVeleroCfg.Plugins = "" - //TODO: Remove this setting when migration path is from 1.13 to higher version - //TODO: or self, because version 1.12 and older versions have no this parameter. - OriginVeleroCfg.WithoutDisableInformerCacheParam = true + if OriginVeleroCfg.SnapshotMoveData { + OriginVeleroCfg.UseNodeAgent = true } + Expect(VeleroInstall(context.Background(), &OriginVeleroCfg, false)).To(Succeed()) if veleroCLI2Version.VeleroVersion != "self" { Expect(CheckVeleroVersion(context.Background(), OriginVeleroCfg.VeleroCLI, @@ -190,10 +204,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) KibishiiData := *DefaultKibishiiData By("Deploy sample workload of Kibishii", func() { - if OriginVeleroCfg.SnapshotMoveData { - KibishiiData.ExpectedNodes = 3 - } - + KibishiiData.ExpectedNodes = kibishiiWorkerCount Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *veleroCfg.DefaultClient, veleroCfg.CloudProvider, migrationNamespace, veleroCfg.RegistryCredentialFile, veleroCfg.Features, veleroCfg.KibishiiDirectory, useVolumeSnapshots, &KibishiiData)).To(Succeed()) @@ -238,54 +249,50 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) // TODO - remove after upload progress monitoring is implemented By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(context.Background(), time.Hour, - migrationNamespace, 2)).To(Succeed()) + migrationNamespace, kibishiiWorkerCount)).To(Succeed()) }) } var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = migrationNamespace - if !OriginVeleroCfg.SnapshotMoveData { - By("Snapshot should be created in cloud object store", func() { - snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, 2, - migrationNamespace, backupName, KibishiiPVCNameList) + if OriginVeleroCfg.SnapshotMoveData { + //VolumeSnapshotContent should be deleted after data movement + _, err := util.CheckVolumeSnapshotCR(*veleroCfg.DefaultClient, map[string]string{"namespace": migrationNamespace}, 0) + Expect(err).NotTo(HaveOccurred(), "VSC count is not as expected 0") + } else { + // the snapshots of AWS may be still in pending status when do the restore, wait for a while + // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 + // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed + if veleroCfg.CloudProvider == Azure && strings.EqualFold(veleroCfg.Features, FeatureCSI) || veleroCfg.CloudProvider == AWS { + By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { + time.Sleep(5 * time.Minute) + }) + } + + By("Snapshot should be created in cloud object store with retain policy", func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, kibishiiWorkerCount, + migrationNamespace, backupName, GetKibishiiPVCNameList(kibishiiWorkerCount)) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) }) - } else { - //TODO: checkpoint for datamover } } - if useVolumeSnapshots && veleroCfg.CloudProvider == Azure && - strings.EqualFold(veleroCfg.Features, FeatureCSI) && - !OriginVeleroCfg.SnapshotMoveData { - By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { - time.Sleep(5 * time.Minute) - }) - } - // the snapshots of AWS may be still in pending status when do the restore, wait for a while - // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 - // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if veleroCfg.CloudProvider == Aws && useVolumeSnapshots && !OriginVeleroCfg.SnapshotMoveData { - fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") - time.Sleep(5 * time.Minute) - } - By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyClusterContext), func() { //Ensure workload of "migrationNamespace" existed in cluster-A ns, err := GetNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace) Expect(ns.Name).To(Equal(migrationNamespace)) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("get namespace in cluster-B err: %v", err)) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("get namespace in source cluster err: %v", err)) //Ensure cluster-B is the target cluster Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed()) _, err = GetNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace) - Expect(err).To(HaveOccurred()) - strings.Contains(fmt.Sprint(err), "namespaces \""+migrationNamespace+"\" not found") + Expect(err).To(HaveOccurred(), fmt.Sprintf("get namespace in dst cluster successfully, it's not as expected: %s", migrationNamespace)) fmt.Println(err) + Expect(strings.Contains(fmt.Sprint(err), "namespaces \""+migrationNamespace+"\" not found")).Should(BeTrue()) veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient veleroCfg.ClusterToInstallVelero = veleroCfg.StandbyClusterName diff --git a/test/e2e/pv-backup/pv-backup-filter.go b/test/e2e/pv-backup/pv-backup-filter.go index a858743f95..90c5cfdea6 100644 --- a/test/e2e/pv-backup/pv-backup-filter.go +++ b/test/e2e/pv-backup/pv-backup-filter.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "strings" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -64,7 +63,6 @@ func (p *PVBackupFiltering) Init() error { } func (p *PVBackupFiltering) CreateResources() error { - p.Ctx, p.CtxCancel = context.WithTimeout(context.Background(), 30*time.Minute) err := InstallStorageClass(p.Ctx, fmt.Sprintf("../testdata/storage-class/%s.yaml", p.VeleroCfg.CloudProvider)) if err != nil { return errors.Wrapf(err, "failed to install storage class for pv backup filtering test") diff --git a/test/e2e/resource-filtering/base.go b/test/e2e/resource-filtering/base.go index 61be331c37..8cfce438d5 100644 --- a/test/e2e/resource-filtering/base.go +++ b/test/e2e/resource-filtering/base.go @@ -19,7 +19,6 @@ package filtering import ( "context" "fmt" - "time" "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -61,7 +60,6 @@ func (f *FilteringCase) Init() error { } func (f *FilteringCase) CreateResources() error { - f.Ctx, f.CtxCancel = context.WithTimeout(context.Background(), 30*time.Minute) for nsNum := 0; nsNum < f.NamespacesTotal; nsNum++ { namespace := fmt.Sprintf("%s-%00000d", f.CaseBaseName, nsNum) fmt.Printf("Creating resources in namespace ...%s\n", namespace) diff --git a/test/e2e/resource-filtering/exclude_label.go b/test/e2e/resource-filtering/exclude_label.go index ecf26878ef..27f7f53e6c 100644 --- a/test/e2e/resource-filtering/exclude_label.go +++ b/test/e2e/resource-filtering/exclude_label.go @@ -17,9 +17,7 @@ limitations under the License. package filtering import ( - "context" "fmt" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -76,7 +74,6 @@ func (e *ExcludeFromBackup) Init() error { } func (e *ExcludeFromBackup) CreateResources() error { - e.Ctx, e.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) namespace := e.CaseBaseName // These 2 labels for resources to be included label1 := map[string]string{ diff --git a/test/e2e/resource-filtering/exclude_namespaces.go b/test/e2e/resource-filtering/exclude_namespaces.go index f4e22e852f..1b8e5da550 100644 --- a/test/e2e/resource-filtering/exclude_namespaces.go +++ b/test/e2e/resource-filtering/exclude_namespaces.go @@ -17,10 +17,8 @@ limitations under the License. package filtering import ( - "context" "fmt" "strings" - "time" "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -108,7 +106,6 @@ func (e *ExcludeNamespaces) Init() error { } func (e *ExcludeNamespaces) CreateResources() error { - e.Ctx, e.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { createNSName := fmt.Sprintf("%s-%00000d", e.CaseBaseName, nsNum) fmt.Printf("Creating namespaces ...%s\n", createNSName) diff --git a/test/e2e/resource-filtering/include_namespaces.go b/test/e2e/resource-filtering/include_namespaces.go index d39c2acb94..d511de2123 100644 --- a/test/e2e/resource-filtering/include_namespaces.go +++ b/test/e2e/resource-filtering/include_namespaces.go @@ -17,10 +17,8 @@ limitations under the License. package filtering import ( - "context" "fmt" "strings" - "time" "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -103,7 +101,6 @@ func (i *IncludeNamespaces) Init() error { } func (i *IncludeNamespaces) CreateResources() error { - i.Ctx, i.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) for nsNum := 0; nsNum < i.NamespacesTotal; nsNum++ { createNSName := fmt.Sprintf("%s-%00000d", i.CaseBaseName, nsNum) fmt.Printf("Creating namespaces ...%s\n", createNSName) diff --git a/test/e2e/resource-filtering/label_selector.go b/test/e2e/resource-filtering/label_selector.go index 21610ec040..1d566b5853 100644 --- a/test/e2e/resource-filtering/label_selector.go +++ b/test/e2e/resource-filtering/label_selector.go @@ -17,10 +17,8 @@ limitations under the License. package filtering import ( - "context" "fmt" "strings" - "time" "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -75,7 +73,6 @@ func (l *LabelSelector) Init() error { } func (l *LabelSelector) CreateResources() error { - l.Ctx, l.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) for nsNum := 0; nsNum < l.NamespacesTotal; nsNum++ { namespace := fmt.Sprintf("%s-%00000d", l.CaseBaseName, nsNum) fmt.Printf("Creating resources in namespace ...%s\n", namespace) diff --git a/test/e2e/resourcemodifiers/resource_modifiers.go b/test/e2e/resourcemodifiers/resource_modifiers.go index f87965d2fa..cc1d6d3f18 100644 --- a/test/e2e/resourcemodifiers/resource_modifiers.go +++ b/test/e2e/resourcemodifiers/resource_modifiers.go @@ -17,10 +17,8 @@ limitations under the License. package resourcemodifiers import ( - "context" "fmt" "strings" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -97,9 +95,6 @@ func (r *ResourceModifiersCase) Init() error { } func (r *ResourceModifiersCase) CreateResources() error { - // It's better to set a global timeout in CreateResources function which is the real beginning of one e2e test - r.Ctx, r.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) - By(fmt.Sprintf("Create configmap %s in namespaces %s for workload\n", r.cmName, r.VeleroCfg.VeleroNamespace), func() { Expect(CreateConfigMapFromYAMLData(r.Client.ClientGo, r.yamlConfig, r.cmName, r.VeleroCfg.VeleroNamespace)).To(Succeed(), fmt.Sprintf("Failed to create configmap %s in namespaces %s for workload\n", r.cmName, r.VeleroCfg.VeleroNamespace)) }) diff --git a/test/e2e/resourcepolicies/resource_policies.go b/test/e2e/resourcepolicies/resource_policies.go index c98adfdb3b..7502f8a0d5 100644 --- a/test/e2e/resourcepolicies/resource_policies.go +++ b/test/e2e/resourcepolicies/resource_policies.go @@ -17,10 +17,8 @@ limitations under the License. package filtering import ( - "context" "fmt" "strings" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -103,9 +101,6 @@ func (r *ResourcePoliciesCase) Init() error { } func (r *ResourcePoliciesCase) CreateResources() error { - // It's better to set a global timeout in CreateResources function which is the real beginning of one e2e test - r.Ctx, r.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) - By(("Installing storage class..."), func() { Expect(InstallTestStorageClasses(fmt.Sprintf("../testdata/storage-class/%s.yaml", r.VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install storage class") }) diff --git a/test/e2e/schedule/ordered_resources.go b/test/e2e/schedule/ordered_resources.go index 4724c7d7c7..8ae7fb56d3 100644 --- a/test/e2e/schedule/ordered_resources.go +++ b/test/e2e/schedule/ordered_resources.go @@ -18,7 +18,6 @@ limitations under the License. //the ordered resources test related to https://github.com/vmware-tanzu/velero/issues/4561 import ( - "context" "fmt" "strings" "time" @@ -70,9 +69,7 @@ func (o *OrderedResources) Init() error { return nil } - func (o *OrderedResources) CreateResources() error { - o.Ctx, o.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) label := map[string]string{ "orderedresources": "true", } diff --git a/test/e2e/schedule/schedule-backup-creation.go b/test/e2e/schedule/schedule-backup-creation.go index 437afde75f..b87169df66 100644 --- a/test/e2e/schedule/schedule-backup-creation.go +++ b/test/e2e/schedule/schedule-backup-creation.go @@ -63,7 +63,6 @@ func (n *ScheduleBackupCreation) Init() error { } func (p *ScheduleBackupCreation) CreateResources() error { - p.Ctx, p.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) By(fmt.Sprintf("Create namespace %s", p.namespace), func() { Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", p.namespace)) diff --git a/test/e2e/schedule/schedule.go b/test/e2e/schedule/schedule.go index 959000c29a..99e4c01ddb 100644 --- a/test/e2e/schedule/schedule.go +++ b/test/e2e/schedule/schedule.go @@ -47,9 +47,7 @@ func (n *ScheduleBackup) Init() error { Expect(n.Period).To(BeNumerically("<", 30)) return nil } - func (n *ScheduleBackup) CreateResources() error { - n.Ctx, n.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) for _, ns := range *n.NSIncluded { By(fmt.Sprintf("Creating namespaces %s ......\n", ns), func() { Expect(CreateNamespace(n.Ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index 89a8fa71d1..9f64f3b1f4 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -18,7 +18,6 @@ package test import ( "context" - "flag" "fmt" "math/rand" "strings" @@ -48,6 +47,7 @@ type VeleroBackupRestoreTest interface { Restore() error Verify() error Clean() error + Start() error GetTestMsg() *TestMSG GetTestCase() *TestCase } @@ -78,52 +78,26 @@ type TestCase struct { func TestFunc(test VeleroBackupRestoreTest) func() { return func() { - Expect(test.Init()).To(Succeed(), "Failed to instantiate test cases") - BeforeEach(func() { - flag.Parse() - // Using the global velero config which covered the installation for most common cases - veleroCfg := test.GetTestCase().VeleroCfg - // TODO: Skip nodeport test until issue https://github.com/kubernetes/kubernetes/issues/114384 fixed - // TODO: Although this issue is closed, but it's not fixed. - // TODO: After bump up k8s version in AWS pipeline, this issue also apply for AWS pipeline. - if (veleroCfg.CloudProvider == Azure || veleroCfg.CloudProvider == Aws) && strings.Contains(test.GetTestCase().CaseBaseName, "nodeport") { - Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS") - } - if InstallVelero { - Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName, veleroCfg)).To(Succeed()) - } - }) - It(test.GetTestMsg().Text, func() { - Expect(RunTestCase(test)).To(Succeed(), test.GetTestMsg().FailedMSG) - }) + TestIt(test) } } func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() { return func() { for k := range tests { - Expect(tests[k].Init()).To(Succeed(), fmt.Sprintf("Failed to instantiate test %s case", tests[k].GetTestMsg().Desc)) - defer tests[k].GetTestCase().CtxCancel() - } - - BeforeEach(func() { - flag.Parse() - if InstallVelero { - Expect(PrepareVelero(context.Background(), tests[0].GetTestCase().CaseBaseName, tests[0].GetTestCase().VeleroCfg)).To(Succeed()) - } - }) - - for k := range tests { - curTest := tests[k] - It(curTest.GetTestMsg().Text, func() { - Expect(RunTestCase(curTest)).To(Succeed(), curTest.GetTestMsg().FailedMSG) - }) + TestIt(tests[k]) } } } +func TestIt(test VeleroBackupRestoreTest) error { + test.Init() + It(test.GetTestMsg().Text, func() { + Expect(RunTestCase(test)).To(Succeed(), test.GetTestMsg().FailedMSG) + }) + return nil +} func (t *TestCase) Init() error { - t.Ctx, t.CtxCancel = context.WithTimeout(context.Background(), 1*time.Hour) t.UUIDgen = t.GenerateUUID() t.VeleroCfg = VeleroCfg t.Client = *t.VeleroCfg.ClientToInstallVelero @@ -141,18 +115,13 @@ func (t *TestCase) CreateResources() error { func (t *TestCase) Backup() error { veleroCfg := t.GetTestCase().VeleroCfg - if err := VeleroBackupExec(t.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, t.BackupArgs); err != nil { - RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, "") - return errors.Wrapf(err, "Failed to backup resources") - } - // the snapshots of AWS may be still in pending status when do the restore, wait for a while - // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 - // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if t.UseVolumeSnapshots { - fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") - time.Sleep(5 * time.Minute) - } + By("Start to backup ......", func() { + Expect(VeleroBackupExec(t.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, t.BackupArgs)).To(Succeed(), func() string { + RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, "") + return "Failed to backup resources" + }) + }) return nil } @@ -170,13 +139,15 @@ func (t *TestCase) Restore() error { } veleroCfg := t.GetTestCase().VeleroCfg + // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if t.UseVolumeSnapshots && veleroCfg.CloudProvider != Vsphere { - fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") - time.Sleep(5 * time.Minute) - } + By("Waiting 5 minutes to make sure the snapshots are ready...", func() { + if t.UseVolumeSnapshots && veleroCfg.CloudProvider != Vsphere { + time.Sleep(5 * time.Minute) + } + }) By("Start to restore ......", func() { if t.RestorePhaseExpect == "" { @@ -194,6 +165,15 @@ func (t *TestCase) Verify() error { return nil } +func (t *TestCase) Start() error { + t.Ctx, t.CtxCancel = context.WithTimeout(context.Background(), 1*time.Hour) + veleroCfg := t.GetTestCase().VeleroCfg + if (veleroCfg.CloudProvider == Azure || veleroCfg.CloudProvider == AWS) && strings.Contains(t.GetTestCase().CaseBaseName, "nodeport") { + Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS") + } + return nil +} + func (t *TestCase) Clean() error { veleroCfg := t.GetTestCase().VeleroCfg if !veleroCfg.Debug { @@ -217,10 +197,17 @@ func (t *TestCase) GetTestCase() *TestCase { } func RunTestCase(test VeleroBackupRestoreTest) error { - fmt.Printf("Running test case %s %s\n", test.GetTestMsg().Desc, time.Now().Format("2006-01-02 15:04:05")) if test == nil { return errors.New("No case should be tested") } + test.Start() + defer test.GetTestCase().CtxCancel() + + fmt.Printf("Running test case %s %s\n", test.GetTestMsg().Desc, time.Now().Format("2006-01-02 15:04:05")) + + if InstallVelero { + Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName, test.GetTestCase().VeleroCfg)).To(Succeed()) + } defer test.Clean() diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 149ae76253..ef9e31c217 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -33,10 +33,7 @@ import ( . "github.com/vmware-tanzu/velero/test/util/velero" ) -const ( - upgradeNamespace = "upgrade-workload" -) - +var upgradeNamespace string var veleroCfg VeleroConfig func BackupUpgradeRestoreWithSnapshots() { @@ -62,6 +59,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC BeforeEach(func() { veleroCfg = VeleroCfg veleroCfg.IsUpgradeTest = true + UUIDgen, err = uuid.NewRandom() + upgradeNamespace = "upgrade-" + UUIDgen.String() if !InstallVelero { Skip("Upgrade test should not be triggered if veleroCfg.InstallVelero is set to false") } @@ -223,7 +222,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if tmpCfg.CloudProvider == Aws && useVolumeSnapshots { + if tmpCfg.CloudProvider == AWS && useVolumeSnapshots { fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") time.Sleep(5 * time.Minute) } diff --git a/test/types.go b/test/types.go index e64c6d677a..6ba28b549b 100644 --- a/test/types.go +++ b/test/types.go @@ -34,11 +34,11 @@ const Kind = "kind" const Azure = "azure" const AzureCSI = "azure-csi" const AwsCSI = "aws-csi" -const Aws = "aws" +const AWS = "aws" const Gcp = "gcp" const Vsphere = "vsphere" -var PublicCloudProviders = []string{Aws, Azure, Gcp, Vsphere} +var PublicCloudProviders = []string{AWS, Azure, Gcp, Vsphere} var LocalCloudProviders = []string{Kind, VanillaZFS} var CloudProviders = append(PublicCloudProviders, LocalCloudProviders...) diff --git a/test/util/csi/common.go b/test/util/csi/common.go index e336336ffc..9c10e962d9 100644 --- a/test/util/csi/common.go +++ b/test/util/csi/common.go @@ -21,6 +21,7 @@ import ( "fmt" "strings" + volumeSnapshotV1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned" "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -52,77 +53,47 @@ func GetClients() (*kubernetes.Clientset, *snapshotterClientSet.Clientset, error return client, snapshotterClient, nil } -func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error) { +func GetCsiSnapshotHandle(client TestClient, apiVersion string, index map[string]string) ([]string, error) { _, snapshotClient, err := GetClients() + + var vscList *volumeSnapshotV1.VolumeSnapshotContentList if err != nil { return nil, err } - vscList, err1 := snapshotClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{}) - if err1 != nil { - return nil, err + switch apiVersion { + // case "v1beta1": + // vscList, err = snapshotterClientSetV1beta1.SnapshotV1beta1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{}) + case "v1": + vscList, err = snapshotClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{}) + default: + errors.New(fmt.Sprintf("API version %s is not valid", apiVersion)) } - var snapshotHandleList []string - for _, i := range vscList.Items { - if i.Status == nil { - fmt.Println("SnapshotHandle Status s nil") - continue - } - if i.Status.SnapshotHandle == nil { - fmt.Println("SnapshotHandle is nil") - continue - } - - if i.Labels == nil { - fmt.Println("VolumeSnapshotContents label is nil") - continue - } - - if i.Labels["velero.io/backup-name"] == backupName { - tmp := strings.Split(*i.Status.SnapshotHandle, "/") - snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1]) - } - } - - if len(snapshotHandleList) == 0 { - fmt.Printf("No VolumeSnapshotContent from backup %s\n", backupName) - } - return snapshotHandleList, nil -} -func GetCsiSnapshotHandleV1(client TestClient, backupName string) ([]string, error) { - _, snapshotClient, err := GetClients() if err != nil { return nil, err } - vscList, err1 := snapshotClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{}) - if err1 != nil { - return nil, err - } var snapshotHandleList []string for _, i := range vscList.Items { if i.Status == nil { - fmt.Println("SnapshotHandle Status s nil") + fmt.Println("VolumeSnapshotContent status is nil") continue } if i.Status.SnapshotHandle == nil { fmt.Println("SnapshotHandle is nil") continue } - - if i.Labels == nil { - fmt.Println("VolumeSnapshotContents label is nil") - continue - } - - if i.Labels["velero.io/backup-name"] == backupName { + if (index["backupNameLabel"] != "" && i.Labels != nil && i.Labels["velero.io/backup-name"] == index["backupNameLabel"]) || + (index["namespace"] != "" && i.Spec.VolumeSnapshotRef.Namespace == index["namespace"]) { tmp := strings.Split(*i.Status.SnapshotHandle, "/") snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1]) } } if len(snapshotHandleList) == 0 { - fmt.Printf("No VolumeSnapshotContent from backup %s\n", backupName) + fmt.Printf("No VolumeSnapshotContent from key %v\n", index) + } else { + fmt.Printf("Volume snapshot content list: %v\n", snapshotHandleList) } return snapshotHandleList, nil } @@ -164,22 +135,35 @@ func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, ba return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace)) } -func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int, apiVersion string) ([]string, error) { +func CheckVolumeSnapshotCR(client TestClient, index map[string]string, expectedCount int) ([]string, error) { var err error var snapshotContentNameList []string - if apiVersion == "v1beta1" { - if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil { - return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") - } - } else if apiVersion == "v1" { - if snapshotContentNameList, err = GetCsiSnapshotHandleV1(client, backupName); err != nil { + + resourceName := "snapshot.storage.k8s.io" + + apiVersion, err := GetAPIVersions(&client, resourceName) + if err != nil { + return nil, err + } + if len(apiVersion) == 0 { + return nil, errors.New("Fail to get APIVersion") + } + // if apiVersion[0] == "v1beta1" { + // if snapshotContentNameList, err = GetCsiSnapshotHandle(client, apiVersion[0], index); err != nil { + // return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") + // } + // } else + if apiVersion[0] == "v1" { + if snapshotContentNameList, err = GetCsiSnapshotHandle(client, apiVersion[0], index); err != nil { return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") } } else { return nil, errors.New("API version is invalid") } - if len(snapshotContentNameList) != expectedCount { - return nil, errors.New(fmt.Sprintf("Snapshot content count %d is not as expect %d", len(snapshotContentNameList), expectedCount)) + if expectedCount >= 0 { + if len(snapshotContentNameList) != expectedCount { + return nil, errors.New(fmt.Sprintf("Snapshot content count %d is not as expect %d", len(snapshotContentNameList), expectedCount)) + } } fmt.Printf("snapshotContentNameList: %v \n", snapshotContentNameList) return snapshotContentNameList, nil diff --git a/test/util/k8s/common.go b/test/util/k8s/common.go index 097e3be2fd..9335815629 100644 --- a/test/util/k8s/common.go +++ b/test/util/k8s/common.go @@ -250,7 +250,6 @@ func GetAPIVersions(client *TestClient, name string) ([]string, error) { return nil, errors.Wrap(err, "Fail to get server API groups") } for _, group := range APIGroup.Groups { - fmt.Println(group.Name) if group.Name == name { for _, v := range group.Versions { fmt.Println(v.Version) @@ -259,7 +258,7 @@ func GetAPIVersions(client *TestClient, name string) ([]string, error) { return version, nil } } - return nil, errors.New("Server API groups is empty") + return nil, errors.New("Fail to get server API groups") } func GetPVByPVCName(client TestClient, namespace, pvcName string) (string, error) { diff --git a/test/util/k8s/namespace.go b/test/util/k8s/namespace.go index e747a4d152..b2a5cae0e6 100644 --- a/test/util/k8s/namespace.go +++ b/test/util/k8s/namespace.go @@ -42,7 +42,7 @@ func CreateNamespace(ctx context.Context, client TestClient, namespace string) e "pod-security.kubernetes.io/enforce": "baseline", "pod-security.kubernetes.io/enforce-version": "latest", } - _, err := client.ClientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + _, err := client.ClientGo.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return nil } @@ -70,7 +70,7 @@ func CreateNamespaceWithAnnotation(ctx context.Context, client TestClient, names "pod-security.kubernetes.io/enforce-version": "latest", } ns.ObjectMeta.Annotations = annotation - _, err := client.ClientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + _, err := client.ClientGo.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return nil } @@ -169,6 +169,9 @@ func CleanupNamespacesFiterdByExcludes(ctx context.Context, client TestClient, e } func CleanupNamespaces(ctx context.Context, client TestClient, CaseBaseName string) error { + if ctx == nil { + fmt.Println("ctx is nil ....") + } namespaces, err := client.ClientGo.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) if err != nil { return errors.Wrap(err, "Could not retrieve namespaces") diff --git a/test/util/k8s/rbac.go b/test/util/k8s/rbac.go index c6fa2559c6..634fdf84f1 100644 --- a/test/util/k8s/rbac.go +++ b/test/util/k8s/rbac.go @@ -34,7 +34,7 @@ func CreateRBACWithBindingSA(ctx context.Context, client TestClient, namespace s }, } - _, err = client.ClientGo.RbacV1().ClusterRoles().Create(ctx, role, metav1.CreateOptions{}) + _, err = client.ClientGo.RbacV1().ClusterRoles().Create(context.TODO(), role, metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { return err diff --git a/test/util/k8s/serviceaccount.go b/test/util/k8s/serviceaccount.go index 24d748aeaf..1af56a798a 100644 --- a/test/util/k8s/serviceaccount.go +++ b/test/util/k8s/serviceaccount.go @@ -23,14 +23,13 @@ import ( "time" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "github.com/vmware-tanzu/velero/pkg/builder" - - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func WaitUntilServiceAccountCreated(ctx context.Context, client TestClient, namespace, serviceAccount string, timeout time.Duration) error { @@ -81,7 +80,7 @@ func CreateServiceAccount(ctx context.Context, client TestClient, namespace stri AutomountServiceAccountToken: nil, } - _, err = client.ClientGo.CoreV1().ServiceAccounts(namespace).Create(ctx, sa, metav1.CreateOptions{}) + _, err = client.ClientGo.CoreV1().ServiceAccounts(namespace).Create(context.TODO(), sa, metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { return err @@ -90,5 +89,5 @@ func CreateServiceAccount(ctx context.Context, client TestClient, namespace stri } func GetServiceAccount(ctx context.Context, client TestClient, namespace string, serviceAccount string) (*corev1.ServiceAccount, error) { - return client.ClientGo.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccount, metav1.GetOptions{}) + return client.ClientGo.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), serviceAccount, metav1.GetOptions{}) } diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index 5046f4f21c..2fe223ea96 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -55,6 +55,14 @@ var KibishiiPodNameList = []string{"kibishii-deployment-0", "kibishii-deployment var KibishiiPVCNameList = []string{"kibishii-data-kibishii-deployment-0", "kibishii-data-kibishii-deployment-1"} var KibishiiStorageClassName = "kibishii-storage-class" +func GetKibishiiPVCNameList(workerCount int) []string { + var kibishiiPVCNameList []string + for i := 0; i < workerCount; i++ { + kibishiiPVCNameList = append(kibishiiPVCNameList, fmt.Sprintf("kibishii-data-kibishii-deployment-%d", i)) + } + return kibishiiPVCNameList +} + // RunKibishiiTests runs kibishii tests on the provider. func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string, useVolumeSnapshots, defaultVolumesToFsBackup bool) error { @@ -211,10 +219,10 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc } } - // the snapshots of AWS may be still in pending status when do the restore, wait for a while - // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 - // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed if useVolumeSnapshots { + // the snapshots of AWS may be still in pending status when do the restore, wait for a while + // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 + // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") time.Sleep(5 * time.Minute) } @@ -248,7 +256,7 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform, veler strings.EqualFold(veleroFeatures, FeatureCSI) { cloudPlatform = AzureCSI } - if strings.EqualFold(cloudPlatform, Aws) && + if strings.EqualFold(cloudPlatform, AWS) && strings.EqualFold(veleroFeatures, FeatureCSI) { cloudPlatform = AwsCSI } diff --git a/test/util/providers/common.go b/test/util/providers/common.go index 1b39f22712..99441d268e 100644 --- a/test/util/providers/common.go +++ b/test/util/providers/common.go @@ -75,7 +75,7 @@ func ObjectsShouldNotBeInBucket(objectStoreProvider, cloudCredentialsFile, bslBu func getProvider(cloudProvider string) (ObjectsInStorage, error) { var s ObjectsInStorage switch cloudProvider { - case Aws, Vsphere: + case AWS, Vsphere: aws := AWSStorage("") s = &aws case Gcp: diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 09fc7f3f77..48e2faf1d9 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -104,7 +104,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste // backup, but needed to pick up the provider plugins earlier. vSphere plugin no longer needs a Volume // Snapshot location specified if veleroCfg.ObjectStoreProvider == "" { - veleroCfg.ObjectStoreProvider = Aws + veleroCfg.ObjectStoreProvider = AWS } if err := configvSpherePlugin(veleroCfg); err != nil { return errors.WithMessagef(err, "Failed to config vsphere plugin") @@ -118,7 +118,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste // For AWS IRSA credential test, AWS IAM service account is required, so if ServiceAccountName and EKSPolicyARN // are both provided, we assume IRSA test is running, otherwise skip this IAM service account creation part. - if veleroCfg.CloudProvider == Aws && veleroInstallOptions.ServiceAccountName != "" { + if veleroCfg.CloudProvider == AWS && veleroInstallOptions.ServiceAccountName != "" { if veleroCfg.EKSPolicyARN == "" { return errors.New("Please provide EKSPolicyARN for IRSA test.") } @@ -155,6 +155,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste }) if err != nil { + time.Sleep(9 * time.Hour) RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", "") return errors.WithMessagef(err, "Failed to install Velero in the cluster") } diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index 21f895fddf..ac55a7c776 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -98,11 +98,12 @@ var pluginsMatrix = map[string]map[string][]string{ "csi": {"velero/velero-plugin-for-csi:v0.6.0"}, }, "v1.13": { - "aws": {"velero/velero-plugin-for-aws:v1.9.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.9.0"}, - "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.9.0"}, - "csi": {"velero/velero-plugin-for-csi:v0.7.0"}, + "aws": {"velero/velero-plugin-for-aws:v1.9.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.9.0"}, + "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.9.0"}, + "csi": {"velero/velero-plugin-for-csi:v0.7.0"}, + "datamover": {"velero/velero-plugin-for-aws:v1.9.0"}, }, "main": { "aws": {"velero/velero-plugin-for-aws:main"}, @@ -129,7 +130,7 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider string, nee var ok bool if slices.Contains(LocalCloudProviders, cloudProvider) { - plugins, ok = cloudMap[Aws] + plugins, ok = cloudMap[AWS] if !ok { return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, cloudProvider) } @@ -1231,24 +1232,16 @@ func GetRepositories(ctx context.Context, veleroNamespace, targetNamespace strin } func GetSnapshotCheckPoint(client TestClient, veleroCfg VeleroConfig, expectCount int, namespaceBackedUp, backupName string, KibishiiPVCNameList []string) (SnapshotCheckPoint, error) { + var err error var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.ExpectCount = expectCount snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp snapshotCheckPoint.PodName = KibishiiPVCNameList - if (veleroCfg.CloudProvider == Azure || veleroCfg.CloudProvider == Aws) && strings.EqualFold(veleroCfg.Features, FeatureCSI) { + if (veleroCfg.CloudProvider == Azure || veleroCfg.CloudProvider == AWS) && strings.EqualFold(veleroCfg.Features, FeatureCSI) { snapshotCheckPoint.EnableCSI = true - resourceName := "snapshot.storage.k8s.io" - - srcVersions, err := GetAPIVersions(veleroCfg.DefaultClient, resourceName) - if err != nil { - return snapshotCheckPoint, err - } - if len(srcVersions) == 0 { - return snapshotCheckPoint, errors.New("Fail to get APIVersion") - } - if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount, srcVersions[0]); err != nil { + if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, map[string]string{"backupNameLabel": backupName}, expectCount); err != nil { return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content") } }