diff --git a/.github/workflows/pr-linter-check.yml b/.github/workflows/pr-linter-check.yml index 36badcaa14..077753c73e 100644 --- a/.github/workflows/pr-linter-check.yml +++ b/.github/workflows/pr-linter-check.yml @@ -16,7 +16,7 @@ jobs: uses: actions/checkout@v4 - name: golangci-lint - uses: golangci/golangci-lint-action@v4 + uses: golangci/golangci-lint-action@v5 with: version: v1.57.2 args: --out-format=colored-line-number diff --git a/pkg/cmd/cli/install/install.go b/pkg/cmd/cli/install/install.go index 48629252d9..c51cee6586 100644 --- a/pkg/cmd/cli/install/install.go +++ b/pkg/cmd/cli/install/install.go @@ -164,9 +164,6 @@ func NewInstallOptions() *Options { DefaultSnapshotMoveData: false, DisableInformerCache: false, ScheduleSkipImmediately: false, - MaintenanceCfg: repository.MaintenanceConfig{ - KeepLatestMaitenanceJobs: repository.DefaultKeepLatestMaitenanceJobs, - }, } } diff --git a/pkg/controller/backup_repository_controller.go b/pkg/controller/backup_repository_controller.go index e4d68d9998..7e298d48dc 100644 --- a/pkg/controller/backup_repository_controller.go +++ b/pkg/controller/backup_repository_controller.go @@ -72,7 +72,7 @@ func (r *BackupRepoReconciler) SetupWithManager(mgr ctrl.Manager) error { s := kube.NewPeriodicalEnqueueSource(r.logger, mgr.GetClient(), &velerov1api.BackupRepositoryList{}, repoSyncPeriod, kube.PeriodicalEnqueueSourceOption{}) return ctrl.NewControllerManagedBy(mgr). - For(&velerov1api.BackupRepository{}). + For(&velerov1api.BackupRepository{}, builder.WithPredicates(kube.SpecChangePredicate{})). WatchesRawSource(s, nil). Watches(&velerov1api.BackupStorageLocation{}, kube.EnqueueRequestsFromMapUpdateFunc(r.invalidateBackupReposForBSL), builder.WithPredicates( diff --git a/site/content/community/_index.md b/site/content/community/_index.md index 19304273d1..c9eebef314 100644 --- a/site/content/community/_index.md +++ b/site/content/community/_index.md @@ -18,6 +18,6 @@ You can follow the work we do, see our milestones, and our backlog on our [GitHu Bi-weekly community meeting alternating every week between Beijing Friendly timezone and EST/Europe Friendly Timezone * Beijing/US friendly - we start at 8am Beijing Time(bound to CST) / 8pm EDT(7pm EST) / 5pm PDT(4pm PST) / 2am CEST(1am CET) - [Convert to your time zone](https://dateful.com/convert/beijing-china?t=8am) * US/Europe friendly - we start at 10am ET(bound to ET) / 7am PT / 3pm CET / 10pm(11pm) CST - [Convert to your time zone](https://dateful.com/convert/est-edt-eastern-time?t=10) -* Read and comment on the [meeting notes](https://hackmd.io/Jq6F5zqZR7S80CeDWUklkA?view) +* Read and comment on the [meeting notes](https://hackmd.io/bxrvgewUQ5ORH10BKUFpxw) * See previous community meetings on our [YouTube Channel](https://www.youtube.com/playlist?list=PL7bmigfV0EqQRysvqvqOtRNk4L5S7uqwM) * Have a question to discuss in the community meeting? Please add it to our [Q&A Discussion board](https://github.com/vmware-tanzu/velero/discussions/categories/community-support-q-a) diff --git a/site/content/docs/main/api-types/schedule.md b/site/content/docs/main/api-types/schedule.md index 2797f70269..e9afea460d 100644 --- a/site/content/docs/main/api-types/schedule.md +++ b/site/content/docs/main/api-types/schedule.md @@ -30,6 +30,8 @@ metadata: namespace: velero # Parameters about the scheduled backup. Required. spec: + # Paused specifies whether the schedule is paused or not + paused: false # Schedule is a Cron expression defining when to run the Backup schedule: 0 7 * * * # Specifies whether to use OwnerReferences on backups created by this Schedule. diff --git a/site/content/docs/v1.10/api-types/schedule.md b/site/content/docs/v1.10/api-types/schedule.md index 4b759501af..623dbf77ee 100644 --- a/site/content/docs/v1.10/api-types/schedule.md +++ b/site/content/docs/v1.10/api-types/schedule.md @@ -30,6 +30,8 @@ metadata: namespace: velero # Parameters about the scheduled backup. Required. spec: + # Paused specifies whether the schedule is paused or not + paused: false # Schedule is a Cron expression defining when to run the Backup schedule: 0 7 * * * # Specifies whether to use OwnerReferences on backups created by this Schedule. diff --git a/site/content/docs/v1.11/api-types/schedule.md b/site/content/docs/v1.11/api-types/schedule.md index 58a4c454b1..278f383aab 100644 --- a/site/content/docs/v1.11/api-types/schedule.md +++ b/site/content/docs/v1.11/api-types/schedule.md @@ -30,6 +30,8 @@ metadata: namespace: velero # Parameters about the scheduled backup. Required. spec: + # Paused specifies whether the schedule is paused or not + paused: false # Schedule is a Cron expression defining when to run the Backup schedule: 0 7 * * * # Specifies whether to use OwnerReferences on backups created by this Schedule. diff --git a/site/content/docs/v1.12/api-types/schedule.md b/site/content/docs/v1.12/api-types/schedule.md index eb8e8fbd80..37f671b5cc 100644 --- a/site/content/docs/v1.12/api-types/schedule.md +++ b/site/content/docs/v1.12/api-types/schedule.md @@ -30,6 +30,8 @@ metadata: namespace: velero # Parameters about the scheduled backup. Required. spec: + # Paused specifies whether the schedule is paused or not + paused: false # Schedule is a Cron expression defining when to run the Backup schedule: 0 7 * * * # Specifies whether to use OwnerReferences on backups created by this Schedule. diff --git a/site/content/docs/v1.13/api-types/schedule.md b/site/content/docs/v1.13/api-types/schedule.md index 2797f70269..e9afea460d 100644 --- a/site/content/docs/v1.13/api-types/schedule.md +++ b/site/content/docs/v1.13/api-types/schedule.md @@ -30,6 +30,8 @@ metadata: namespace: velero # Parameters about the scheduled backup. Required. spec: + # Paused specifies whether the schedule is paused or not + paused: false # Schedule is a Cron expression defining when to run the Backup schedule: 0 7 * * * # Specifies whether to use OwnerReferences on backups created by this Schedule. diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 409773264f..0e5a4d653d 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -139,7 +139,7 @@ func TTLTest() { fmt.Sprintf("Failed to delete namespace %s", BackupCfg.BackupName)) }) - if veleroCfg.CloudProvider == Aws && useVolumeSnapshots { + if veleroCfg.CloudProvider == AWS && useVolumeSnapshots { fmt.Println("Waiting 7 minutes to make sure the snapshots are ready...") time.Sleep(7 * time.Minute) } diff --git a/test/e2e/basic/backup-volume-info/base.go b/test/e2e/basic/backup-volume-info/base.go index b23fe15671..e6ba12ce1a 100644 --- a/test/e2e/basic/backup-volume-info/base.go +++ b/test/e2e/basic/backup-volume-info/base.go @@ -17,7 +17,6 @@ limitations under the License. package basic import ( - "context" "fmt" "strconv" "strings" @@ -45,26 +44,6 @@ type BackupVolumeInfo struct { func (v *BackupVolumeInfo) Init() error { v.TestCase.Init() - - BeforeEach(func() { - if v.VeleroCfg.CloudProvider == Vsphere && (!strings.Contains(v.CaseBaseName, "fs-upload") && !strings.Contains(v.CaseBaseName, "skipped")) { - fmt.Printf("Skip snapshot case %s for vsphere environment.\n", v.CaseBaseName) - Skip("Skip snapshot case due to vsphere environment doesn't cover the CSI test, and it doesn't have a Velero native snapshot plugin.") - } - - if strings.Contains(v.VeleroCfg.Features, FeatureCSI) { - if strings.Contains(v.CaseBaseName, "native-snapshot") { - fmt.Printf("Skip native snapshot case %s when the CSI feature is enabled.\n", v.CaseBaseName) - Skip("Skip native snapshot case due to CSI feature is enabled.") - } - } else { - if strings.Contains(v.CaseBaseName, "csi") { - fmt.Printf("Skip CSI related case %s when the CSI feature is not enabled.\n", v.CaseBaseName) - Skip("Skip CSI cases due to CSI feature is not enabled.") - } - } - }) - v.CaseBaseName = v.CaseBaseName + v.UUIDgen v.BackupName = "backup-" + v.CaseBaseName v.RestoreName = "restore-" + v.CaseBaseName @@ -98,8 +77,27 @@ func (v *BackupVolumeInfo) Init() error { return nil } +func (v *BackupVolumeInfo) Start() error { + if v.VeleroCfg.CloudProvider == Vsphere && (!strings.Contains(v.CaseBaseName, "fs-upload") && !strings.Contains(v.CaseBaseName, "skipped")) { + fmt.Printf("Skip snapshot case %s for vsphere environment.\n", v.CaseBaseName) + Skip("Skip snapshot case due to vsphere environment doesn't cover the CSI test, and it doesn't have a Velero native snapshot plugin.") + } + + if strings.Contains(v.VeleroCfg.Features, FeatureCSI) { + if strings.Contains(v.CaseBaseName, "native-snapshot") { + fmt.Printf("Skip native snapshot case %s when the CSI feature is enabled.\n", v.CaseBaseName) + Skip("Skip native snapshot case due to CSI feature is enabled.") + } + } else { + if strings.Contains(v.CaseBaseName, "csi") { + fmt.Printf("Skip CSI related case %s when the CSI feature is not enabled.\n", v.CaseBaseName) + Skip("Skip CSI cases due to CSI feature is not enabled.") + } + } + v.TestCase.Start() + return nil +} func (v *BackupVolumeInfo) CreateResources() error { - v.Ctx, v.CtxCancel = context.WithTimeout(context.Background(), v.TimeoutDuration) labels := map[string]string{ "volume-info": "true", } diff --git a/test/e2e/basic/backup-volume-info/native_snapshot.go b/test/e2e/basic/backup-volume-info/native_snapshot.go index fb3a6f3d8c..c8ec0be1b5 100644 --- a/test/e2e/basic/backup-volume-info/native_snapshot.go +++ b/test/e2e/basic/backup-volume-info/native_snapshot.go @@ -30,7 +30,8 @@ var NativeSnapshotVolumeInfoTest func() = TestFunc(&NativeSnapshotVolumeInfo{ BackupVolumeInfo{ SnapshotVolumes: true, TestCase: TestCase{ - CaseBaseName: "native-snapshot-volumeinfo", + UseVolumeSnapshots: true, + CaseBaseName: "native-snapshot-volumeinfo", TestMsg: &TestMSG{ Desc: "Test backup's VolumeInfo metadata content for native snapshot case.", Text: "The VolumeInfo should be generated, and the NativeSnapshotInfo structure should not be nil.", @@ -54,7 +55,7 @@ func (n *NativeSnapshotVolumeInfo) Verify() error { BackupObjectsPrefix+"/"+n.BackupName, ) - Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Fail to get VolumeInfo metadata in the Backup Repository.")) + Expect(err).ShouldNot(HaveOccurred(), "Fail to get VolumeInfo metadata in the Backup Repository.") fmt.Printf("The VolumeInfo metadata content: %+v\n", *volumeInfo[0]) Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true)) diff --git a/test/e2e/basic/namespace-mapping.go b/test/e2e/basic/namespace-mapping.go index c85fe247e0..48aa0cdd5c 100644 --- a/test/e2e/basic/namespace-mapping.go +++ b/test/e2e/basic/namespace-mapping.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "strings" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -86,7 +85,6 @@ func (n *NamespaceMapping) Init() error { } func (n *NamespaceMapping) CreateResources() error { - n.Ctx, n.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) for index, ns := range *n.NSIncluded { n.kibishiiData.Levels = len(*n.NSIncluded) + index By(fmt.Sprintf("Creating namespaces ...%s\n", ns), func() { diff --git a/test/e2e/basic/nodeport.go b/test/e2e/basic/nodeport.go index 846edb9a2a..a127d0d391 100644 --- a/test/e2e/basic/nodeport.go +++ b/test/e2e/basic/nodeport.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "strings" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -61,9 +60,8 @@ func (n *NodePort) Init() error { } return nil } -func (n *NodePort) CreateResources() error { - n.Ctx, n.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) +func (n *NodePort) CreateResources() error { for _, ns := range *n.NSIncluded { By(fmt.Sprintf("Creating service %s in namespaces %s ......\n", n.serviceName, ns), func() { Expect(CreateNamespace(n.Ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) diff --git a/test/e2e/basic/pvc-selected-node-changing.go b/test/e2e/basic/pvc-selected-node-changing.go index e073946158..073c874df6 100644 --- a/test/e2e/basic/pvc-selected-node-changing.go +++ b/test/e2e/basic/pvc-selected-node-changing.go @@ -3,7 +3,6 @@ package basic import ( "context" "fmt" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -64,7 +63,6 @@ func (p *PVCSelectedNodeChanging) Init() error { } func (p *PVCSelectedNodeChanging) CreateResources() error { - p.Ctx, p.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) By(fmt.Sprintf("Create namespace %s", p.namespace), func() { Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", p.namespace)) diff --git a/test/e2e/basic/resources-check/namespaces.go b/test/e2e/basic/resources-check/namespaces.go index e4a88d4a77..b99154a563 100644 --- a/test/e2e/basic/resources-check/namespaces.go +++ b/test/e2e/basic/resources-check/namespaces.go @@ -84,7 +84,6 @@ func (m *MultiNSBackup) Init() error { } func (m *MultiNSBackup) CreateResources() error { - m.Ctx, m.CtxCancel = context.WithTimeout(context.Background(), m.TimeoutDuration) fmt.Printf("Creating namespaces ...\n") labels := map[string]string{ "ns-test": "true", diff --git a/test/e2e/basic/resources-check/namespaces_annotation.go b/test/e2e/basic/resources-check/namespaces_annotation.go index c8642c22c7..e698d48185 100644 --- a/test/e2e/basic/resources-check/namespaces_annotation.go +++ b/test/e2e/basic/resources-check/namespaces_annotation.go @@ -17,10 +17,8 @@ limitations under the License. package basic import ( - "context" "fmt" "strings" - "time" "github.com/pkg/errors" @@ -63,7 +61,6 @@ func (n *NSAnnotationCase) Init() error { } func (n *NSAnnotationCase) CreateResources() error { - n.Ctx, n.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) for nsNum := 0; nsNum < n.NamespacesTotal; nsNum++ { createNSName := fmt.Sprintf("%s-%00000d", n.CaseBaseName, nsNum) createAnnotationName := fmt.Sprintf("annotation-%s-%00000d", n.CaseBaseName, nsNum) diff --git a/test/e2e/basic/resources-check/rbac.go b/test/e2e/basic/resources-check/rbac.go index 37e2ce5fbc..7afedd13c6 100644 --- a/test/e2e/basic/resources-check/rbac.go +++ b/test/e2e/basic/resources-check/rbac.go @@ -33,10 +33,8 @@ limitations under the License. package basic import ( - "context" "fmt" "strings" - "time" "github.com/pkg/errors" @@ -79,7 +77,6 @@ func (r *RBACCase) Init() error { } func (r *RBACCase) CreateResources() error { - r.Ctx, r.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) for nsNum := 0; nsNum < r.NamespacesTotal; nsNum++ { createNSName := fmt.Sprintf("%s-%00000d", r.CaseBaseName, nsNum) fmt.Printf("Creating namespaces ...%s\n", createNSName) diff --git a/test/e2e/basic/storage-class-changing.go b/test/e2e/basic/storage-class-changing.go index fae10c9f89..8bd344700c 100644 --- a/test/e2e/basic/storage-class-changing.go +++ b/test/e2e/basic/storage-class-changing.go @@ -3,7 +3,6 @@ package basic import ( "context" "fmt" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -67,11 +66,11 @@ func (s *StorageClasssChanging) Init() error { } return nil } + func (s *StorageClasssChanging) CreateResources() error { label := map[string]string{ "app": "test", } - s.Ctx, s.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) By(("Installing storage class..."), func() { Expect(InstallTestStorageClasses(fmt.Sprintf("../testdata/storage-class/%s.yaml", s.VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install storage class") diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index f84a3f7a54..c57b04b469 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -27,6 +27,7 @@ import ( . "github.com/onsi/gomega" . "github.com/vmware-tanzu/velero/test" + util "github.com/vmware-tanzu/velero/test/util/csi" . "github.com/vmware-tanzu/velero/test/util/k8s" . "github.com/vmware-tanzu/velero/test/util/kibishii" . "github.com/vmware-tanzu/velero/test/util/providers" @@ -54,9 +55,11 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) var ( backupName, restoreName string backupScName, restoreScName string + kibishiiWorkerCount int err error ) BeforeEach(func() { + kibishiiWorkerCount = 3 veleroCfg = VeleroCfg UUIDgen, err = uuid.NewRandom() migrationNamespace = "migration-" + UUIDgen.String() @@ -116,10 +119,6 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) if !useVolumeSnapshots { Skip("FSB migration test is not needed in data mover scenario") } - // TODO: remove this block once Velero version in cluster A is great than V1.11 for all migration path. - if veleroCLI2Version.VeleroVersion != "self" { - Skip(fmt.Sprintf("Only V1.12 support data mover scenario instead of %s", veleroCLI2Version.VeleroVersion)) - } } oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60) defer ctxCancel() @@ -128,20 +127,46 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) Expect(err).To(Succeed()) supportUploaderType, err := IsSupportUploaderType(veleroCLI2Version.VeleroVersion) Expect(err).To(Succeed()) + + OriginVeleroCfg := veleroCfg if veleroCLI2Version.VeleroCLI == "" { //Assume tag of velero server image is identical to velero CLI version //Download velero CLI if it's empty according to velero CLI version By(fmt.Sprintf("Install the expected version Velero CLI (%s) for installing Velero", veleroCLI2Version.VeleroVersion), func() { + //"self" represents 1.14.x and future versions if veleroCLI2Version.VeleroVersion == "self" { veleroCLI2Version.VeleroCLI = veleroCfg.VeleroCLI } else { + fmt.Printf("Using default images address of Velero CLI %s\n", veleroCLI2Version.VeleroVersion) + OriginVeleroCfg.VeleroImage = "" + OriginVeleroCfg.RestoreHelperImage = "" + OriginVeleroCfg.Plugins = "" + + // It is for v1.13.x migration scenario only, because since v1.14, nightly CI won't + // pass velero-plugin-for-csi to E2E test anymore, and velero installation will not + // fetch velero-plugin-for-csi automatically, so add it as hardcode below. + // TODO: remove this section from future version like v1.15, e.g. + if OriginVeleroCfg.CloudProvider == Azure { + OriginVeleroCfg.Plugins = "velero/velero-plugin-for-microsoft-azure:v1.9.0" + } + if OriginVeleroCfg.CloudProvider == AWS { + OriginVeleroCfg.Plugins = "velero/velero-plugin-for-aws:v1.9.0" + } + if strings.Contains(OriginVeleroCfg.Features, FeatureCSI) { + OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + ",velero/velero-plugin-for-csi:v0.7.0" + } + if OriginVeleroCfg.SnapshotMoveData { + if OriginVeleroCfg.CloudProvider == Azure { + OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + ",velero/velero-plugin-for-aws:v1.9.0" + } + } veleroCLI2Version.VeleroCLI, err = InstallVeleroCLI(veleroCLI2Version.VeleroVersion) Expect(err).To(Succeed()) } }) } - OriginVeleroCfg := veleroCfg + By(fmt.Sprintf("Install Velero in cluster-A (%s) to backup workload", veleroCfg.DefaultClusterContext), func() { Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) OriginVeleroCfg.MigrateFromVeleroVersion = veleroCLI2Version.VeleroVersion @@ -156,21 +181,10 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) Expect(err).To(Succeed(), "Fail to get Velero version") OriginVeleroCfg.VeleroVersion = version - // self represents v1.12 - if veleroCLI2Version.VeleroVersion == "self" { - if OriginVeleroCfg.SnapshotMoveData { - OriginVeleroCfg.UseNodeAgent = true - } - } else { - Expect(err).To(Succeed()) - fmt.Printf("Using default images address of Velero CLI %s\n", veleroCLI2Version.VeleroVersion) - OriginVeleroCfg.VeleroImage = "" - OriginVeleroCfg.RestoreHelperImage = "" - OriginVeleroCfg.Plugins = "" - //TODO: Remove this setting when migration path is from 1.13 to higher version - //TODO: or self, because version 1.12 and older versions have no this parameter. - OriginVeleroCfg.WithoutDisableInformerCacheParam = true + if OriginVeleroCfg.SnapshotMoveData { + OriginVeleroCfg.UseNodeAgent = true } + Expect(VeleroInstall(context.Background(), &OriginVeleroCfg, false)).To(Succeed()) if veleroCLI2Version.VeleroVersion != "self" { Expect(CheckVeleroVersion(context.Background(), OriginVeleroCfg.VeleroCLI, @@ -190,10 +204,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) KibishiiData := *DefaultKibishiiData By("Deploy sample workload of Kibishii", func() { - if OriginVeleroCfg.SnapshotMoveData { - KibishiiData.ExpectedNodes = 3 - } - + KibishiiData.ExpectedNodes = kibishiiWorkerCount Expect(KibishiiPrepareBeforeBackup(oneHourTimeout, *veleroCfg.DefaultClient, veleroCfg.CloudProvider, migrationNamespace, veleroCfg.RegistryCredentialFile, veleroCfg.Features, veleroCfg.KibishiiDirectory, useVolumeSnapshots, &KibishiiData)).To(Succeed()) @@ -238,54 +249,50 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) // TODO - remove after upload progress monitoring is implemented By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(context.Background(), time.Hour, - migrationNamespace, 2)).To(Succeed()) + migrationNamespace, kibishiiWorkerCount)).To(Succeed()) }) } var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = migrationNamespace - if !OriginVeleroCfg.SnapshotMoveData { - By("Snapshot should be created in cloud object store", func() { - snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, 2, - migrationNamespace, backupName, KibishiiPVCNameList) + if OriginVeleroCfg.SnapshotMoveData { + //VolumeSnapshotContent should be deleted after data movement + _, err := util.CheckVolumeSnapshotCR(*veleroCfg.DefaultClient, map[string]string{"namespace": migrationNamespace}, 0) + Expect(err).NotTo(HaveOccurred(), "VSC count is not as expected 0") + } else { + // the snapshots of AWS may be still in pending status when do the restore, wait for a while + // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 + // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed + if veleroCfg.CloudProvider == Azure && strings.EqualFold(veleroCfg.Features, FeatureCSI) || veleroCfg.CloudProvider == AWS { + By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { + time.Sleep(5 * time.Minute) + }) + } + + By("Snapshot should be created in cloud object store with retain policy", func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, kibishiiWorkerCount, + migrationNamespace, backupName, GetKibishiiPVCNameList(kibishiiWorkerCount)) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) }) - } else { - //TODO: checkpoint for datamover } } - if useVolumeSnapshots && veleroCfg.CloudProvider == Azure && - strings.EqualFold(veleroCfg.Features, FeatureCSI) && - !OriginVeleroCfg.SnapshotMoveData { - By("Sleep 5 minutes to avoid snapshot recreated by unknown reason ", func() { - time.Sleep(5 * time.Minute) - }) - } - // the snapshots of AWS may be still in pending status when do the restore, wait for a while - // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 - // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if veleroCfg.CloudProvider == Aws && useVolumeSnapshots && !OriginVeleroCfg.SnapshotMoveData { - fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") - time.Sleep(5 * time.Minute) - } - By(fmt.Sprintf("Install Velero in cluster-B (%s) to restore workload", veleroCfg.StandbyClusterContext), func() { //Ensure workload of "migrationNamespace" existed in cluster-A ns, err := GetNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace) Expect(ns.Name).To(Equal(migrationNamespace)) - Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("get namespace in cluster-B err: %v", err)) + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("get namespace in source cluster err: %v", err)) //Ensure cluster-B is the target cluster Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed()) _, err = GetNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace) - Expect(err).To(HaveOccurred()) - strings.Contains(fmt.Sprint(err), "namespaces \""+migrationNamespace+"\" not found") + Expect(err).To(HaveOccurred(), fmt.Sprintf("get namespace in dst cluster successfully, it's not as expected: %s", migrationNamespace)) fmt.Println(err) + Expect(strings.Contains(fmt.Sprint(err), "namespaces \""+migrationNamespace+"\" not found")).Should(BeTrue()) veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient veleroCfg.ClusterToInstallVelero = veleroCfg.StandbyClusterName diff --git a/test/e2e/pv-backup/pv-backup-filter.go b/test/e2e/pv-backup/pv-backup-filter.go index a858743f95..90c5cfdea6 100644 --- a/test/e2e/pv-backup/pv-backup-filter.go +++ b/test/e2e/pv-backup/pv-backup-filter.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "strings" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -64,7 +63,6 @@ func (p *PVBackupFiltering) Init() error { } func (p *PVBackupFiltering) CreateResources() error { - p.Ctx, p.CtxCancel = context.WithTimeout(context.Background(), 30*time.Minute) err := InstallStorageClass(p.Ctx, fmt.Sprintf("../testdata/storage-class/%s.yaml", p.VeleroCfg.CloudProvider)) if err != nil { return errors.Wrapf(err, "failed to install storage class for pv backup filtering test") diff --git a/test/e2e/resource-filtering/base.go b/test/e2e/resource-filtering/base.go index 61be331c37..8cfce438d5 100644 --- a/test/e2e/resource-filtering/base.go +++ b/test/e2e/resource-filtering/base.go @@ -19,7 +19,6 @@ package filtering import ( "context" "fmt" - "time" "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -61,7 +60,6 @@ func (f *FilteringCase) Init() error { } func (f *FilteringCase) CreateResources() error { - f.Ctx, f.CtxCancel = context.WithTimeout(context.Background(), 30*time.Minute) for nsNum := 0; nsNum < f.NamespacesTotal; nsNum++ { namespace := fmt.Sprintf("%s-%00000d", f.CaseBaseName, nsNum) fmt.Printf("Creating resources in namespace ...%s\n", namespace) diff --git a/test/e2e/resource-filtering/exclude_label.go b/test/e2e/resource-filtering/exclude_label.go index ecf26878ef..27f7f53e6c 100644 --- a/test/e2e/resource-filtering/exclude_label.go +++ b/test/e2e/resource-filtering/exclude_label.go @@ -17,9 +17,7 @@ limitations under the License. package filtering import ( - "context" "fmt" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -76,7 +74,6 @@ func (e *ExcludeFromBackup) Init() error { } func (e *ExcludeFromBackup) CreateResources() error { - e.Ctx, e.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) namespace := e.CaseBaseName // These 2 labels for resources to be included label1 := map[string]string{ diff --git a/test/e2e/resource-filtering/exclude_namespaces.go b/test/e2e/resource-filtering/exclude_namespaces.go index f4e22e852f..1b8e5da550 100644 --- a/test/e2e/resource-filtering/exclude_namespaces.go +++ b/test/e2e/resource-filtering/exclude_namespaces.go @@ -17,10 +17,8 @@ limitations under the License. package filtering import ( - "context" "fmt" "strings" - "time" "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -108,7 +106,6 @@ func (e *ExcludeNamespaces) Init() error { } func (e *ExcludeNamespaces) CreateResources() error { - e.Ctx, e.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) for nsNum := 0; nsNum < e.NamespacesTotal; nsNum++ { createNSName := fmt.Sprintf("%s-%00000d", e.CaseBaseName, nsNum) fmt.Printf("Creating namespaces ...%s\n", createNSName) diff --git a/test/e2e/resource-filtering/include_namespaces.go b/test/e2e/resource-filtering/include_namespaces.go index d39c2acb94..d511de2123 100644 --- a/test/e2e/resource-filtering/include_namespaces.go +++ b/test/e2e/resource-filtering/include_namespaces.go @@ -17,10 +17,8 @@ limitations under the License. package filtering import ( - "context" "fmt" "strings" - "time" "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -103,7 +101,6 @@ func (i *IncludeNamespaces) Init() error { } func (i *IncludeNamespaces) CreateResources() error { - i.Ctx, i.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) for nsNum := 0; nsNum < i.NamespacesTotal; nsNum++ { createNSName := fmt.Sprintf("%s-%00000d", i.CaseBaseName, nsNum) fmt.Printf("Creating namespaces ...%s\n", createNSName) diff --git a/test/e2e/resource-filtering/label_selector.go b/test/e2e/resource-filtering/label_selector.go index 21610ec040..1d566b5853 100644 --- a/test/e2e/resource-filtering/label_selector.go +++ b/test/e2e/resource-filtering/label_selector.go @@ -17,10 +17,8 @@ limitations under the License. package filtering import ( - "context" "fmt" "strings" - "time" "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -75,7 +73,6 @@ func (l *LabelSelector) Init() error { } func (l *LabelSelector) CreateResources() error { - l.Ctx, l.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) for nsNum := 0; nsNum < l.NamespacesTotal; nsNum++ { namespace := fmt.Sprintf("%s-%00000d", l.CaseBaseName, nsNum) fmt.Printf("Creating resources in namespace ...%s\n", namespace) diff --git a/test/e2e/resourcemodifiers/resource_modifiers.go b/test/e2e/resourcemodifiers/resource_modifiers.go index f87965d2fa..cc1d6d3f18 100644 --- a/test/e2e/resourcemodifiers/resource_modifiers.go +++ b/test/e2e/resourcemodifiers/resource_modifiers.go @@ -17,10 +17,8 @@ limitations under the License. package resourcemodifiers import ( - "context" "fmt" "strings" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -97,9 +95,6 @@ func (r *ResourceModifiersCase) Init() error { } func (r *ResourceModifiersCase) CreateResources() error { - // It's better to set a global timeout in CreateResources function which is the real beginning of one e2e test - r.Ctx, r.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) - By(fmt.Sprintf("Create configmap %s in namespaces %s for workload\n", r.cmName, r.VeleroCfg.VeleroNamespace), func() { Expect(CreateConfigMapFromYAMLData(r.Client.ClientGo, r.yamlConfig, r.cmName, r.VeleroCfg.VeleroNamespace)).To(Succeed(), fmt.Sprintf("Failed to create configmap %s in namespaces %s for workload\n", r.cmName, r.VeleroCfg.VeleroNamespace)) }) diff --git a/test/e2e/resourcepolicies/resource_policies.go b/test/e2e/resourcepolicies/resource_policies.go index c98adfdb3b..7502f8a0d5 100644 --- a/test/e2e/resourcepolicies/resource_policies.go +++ b/test/e2e/resourcepolicies/resource_policies.go @@ -17,10 +17,8 @@ limitations under the License. package filtering import ( - "context" "fmt" "strings" - "time" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -103,9 +101,6 @@ func (r *ResourcePoliciesCase) Init() error { } func (r *ResourcePoliciesCase) CreateResources() error { - // It's better to set a global timeout in CreateResources function which is the real beginning of one e2e test - r.Ctx, r.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) - By(("Installing storage class..."), func() { Expect(InstallTestStorageClasses(fmt.Sprintf("../testdata/storage-class/%s.yaml", r.VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install storage class") }) diff --git a/test/e2e/schedule/ordered_resources.go b/test/e2e/schedule/ordered_resources.go index 4724c7d7c7..8ae7fb56d3 100644 --- a/test/e2e/schedule/ordered_resources.go +++ b/test/e2e/schedule/ordered_resources.go @@ -18,7 +18,6 @@ limitations under the License. //the ordered resources test related to https://github.com/vmware-tanzu/velero/issues/4561 import ( - "context" "fmt" "strings" "time" @@ -70,9 +69,7 @@ func (o *OrderedResources) Init() error { return nil } - func (o *OrderedResources) CreateResources() error { - o.Ctx, o.CtxCancel = context.WithTimeout(context.Background(), 10*time.Minute) label := map[string]string{ "orderedresources": "true", } diff --git a/test/e2e/schedule/schedule-backup-creation.go b/test/e2e/schedule/schedule-backup-creation.go index 437afde75f..b87169df66 100644 --- a/test/e2e/schedule/schedule-backup-creation.go +++ b/test/e2e/schedule/schedule-backup-creation.go @@ -63,7 +63,6 @@ func (n *ScheduleBackupCreation) Init() error { } func (p *ScheduleBackupCreation) CreateResources() error { - p.Ctx, p.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) By(fmt.Sprintf("Create namespace %s", p.namespace), func() { Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", p.namespace)) diff --git a/test/e2e/schedule/schedule.go b/test/e2e/schedule/schedule.go index 959000c29a..99e4c01ddb 100644 --- a/test/e2e/schedule/schedule.go +++ b/test/e2e/schedule/schedule.go @@ -47,9 +47,7 @@ func (n *ScheduleBackup) Init() error { Expect(n.Period).To(BeNumerically("<", 30)) return nil } - func (n *ScheduleBackup) CreateResources() error { - n.Ctx, n.CtxCancel = context.WithTimeout(context.Background(), 60*time.Minute) for _, ns := range *n.NSIncluded { By(fmt.Sprintf("Creating namespaces %s ......\n", ns), func() { Expect(CreateNamespace(n.Ctx, n.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) diff --git a/test/e2e/test/test.go b/test/e2e/test/test.go index 89a8fa71d1..9f64f3b1f4 100644 --- a/test/e2e/test/test.go +++ b/test/e2e/test/test.go @@ -18,7 +18,6 @@ package test import ( "context" - "flag" "fmt" "math/rand" "strings" @@ -48,6 +47,7 @@ type VeleroBackupRestoreTest interface { Restore() error Verify() error Clean() error + Start() error GetTestMsg() *TestMSG GetTestCase() *TestCase } @@ -78,52 +78,26 @@ type TestCase struct { func TestFunc(test VeleroBackupRestoreTest) func() { return func() { - Expect(test.Init()).To(Succeed(), "Failed to instantiate test cases") - BeforeEach(func() { - flag.Parse() - // Using the global velero config which covered the installation for most common cases - veleroCfg := test.GetTestCase().VeleroCfg - // TODO: Skip nodeport test until issue https://github.com/kubernetes/kubernetes/issues/114384 fixed - // TODO: Although this issue is closed, but it's not fixed. - // TODO: After bump up k8s version in AWS pipeline, this issue also apply for AWS pipeline. - if (veleroCfg.CloudProvider == Azure || veleroCfg.CloudProvider == Aws) && strings.Contains(test.GetTestCase().CaseBaseName, "nodeport") { - Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS") - } - if InstallVelero { - Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName, veleroCfg)).To(Succeed()) - } - }) - It(test.GetTestMsg().Text, func() { - Expect(RunTestCase(test)).To(Succeed(), test.GetTestMsg().FailedMSG) - }) + TestIt(test) } } func TestFuncWithMultiIt(tests []VeleroBackupRestoreTest) func() { return func() { for k := range tests { - Expect(tests[k].Init()).To(Succeed(), fmt.Sprintf("Failed to instantiate test %s case", tests[k].GetTestMsg().Desc)) - defer tests[k].GetTestCase().CtxCancel() - } - - BeforeEach(func() { - flag.Parse() - if InstallVelero { - Expect(PrepareVelero(context.Background(), tests[0].GetTestCase().CaseBaseName, tests[0].GetTestCase().VeleroCfg)).To(Succeed()) - } - }) - - for k := range tests { - curTest := tests[k] - It(curTest.GetTestMsg().Text, func() { - Expect(RunTestCase(curTest)).To(Succeed(), curTest.GetTestMsg().FailedMSG) - }) + TestIt(tests[k]) } } } +func TestIt(test VeleroBackupRestoreTest) error { + test.Init() + It(test.GetTestMsg().Text, func() { + Expect(RunTestCase(test)).To(Succeed(), test.GetTestMsg().FailedMSG) + }) + return nil +} func (t *TestCase) Init() error { - t.Ctx, t.CtxCancel = context.WithTimeout(context.Background(), 1*time.Hour) t.UUIDgen = t.GenerateUUID() t.VeleroCfg = VeleroCfg t.Client = *t.VeleroCfg.ClientToInstallVelero @@ -141,18 +115,13 @@ func (t *TestCase) CreateResources() error { func (t *TestCase) Backup() error { veleroCfg := t.GetTestCase().VeleroCfg - if err := VeleroBackupExec(t.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, t.BackupArgs); err != nil { - RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, "") - return errors.Wrapf(err, "Failed to backup resources") - } - // the snapshots of AWS may be still in pending status when do the restore, wait for a while - // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 - // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if t.UseVolumeSnapshots { - fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") - time.Sleep(5 * time.Minute) - } + By("Start to backup ......", func() { + Expect(VeleroBackupExec(t.Ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, t.BackupArgs)).To(Succeed(), func() string { + RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, t.BackupName, "") + return "Failed to backup resources" + }) + }) return nil } @@ -170,13 +139,15 @@ func (t *TestCase) Restore() error { } veleroCfg := t.GetTestCase().VeleroCfg + // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if t.UseVolumeSnapshots && veleroCfg.CloudProvider != Vsphere { - fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") - time.Sleep(5 * time.Minute) - } + By("Waiting 5 minutes to make sure the snapshots are ready...", func() { + if t.UseVolumeSnapshots && veleroCfg.CloudProvider != Vsphere { + time.Sleep(5 * time.Minute) + } + }) By("Start to restore ......", func() { if t.RestorePhaseExpect == "" { @@ -194,6 +165,15 @@ func (t *TestCase) Verify() error { return nil } +func (t *TestCase) Start() error { + t.Ctx, t.CtxCancel = context.WithTimeout(context.Background(), 1*time.Hour) + veleroCfg := t.GetTestCase().VeleroCfg + if (veleroCfg.CloudProvider == Azure || veleroCfg.CloudProvider == AWS) && strings.Contains(t.GetTestCase().CaseBaseName, "nodeport") { + Skip("Skip due to issue https://github.com/kubernetes/kubernetes/issues/114384 on AKS") + } + return nil +} + func (t *TestCase) Clean() error { veleroCfg := t.GetTestCase().VeleroCfg if !veleroCfg.Debug { @@ -217,10 +197,17 @@ func (t *TestCase) GetTestCase() *TestCase { } func RunTestCase(test VeleroBackupRestoreTest) error { - fmt.Printf("Running test case %s %s\n", test.GetTestMsg().Desc, time.Now().Format("2006-01-02 15:04:05")) if test == nil { return errors.New("No case should be tested") } + test.Start() + defer test.GetTestCase().CtxCancel() + + fmt.Printf("Running test case %s %s\n", test.GetTestMsg().Desc, time.Now().Format("2006-01-02 15:04:05")) + + if InstallVelero { + Expect(PrepareVelero(context.Background(), test.GetTestCase().CaseBaseName, test.GetTestCase().VeleroCfg)).To(Succeed()) + } defer test.Clean() diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 149ae76253..ef9e31c217 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -33,10 +33,7 @@ import ( . "github.com/vmware-tanzu/velero/test/util/velero" ) -const ( - upgradeNamespace = "upgrade-workload" -) - +var upgradeNamespace string var veleroCfg VeleroConfig func BackupUpgradeRestoreWithSnapshots() { @@ -62,6 +59,8 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC BeforeEach(func() { veleroCfg = VeleroCfg veleroCfg.IsUpgradeTest = true + UUIDgen, err = uuid.NewRandom() + upgradeNamespace = "upgrade-" + UUIDgen.String() if !InstallVelero { Skip("Upgrade test should not be triggered if veleroCfg.InstallVelero is set to false") } @@ -223,7 +222,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC // the snapshots of AWS may be still in pending status when do the restore, wait for a while // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed - if tmpCfg.CloudProvider == Aws && useVolumeSnapshots { + if tmpCfg.CloudProvider == AWS && useVolumeSnapshots { fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") time.Sleep(5 * time.Minute) } diff --git a/test/types.go b/test/types.go index e64c6d677a..6ba28b549b 100644 --- a/test/types.go +++ b/test/types.go @@ -34,11 +34,11 @@ const Kind = "kind" const Azure = "azure" const AzureCSI = "azure-csi" const AwsCSI = "aws-csi" -const Aws = "aws" +const AWS = "aws" const Gcp = "gcp" const Vsphere = "vsphere" -var PublicCloudProviders = []string{Aws, Azure, Gcp, Vsphere} +var PublicCloudProviders = []string{AWS, Azure, Gcp, Vsphere} var LocalCloudProviders = []string{Kind, VanillaZFS} var CloudProviders = append(PublicCloudProviders, LocalCloudProviders...) diff --git a/test/util/csi/common.go b/test/util/csi/common.go index e336336ffc..9c10e962d9 100644 --- a/test/util/csi/common.go +++ b/test/util/csi/common.go @@ -21,6 +21,7 @@ import ( "fmt" "strings" + volumeSnapshotV1 "github.com/kubernetes-csi/external-snapshotter/client/v7/apis/volumesnapshot/v1" snapshotterClientSet "github.com/kubernetes-csi/external-snapshotter/client/v7/clientset/versioned" "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -52,77 +53,47 @@ func GetClients() (*kubernetes.Clientset, *snapshotterClientSet.Clientset, error return client, snapshotterClient, nil } -func GetCsiSnapshotHandle(client TestClient, backupName string) ([]string, error) { +func GetCsiSnapshotHandle(client TestClient, apiVersion string, index map[string]string) ([]string, error) { _, snapshotClient, err := GetClients() + + var vscList *volumeSnapshotV1.VolumeSnapshotContentList if err != nil { return nil, err } - vscList, err1 := snapshotClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{}) - if err1 != nil { - return nil, err + switch apiVersion { + // case "v1beta1": + // vscList, err = snapshotterClientSetV1beta1.SnapshotV1beta1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{}) + case "v1": + vscList, err = snapshotClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{}) + default: + errors.New(fmt.Sprintf("API version %s is not valid", apiVersion)) } - var snapshotHandleList []string - for _, i := range vscList.Items { - if i.Status == nil { - fmt.Println("SnapshotHandle Status s nil") - continue - } - if i.Status.SnapshotHandle == nil { - fmt.Println("SnapshotHandle is nil") - continue - } - - if i.Labels == nil { - fmt.Println("VolumeSnapshotContents label is nil") - continue - } - - if i.Labels["velero.io/backup-name"] == backupName { - tmp := strings.Split(*i.Status.SnapshotHandle, "/") - snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1]) - } - } - - if len(snapshotHandleList) == 0 { - fmt.Printf("No VolumeSnapshotContent from backup %s\n", backupName) - } - return snapshotHandleList, nil -} -func GetCsiSnapshotHandleV1(client TestClient, backupName string) ([]string, error) { - _, snapshotClient, err := GetClients() if err != nil { return nil, err } - vscList, err1 := snapshotClient.SnapshotV1().VolumeSnapshotContents().List(context.TODO(), metav1.ListOptions{}) - if err1 != nil { - return nil, err - } var snapshotHandleList []string for _, i := range vscList.Items { if i.Status == nil { - fmt.Println("SnapshotHandle Status s nil") + fmt.Println("VolumeSnapshotContent status is nil") continue } if i.Status.SnapshotHandle == nil { fmt.Println("SnapshotHandle is nil") continue } - - if i.Labels == nil { - fmt.Println("VolumeSnapshotContents label is nil") - continue - } - - if i.Labels["velero.io/backup-name"] == backupName { + if (index["backupNameLabel"] != "" && i.Labels != nil && i.Labels["velero.io/backup-name"] == index["backupNameLabel"]) || + (index["namespace"] != "" && i.Spec.VolumeSnapshotRef.Namespace == index["namespace"]) { tmp := strings.Split(*i.Status.SnapshotHandle, "/") snapshotHandleList = append(snapshotHandleList, tmp[len(tmp)-1]) } } if len(snapshotHandleList) == 0 { - fmt.Printf("No VolumeSnapshotContent from backup %s\n", backupName) + fmt.Printf("No VolumeSnapshotContent from key %v\n", index) + } else { + fmt.Printf("Volume snapshot content list: %v\n", snapshotHandleList) } return snapshotHandleList, nil } @@ -164,22 +135,35 @@ func GetVolumeSnapshotContentNameByPod(client TestClient, podName, namespace, ba return "", errors.New(fmt.Sprintf("Fail to get VolumeSnapshotContentName for pod %s under namespace %s", podName, namespace)) } -func CheckVolumeSnapshotCR(client TestClient, backupName string, expectedCount int, apiVersion string) ([]string, error) { +func CheckVolumeSnapshotCR(client TestClient, index map[string]string, expectedCount int) ([]string, error) { var err error var snapshotContentNameList []string - if apiVersion == "v1beta1" { - if snapshotContentNameList, err = GetCsiSnapshotHandle(client, backupName); err != nil { - return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") - } - } else if apiVersion == "v1" { - if snapshotContentNameList, err = GetCsiSnapshotHandleV1(client, backupName); err != nil { + + resourceName := "snapshot.storage.k8s.io" + + apiVersion, err := GetAPIVersions(&client, resourceName) + if err != nil { + return nil, err + } + if len(apiVersion) == 0 { + return nil, errors.New("Fail to get APIVersion") + } + // if apiVersion[0] == "v1beta1" { + // if snapshotContentNameList, err = GetCsiSnapshotHandle(client, apiVersion[0], index); err != nil { + // return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") + // } + // } else + if apiVersion[0] == "v1" { + if snapshotContentNameList, err = GetCsiSnapshotHandle(client, apiVersion[0], index); err != nil { return nil, errors.Wrap(err, "Fail to get Azure CSI snapshot content") } } else { return nil, errors.New("API version is invalid") } - if len(snapshotContentNameList) != expectedCount { - return nil, errors.New(fmt.Sprintf("Snapshot content count %d is not as expect %d", len(snapshotContentNameList), expectedCount)) + if expectedCount >= 0 { + if len(snapshotContentNameList) != expectedCount { + return nil, errors.New(fmt.Sprintf("Snapshot content count %d is not as expect %d", len(snapshotContentNameList), expectedCount)) + } } fmt.Printf("snapshotContentNameList: %v \n", snapshotContentNameList) return snapshotContentNameList, nil diff --git a/test/util/k8s/common.go b/test/util/k8s/common.go index 097e3be2fd..9335815629 100644 --- a/test/util/k8s/common.go +++ b/test/util/k8s/common.go @@ -250,7 +250,6 @@ func GetAPIVersions(client *TestClient, name string) ([]string, error) { return nil, errors.Wrap(err, "Fail to get server API groups") } for _, group := range APIGroup.Groups { - fmt.Println(group.Name) if group.Name == name { for _, v := range group.Versions { fmt.Println(v.Version) @@ -259,7 +258,7 @@ func GetAPIVersions(client *TestClient, name string) ([]string, error) { return version, nil } } - return nil, errors.New("Server API groups is empty") + return nil, errors.New("Fail to get server API groups") } func GetPVByPVCName(client TestClient, namespace, pvcName string) (string, error) { diff --git a/test/util/k8s/namespace.go b/test/util/k8s/namespace.go index e747a4d152..b2a5cae0e6 100644 --- a/test/util/k8s/namespace.go +++ b/test/util/k8s/namespace.go @@ -42,7 +42,7 @@ func CreateNamespace(ctx context.Context, client TestClient, namespace string) e "pod-security.kubernetes.io/enforce": "baseline", "pod-security.kubernetes.io/enforce-version": "latest", } - _, err := client.ClientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + _, err := client.ClientGo.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return nil } @@ -70,7 +70,7 @@ func CreateNamespaceWithAnnotation(ctx context.Context, client TestClient, names "pod-security.kubernetes.io/enforce-version": "latest", } ns.ObjectMeta.Annotations = annotation - _, err := client.ClientGo.CoreV1().Namespaces().Create(ctx, ns, metav1.CreateOptions{}) + _, err := client.ClientGo.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{}) if apierrors.IsAlreadyExists(err) { return nil } @@ -169,6 +169,9 @@ func CleanupNamespacesFiterdByExcludes(ctx context.Context, client TestClient, e } func CleanupNamespaces(ctx context.Context, client TestClient, CaseBaseName string) error { + if ctx == nil { + fmt.Println("ctx is nil ....") + } namespaces, err := client.ClientGo.CoreV1().Namespaces().List(ctx, metav1.ListOptions{}) if err != nil { return errors.Wrap(err, "Could not retrieve namespaces") diff --git a/test/util/k8s/rbac.go b/test/util/k8s/rbac.go index c6fa2559c6..634fdf84f1 100644 --- a/test/util/k8s/rbac.go +++ b/test/util/k8s/rbac.go @@ -34,7 +34,7 @@ func CreateRBACWithBindingSA(ctx context.Context, client TestClient, namespace s }, } - _, err = client.ClientGo.RbacV1().ClusterRoles().Create(ctx, role, metav1.CreateOptions{}) + _, err = client.ClientGo.RbacV1().ClusterRoles().Create(context.TODO(), role, metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { return err diff --git a/test/util/k8s/serviceaccount.go b/test/util/k8s/serviceaccount.go index 24d748aeaf..1af56a798a 100644 --- a/test/util/k8s/serviceaccount.go +++ b/test/util/k8s/serviceaccount.go @@ -23,14 +23,13 @@ import ( "time" "github.com/pkg/errors" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "github.com/vmware-tanzu/velero/pkg/builder" - - corev1 "k8s.io/api/core/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func WaitUntilServiceAccountCreated(ctx context.Context, client TestClient, namespace, serviceAccount string, timeout time.Duration) error { @@ -81,7 +80,7 @@ func CreateServiceAccount(ctx context.Context, client TestClient, namespace stri AutomountServiceAccountToken: nil, } - _, err = client.ClientGo.CoreV1().ServiceAccounts(namespace).Create(ctx, sa, metav1.CreateOptions{}) + _, err = client.ClientGo.CoreV1().ServiceAccounts(namespace).Create(context.TODO(), sa, metav1.CreateOptions{}) if err != nil && !apierrors.IsAlreadyExists(err) { return err @@ -90,5 +89,5 @@ func CreateServiceAccount(ctx context.Context, client TestClient, namespace stri } func GetServiceAccount(ctx context.Context, client TestClient, namespace string, serviceAccount string) (*corev1.ServiceAccount, error) { - return client.ClientGo.CoreV1().ServiceAccounts(namespace).Get(ctx, serviceAccount, metav1.GetOptions{}) + return client.ClientGo.CoreV1().ServiceAccounts(namespace).Get(context.TODO(), serviceAccount, metav1.GetOptions{}) } diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index 5046f4f21c..2fe223ea96 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -55,6 +55,14 @@ var KibishiiPodNameList = []string{"kibishii-deployment-0", "kibishii-deployment var KibishiiPVCNameList = []string{"kibishii-data-kibishii-deployment-0", "kibishii-data-kibishii-deployment-1"} var KibishiiStorageClassName = "kibishii-storage-class" +func GetKibishiiPVCNameList(workerCount int) []string { + var kibishiiPVCNameList []string + for i := 0; i < workerCount; i++ { + kibishiiPVCNameList = append(kibishiiPVCNameList, fmt.Sprintf("kibishii-data-kibishii-deployment-%d", i)) + } + return kibishiiPVCNameList +} + // RunKibishiiTests runs kibishii tests on the provider. func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string, useVolumeSnapshots, defaultVolumesToFsBackup bool) error { @@ -211,10 +219,10 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc } } - // the snapshots of AWS may be still in pending status when do the restore, wait for a while - // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 - // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed if useVolumeSnapshots { + // the snapshots of AWS may be still in pending status when do the restore, wait for a while + // to avoid this https://github.com/vmware-tanzu/velero/issues/1799 + // TODO remove this after https://github.com/vmware-tanzu/velero/issues/3533 is fixed fmt.Println("Waiting 5 minutes to make sure the snapshots are ready...") time.Sleep(5 * time.Minute) } @@ -248,7 +256,7 @@ func installKibishii(ctx context.Context, namespace string, cloudPlatform, veler strings.EqualFold(veleroFeatures, FeatureCSI) { cloudPlatform = AzureCSI } - if strings.EqualFold(cloudPlatform, Aws) && + if strings.EqualFold(cloudPlatform, AWS) && strings.EqualFold(veleroFeatures, FeatureCSI) { cloudPlatform = AwsCSI } diff --git a/test/util/providers/common.go b/test/util/providers/common.go index 1b39f22712..99441d268e 100644 --- a/test/util/providers/common.go +++ b/test/util/providers/common.go @@ -75,7 +75,7 @@ func ObjectsShouldNotBeInBucket(objectStoreProvider, cloudCredentialsFile, bslBu func getProvider(cloudProvider string) (ObjectsInStorage, error) { var s ObjectsInStorage switch cloudProvider { - case Aws, Vsphere: + case AWS, Vsphere: aws := AWSStorage("") s = &aws case Gcp: diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 09fc7f3f77..48e2faf1d9 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -104,7 +104,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste // backup, but needed to pick up the provider plugins earlier. vSphere plugin no longer needs a Volume // Snapshot location specified if veleroCfg.ObjectStoreProvider == "" { - veleroCfg.ObjectStoreProvider = Aws + veleroCfg.ObjectStoreProvider = AWS } if err := configvSpherePlugin(veleroCfg); err != nil { return errors.WithMessagef(err, "Failed to config vsphere plugin") @@ -118,7 +118,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste // For AWS IRSA credential test, AWS IAM service account is required, so if ServiceAccountName and EKSPolicyARN // are both provided, we assume IRSA test is running, otherwise skip this IAM service account creation part. - if veleroCfg.CloudProvider == Aws && veleroInstallOptions.ServiceAccountName != "" { + if veleroCfg.CloudProvider == AWS && veleroInstallOptions.ServiceAccountName != "" { if veleroCfg.EKSPolicyARN == "" { return errors.New("Please provide EKSPolicyARN for IRSA test.") } @@ -155,6 +155,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste }) if err != nil { + time.Sleep(9 * time.Hour) RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", "") return errors.WithMessagef(err, "Failed to install Velero in the cluster") } diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index 21f895fddf..ac55a7c776 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -98,11 +98,12 @@ var pluginsMatrix = map[string]map[string][]string{ "csi": {"velero/velero-plugin-for-csi:v0.6.0"}, }, "v1.13": { - "aws": {"velero/velero-plugin-for-aws:v1.9.0"}, - "azure": {"velero/velero-plugin-for-microsoft-azure:v1.9.0"}, - "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"}, - "gcp": {"velero/velero-plugin-for-gcp:v1.9.0"}, - "csi": {"velero/velero-plugin-for-csi:v0.7.0"}, + "aws": {"velero/velero-plugin-for-aws:v1.9.0"}, + "azure": {"velero/velero-plugin-for-microsoft-azure:v1.9.0"}, + "vsphere": {"vsphereveleroplugin/velero-plugin-for-vsphere:v1.5.2"}, + "gcp": {"velero/velero-plugin-for-gcp:v1.9.0"}, + "csi": {"velero/velero-plugin-for-csi:v0.7.0"}, + "datamover": {"velero/velero-plugin-for-aws:v1.9.0"}, }, "main": { "aws": {"velero/velero-plugin-for-aws:main"}, @@ -129,7 +130,7 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider string, nee var ok bool if slices.Contains(LocalCloudProviders, cloudProvider) { - plugins, ok = cloudMap[Aws] + plugins, ok = cloudMap[AWS] if !ok { return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, cloudProvider) } @@ -1231,24 +1232,16 @@ func GetRepositories(ctx context.Context, veleroNamespace, targetNamespace strin } func GetSnapshotCheckPoint(client TestClient, veleroCfg VeleroConfig, expectCount int, namespaceBackedUp, backupName string, KibishiiPVCNameList []string) (SnapshotCheckPoint, error) { + var err error var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.ExpectCount = expectCount snapshotCheckPoint.NamespaceBackedUp = namespaceBackedUp snapshotCheckPoint.PodName = KibishiiPVCNameList - if (veleroCfg.CloudProvider == Azure || veleroCfg.CloudProvider == Aws) && strings.EqualFold(veleroCfg.Features, FeatureCSI) { + if (veleroCfg.CloudProvider == Azure || veleroCfg.CloudProvider == AWS) && strings.EqualFold(veleroCfg.Features, FeatureCSI) { snapshotCheckPoint.EnableCSI = true - resourceName := "snapshot.storage.k8s.io" - - srcVersions, err := GetAPIVersions(veleroCfg.DefaultClient, resourceName) - if err != nil { - return snapshotCheckPoint, err - } - if len(srcVersions) == 0 { - return snapshotCheckPoint, errors.New("Fail to get APIVersion") - } - if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, backupName, expectCount, srcVersions[0]); err != nil { + if snapshotCheckPoint.SnapshotIDList, err = util.CheckVolumeSnapshotCR(client, map[string]string{"backupNameLabel": backupName}, expectCount); err != nil { return snapshotCheckPoint, errors.Wrapf(err, "Fail to get Azure CSI snapshot content") } }