diff --git a/.github/workflows/e2e-test-kind.yaml b/.github/workflows/e2e-test-kind.yaml index 3640de5998..5fcefa25b0 100644 --- a/.github/workflows/e2e-test-kind.yaml +++ b/.github/workflows/e2e-test-kind.yaml @@ -67,6 +67,16 @@ jobs: - 1.23.6 - 1.24.0 - 1.25.3 + focus: + # tests to focus on, use `|` to concatenate multiple regexes to run on the same job + # ordered according to e2e_suite_test.go order + - Basic\]\[ClusterResource + - ResourceFiltering + - ResourceModifier|Backups|PrivilegesMgmt\]\[SSR + - Schedule\]\[OrderedResources + - NamespaceMapping\]\[Single\]\[Restic|NamespaceMapping\]\[Multiple\]\[Restic + - Basic\]\[Nodeport + - Basic\]\[StorageClass fail-fast: false steps: - name: Set up Go @@ -123,7 +133,8 @@ jobs: CREDS_FILE=/tmp/credential BSL_BUCKET=bucket \ ADDITIONAL_OBJECT_STORE_PROVIDER=aws ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \ ADDITIONAL_CREDS_FILE=/tmp/credential ADDITIONAL_BSL_BUCKET=additional-bucket \ - GINKGO_FOCUS='Basic\]\[ClusterResource' VELERO_IMAGE=velero:pr-test \ + GINKGO_FOCUS='${{ matrix.focus }}' VELERO_IMAGE=velero:pr-test \ + GINKGO_SKIP='SKIP_KIND|pv-backup|Restic|Snapshot|LongTime' \ make -C test/e2e run timeout-minutes: 30 - name: Upload debug bundle diff --git a/Makefile b/Makefile index 3fce3e9242..41c4f3ef56 100644 --- a/Makefile +++ b/Makefile @@ -108,6 +108,7 @@ platform_temp = $(subst -, ,$(ARCH)) GOOS = $(word 1, $(platform_temp)) GOARCH = $(word 2, $(platform_temp)) GOPROXY ?= https://proxy.golang.org +GOBIN=$$(pwd)/.go/bin # If you want to build all binaries, see the 'all-build' rule. # If you want to build all containers, see the 'all-containers' rule. @@ -129,6 +130,7 @@ local: build-dirs # Add DEBUG=1 to enable debug locally GOOS=$(GOOS) \ GOARCH=$(GOARCH) \ + GOBIN=$(GOBIN) \ VERSION=$(VERSION) \ REGISTRY=$(REGISTRY) \ PKG=$(PKG) \ @@ -145,6 +147,7 @@ _output/bin/$(GOOS)/$(GOARCH)/$(BIN): build-dirs $(MAKE) shell CMD="-c '\ GOOS=$(GOOS) \ GOARCH=$(GOARCH) \ + GOBIN=$(GOBIN) \ VERSION=$(VERSION) \ REGISTRY=$(REGISTRY) \ PKG=$(PKG) \ diff --git a/hack/build.sh b/hack/build.sh index 47052e81ec..064b583ef4 100755 --- a/hack/build.sh +++ b/hack/build.sh @@ -32,6 +32,10 @@ if [[ -z "${GOOS}" ]]; then echo "GOOS must be set" exit 1 fi +if [[ -z "${GOBIN}" ]]; then + echo "GOBIN must be set" + exit 1 +fi if [[ -z "${GOARCH}" ]]; then echo "GOARCH must be set" exit 1 diff --git a/test/e2e/Makefile b/test/e2e/Makefile index 3bc3cbc7a1..13df7733eb 100644 --- a/test/e2e/Makefile +++ b/test/e2e/Makefile @@ -41,8 +41,10 @@ help: ## Display this help TOOLS_DIR := $(REPO_ROOT)/hack/tools BIN_DIR := bin +# Try to not modify PATH if possible +GOBIN := $(REPO_ROOT)/.go/bin TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR) -GINKGO := $(shell go env GOPATH)/bin/ginkgo +GINKGO := $(GOBIN)/ginkgo KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin GINKGO_FOCUS ?= @@ -113,9 +115,13 @@ STANDBY_CLUSTER_NAME ?= EKS_POLICY_ARN ?= +# Make sure ginkgo is in $GOBIN .PHONY:ginkgo -ginkgo: # Make sure ginkgo is in $GOPATH/bin - go install github.com/onsi/ginkgo/ginkgo@v1.16.5 +ginkgo: ${GOBIN}/ginkgo + +# This target does not run if ginkgo is already in $GOBIN +${GOBIN}/ginkgo: + GOBIN=${GOBIN} go install github.com/onsi/ginkgo/ginkgo@v1.16.5 .PHONY: run run: ginkgo diff --git a/test/e2e/README.md b/test/e2e/README.md index cb1d001fdd..10a43deeb2 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -66,7 +66,7 @@ the object-store-provider to be specified. 1. `-features`: Comma-separated list of features to enable for this Velero process. 1. `-registry-credential-file`: File containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. This credential will be loaded in Velero server pod to help on Docker Hub rate limit issue. 1. `-kibishii-directory`: The file directory or URL path to install Kibishii. It's configurable in case the default path is not accessible for your own test environment. -1. `-debug-e2e-test`: A Switch for enable or disable test data cleaning action. +1. `-debug-e2e-test`: A Switch for enable or disable test data cleaning action. 1. `-garbage-collection-frequency`: frequency of garbage collection. It is a parameter for Velero installation. Optional. 1. `-velero-server-debug-mode`: A switch for enable or disable having debug log of Velero server. 1. `-default-cluster-context`: Default (source) cluster's kube config context, it's for migration test. @@ -101,7 +101,7 @@ Below is a mapping between `make` variables to E2E configuration flags. 1. `VELERO_NAMESPACE `: the `-velero-namespace`. Optional. 1. `PLUGINS `: the `-plugins`. Optional. 1. `BSL_PREFIX`: `-prefix`. Optional. -1. `BSL_CONFIG`: `-bsl-config`. Optional. +1. `BSL_CONFIG`: `-bsl-config`. Optional. Example: BSL_CONFIG="region=us-east-1". May be required for some object store provider 1. `VSL_CONFIG`: `-vsl-config`. Optional. 1. `UPGRADE_FROM_VELERO_CLI `: `-upgrade-from-velero-cli`. Optional. 1. `UPGRADE_FROM_VELERO_VERSION `: `-upgrade-from-velero-version`. Optional. @@ -136,9 +136,18 @@ Below is a mapping between `make` variables to E2E configuration flags. Basic examples: 1. Run Velero tests in a kind cluster with AWS (or Minio) as the storage provider: + + Start kind cluster + ```bash + kind create cluster + ``` ```bash BSL_PREFIX= BSL_BUCKET= CREDS_FILE=/path/to/aws-creds CLOUD_PROVIDER=kind OBJECT_STORE_PROVIDER=aws make test-e2e ``` + Stop kind cluster + ```bash + kind delete cluster + ``` 1. Run Velero tests in an AWS cluster: ```bash BSL_PREFIX= BSL_BUCKET= CREDS_FILE=/path/to/aws-creds CLOUD_PROVIDER=aws make test-e2e @@ -303,4 +312,8 @@ The TestFunc function concatenate all the flows in a test. It will reduce the frequency of velero installation by installing and checking Velero with the global Velero. It's Need to explicit reinstall Velero if the case has specicial configuration, such as API Group test case we need to enable feature EnableCSI. ### Tips -Look for the ⛵ emoji printed at the end of each install and uninstall log. There should not be two install/unintall in a row, and there should be tests between an install and an uninstall. \ No newline at end of file +Look for the ⛵ emoji printed at the end of each install and uninstall log. There should not be two install/unintall in a row, and there should be tests between an install and an uninstall. + +#### Troubleshooting + +If velero log shows `level=error msg="Failed to get bucket region, bucket: xbucket, error: operation error S3: HeadBucket, failed to resolve service endpoint, endpoint rule error, A region must be set when sending requests to S3." backup-storage-location=velero/default cmd=/plugins/velero-plugin-for-aws controller=backup-storage-location logSource="/go/src/velero-plugin-for-aws/velero-plugin-for-aws/object_store.go:136" pluginName=velero-plugin-for-aws`, it means you need to set `BSL_CONFIG` to include `region=`. diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index 7fd31c7b97..0703c7ce01 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -74,8 +74,12 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) { veleroCfg.KibishiiDirectory = veleroCfg.KibishiiDirectory + backupRestoreTestConfig.kibishiiPatchSubDir veleroCfg.UseVolumeSnapshots = useVolumeSnapshots veleroCfg.UseNodeAgent = !useVolumeSnapshots - if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" { - Skip("Volume snapshots not supported on kind") + if veleroCfg.CloudProvider == "kind" { + Skip("Volume snapshots plugin and File System Backups are not supported on kind") + // on kind cluster snapshots are not supported since there is no velero snapshot plugin for kind volumes. + // and PodVolumeBackups are not supported because PVB creation gets skipped for hostpath volumes, which are the only + // volumes created on kind clusters using the default storage class and provisioner (provisioner: rancher.io/local-path) + // This test suite checks for volume snapshots and PVBs generated from FileSystemBackups, so skip it on kind clusters } // [SKIP]: Static provisioning for vSphere CSI driver works differently from other drivers. // For vSphere CSI, after you create a PV specifying an existing volume handle, CSI diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index 76b078532b..ff929e1f52 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -86,6 +86,9 @@ func backup_deletion_test(useVolumeSnapshots bool) { // runUpgradeTests runs upgrade test on the provider by kibishii. func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupName, backupLocation string, useVolumeSnapshots bool, kibishiiDirectory string) error { + if useVolumeSnapshots && veleroCfg.CloudProvider == "kind" { + Skip("Volume snapshots not supported on kind") + } oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60) defer ctxCancel() veleroCLI := veleroCfg.VeleroCLI @@ -111,7 +114,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam registryCredentialFile, veleroFeatures, kibishiiDirectory, useVolumeSnapshots, DefaultKibishiiData); err != nil { return errors.Wrapf(err, "Failed to install and prepare data for kibishii %s", deletionTest) } - err := ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1) + err := ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix, 1) if err != nil { return err } @@ -139,7 +142,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam return errors.Wrapf(err, "Error waiting for uploads to complete") } } - err = ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) + err = ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) if err != nil { return err } @@ -167,7 +170,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam } } - err = ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5) + err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5) if err != nil { return err } @@ -194,12 +197,12 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupNam }) }) - err = DeleteObjectsInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) + err = DeleteObjectsInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) if err != nil { return err } - err = ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1) + err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1) if err != nil { return err } diff --git a/test/e2e/backups/sync_backups.go b/test/e2e/backups/sync_backups.go index 9a5c81113d..65e1a7d87b 100644 --- a/test/e2e/backups/sync_backups.go +++ b/test/e2e/backups/sync_backups.go @@ -148,13 +148,13 @@ func BackupsSyncTest() { }) By(fmt.Sprintf("Delete %s backup files in object store", test.backupName), func() { - err = DeleteObjectsInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + err = DeleteObjectsInBucket(VeleroCfg.ObjectStoreProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName, BackupObjectsPrefix) Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete object in bucket %s with err %v", test.backupName, err)) }) By(fmt.Sprintf("Check %s backup files in object store is deleted", test.backupName), func() { - err = ObjectsShouldNotBeInBucket(VeleroCfg.CloudProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, + err = ObjectsShouldNotBeInBucket(VeleroCfg.ObjectStoreProvider, VeleroCfg.CloudCredentialsFile, VeleroCfg.BSLBucket, VeleroCfg.BSLPrefix, VeleroCfg.BSLConfig, test.backupName, BackupObjectsPrefix, 1) Expect(err).To(Succeed(), fmt.Sprintf("Failed to delete object in bucket %s with err %v", test.backupName, err)) }) diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 9c353eb346..b66af4e2eb 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -155,7 +155,7 @@ func TTLTest() { }) By("Associated Restores should be created", func() { - Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, + Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.restoreName, RestoreObjectsPrefix)).NotTo(HaveOccurred(), "Fail to get restore object") @@ -179,7 +179,7 @@ func TTLTest() { }) By("Backup file from cloud object storage should be deleted", func() { - Expect(ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, + Expect(ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.backupName, BackupObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") @@ -194,7 +194,7 @@ func TTLTest() { }) By("Associated Restores should be deleted", func() { - Expect(ObjectsShouldNotBeInBucket(veleroCfg.CloudProvider, + Expect(ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, test.restoreName, RestoreObjectsPrefix, 5)).NotTo(HaveOccurred(), "Fail to get restore object") diff --git a/test/e2e/basic/namespace-mapping.go b/test/e2e/basic/namespace-mapping.go index 27e74af656..5cc18921af 100644 --- a/test/e2e/basic/namespace-mapping.go +++ b/test/e2e/basic/namespace-mapping.go @@ -38,6 +38,9 @@ func (n *NamespaceMapping) Init() error { n.VeleroCfg.UseVolumeSnapshots = n.UseVolumeSnapshots n.VeleroCfg.UseNodeAgent = !n.UseVolumeSnapshots n.kibishiiData = &KibishiiData{Levels: 2, DirsPerLevel: 10, FilesPerLevel: 10, FileLength: 1024, BlockSize: 1024, PassNum: 0, ExpectedNodes: 2} + if n.VeleroCfg.CloudProvider == "kind" { + n.kibishiiData = &KibishiiData{Levels: 0, DirsPerLevel: 0, FilesPerLevel: 0, FileLength: 0, BlockSize: 0, PassNum: 0, ExpectedNodes: 2} + } backupType := "restic" if n.UseVolumeSnapshots { backupType = "snapshot" @@ -67,7 +70,11 @@ func (n *NamespaceMapping) Init() error { "create", "--namespace", n.VeleroCfg.VeleroNamespace, "backup", n.BackupName, "--include-namespaces", strings.Join(*n.NSIncluded, ","), "--wait", } - if n.UseVolumeSnapshots { + if VeleroCfg.CloudProvider == "kind" { + // don't test volume snapshotter or file system backup on kind + n.BackupArgs = append(n.BackupArgs, "--snapshot-volumes=false") + n.UseVolumeSnapshots = false + } else if n.UseVolumeSnapshots { n.BackupArgs = append(n.BackupArgs, "--snapshot-volumes") } else { n.BackupArgs = append(n.BackupArgs, "--snapshot-volumes=false") diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 4dc7a0561f..5cd9f9d87b 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -276,7 +276,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { Expect(cmp.Diff(backupsInBsl1AndBsl2, backupsBeforeDel, cmpopts.SortSlices(less))).Should(BeEmpty()) By(fmt.Sprintf("Backup1 %s should exist in cloud object store before bsl deletion", backupName_1), func() { - Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, + Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName_1, BackupObjectsPrefix)).To(Succeed()) }) @@ -295,14 +295,14 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) By(fmt.Sprintf("Backup1 %s should still exist in cloud object store after bsl deletion", backupName_1), func() { - Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.CloudCredentialsFile, + Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName_1, BackupObjectsPrefix)).To(Succeed()) }) // TODO: Choose additional BSL to be deleted as an new test case // By(fmt.Sprintf("Backup %s should still exist in cloud object store", backupName_2), func() { - // Expect(ObjectsShouldBeInBucket(veleroCfg.CloudProvider, veleroCfg.AdditionalBSLCredentials, + // Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.AdditionalBSLCredentials, // veleroCfg.AdditionalBSLBucket, veleroCfg.AdditionalBSLPrefix, veleroCfg.AdditionalBSLConfig, // backupName_2, BackupObjectsPrefix)).To(Succeed()) // }) diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 99dd7b16de..7c7da5c5c6 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -92,7 +92,7 @@ func init() { flag.StringVar(&VeleroCfg.DataMoverPlugin, "data-mover-plugin", "", "customized plugin for data mover.") flag.StringVar(&VeleroCfg.StandbyClusterCloudProvider, "standby-cluster-cloud-provider", "", "cloud provider for standby cluster.") flag.StringVar(&VeleroCfg.StandbyClusterPlugins, "standby-cluster-plugins", "", "plugins provider for standby cluster.") - flag.StringVar(&VeleroCfg.StandbyClusterOjbectStoreProvider, "standby-cluster-object-store-provider", "", "object store provider for standby cluster.") + flag.StringVar(&VeleroCfg.StandbyClusterObjectStoreProvider, "standby-cluster-object-store-provider", "", "object store provider for standby cluster.") flag.BoolVar(&VeleroCfg.DebugVeleroPodRestart, "debug-velero-pod-restart", false, "a switch for debugging velero pod restart.") flag.BoolVar(&VeleroCfg.DisableInformerCache, "disable-informer-cache", false, "a switch for disable informer cache.") flag.StringVar(&VeleroCfg.DefaultClusterName, "default-cluster-name", "", "default cluster's name in kube config file, it's for EKS IRSA test.") @@ -103,8 +103,8 @@ func init() { } -var _ = Describe("[APIGroup][APIVersion] Velero tests with various CRD API group versions", APIGropuVersionsTest) -var _ = Describe("[APIGroup][APIExtensions] CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", APIExtensionsVersionsTest) +var _ = Describe("[APIGroup][APIVersion][SKIP_KIND] Velero tests with various CRD API group versions", APIGropuVersionsTest) +var _ = Describe("[APIGroup][APIExtensions][SKIP_KIND] CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", APIExtensionsVersionsTest) // Test backup and restore of Kibishi using restic var _ = Describe("[Basic][Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic) @@ -134,18 +134,18 @@ var _ = Describe("[ResourceFiltering][IncludeNamespaces][Restore] Velero test on var _ = Describe("[ResourceFiltering][IncludeResources][Backup] Velero test on include resources from the cluster backup", BackupWithIncludeResources) var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on include resources from the cluster restore", RestoreWithIncludeResources) var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector) -var _ = Describe("[ResourceFiltering][ResourcePolicies] Velero test on skip backup of volume by resource policies", ResourcePoliciesTest) +var _ = Describe("[ResourceFiltering][ResourcePolicies][Restic] Velero test on skip backup of volume by resource policies", ResourcePoliciesTest) var _ = Describe("[ResourceModifier][Restore] Velero test on resource modifiers from the cluster restore", ResourceModifiersTest) var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic) var _ = Describe("[Backups][Deletion][Snapshot] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots) -var _ = Describe("[Backups][TTL][LongTime] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest) +var _ = Describe("[Backups][TTL][LongTime][Snapshot] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest) var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest) var _ = Describe("[Schedule][BR][Pause][LongTime] Backup will be created periodly by schedule defined by a Cron expression", ScheduleBackupTest) -var _ = Describe("[Schedule][OrederedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources) -var _ = Describe("[Schedule][BackupCreation] Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", ScheduleBackupCreationTest) +var _ = Describe("[Schedule][OrderedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources) +var _ = Describe("[Schedule][BackupCreation][SKIP_KIND] Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", ScheduleBackupCreationTest) var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest) @@ -165,7 +165,7 @@ var _ = Describe("[pv-backup][Opt-Out] Backup resources should follow the specif var _ = Describe("[Basic][Nodeport] Service nodeport reservation during restore is configurable", NodePortTest) var _ = Describe("[Basic][StorageClass] Storage class of persistent volumes and persistent volume claims can be changed during restores", StorageClasssChangingTest) -var _ = Describe("[Basic][SelectedNode] Node selectors of persistent volume claims can be changed during restores", PVCSelectedNodeChangingTest) +var _ = Describe("[Basic][SelectedNode][SKIP_KIND] Node selectors of persistent volume claims can be changed during restores", PVCSelectedNodeChangingTest) func GetKubeconfigContext() error { var err error @@ -206,6 +206,18 @@ func TestE2e(t *testing.T) { t.Skip("Skipping E2E tests") } + if VeleroCfg.CloudProvider != "kind" { + fmt.Println("For cloud platforms, object store plugin provider will be set as cloud provider") + // If ObjectStoreProvider is not provided, then using the value same as CloudProvider + if VeleroCfg.ObjectStoreProvider == "" { + VeleroCfg.ObjectStoreProvider = VeleroCfg.CloudProvider + } + } else { + if VeleroCfg.ObjectStoreProvider == "" { + t.Error(errors.New("No object store provider specified - must be specified when using kind as the cloud provider")) // Must have an object store provider + } + } + var err error if err = GetKubeconfigContext(); err != nil { fmt.Println(err) diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 7395970a5b..56f0a9e6b7 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -298,7 +298,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) // For SnapshotMoveData pipelines, we should use standby clustr setting for Velero installation // In nightly CI, StandbyClusterPlugins is set properly if pipeline is for SnapshotMoveData. veleroCfg.Plugins = veleroCfg.StandbyClusterPlugins - veleroCfg.ObjectStoreProvider = veleroCfg.StandbyClusterOjbectStoreProvider + veleroCfg.ObjectStoreProvider = veleroCfg.StandbyClusterObjectStoreProvider } Expect(VeleroInstall(context.Background(), &veleroCfg, true)).To(Succeed()) diff --git a/test/perf/Makefile b/test/perf/Makefile index 759d19aba8..7500d0b1bf 100644 --- a/test/perf/Makefile +++ b/test/perf/Makefile @@ -41,8 +41,10 @@ help: ## Display this help TOOLS_DIR := $(REPO_ROOT)/hack/tools BIN_DIR := bin +# Try to not modify PATH if possible +GOBIN := $(REPO_ROOT)/.go/bin TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR) -GINKGO := $(GOPATH)/bin/ginkgo +GINKGO := $(GOBIN)/ginkgo KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin GINKGO_FOCUS ?= @@ -88,9 +90,13 @@ VELERO_POD_CPU_REQUEST ?= 2 VELERO_POD_MEM_REQUEST ?= 2Gi POD_VOLUME_OPERATION_TIMEOUT ?= 6h +# Make sure ginkgo is in $GOBIN .PHONY:ginkgo -ginkgo: # Make sure ginkgo is in $GOPATH/bin - go install github.com/onsi/ginkgo/ginkgo@v1.16.5 +ginkgo: ${GOBIN}/ginkgo + +# This target does not run if ginkgo is already in $GOBIN +${GOBIN}/ginkgo: + GOBIN=${GOBIN} go install github.com/onsi/ginkgo/ginkgo@v1.16.5 .PHONY: run run: ginkgo diff --git a/test/testdata/storage-class/kind.yaml b/test/testdata/storage-class/kind.yaml new file mode 100644 index 0000000000..b794b9332f --- /dev/null +++ b/test/testdata/storage-class/kind.yaml @@ -0,0 +1,10 @@ +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: e2e-storage-class +parameters: + type: pd-standard +provisioner: rancher.io/local-path +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer \ No newline at end of file diff --git a/test/types.go b/test/types.go index 0d27e96bea..4b1d78485e 100644 --- a/test/types.go +++ b/test/types.go @@ -85,7 +85,7 @@ type VeleroConfig struct { DataMoverPlugin string StandbyClusterCloudProvider string StandbyClusterPlugins string - StandbyClusterOjbectStoreProvider string + StandbyClusterObjectStoreProvider string DebugVeleroPodRestart bool IsUpgradeTest bool WithoutDisableInformerCacheParam bool diff --git a/test/util/k8s/serviceaccount.go b/test/util/k8s/serviceaccount.go index 5013a628d9..24d748aeaf 100644 --- a/test/util/k8s/serviceaccount.go +++ b/test/util/k8s/serviceaccount.go @@ -47,8 +47,16 @@ func WaitUntilServiceAccountCreated(ctx context.Context, client TestClient, name } func PatchServiceAccountWithImagePullSecret(ctx context.Context, client TestClient, namespace, serviceAccount, dockerCredentialFile string) error { + if dockerCredentialFile == "" { + // use the default docker credential file in the home directory + dockerCredentialFile = os.Getenv("HOME") + "/.docker/config.json" + } + // if file do not exist, do not patch the service account, just return credential, err := os.ReadFile(dockerCredentialFile) if err != nil { + if os.IsNotExist(err) { + return nil + } return errors.Wrapf(err, "failed to read the docker credential file %q", dockerCredentialFile) } secretName := "image-pull-secret" diff --git a/test/util/providers/common.go b/test/util/providers/common.go index 2576d53577..13b1cf8eef 100644 --- a/test/util/providers/common.go +++ b/test/util/providers/common.go @@ -34,24 +34,24 @@ type ObjectsInStorage interface { IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error } -func ObjectsShouldBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error { +func ObjectsShouldBeInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error { fmt.Printf("|| VERIFICATION || - %s should exist in storage [%s %s]\n", backupName, bslPrefix, subPrefix) - exist, err := IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix) + exist, err := IsObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix) if !exist { return errors.Wrap(err, fmt.Sprintf("|| UNEXPECTED ||Backup object %s is not exist in object store after backup as expected\n", backupName)) } fmt.Printf("|| EXPECTED || - Backup %s exist in object storage bucket %s\n", backupName, bslBucket) return nil } -func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string, retryTimes int) error { +func ObjectsShouldNotBeInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string, retryTimes int) error { var err error var exist bool fmt.Printf("|| VERIFICATION || - %s %s should not exist in object store %s\n", subPrefix, backupName, bslPrefix) if cloudCredentialsFile == "" { - return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider)) + return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", objectStoreProvider)) } for i := 0; i < retryTimes; i++ { - exist, err = IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix) + exist, err = IsObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix) if err != nil { return errors.Wrapf(err, "|| UNEXPECTED || - Failed to get backup %s in object store\n", backupName) } @@ -63,6 +63,10 @@ func ObjectsShouldNotBeInBucket(cloudProvider, cloudCredentialsFile, bslBucket, } return errors.New(fmt.Sprintf("|| UNEXPECTED ||Backup object %s still exist in object store after backup deletion\n", backupName)) } + +// This function returns a storage interface based on the cloud provider for querying objects and snapshots +// When cloudProvider is kind, pass in object storage provider instead. For example, "aws". +// Snapshots are not supported on kind. func getProvider(cloudProvider string) (ObjectsInStorage, error) { var s ObjectsInStorage switch cloudProvider { @@ -89,24 +93,24 @@ func getFullPrefix(bslPrefix, subPrefix string) string { } return bslPrefix } -func IsObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) (bool, error) { +func IsObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) (bool, error) { bslPrefix = getFullPrefix(bslPrefix, subPrefix) - s, err := getProvider(cloudProvider) + s, err := getProvider(objectStoreProvider) if err != nil { - return false, errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) + return false, errors.Wrapf(err, fmt.Sprintf("Object store provider %s is not valid", objectStoreProvider)) } return s.IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName) } -func DeleteObjectsInBucket(cloudProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error { +func DeleteObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error { bslPrefix = getFullPrefix(bslPrefix, subPrefix) fmt.Printf("|| VERIFICATION || - Delete backup %s in storage %s\n", backupName, bslPrefix) if cloudCredentialsFile == "" { - return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider)) + return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", objectStoreProvider)) } - s, err := getProvider(cloudProvider) + s, err := getProvider(objectStoreProvider) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) + return errors.Wrapf(err, fmt.Sprintf("Object store provider %s is not valid", objectStoreProvider)) } err = s.DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName) if err != nil { diff --git a/test/util/velero/install.go b/test/util/velero/install.go index cdb61e7409..5a19eb064a 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -75,6 +75,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *VeleroConfig, isStandbyCluste if isStandbyCluster { veleroCfg.CloudProvider = veleroCfg.StandbyClusterCloudProvider } + if veleroCfg.CloudProvider != "kind" { fmt.Println("For cloud platforms, object store plugin provider will be set as cloud provider") // If ObjectStoreProvider is not provided, then using the value same as CloudProvider @@ -278,14 +279,11 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options if len(options.Features) > 0 { args = append(args, "--features", options.Features) - if strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots { - if strings.EqualFold(cloudProvider, "azure") { - fmt.Println("Start to install Azure VolumeSnapshotClass ...") - if err := KubectlApplyByFile(ctx, "../util/csi/AzureVolumeSnapshotClass.yaml"); err != nil { - return err - } + if strings.EqualFold(cloudProvider, "azure") && strings.EqualFold(options.Features, FeatureCSI) && options.UseVolumeSnapshots { + fmt.Println("Start to install Azure VolumeSnapshotClass ...") + if err := KubectlApplyByFile(ctx, "../util/csi/AzureVolumeSnapshotClass.yaml"); err != nil { + return err } - } } if options.GarbageCollectionFrequency > 0 { diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index 91f3f8b04e..29d0afc140 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -26,6 +26,7 @@ import ( "net/http" "os" "os/exec" + "os/user" "path/filepath" "reflect" "regexp" @@ -184,6 +185,14 @@ func getProviderVeleroInstallOptions(veleroCfg *VeleroConfig, io.ProviderName = veleroCfg.ObjectStoreProvider if veleroCfg.CloudCredentialsFile != "" { + // Expand home directory if it is specified. https://stackoverflow.com/a/17617721 + if strings.HasPrefix(veleroCfg.CloudCredentialsFile, "~/") { + usr, _ := user.Current() + dir := usr.HomeDir + // Use strings.HasPrefix so we don't match paths like + // "/something/~/something/" + veleroCfg.CloudCredentialsFile = filepath.Join(dir, veleroCfg.CloudCredentialsFile[2:]) + } realPath, err := filepath.Abs(veleroCfg.CloudCredentialsFile) if err != nil { return nil, err @@ -927,7 +936,7 @@ func getVeleroCliTarball(cliTarballUrl string) (*os.File, error) { } func DeleteBackupResource(ctx context.Context, veleroCLI string, backupName string) error { - args := []string{"backup", "delete", backupName, "--confirm"} + args := []string{"--namespace", VeleroCfg.VeleroNamespace, "backup", "delete", backupName, "--confirm"} cmd := exec.CommandContext(ctx, veleroCLI, args...) fmt.Println("Delete backup Command:" + cmd.String()) @@ -939,7 +948,7 @@ func DeleteBackupResource(ctx context.Context, veleroCLI string, backupName stri output := strings.Replace(stdout, "\n", " ", -1) fmt.Println("Backup delete command output:" + output) - args = []string{"backup", "get", backupName} + args = []string{"--namespace", VeleroCfg.VeleroNamespace, "backup", "get", backupName} retryTimes := 5 for i := 1; i < retryTimes+1; i++ { @@ -958,13 +967,13 @@ func DeleteBackupResource(ctx context.Context, veleroCLI string, backupName stri return nil } -func GetBackup(ctx context.Context, veleroCLI string, backupName string) (string, string, error) { - args := []string{"backup", "get", backupName} +func GetBackup(ctx context.Context, veleroCLI, backupName string) (string, string, error) { + args := []string{"--namespace", VeleroCfg.VeleroNamespace, "backup", "get", backupName} cmd := exec.CommandContext(ctx, veleroCLI, args...) return veleroexec.RunCommand(cmd) } -func IsBackupExist(ctx context.Context, veleroCLI string, backupName string) (bool, error) { +func IsBackupExist(ctx context.Context, veleroCLI, backupName string) (bool, error) { out, outerr, err := GetBackup(ctx, veleroCLI, backupName) if err != nil { if strings.Contains(outerr, "not found") { @@ -977,7 +986,7 @@ func IsBackupExist(ctx context.Context, veleroCLI string, backupName string) (bo return true, nil } -func WaitBackupDeleted(ctx context.Context, veleroCLI string, backupName string, timeout time.Duration) error { +func WaitBackupDeleted(ctx context.Context, veleroCLI, backupName string, timeout time.Duration) error { return wait.PollImmediate(10*time.Second, timeout, func() (bool, error) { if exist, err := IsBackupExist(ctx, veleroCLI, backupName); err != nil { return false, err @@ -992,7 +1001,7 @@ func WaitBackupDeleted(ctx context.Context, veleroCLI string, backupName string, }) } -func WaitForExpectedStateOfBackup(ctx context.Context, veleroCLI string, backupName string, +func WaitForExpectedStateOfBackup(ctx context.Context, veleroCLI, backupName string, timeout time.Duration, existing bool) error { return wait.PollImmediate(10*time.Second, timeout, func() (bool, error) { if exist, err := IsBackupExist(ctx, veleroCLI, backupName); err != nil { @@ -1013,7 +1022,7 @@ func WaitForExpectedStateOfBackup(ctx context.Context, veleroCLI string, backupN }) } -func WaitForBackupToBeCreated(ctx context.Context, veleroCLI string, backupName string, timeout time.Duration) error { +func WaitForBackupToBeCreated(ctx context.Context, veleroCLI, backupName string, timeout time.Duration) error { return WaitForExpectedStateOfBackup(ctx, veleroCLI, backupName, timeout, true) }