From d9db13fcbebf67382ee3eca14010419def542537 Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Tue, 23 Jul 2024 14:39:01 +0800 Subject: [PATCH] Use labels instead of regex to filter E2E test cases. Signed-off-by: Xun Jiang --- .github/workflows/e2e-test-kind.yaml | 30 ++-- test/Makefile | 11 +- .../api-group/enable_api_group_versions.go | 2 +- test/e2e/e2e_suite_test.go | 169 +++++++++++------- test/perf/e2e_suite_test.go | 9 +- 5 files changed, 136 insertions(+), 85 deletions(-) diff --git a/.github/workflows/e2e-test-kind.yaml b/.github/workflows/e2e-test-kind.yaml index 57d4998237..0975aa37fd 100644 --- a/.github/workflows/e2e-test-kind.yaml +++ b/.github/workflows/e2e-test-kind.yaml @@ -70,13 +70,10 @@ jobs: focus: # tests to focus on, use `|` to concatenate multiple regexes to run on the same job # ordered according to e2e_suite_test.go order - - Basic\]\[ClusterResource - - ResourceFiltering - - ResourceModifier|Backups|PrivilegesMgmt\]\[SSR - - Schedule\]\[OrderedResources - - NamespaceMapping\]\[Single\]\[Restic|NamespaceMapping\]\[Multiple\]\[Restic - - Basic\]\[Nodeport - - Basic\]\[StorageClass + - Basic && (ClusterResource || NodePort || StorageClass) + - ResourceFiltering && !Restic + - ResourceModifier || (Backups && BackupsSync) || PrivilegesMgmt || OrderedResources + - (NamespaceMapping && Single && Restic) || (NamespaceMapping && Multiple && Restic) fail-fast: false steps: - name: Set up Go @@ -128,13 +125,18 @@ jobs: curl -LO https://dl.k8s.io/release/v${{ matrix.k8s }}/bin/linux/amd64/kubectl sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl - GOPATH=~/go CLOUD_PROVIDER=kind \ - OBJECT_STORE_PROVIDER=aws BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \ - CREDS_FILE=/tmp/credential BSL_BUCKET=bucket \ - ADDITIONAL_OBJECT_STORE_PROVIDER=aws ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \ - ADDITIONAL_CREDS_FILE=/tmp/credential ADDITIONAL_BSL_BUCKET=additional-bucket \ - GINKGO_FOCUS='${{ matrix.focus }}' VELERO_IMAGE=velero:pr-test \ - GINKGO_SKIP='SKIP_KIND|pv-backup|Restic|Snapshot|LongTime' \ + GOPATH=~/go \ + CLOUD_PROVIDER=kind \ + OBJECT_STORE_PROVIDER=aws \ + BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \ + CREDS_FILE=/tmp/credential \ + BSL_BUCKET=bucket \ + ADDITIONAL_OBJECT_STORE_PROVIDER=aws \ + ADDITIONAL_BSL_CONFIG=region=minio,s3ForcePathStyle="true",s3Url=http://$(hostname -i):9000 \ + ADDITIONAL_CREDS_FILE=/tmp/credential \ + ADDITIONAL_BSL_BUCKET=additional-bucket \ + VELERO_IMAGE=velero:pr-test \ + GINKGO_LABELs='${{ matrix.focus }}' \ make -C test/ run-e2e timeout-minutes: 30 - name: Upload debug bundle diff --git a/test/Makefile b/test/Makefile index fbc0e08e35..e8a2dfb788 100644 --- a/test/Makefile +++ b/test/Makefile @@ -47,10 +47,9 @@ TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR) GINKGO := $(GOBIN)/ginkgo KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin -GINKGO_FOCUS ?= -GINKGO_SKIP ?= -SKIP_STR := $(foreach var, $(subst ., ,$(GINKGO_SKIP)),-skip "$(var)") -FOCUS_STR := $(foreach var, $(subst ., ,$(GINKGO_FOCUS)),-focus "$(var)") +# Please reference to this document for Ginkgo label spec format. +# https://onsi.github.io/ginkgo/#spec-labels +GINKGO_LABELS ?= VELERO_CLI ?=$$(pwd)/../_output/bin/$(GOOS)/$(GOARCH)/velero VELERO_IMAGE ?= velero/velero:main PLUGINS ?= @@ -166,7 +165,7 @@ run-e2e: ginkgo (echo "Bucket to store the backups from E2E tests is required, please re-run with BSL_BUCKET="; exit 1 ) @[ "${CLOUD_PROVIDER}" ] && echo "Using cloud provider ${CLOUD_PROVIDER}" || \ (echo "Cloud provider for target cloud/plugin provider is required, please rerun with CLOUD_PROVIDER="; exit 1) - @$(GINKGO) run -v $(FOCUS_STR) $(SKIP_STR) --junit-report report.xml ./e2e -- $(COMMON_ARGS) \ + @$(GINKGO) -v --label-filter=\"$(GINKGO_LABELS)\" ./e2e -- $(COMMON_ARGS) \ -upgrade-from-velero-cli=$(UPGRADE_FROM_VELERO_CLI) \ -upgrade-from-velero-version=$(UPGRADE_FROM_VELERO_VERSION) \ -migrate-from-velero-cli=$(MIGRATE_FROM_VELERO_CLI) \ @@ -200,7 +199,7 @@ run-perf: ginkgo (echo "Bucket to store the backups from E2E tests is required, please re-run with BSL_BUCKET="; exit 1 ) @[ "${CLOUD_PROVIDER}" ] && echo "Using cloud provider ${CLOUD_PROVIDER}" || \ (echo "Cloud provider for target cloud/plugin provider is required, please rerun with CLOUD_PROVIDER="; exit 1) - @$(GINKGO) run -v $(FOCUS_STR) $(SKIP_STR) --junit-report report.xml ./perf -- $(COMMON_ARGS) \ + @$(GINKGO) -v --label-filter=\"$(GINKGO_LABELS)\" ./perf -- $(COMMON_ARGS) \ -nfs-server-path=$(NFS_SERVER_PATH) \ -test-case-describe=$(TEST_CASE_DESCRIBE) \ -backup-for-restore=$(BACKUP_FOR_RESTORE) \ diff --git a/test/e2e/basic/api-group/enable_api_group_versions.go b/test/e2e/basic/api-group/enable_api_group_versions.go index 4fc17ed975..7f67dbf6e1 100644 --- a/test/e2e/basic/api-group/enable_api_group_versions.go +++ b/test/e2e/basic/api-group/enable_api_group_versions.go @@ -53,7 +53,7 @@ type apiGropuVersionsTest struct { want map[string]map[string]string } -func APIGropuVersionsTest() { +func APIGroupVersionsTest() { var ( group string err error diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 45b093b13b..9b4201c018 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -112,79 +112,126 @@ func init() { // caused by no expected snapshot found. If we use retain as reclaim policy, then this label can be ignored, all test // cases can be executed as expected successful result. -var _ = Describe("[APIGroup][APIVersion][SKIP_KIND] Velero tests with various CRD API group versions", APIGropuVersionsTest) -var _ = Describe("[APIGroup][APIExtensions][SKIP_KIND] CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", APIExtensionsVersionsTest) +var _ = Describe("Velero tests with various CRD API group versions", + Label("APIGroup", "APIVersion", "SKIP_KIND"), APIGroupVersionsTest) +var _ = Describe("CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", + Label("APIGroup", "APIExtensions", "SKIP_KIND"), APIExtensionsVersionsTest) -// Test backup and restore of Kibishi using restic -var _ = Describe("[Basic][Restic] Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupRestoreWithRestic) +// Test backup and restore of Kibishii using restic +var _ = Describe("Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", + Label("Basic", "Restic"), BackupRestoreWithRestic) -var _ = Describe("[Basic][Snapshot][SkipVanillaZfs] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreWithSnapshots) +var _ = Describe("Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", + Label("Basic", "Snapshot", "SkipVanillaZfs"), BackupRestoreWithSnapshots) -var _ = Describe("[Basic][Snapshot][RetainPV] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreRetainedPVWithSnapshots) +var _ = Describe("Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", + Label("Basic", "Snapshot", "RetainPV"), BackupRestoreRetainedPVWithSnapshots) -var _ = Describe("[Basic][Restic][RetainPV] Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupRestoreRetainedPVWithRestic) +var _ = Describe("Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", + Label("Basic", "Restic", "RetainPV"), BackupRestoreRetainedPVWithRestic) -var _ = Describe("[Basic][ClusterResource] Backup/restore of cluster resources", ResourcesCheckTest) +var _ = Describe("Backup/restore of cluster resources", + Label("Basic", "ClusterResource"), ResourcesCheckTest) -var _ = Describe("[Scale][LongTime] Backup/restore of 2500 namespaces", MultiNSBackupRestore) +var _ = Describe("Service NodePort reservation during restore is configurable", + Label("Basic", "NodePort"), NodePortTest) -// Upgrade test by Kibishi using restic -var _ = Describe("[Upgrade][Restic] Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", BackupUpgradeRestoreWithRestic) -var _ = Describe("[Upgrade][Snapshot][SkipVanillaZfs] Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", BackupUpgradeRestoreWithSnapshots) +var _ = Describe("Storage class of persistent volumes and persistent volume claims can be changed during restores", + Label("Basic", "StorageClass"), StorageClasssChangingTest) -// test filter objects by namespace, type, or labels when backup or restore. -var _ = Describe("[ResourceFiltering][ExcludeFromBackup] Resources with the label velero.io/exclude-from-backup=true are not included in backup", ExcludeFromBackupTest) -var _ = Describe("[ResourceFiltering][ExcludeNamespaces][Backup] Velero test on exclude namespace from the cluster backup", BackupWithExcludeNamespaces) -var _ = Describe("[ResourceFiltering][ExcludeNamespaces][Restore] Velero test on exclude namespace from the cluster restore", RestoreWithExcludeNamespaces) -var _ = Describe("[ResourceFiltering][ExcludeResources][Backup] Velero test on exclude resources from the cluster backup", BackupWithExcludeResources) -var _ = Describe("[ResourceFiltering][ExcludeResources][Restore] Velero test on exclude resources from the cluster restore", RestoreWithExcludeResources) -var _ = Describe("[ResourceFiltering][IncludeNamespaces][Backup] Velero test on include namespace from the cluster backup", BackupWithIncludeNamespaces) -var _ = Describe("[ResourceFiltering][IncludeNamespaces][Restore] Velero test on include namespace from the cluster restore", RestoreWithIncludeNamespaces) -var _ = Describe("[ResourceFiltering][IncludeResources][Backup] Velero test on include resources from the cluster backup", BackupWithIncludeResources) -var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on include resources from the cluster restore", RestoreWithIncludeResources) -var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector) -var _ = Describe("[ResourceFiltering][ResourcePolicies][Restic] Velero test on skip backup of volume by resource policies", ResourcePoliciesTest) - -// backup VolumeInfo test -var _ = Describe("[BackupVolumeInfo][SkippedVolume]", SkippedVolumeInfoTest) -var _ = Describe("[BackupVolumeInfo][FilesystemUpload]", FilesystemUploadVolumeInfoTest) -var _ = Describe("[BackupVolumeInfo][CSIDataMover]", CSIDataMoverVolumeInfoTest) -var _ = Describe("[BackupVolumeInfo][CSISnapshot]", CSISnapshotVolumeInfoTest) -var _ = Describe("[BackupVolumeInfo][NativeSnapshot]", NativeSnapshotVolumeInfoTest) - -var _ = Describe("[ResourceModifier][Restore] Velero test on resource modifiers from the cluster restore", ResourceModifiersTest) - -var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic) -var _ = Describe("[Backups][Deletion][Snapshot][SkipVanillaZfs] Velero tests of snapshot backup deletion", BackupDeletionWithSnapshots) -var _ = Describe("[Backups][TTL][LongTime][Snapshot][SkipVanillaZfs] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", TTLTest) -var _ = Describe("[Backups][BackupsSync] Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", BackupsSyncTest) - -var _ = Describe("[Schedule][BR][Pause][LongTime] Backup will be created periodly by schedule defined by a Cron expression", ScheduleBackupTest) -var _ = Describe("[Schedule][OrderedResources] Backup resources should follow the specific order in schedule", ScheduleOrderedResources) -var _ = Describe("[Schedule][BackupCreation][SKIP_KIND] Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", ScheduleBackupCreationTest) +var _ = Describe("Node selectors of persistent volume claims can be changed during restores", + Label("Basic", "SelectedNode", "SKIP_KIND"), PVCSelectedNodeChangingTest) -var _ = Describe("[PrivilegesMgmt][SSR] Velero test on ssr object when controller namespace mix-ups", SSRTest) +var _ = Describe("Backup/restore of 2500 namespaces", + Label("Scale", "LongTime"), MultiNSBackupRestore) -var _ = Describe("[BSL][Deletion][Snapshot][SkipVanillaZfs] Local backups will be deleted once the corresponding backup storage location is deleted", BslDeletionWithSnapshots) -var _ = Describe("[BSL][Deletion][Restic] Local backups and restic repos will be deleted once the corresponding backup storage location is deleted", BslDeletionWithRestic) +// Upgrade test by Kibishii using Restic +var _ = Describe("Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", + Label("Upgrade", "Restic"), BackupUpgradeRestoreWithRestic) +var _ = Describe("Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", + Label("Upgrade", "Snapshot", "SkipVanillaZfs"), BackupUpgradeRestoreWithSnapshots) -var _ = Describe("[Migration][Restic] Migrate resources between clusters by Restic", MigrationWithRestic) -var _ = Describe("[Migration][Snapshot][SkipVanillaZfs] Migrate resources between clusters by snapshot", MigrationWithSnapshots) - -var _ = Describe("[NamespaceMapping][Single][Restic] Backup resources should follow the specific order in schedule", OneNamespaceMappingResticTest) -var _ = Describe("[NamespaceMapping][Multiple][Restic] Backup resources should follow the specific order in schedule", MultiNamespacesMappingResticTest) -var _ = Describe("[NamespaceMapping][Single][Snapshot][SkipVanillaZfs] Backup resources should follow the specific order in schedule", OneNamespaceMappingSnapshotTest) -var _ = Describe("[NamespaceMapping][Multiple][Snapshot]SkipVanillaZfs] Backup resources should follow the specific order in schedule", MultiNamespacesMappingSnapshotTest) - -var _ = Describe("Backup resources should follow the specific order in schedule", Label("pv-backup", "Opt-In"), OptInPVBackupTest) -var _ = Describe("Backup resources should follow the specific order in schedule", Label("pv-backup", "Opt-Out"), OptOutPVBackupTest) - -var _ = Describe("[Basic][Nodeport] Service nodeport reservation during restore is configurable", NodePortTest) -var _ = Describe("[Basic][StorageClass] Storage class of persistent volumes and persistent volume claims can be changed during restores", StorageClasssChangingTest) -var _ = Describe("[Basic][SelectedNode][SKIP_KIND] Node selectors of persistent volume claims can be changed during restores", PVCSelectedNodeChangingTest) +// test filter objects by namespace, type, or labels when backup or restore. +var _ = Describe("Resources with the label velero.io/exclude-from-backup=true are not included in backup", + Label("ResourceFiltering", "ExcludeFromBackup"), ExcludeFromBackupTest) +var _ = Describe("Velero test on exclude namespace from the cluster backup", + Label("ResourceFiltering", "ExcludeNamespaces", "Backup"), BackupWithExcludeNamespaces) +var _ = Describe("Velero test on exclude namespace from the cluster restore", + Label("ResourceFiltering", "ExcludeNamespaces", "Restore"), RestoreWithExcludeNamespaces) +var _ = Describe("Velero test on exclude resources from the cluster backup", + Label("ResourceFiltering", "ExcludeResources", "Backup"), BackupWithExcludeResources) +var _ = Describe("Velero test on exclude resources from the cluster restore", + Label("ResourceFiltering", "ExcludeResources", "Restore"), RestoreWithExcludeResources) +var _ = Describe("Velero test on include namespace from the cluster backup", + Label("ResourceFiltering", "IncludeNamespaces", "Backup"), BackupWithIncludeNamespaces) +var _ = Describe("Velero test on include namespace from the cluster restore", + Label("ResourceFiltering", "IncludeNamespaces", "Restore"), RestoreWithIncludeNamespaces) +var _ = Describe("Velero test on include resources from the cluster backup", + Label("ResourceFiltering", "IncludeResources", "Backup"), BackupWithIncludeResources) +var _ = Describe("Velero test on include resources from the cluster restore", + Label("ResourceFiltering", "IncludeResources", "Restore"), RestoreWithIncludeResources) +var _ = Describe("Velero test on backup include resources matching the label selector", + Label("ResourceFiltering", "LabelSelector"), BackupWithLabelSelector) +var _ = Describe("Velero test on skip backup of volume by resource policies", + Label("ResourceFiltering", "ResourcePolicies", "Restic"), ResourcePoliciesTest) -var _ = Describe("[UploaderConfig][ParallelFilesUpload] Velero test on parallel files upload", ParallelFilesUploadTest) -var _ = Describe("[UploaderConfig][ParallelFilesDownload] Velero test on parallel files download", ParallelFilesDownloadTest) +// backup VolumeInfo test +var _ = Describe("", Label("BackupVolumeInfo", "SkippedVolume"), SkippedVolumeInfoTest) +var _ = Describe("", Label("BackupVolumeInfo", "FilesystemUpload"), FilesystemUploadVolumeInfoTest) +var _ = Describe("", Label("BackupVolumeInfo", "CSIDataMover"), CSIDataMoverVolumeInfoTest) +var _ = Describe("", Label("BackupVolumeInfo", "CSISnapshot"), CSISnapshotVolumeInfoTest) +var _ = Describe("", Label("BackupVolumeInfo", "NativeSnapshot"), NativeSnapshotVolumeInfoTest) + +var _ = Describe("Velero test on resource modifiers from the cluster restore", + Label("ResourceModifier", "Restore"), ResourceModifiersTest) + +var _ = Describe("Velero tests of Restic backup deletion", + Label("Backups", "Deletion", "Restic"), BackupDeletionWithRestic) +var _ = Describe("Velero tests of snapshot backup deletion", + Label("Backups", "Deletion", "Snapshot", "SkipVanillaZfs"), BackupDeletionWithSnapshots) +var _ = Describe("Local backups and Restic repos will be deleted once the corresponding backup storage location is deleted", + Label("Backups", "TTL", "LongTime", "Snapshot", "SkipVanillaZfs"), TTLTest) +var _ = Describe("Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", + Label("Backups", "BackupsSync"), BackupsSyncTest) + +var _ = Describe("Backup will be created periodically by schedule defined by a Cron expression", + Label("Schedule", "BR", "Pause", "LongTime"), ScheduleBackupTest) +var _ = Describe("Backup resources should follow the specific order in schedule", + Label("Schedule", "OrderedResources"), ScheduleOrderedResources) +var _ = Describe("Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", + Label("Schedule", "BackupCreation", "SKIP_KIND"), ScheduleBackupCreationTest) + +var _ = Describe("Velero test on ssr object when controller namespace mix-ups", + Label("PrivilegesMgmt", "SSR"), SSRTest) + +var _ = Describe("Local backups will be deleted once the corresponding backup storage location is deleted", + Label("BSL", "Deletion", "Snapshot", "SkipVanillaZfs"), BslDeletionWithSnapshots) +var _ = Describe("Local backups and Restic repos will be deleted once the corresponding backup storage location is deleted", + Label("BSL", "Deletion", "Restic"), BslDeletionWithRestic) + +var _ = Describe("Migrate resources between clusters by Restic", + Label("Migration", "Restic"), MigrationWithRestic) +var _ = Describe("Migrate resources between clusters by snapshot", + Label("Migration", "Snapshot", "SkipVanillaZfs"), MigrationWithSnapshots) + +var _ = Describe("Backup resources should follow the specific order in schedule", + Label("NamespaceMapping", "Single", "Restic"), OneNamespaceMappingResticTest) +var _ = Describe("Backup resources should follow the specific order in schedule", + Label("NamespaceMapping", "Multiple", "Restic"), MultiNamespacesMappingResticTest) +var _ = Describe("Backup resources should follow the specific order in schedule", + Label("NamespaceMapping", "Single", "Snapshot", "SkipVanillaZfs"), OneNamespaceMappingSnapshotTest) +var _ = Describe("Backup resources should follow the specific order in schedule", + Label("NamespaceMapping", "Multiple", "Snapshot", "SkipVanillaZfs"), MultiNamespacesMappingSnapshotTest) + +var _ = Describe("Backup resources should follow the specific order in schedule", + Label("PVBackup", "OptIN"), OptInPVBackupTest) +var _ = Describe("Backup resources should follow the specific order in schedule", + Label("PVBackup", "OptOUT"), OptOutPVBackupTest) + +var _ = Describe("Velero test on parallel files upload", + Label("UploaderConfig", "ParallelFilesUpload"), ParallelFilesUploadTest) +var _ = Describe("Velero test on parallel files download", + Label("UploaderConfig", "ParallelFilesDownload"), ParallelFilesDownloadTest) func GetKubeconfigContext() error { var err error diff --git a/test/perf/e2e_suite_test.go b/test/perf/e2e_suite_test.go index a2aa85d63c..e047749c22 100644 --- a/test/perf/e2e_suite_test.go +++ b/test/perf/e2e_suite_test.go @@ -94,11 +94,14 @@ func initConfig() error { return nil } -var _ = Describe("[PerformanceTest][BackupAndRestore] Velero test on both backup and restore resources", test.TestFunc(&basic.BasicTest{})) +var _ = Describe("Velero test on both backup and restore resources", + Label("PerformanceTest", "BackupAndRestore"), test.TestFunc(&basic.BasicTest{})) -var _ = Describe("[PerformanceTest][Backup] Velero test on only backup resources", test.TestFunc(&backup.BackupTest{})) +var _ = Describe("Velero test on only backup resources", + Label("PerformanceTest", "Backup"), test.TestFunc(&backup.BackupTest{})) -var _ = Describe("[PerformanceTest][Restore] Velero test on only restore resources", test.TestFunc(&restore.RestoreTest{})) +var _ = Describe("Velero test on only restore resources", + Label("PerformanceTest", "Restore"), test.TestFunc(&restore.RestoreTest{})) func TestE2e(t *testing.T) { flag.Parse()