From e5354e123b70877cc49e8d2df7ca02fc12d71394 Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Thu, 14 Nov 2024 11:07:05 +0800 Subject: [PATCH] Modify the StorageClass install and delete code. * Only install and uninstall SC and VSC once for default cluster. * Install and uninstall SC and VSC for standby cluster on migration case. * Refactor the StorageClass and VolumeSnapshotClass YAMLs. * Prettify the e2e_suite_test.go Signed-off-by: Xun Jiang --- test/e2e/backup/backup.go | 2 +- test/e2e/backups/sync_backups.go | 4 +- test/e2e/backups/ttl.go | 2 +- .../api-group/enable_api_group_extentions.go | 6 +- .../api-group/enable_api_group_versions.go | 2 +- test/e2e/basic/backup-volume-info/base.go | 14 +- .../backup-volume-info/csi_data_mover.go | 3 +- .../basic/backup-volume-info/csi_snapshot.go | 3 +- .../backup-volume-info/filesystem_upload.go | 3 +- .../backup-volume-info/native_snapshot.go | 3 +- .../backup-volume-info/skipped_volumes.go | 3 +- test/e2e/basic/pvc-selected-node-changing.go | 8 - test/e2e/basic/storage-class-changing.go | 15 +- test/e2e/e2e_suite_test.go | 800 ++++++++++++++---- test/e2e/migration/migration.go | 94 +- .../parallel_files_download.go | 6 - .../parallel_files_upload.go | 6 - test/e2e/pv-backup/pv-backup-filter.go | 5 - .../resourcemodifiers/resource_modifiers.go | 2 +- .../e2e/resourcepolicies/resource_policies.go | 22 +- test/e2e/schedule/schedule-backup-creation.go | 27 +- test/e2e/upgrade/upgrade.go | 6 +- test/perf/e2e_suite_test.go | 2 +- test/testdata/storage-class/README.md | 12 + .../{aws-csi.yaml => aws-legecy.yaml} | 4 +- test/testdata/storage-class/aws.yaml | 2 +- .../{azure-csi.yaml => azure-legacy.yaml} | 4 +- test/testdata/storage-class/azure.yaml | 2 +- .../{gcp-csi.yaml => gcp-legacy.yaml} | 4 +- test/testdata/storage-class/gcp.yaml | 2 +- .../storage-class/vanilla-zfs-csi.yaml | 13 - test/testdata/storage-class/vanilla-zfs.yaml | 2 +- .../{vsphere-csi.yaml => vsphere-legacy.yaml} | 5 +- test/testdata/storage-class/vsphere.yaml | 3 +- .../volume-snapshot-class/vanilla-zfs.yaml | 2 +- .../volume-snapshot-class/vsphere.yaml | 2 +- test/types.go | 5 +- test/util/k8s/configmap.go | 4 +- test/util/velero/install.go | 49 +- test/util/velero/velero_utils.go | 32 +- 40 files changed, 813 insertions(+), 372 deletions(-) create mode 100644 test/testdata/storage-class/README.md rename test/testdata/storage-class/{aws-csi.yaml => aws-legecy.yaml} (70%) rename test/testdata/storage-class/{azure-csi.yaml => azure-legacy.yaml} (76%) rename test/testdata/storage-class/{gcp-csi.yaml => gcp-legacy.yaml} (78%) delete mode 100644 test/testdata/storage-class/vanilla-zfs-csi.yaml rename test/testdata/storage-class/{vsphere-csi.yaml => vsphere-legacy.yaml} (53%) diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index cfdc897244..2f252171b1 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -110,7 +110,7 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) { if InstallVelero { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() - err = VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace) + err = VeleroUninstall(ctx, veleroCfg) Expect(err).To(Succeed()) } } diff --git a/test/e2e/backups/sync_backups.go b/test/e2e/backups/sync_backups.go index e4532fea62..447fc735b9 100644 --- a/test/e2e/backups/sync_backups.go +++ b/test/e2e/backups/sync_backups.go @@ -72,7 +72,7 @@ func BackupsSyncTest() { if InstallVelero { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() - Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed()) + Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed()) } } }) @@ -108,7 +108,7 @@ func BackupsSyncTest() { }) By("Uninstall velero", func() { - Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed()) + Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed()) }) By("Install velero", func() { diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 5135c4b0b4..c2c2224e9d 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -84,7 +84,7 @@ func TTLTest() { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() if InstallVelero { - Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).To(Succeed()) + Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed()) } Expect(DeleteNamespace(ctx, client, test.testNS, false)).To(Succeed(), fmt.Sprintf("Failed to delete the namespace %s", test.testNS)) } diff --git a/test/e2e/basic/api-group/enable_api_group_extentions.go b/test/e2e/basic/api-group/enable_api_group_extentions.go index 586fdeafa1..546d2f7212 100644 --- a/test/e2e/basic/api-group/enable_api_group_extentions.go +++ b/test/e2e/basic/api-group/enable_api_group_extentions.go @@ -83,13 +83,11 @@ func APIExtensionsVersionsTest() { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) - Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace)).To(Succeed()) + Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed()) Expect(DeleteCRDByName(context.Background(), crdName)).To(Succeed()) Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed()) - Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace)).To(Succeed()) + Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed()) Expect(DeleteCRDByName(context.Background(), crdName)).To(Succeed()) }) } diff --git a/test/e2e/basic/api-group/enable_api_group_versions.go b/test/e2e/basic/api-group/enable_api_group_versions.go index cb6606672c..c0eb770626 100644 --- a/test/e2e/basic/api-group/enable_api_group_versions.go +++ b/test/e2e/basic/api-group/enable_api_group_versions.go @@ -100,7 +100,7 @@ func APIGroupVersionsTest() { }) if InstallVelero { By("Uninstall Velero in api group version case", func() { - Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace)).NotTo(HaveOccurred()) + Expect(VeleroUninstall(ctx, veleroCfg)).NotTo(HaveOccurred()) }) } } diff --git a/test/e2e/basic/backup-volume-info/base.go b/test/e2e/basic/backup-volume-info/base.go index af54763198..15e92699e6 100644 --- a/test/e2e/basic/backup-volume-info/base.go +++ b/test/e2e/basic/backup-volume-info/base.go @@ -31,7 +31,6 @@ import ( . "github.com/vmware-tanzu/velero/test/e2e/test" . "github.com/vmware-tanzu/velero/test/util/common" . "github.com/vmware-tanzu/velero/test/util/k8s" - . "github.com/vmware-tanzu/velero/test/util/velero" ) type BackupVolumeInfo struct { @@ -108,9 +107,6 @@ func (v *BackupVolumeInfo) CreateResources() error { return errors.Wrapf(err, "Failed to create namespace %s", createNSName) } - // Install StorageClass - Expect(InstallTestStorageClasses(fmt.Sprintf("../testdata/storage-class/%s-csi.yaml", v.VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install StorageClass") - // Create deployment fmt.Printf("Creating deployment in namespaces ...%s\n", createNSName) // Make sure PVC count is great than 3 to allow both empty volumes and file populated volumes exist per pod @@ -120,7 +116,7 @@ func (v *BackupVolumeInfo) CreateResources() error { var vols []*v1.Volume for i := 0; i <= pvcCount-1; i++ { pvcName := fmt.Sprintf("volume-info-pvc-%d", i) - pvc, err := CreatePVC(v.Client, createNSName, pvcName, CSIStorageClassName, nil) + pvc, err := CreatePVC(v.Client, createNSName, pvcName, StorageClassName, nil) Expect(err).To(Succeed()) volumeName := fmt.Sprintf("volume-info-pv-%d", i) vols = append(vols, CreateVolumes(pvc.Name, []string{volumeName})...) @@ -159,11 +155,3 @@ func (v *BackupVolumeInfo) Destroy() error { return WaitAllSelectedNSDeleted(v.Ctx, v.Client, "ns-test=true") } - -func (v *BackupVolumeInfo) cleanResource() error { - if err := DeleteStorageClass(v.Ctx, v.Client, CSIStorageClassName); err != nil { - return errors.Wrap(err, "fail to delete the StorageClass") - } - - return nil -} diff --git a/test/e2e/basic/backup-volume-info/csi_data_mover.go b/test/e2e/basic/backup-volume-info/csi_data_mover.go index 7e08114b5c..de1ca5c423 100644 --- a/test/e2e/basic/backup-volume-info/csi_data_mover.go +++ b/test/e2e/basic/backup-volume-info/csi_data_mover.go @@ -61,6 +61,5 @@ func (c *CSIDataMoverVolumeInfo) Verify() error { Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true)) Expect(volumeInfo[0].SnapshotDataMovementInfo).NotTo(BeNil()) - // Clean SC and VSC - return c.cleanResource() + return nil } diff --git a/test/e2e/basic/backup-volume-info/csi_snapshot.go b/test/e2e/basic/backup-volume-info/csi_snapshot.go index 0bc3a7111b..ef476421c0 100644 --- a/test/e2e/basic/backup-volume-info/csi_snapshot.go +++ b/test/e2e/basic/backup-volume-info/csi_snapshot.go @@ -60,6 +60,5 @@ func (c *CSISnapshotVolumeInfo) Verify() error { Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true)) Expect(volumeInfo[0].CSISnapshotInfo).NotTo(BeNil()) - // Clean SC and VSC - return c.cleanResource() + return nil } diff --git a/test/e2e/basic/backup-volume-info/filesystem_upload.go b/test/e2e/basic/backup-volume-info/filesystem_upload.go index bab85ca126..d58eee5ada 100644 --- a/test/e2e/basic/backup-volume-info/filesystem_upload.go +++ b/test/e2e/basic/backup-volume-info/filesystem_upload.go @@ -60,6 +60,5 @@ func (f *FilesystemUploadVolumeInfo) Verify() error { Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true)) Expect(volumeInfo[0].PVBInfo).NotTo(BeNil()) - // Clean SC and VSC - return f.cleanResource() + return nil } diff --git a/test/e2e/basic/backup-volume-info/native_snapshot.go b/test/e2e/basic/backup-volume-info/native_snapshot.go index c8ec0be1b5..b4200f85ca 100644 --- a/test/e2e/basic/backup-volume-info/native_snapshot.go +++ b/test/e2e/basic/backup-volume-info/native_snapshot.go @@ -61,6 +61,5 @@ func (n *NativeSnapshotVolumeInfo) Verify() error { Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true)) Expect(volumeInfo[0].NativeSnapshotInfo).NotTo(BeNil()) - // Clean SC and VSC - return n.cleanResource() + return nil } diff --git a/test/e2e/basic/backup-volume-info/skipped_volumes.go b/test/e2e/basic/backup-volume-info/skipped_volumes.go index f6fc2e226b..0f1ccec0c2 100644 --- a/test/e2e/basic/backup-volume-info/skipped_volumes.go +++ b/test/e2e/basic/backup-volume-info/skipped_volumes.go @@ -60,6 +60,5 @@ func (s *SkippedVolumeInfo) Verify() error { Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true)) Expect(volumeInfo[0].Skipped).To(BeIdenticalTo(true)) - // Clean SC and VSC - return s.cleanResource() + return nil } diff --git a/test/e2e/basic/pvc-selected-node-changing.go b/test/e2e/basic/pvc-selected-node-changing.go index 65806d04bc..e200929c11 100644 --- a/test/e2e/basic/pvc-selected-node-changing.go +++ b/test/e2e/basic/pvc-selected-node-changing.go @@ -68,14 +68,6 @@ func (p *PVCSelectedNodeChanging) CreateResources() error { fmt.Sprintf("Failed to create namespace %s", p.namespace)) }) - By(fmt.Sprintf("Create a storage class %s.", StorageClassName), func() { - Expect(InstallStorageClass(context.Background(), fmt.Sprintf("../testdata/storage-class/%s.yaml", p.VeleroCfg.CloudProvider))).To(Succeed()) - }) - - By(fmt.Sprintf("Create a storage class %s.", StorageClassName), func() { - Expect(InstallTestStorageClasses(fmt.Sprintf("../testdata/storage-class/%s.yaml", p.VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install storage class") - }) - By(fmt.Sprintf("Create pod %s in namespace %s", p.podName, p.namespace), func() { nodeNameList, err := GetWorkerNodes(p.Ctx) Expect(err).To(Succeed()) diff --git a/test/e2e/basic/storage-class-changing.go b/test/e2e/basic/storage-class-changing.go index fc20d09f5b..e2e5a02967 100644 --- a/test/e2e/basic/storage-class-changing.go +++ b/test/e2e/basic/storage-class-changing.go @@ -18,7 +18,7 @@ type StorageClasssChanging struct { TestCase labels map[string]string data map[string]string - configmaptName string + cmName string namespace string srcStorageClass string desStorageClass string @@ -51,7 +51,7 @@ func (s *StorageClasssChanging) Init() error { s.labels = map[string]string{"velero.io/change-storage-class": "RestoreItemAction", "velero.io/plugin-config": ""} s.data = map[string]string{s.srcStorageClass: s.desStorageClass} - s.configmaptName = "change-storage-class-config" + s.cmName = "change-storage-class-config" s.volume = "volume-1" s.pvcName = fmt.Sprintf("pvc-%s", s.volume) s.podName = "pod-1" @@ -72,10 +72,6 @@ func (s *StorageClasssChanging) CreateResources() error { "app": "test", } - By(("Installing storage class..."), func() { - Expect(InstallTestStorageClasses(fmt.Sprintf("../testdata/storage-class/%s.yaml", s.VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install storage class") - }) - By(fmt.Sprintf("Create namespace %s", s.namespace), func() { Expect(CreateNamespace(s.Ctx, s.Client, s.namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", s.namespace)) @@ -94,8 +90,8 @@ func (s *StorageClasssChanging) CreateResources() error { Expect(err).To(Succeed()) }) - By(fmt.Sprintf("Create ConfigMap %s in namespace %s", s.configmaptName, s.VeleroCfg.VeleroNamespace), func() { - _, err := CreateConfigMap(s.Client.ClientGo, s.VeleroCfg.VeleroNamespace, s.configmaptName, s.labels, s.data) + By(fmt.Sprintf("Create ConfigMap %s in namespace %s", s.cmName, s.VeleroCfg.VeleroNamespace), func() { + _, err := CreateConfigMap(s.Client.ClientGo, s.VeleroCfg.VeleroNamespace, s.cmName, s.labels, s.data) Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", s.VeleroCfg.VeleroNamespace)) }) return nil @@ -149,8 +145,7 @@ func (s *StorageClasssChanging) Clean() error { Expect(CleanupNamespacesWithPoll(s.Ctx, s.Client, s.CaseBaseName)).To(Succeed(), fmt.Sprintf("Failed to delete namespace %s", s.CaseBaseName)) }) - DeleteConfigmap(s.Client.ClientGo, s.VeleroCfg.VeleroNamespace, s.configmaptName) - DeleteStorageClass(s.Ctx, s.Client, s.desStorageClass) + DeleteConfigMap(s.Client.ClientGo, s.VeleroCfg.VeleroNamespace, s.cmName) s.TestCase.Clean() } diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 36cb0b0e5d..790019ba29 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package e2e_test +package e2e import ( "context" @@ -22,6 +22,7 @@ import ( "flag" "fmt" "slices" + "strings" "testing" "time" @@ -29,7 +30,7 @@ import ( . "github.com/onsi/gomega" "github.com/vmware-tanzu/velero/pkg/cmd/cli/install" - . "github.com/vmware-tanzu/velero/test" + "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/e2e/backup" . "github.com/vmware-tanzu/velero/test/e2e/backups" . "github.com/vmware-tanzu/velero/test/e2e/basic" @@ -48,62 +49,300 @@ import ( . "github.com/vmware-tanzu/velero/test/e2e/scale" . "github.com/vmware-tanzu/velero/test/e2e/schedule" . "github.com/vmware-tanzu/velero/test/e2e/upgrade" - . "github.com/vmware-tanzu/velero/test/util/k8s" - . "github.com/vmware-tanzu/velero/test/util/velero" + "github.com/vmware-tanzu/velero/test/util/k8s" + veleroutil "github.com/vmware-tanzu/velero/test/util/velero" ) func init() { - VeleroCfg.Options = install.Options{} - flag.StringVar(&VeleroCfg.CloudProvider, "cloud-provider", "", "cloud that Velero will be installed into. Required.") - flag.StringVar(&VeleroCfg.ObjectStoreProvider, "object-store-provider", "", "provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.") - flag.StringVar(&VeleroCfg.BSLBucket, "bucket", "", "name of the object storage bucket where backups from e2e tests should be stored. Required.") - flag.StringVar(&VeleroCfg.CloudCredentialsFile, "credentials-file", "", "file containing credentials for backup and volume provider. Required.") - flag.StringVar(&VeleroCfg.VeleroCLI, "velerocli", "velero", "path to the velero application to use.") - flag.StringVar(&VeleroCfg.VeleroImage, "velero-image", "velero/velero:main", "image for the velero server to be tested.") - flag.StringVar(&VeleroCfg.Plugins, "plugins", "", "provider plugins to be tested.") - flag.StringVar(&VeleroCfg.AddBSLPlugins, "additional-bsl-plugins", "", "additional plugins to be tested.") - flag.StringVar(&VeleroCfg.VeleroVersion, "velero-version", "main", "image version for the velero server to be tested with.") - flag.StringVar(&VeleroCfg.RestoreHelperImage, "restore-helper-image", "", "image for the velero restore helper to be tested.") - flag.StringVar(&VeleroCfg.UpgradeFromVeleroCLI, "upgrade-from-velero-cli", "", "comma-separated list of velero application for the pre-upgrade velero server.") - flag.StringVar(&VeleroCfg.UpgradeFromVeleroVersion, "upgrade-from-velero-version", "v1.7.1", "comma-separated list of Velero version to be tested with for the pre-upgrade velero server.") - flag.StringVar(&VeleroCfg.MigrateFromVeleroCLI, "migrate-from-velero-cli", "", "comma-separated list of velero application on source cluster.") - flag.StringVar(&VeleroCfg.MigrateFromVeleroVersion, "migrate-from-velero-version", "self", "comma-separated list of Velero version to be tested with on source cluster.") - flag.StringVar(&VeleroCfg.BSLConfig, "bsl-config", "", "configuration to use for the backup storage location. Format is key1=value1,key2=value2") - flag.StringVar(&VeleroCfg.BSLPrefix, "prefix", "", "prefix under which all Velero data should be stored within the bucket. Optional.") - flag.StringVar(&VeleroCfg.VSLConfig, "vsl-config", "", "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2") - flag.StringVar(&VeleroCfg.VeleroNamespace, "velero-namespace", "velero", "namespace to install Velero into") - flag.BoolVar(&InstallVelero, "install-velero", true, "install/uninstall velero during the test. Optional.") - flag.BoolVar(&VeleroCfg.UseNodeAgent, "use-node-agent", true, "whether deploy node agent daemonset velero during the test. Optional.") - flag.BoolVar(&VeleroCfg.UseVolumeSnapshots, "use-volume-snapshots", true, "whether or not to create snapshot location automatically. Set to false if you do not plan to create volume snapshots via a storage provider.") - flag.StringVar(&VeleroCfg.RegistryCredentialFile, "registry-credential-file", "", "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.") - flag.StringVar(&VeleroCfg.KibishiiDirectory, "kibishii-directory", "github.com/vmware-tanzu-experiments/distributed-data-generator/kubernetes/yaml/", "file directory or URL path to install Kibishii. Optional.") - //vmware-tanzu-experiments + test.VeleroCfg.Options = install.Options{} + flag.StringVar( + &test.VeleroCfg.CloudProvider, + "cloud-provider", + "", + "cloud that Velero will be installed into. Required.", + ) + flag.StringVar( + &test.VeleroCfg.ObjectStoreProvider, + "object-store-provider", + "", + "provider of object store plugin. Required if cloud-provider is kind, otherwise ignored.", + ) + flag.StringVar( + &test.VeleroCfg.BSLBucket, + "bucket", + "", + "name of the object storage bucket where backups from e2e tests should be stored. Required.", + ) + flag.StringVar( + &test.VeleroCfg.CloudCredentialsFile, + "credentials-file", + "", + "file containing credentials for backup and volume provider. Required.", + ) + flag.StringVar( + &test.VeleroCfg.VeleroCLI, + "velerocli", + "velero", + "path to the velero application to use.", + ) + flag.StringVar( + &test.VeleroCfg.VeleroImage, + "velero-image", + "velero/velero:main", + "image for the velero server to be tested.", + ) + flag.StringVar( + &test.VeleroCfg.Plugins, + "plugins", + "", + "provider plugins to be tested.", + ) + flag.StringVar( + &test.VeleroCfg.AddBSLPlugins, + "additional-bsl-plugins", + "", + "additional plugins to be tested.", + ) + flag.StringVar( + &test.VeleroCfg.VeleroVersion, + "velero-version", + "main", + "image version for the velero server to be tested with.", + ) + flag.StringVar( + &test.VeleroCfg.RestoreHelperImage, + "restore-helper-image", + "", + "image for the velero restore helper to be tested.", + ) + flag.StringVar( + &test.VeleroCfg.UpgradeFromVeleroCLI, + "upgrade-from-velero-cli", + "", + "comma-separated list of velero application for the pre-upgrade velero server.", + ) + flag.StringVar( + &test.VeleroCfg.UpgradeFromVeleroVersion, + "upgrade-from-velero-version", + "v1.7.1", + "comma-separated list of Velero version to be tested with for the pre-upgrade velero server.", + ) + flag.StringVar( + &test.VeleroCfg.MigrateFromVeleroCLI, + "migrate-from-velero-cli", + "", + "comma-separated list of velero application on source cluster.", + ) + flag.StringVar( + &test.VeleroCfg.MigrateFromVeleroVersion, + "migrate-from-velero-version", + "self", + "comma-separated list of Velero version to be tested with on source cluster.", + ) + flag.StringVar( + &test.VeleroCfg.BSLConfig, + "bsl-config", + "", "configuration to use for the backup storage location. Format is key1=value1,key2=value2") + flag.StringVar( + &test.VeleroCfg.BSLPrefix, + "prefix", + "", + "prefix under which all Velero data should be stored within the bucket. Optional.", + ) + flag.StringVar( + &test.VeleroCfg.VSLConfig, + "vsl-config", + "", + "configuration to use for the volume snapshot location. Format is key1=value1,key2=value2", + ) + flag.StringVar( + &test.VeleroCfg.VeleroNamespace, + "velero-namespace", + "velero", + "namespace to install Velero into", + ) + flag.BoolVar( + &test.InstallVelero, + "install-velero", + true, + "install/uninstall velero during the test. Optional.", + ) + flag.BoolVar( + &test.VeleroCfg.UseNodeAgent, + "use-node-agent", + true, + "whether deploy node agent daemonset velero during the test. Optional.", + ) + flag.BoolVar( + &test.VeleroCfg.UseVolumeSnapshots, + "use-volume-snapshots", + true, + "whether or not to create snapshot location automatically. Set to false if you do not plan to create volume snapshots via a storage provider.", + ) + flag.StringVar( + &test.VeleroCfg.RegistryCredentialFile, + "registry-credential-file", + "", + "file containing credential for the image registry, follows the same format rules as the ~/.docker/config.json file. Optional.", + ) + flag.StringVar( + &test.VeleroCfg.KibishiiDirectory, + "kibishii-directory", + "github.com/vmware-tanzu-experiments/distributed-data-generator/kubernetes/yaml/", + "file directory or URL path to install Kibishii. Optional.", + ) + // Flags to create an additional BSL for multiple credentials test - flag.StringVar(&VeleroCfg.AdditionalBSLProvider, "additional-bsl-object-store-provider", "", "provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.") - flag.StringVar(&VeleroCfg.AdditionalBSLBucket, "additional-bsl-bucket", "", "name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.") - flag.StringVar(&VeleroCfg.AdditionalBSLPrefix, "additional-bsl-prefix", "", "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.") - flag.StringVar(&VeleroCfg.AdditionalBSLConfig, "additional-bsl-config", "", "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2") - flag.StringVar(&VeleroCfg.AdditionalBSLCredentials, "additional-bsl-credentials-file", "", "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.") - flag.StringVar(&VeleroCfg.Features, "features", "", "comma-separated list of features to enable for this Velero process.") - flag.StringVar(&VeleroCfg.GCFrequency, "garbage-collection-frequency", "", "frequency of garbage collection.") - flag.StringVar(&VeleroCfg.DefaultClusterContext, "default-cluster-context", "", "default cluster's kube config context, it's for migration test.") - flag.StringVar(&VeleroCfg.StandbyClusterContext, "standby-cluster-context", "", "standby cluster's kube config context, it's for migration test.") - flag.StringVar(&VeleroCfg.UploaderType, "uploader-type", "", "type of uploader for persistent volume backup.") - flag.BoolVar(&VeleroCfg.VeleroServerDebugMode, "velero-server-debug-mode", false, "a switch for enable or disable having debug log of Velero server.") - flag.BoolVar(&VeleroCfg.SnapshotMoveData, "snapshot-move-data", false, "a Switch for taking backup with Velero's data mover, if data-mover-plugin is not provided, using built-in plugin") - flag.StringVar(&VeleroCfg.DataMoverPlugin, "data-mover-plugin", "", "customized plugin for data mover.") - flag.StringVar(&VeleroCfg.StandbyClusterCloudProvider, "standby-cluster-cloud-provider", "", "cloud provider for standby cluster.") - flag.StringVar(&VeleroCfg.StandbyClusterPlugins, "standby-cluster-plugins", "", "plugins provider for standby cluster.") - flag.StringVar(&VeleroCfg.StandbyClusterObjectStoreProvider, "standby-cluster-object-store-provider", "", "object store provider for standby cluster.") - flag.BoolVar(&VeleroCfg.DebugVeleroPodRestart, "debug-velero-pod-restart", false, "a switch for debugging velero pod restart.") - flag.BoolVar(&VeleroCfg.DisableInformerCache, "disable-informer-cache", false, "a switch for disable informer cache.") - flag.StringVar(&VeleroCfg.DefaultClusterName, "default-cluster-name", "", "default cluster's name in kube config file, it's for EKS IRSA test.") - flag.StringVar(&VeleroCfg.StandbyClusterName, "standby-cluster-name", "", "standby cluster's name in kube config file, it's for EKS IRSA test.") - flag.StringVar(&VeleroCfg.EKSPolicyARN, "eks-policy-arn", "", "EKS plicy ARN for creating AWS IAM service account.") - flag.StringVar(&VeleroCfg.DefaultCLSServiceAccountName, "default-cls-service-account-name", "", "default cluster service account name.") - flag.StringVar(&VeleroCfg.StandbyCLSServiceAccountName, "standby-cls-service-account-name", "", "standby cluster service account name.") - flag.BoolVar(&VeleroCfg.FailFast, "fail-fast", true, "a switch for failing fast on meeting error.") - flag.BoolVar(&VeleroCfg.HasVspherePlugin, "has-vsphere-plugin", false, "a switch for installing vSphere plugin.") + flag.StringVar( + &test.VeleroCfg.AdditionalBSLProvider, + "additional-bsl-object-store-provider", + "", + "provider of object store plugin for additional backup storage location. Required if testing multiple credentials support.", + ) + flag.StringVar( + &test.VeleroCfg.AdditionalBSLBucket, + "additional-bsl-bucket", + "", + "name of the object storage bucket for additional backup storage location. Required if testing multiple credentials support.", + ) + flag.StringVar( + &test.VeleroCfg.AdditionalBSLPrefix, + "additional-bsl-prefix", + "", + "prefix under which all Velero data should be stored within the bucket for additional backup storage location. Optional.", + ) + flag.StringVar( + &test.VeleroCfg.AdditionalBSLConfig, + "additional-bsl-config", + "", + "configuration to use for the additional backup storage location. Format is key1=value1,key2=value2", + ) + flag.StringVar( + &test.VeleroCfg.AdditionalBSLCredentials, + "additional-bsl-credentials-file", + "", + "file containing credentials for additional backup storage location provider. Required if testing multiple credentials support.", + ) + flag.StringVar( + &test.VeleroCfg.Features, + "features", + "", + "comma-separated list of features to enable for this Velero process.", + ) + flag.StringVar( + &test.VeleroCfg.GCFrequency, + "garbage-collection-frequency", + "", + "frequency of garbage collection.", + ) + flag.StringVar( + &test.VeleroCfg.DefaultClusterContext, + "default-cluster-context", + "", + "default cluster's kube config context, it's for migration test.", + ) + flag.StringVar( + &test.VeleroCfg.StandbyClusterContext, + "standby-cluster-context", + "", + "standby cluster's kube config context, it's for migration test.", + ) + flag.StringVar( + &test.VeleroCfg.UploaderType, + "uploader-type", + "", + "type of uploader for persistent volume backup.", + ) + flag.BoolVar( + &test.VeleroCfg.VeleroServerDebugMode, + "velero-server-debug-mode", + false, + "a switch for enable or disable having debug log of Velero server.", + ) + flag.BoolVar( + &test.VeleroCfg.SnapshotMoveData, + "snapshot-move-data", + false, + "a Switch for taking backup with Velero's data mover, if data-mover-plugin is not provided, using built-in plugin", + ) + flag.StringVar( + &test.VeleroCfg.DataMoverPlugin, + "data-mover-plugin", + "", + "customized plugin for data mover.", + ) + flag.StringVar( + &test.VeleroCfg.StandbyClusterCloudProvider, + "standby-cluster-cloud-provider", + "", + "cloud provider for standby cluster.", + ) + flag.StringVar( + &test.VeleroCfg.StandbyClusterPlugins, + "standby-cluster-plugins", + "", + "plugins provider for standby cluster.", + ) + flag.StringVar( + &test.VeleroCfg.StandbyClusterObjectStoreProvider, + "standby-cluster-object-store-provider", + "", + "object store provider for standby cluster.", + ) + flag.BoolVar( + &test.VeleroCfg.DebugVeleroPodRestart, + "debug-velero-pod-restart", + false, + "a switch for debugging velero pod restart.", + ) + flag.BoolVar( + &test.VeleroCfg.DisableInformerCache, + "disable-informer-cache", + false, + "a switch for disable informer cache.", + ) + flag.StringVar( + &test.VeleroCfg.DefaultClusterName, + "default-cluster-name", + "", + "default cluster's name in kube config file, it's for EKS IRSA test.", + ) + flag.StringVar( + &test.VeleroCfg.StandbyClusterName, + "standby-cluster-name", + "", + "standby cluster's name in kube config file, it's for EKS IRSA test.", + ) + flag.StringVar( + &test.VeleroCfg.EKSPolicyARN, + "eks-policy-arn", + "", + "EKS plicy ARN for creating AWS IAM service account.", + ) + flag.StringVar( + &test.VeleroCfg.DefaultCLSServiceAccountName, + "default-cls-service-account-name", + "", + "default cluster service account name.", + ) + flag.StringVar( + &test.VeleroCfg.StandbyCLSServiceAccountName, + "standby-cls-service-account-name", + "", + "standby cluster service account name.", + ) + flag.BoolVar( + &test.VeleroCfg.FailFast, + "fail-fast", + true, + "a switch for failing fast on meeting error.", + ) + flag.BoolVar( + &test.VeleroCfg.HasVspherePlugin, + "has-vsphere-plugin", + false, + "a switch for installing vSphere plugin.", + ) } // Add label [SkipVanillaZfs]: @@ -113,147 +352,302 @@ func init() { // caused by no expected snapshot found. If we use retain as reclaim policy, then this label can be ignored, all test // cases can be executed as expected successful result. -var _ = Describe("Velero tests with various CRD API group versions", - Label("APIGroup", "APIVersion", "SKIP_KIND", "LongTime"), APIGroupVersionsTest) -var _ = Describe("CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", - Label("APIGroup", "APIExtensions", "SKIP_KIND"), APIExtensionsVersionsTest) +var _ = Describe( + "Velero tests with various CRD API group versions", + Label("APIGroup", "APIVersion", "SKIP_KIND", "LongTime"), + APIGroupVersionsTest, +) +var _ = Describe( + "CRD of apiextentions v1beta1 should be B/R successfully from cluster(k8s version < 1.22) to cluster(k8s version >= 1.22)", + Label("APIGroup", "APIExtensions", "SKIP_KIND"), + APIExtensionsVersionsTest, +) // Test backup and restore of Kibishii using restic -var _ = Describe("Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", - Label("Basic", "Restic"), BackupRestoreWithRestic) +var _ = Describe( + "Velero tests on cluster using the plugin provider for object storage and Restic for volume backups", + Label("Basic", "Restic"), + BackupRestoreWithRestic, +) -var _ = Describe("Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", - Label("Basic", "Snapshot", "SkipVanillaZfs"), BackupRestoreWithSnapshots) +var _ = Describe( + "Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", + Label("Basic", "Snapshot", "SkipVanillaZfs"), + BackupRestoreWithSnapshots, +) -var _ = Describe("Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", - Label("Basic", "Snapshot", "RetainPV"), BackupRestoreRetainedPVWithSnapshots) +var _ = Describe( + "Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", + Label("Basic", "Snapshot", "RetainPV"), + BackupRestoreRetainedPVWithSnapshots, +) -var _ = Describe("Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", - Label("Basic", "Restic", "RetainPV"), BackupRestoreRetainedPVWithRestic) +var _ = Describe( + "Velero tests on cluster using the plugin provider for object storage and snapshots for volume backups", + Label("Basic", "Restic", "RetainPV"), + BackupRestoreRetainedPVWithRestic, +) -var _ = Describe("Backup/restore of cluster resources", - Label("Basic", "ClusterResource"), ResourcesCheckTest) +var _ = Describe( + "Backup/restore of cluster resources", + Label("Basic", "ClusterResource"), + ResourcesCheckTest, +) -var _ = Describe("Service NodePort reservation during restore is configurable", - Label("Basic", "NodePort"), NodePortTest) +var _ = Describe( + "Service NodePort reservation during restore is configurable", + Label("Basic", "NodePort"), + NodePortTest, +) -var _ = Describe("Storage class of persistent volumes and persistent volume claims can be changed during restores", - Label("Basic", "StorageClass"), StorageClasssChangingTest) +var _ = Describe( + "Storage class of persistent volumes and persistent volume claims can be changed during restores", + Label("Basic", "StorageClass"), + StorageClasssChangingTest, +) -var _ = Describe("Node selectors of persistent volume claims can be changed during restores", - Label("Basic", "SelectedNode", "SKIP_KIND"), PVCSelectedNodeChangingTest) +var _ = Describe( + "Node selectors of persistent volume claims can be changed during restores", + Label("Basic", "SelectedNode", "SKIP_KIND"), + PVCSelectedNodeChangingTest, +) -var _ = Describe("Backup/restore of 2500 namespaces", - Label("Scale", "LongTime"), MultiNSBackupRestore) +var _ = Describe( + "Backup/restore of 2500 namespaces", + Label("Scale", "LongTime"), + MultiNSBackupRestore, +) // Upgrade test by Kibishii using Restic -var _ = Describe("Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", - Label("Upgrade", "Restic"), BackupUpgradeRestoreWithRestic) -var _ = Describe("Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", - Label("Upgrade", "Snapshot", "SkipVanillaZfs"), BackupUpgradeRestoreWithSnapshots) +var _ = Describe( + "Velero upgrade tests on cluster using the plugin provider for object storage and Restic for volume backups", + Label("Upgrade", "Restic"), + BackupUpgradeRestoreWithRestic, +) +var _ = Describe( + "Velero upgrade tests on cluster using the plugin provider for object storage and snapshots for volume backups", + Label("Upgrade", "Snapshot", "SkipVanillaZfs"), + BackupUpgradeRestoreWithSnapshots, +) // test filter objects by namespace, type, or labels when backup or restore. -var _ = Describe("Resources with the label velero.io/exclude-from-backup=true are not included in backup", - Label("ResourceFiltering", "ExcludeFromBackup"), ExcludeFromBackupTest) -var _ = Describe("Velero test on exclude namespace from the cluster backup", - Label("ResourceFiltering", "ExcludeNamespaces", "Backup"), BackupWithExcludeNamespaces) -var _ = Describe("Velero test on exclude namespace from the cluster restore", - Label("ResourceFiltering", "ExcludeNamespaces", "Restore"), RestoreWithExcludeNamespaces) -var _ = Describe("Velero test on exclude resources from the cluster backup", - Label("ResourceFiltering", "ExcludeResources", "Backup"), BackupWithExcludeResources) -var _ = Describe("Velero test on exclude resources from the cluster restore", - Label("ResourceFiltering", "ExcludeResources", "Restore"), RestoreWithExcludeResources) -var _ = Describe("Velero test on include namespace from the cluster backup", - Label("ResourceFiltering", "IncludeNamespaces", "Backup"), BackupWithIncludeNamespaces) -var _ = Describe("Velero test on include namespace from the cluster restore", - Label("ResourceFiltering", "IncludeNamespaces", "Restore"), RestoreWithIncludeNamespaces) -var _ = Describe("Velero test on include resources from the cluster backup", - Label("ResourceFiltering", "IncludeResources", "Backup"), BackupWithIncludeResources) -var _ = Describe("Velero test on include resources from the cluster restore", - Label("ResourceFiltering", "IncludeResources", "Restore"), RestoreWithIncludeResources) -var _ = Describe("Velero test on backup include resources matching the label selector", - Label("ResourceFiltering", "LabelSelector"), BackupWithLabelSelector) -var _ = Describe("Velero test on skip backup of volume by resource policies", - Label("ResourceFiltering", "ResourcePolicies", "Restic"), ResourcePoliciesTest) +var _ = Describe( + "Resources with the label velero.io/exclude-from-backup=true are not included in backup", + Label("ResourceFiltering", "ExcludeFromBackup"), + ExcludeFromBackupTest, +) +var _ = Describe( + "Velero test on exclude namespace from the cluster backup", + Label("ResourceFiltering", "ExcludeNamespaces", "Backup"), + BackupWithExcludeNamespaces, +) +var _ = Describe( + "Velero test on exclude namespace from the cluster restore", + Label("ResourceFiltering", "ExcludeNamespaces", "Restore"), + RestoreWithExcludeNamespaces, +) +var _ = Describe( + "Velero test on exclude resources from the cluster backup", + Label("ResourceFiltering", "ExcludeResources", "Backup"), + BackupWithExcludeResources, +) +var _ = Describe( + "Velero test on exclude resources from the cluster restore", + Label("ResourceFiltering", "ExcludeResources", "Restore"), + RestoreWithExcludeResources, +) +var _ = Describe( + "Velero test on include namespace from the cluster backup", + Label("ResourceFiltering", "IncludeNamespaces", "Backup"), + BackupWithIncludeNamespaces, +) +var _ = Describe( + "Velero test on include namespace from the cluster restore", + Label("ResourceFiltering", "IncludeNamespaces", "Restore"), + RestoreWithIncludeNamespaces, +) +var _ = Describe( + "Velero test on include resources from the cluster backup", + Label("ResourceFiltering", "IncludeResources", "Backup"), + BackupWithIncludeResources, +) +var _ = Describe( + "Velero test on include resources from the cluster restore", + Label("ResourceFiltering", "IncludeResources", "Restore"), + RestoreWithIncludeResources, +) +var _ = Describe( + "Velero test on backup include resources matching the label selector", + Label("ResourceFiltering", "LabelSelector"), + BackupWithLabelSelector, +) +var _ = Describe( + "Velero test on skip backup of volume by resource policies", + Label("ResourceFiltering", "ResourcePolicies", "Restic"), + ResourcePoliciesTest, +) // backup VolumeInfo test -var _ = Describe("", Label("BackupVolumeInfo", "SkippedVolume"), SkippedVolumeInfoTest) -var _ = Describe("", Label("BackupVolumeInfo", "FilesystemUpload"), FilesystemUploadVolumeInfoTest) -var _ = Describe("", Label("BackupVolumeInfo", "CSIDataMover"), CSIDataMoverVolumeInfoTest) -var _ = Describe("", Label("BackupVolumeInfo", "CSISnapshot"), CSISnapshotVolumeInfoTest) -var _ = Describe("", Label("BackupVolumeInfo", "NativeSnapshot"), NativeSnapshotVolumeInfoTest) - -var _ = Describe("Velero test on resource modifiers from the cluster restore", - Label("ResourceModifier", "Restore"), ResourceModifiersTest) - -var _ = Describe("Velero tests of Restic backup deletion", - Label("Backups", "Deletion", "Restic"), BackupDeletionWithRestic) -var _ = Describe("Velero tests of snapshot backup deletion", - Label("Backups", "Deletion", "Snapshot", "SkipVanillaZfs"), BackupDeletionWithSnapshots) -var _ = Describe("Local backups and Restic repos will be deleted once the corresponding backup storage location is deleted", - Label("Backups", "TTL", "LongTime", "Snapshot", "SkipVanillaZfs"), TTLTest) -var _ = Describe("Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", - Label("Backups", "BackupsSync"), BackupsSyncTest) - -var _ = Describe("Backup will be created periodically by schedule defined by a Cron expression", - Label("Schedule", "BR", "Pause", "LongTime"), ScheduleBackupTest) -var _ = Describe("Backup resources should follow the specific order in schedule", - Label("Schedule", "OrderedResources", "LongTime"), ScheduleOrderedResources) -var _ = Describe("Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", - Label("Schedule", "BackupCreation", "SKIP_KIND", "LongTime"), ScheduleBackupCreationTest) - -var _ = Describe("Velero test on ssr object when controller namespace mix-ups", - Label("PrivilegesMgmt", "SSR"), SSRTest) - -var _ = Describe("Local backups will be deleted once the corresponding backup storage location is deleted", - Label("BSL", "Deletion", "Snapshot", "SkipVanillaZfs"), BslDeletionWithSnapshots) -var _ = Describe("Local backups and Restic repos will be deleted once the corresponding backup storage location is deleted", - Label("BSL", "Deletion", "Restic"), BslDeletionWithRestic) - -var _ = Describe("Migrate resources between clusters by Restic", - Label("Migration", "Restic"), MigrationWithRestic) -var _ = Describe("Migrate resources between clusters by snapshot", - Label("Migration", "Snapshot", "SkipVanillaZfs"), MigrationWithSnapshots) - -var _ = Describe("Backup resources should follow the specific order in schedule", - Label("NamespaceMapping", "Single", "Restic"), OneNamespaceMappingResticTest) -var _ = Describe("Backup resources should follow the specific order in schedule", - Label("NamespaceMapping", "Multiple", "Restic"), MultiNamespacesMappingResticTest) -var _ = Describe("Backup resources should follow the specific order in schedule", - Label("NamespaceMapping", "Single", "Snapshot", "SkipVanillaZfs"), OneNamespaceMappingSnapshotTest) -var _ = Describe("Backup resources should follow the specific order in schedule", - Label("NamespaceMapping", "Multiple", "Snapshot", "SkipVanillaZfs"), MultiNamespacesMappingSnapshotTest) - -var _ = Describe("Backup resources should follow the specific order in schedule", - Label("PVBackup", "OptIn"), OptInPVBackupTest) -var _ = Describe("Backup resources should follow the specific order in schedule", - Label("PVBackup", "OptOut"), OptOutPVBackupTest) - -var _ = Describe("Velero test on parallel files upload", - Label("UploaderConfig", "ParallelFilesUpload"), ParallelFilesUploadTest) -var _ = Describe("Velero test on parallel files download", - Label("UploaderConfig", "ParallelFilesDownload"), ParallelFilesDownloadTest) +var _ = Describe( + "", + Label("BackupVolumeInfo", "SkippedVolume"), + SkippedVolumeInfoTest, +) +var _ = Describe( + "", + Label("BackupVolumeInfo", "FilesystemUpload"), + FilesystemUploadVolumeInfoTest, +) +var _ = Describe( + "", + Label("BackupVolumeInfo", "CSIDataMover"), + CSIDataMoverVolumeInfoTest, +) +var _ = Describe( + "", + Label("BackupVolumeInfo", "CSISnapshot"), + CSISnapshotVolumeInfoTest, +) +var _ = Describe( + "", + Label("BackupVolumeInfo", "NativeSnapshot"), + NativeSnapshotVolumeInfoTest, +) + +var _ = Describe( + "Velero test on resource modifiers from the cluster restore", + Label("ResourceModifier", "Restore"), + ResourceModifiersTest, +) + +var _ = Describe( + "Velero tests of Restic backup deletion", + Label("Backups", "Deletion", "Restic"), + BackupDeletionWithRestic, +) +var _ = Describe( + "Velero tests of snapshot backup deletion", + Label("Backups", "Deletion", "Snapshot", "SkipVanillaZfs"), + BackupDeletionWithSnapshots, +) +var _ = Describe( + "Local backups and Restic repos will be deleted once the corresponding backup storage location is deleted", + Label("Backups", "TTL", "LongTime", "Snapshot", "SkipVanillaZfs"), + TTLTest, +) +var _ = Describe( + "Backups in object storage are synced to a new Velero and deleted backups in object storage are synced to be deleted in Velero", + Label("Backups", "BackupsSync"), + BackupsSyncTest, +) + +var _ = Describe( + "Backup will be created periodically by schedule defined by a Cron expression", + Label("Schedule", "BR", "Pause", "LongTime"), + ScheduleBackupTest, +) +var _ = Describe( + "Backup resources should follow the specific order in schedule", + Label("Schedule", "OrderedResources", "LongTime"), + ScheduleOrderedResources, +) +var _ = Describe( + "Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", + Label("Schedule", "BackupCreation", "SKIP_KIND", "LongTime"), + ScheduleBackupCreationTest, +) + +var _ = Describe( + "Velero test on ssr object when controller namespace mix-ups", + Label("PrivilegesMgmt", "SSR"), + SSRTest, +) + +var _ = Describe( + "Local backups will be deleted once the corresponding backup storage location is deleted", + Label("BSL", "Deletion", "Snapshot", "SkipVanillaZfs"), + BslDeletionWithSnapshots, +) +var _ = Describe( + "Local backups and Restic repos will be deleted once the corresponding backup storage location is deleted", + Label("BSL", "Deletion", "Restic"), + BslDeletionWithRestic, +) + +var _ = Describe( + "Migrate resources between clusters by Restic", + Label("Migration", "Restic"), + MigrationWithRestic, +) +var _ = Describe( + "Migrate resources between clusters by snapshot", + Label("Migration", "Snapshot", "SkipVanillaZfs"), + MigrationWithSnapshots, +) + +var _ = Describe( + "Backup resources should follow the specific order in schedule", + Label("NamespaceMapping", "Single", "Restic"), + OneNamespaceMappingResticTest, +) +var _ = Describe( + "Backup resources should follow the specific order in schedule", + Label("NamespaceMapping", "Multiple", "Restic"), + MultiNamespacesMappingResticTest, +) +var _ = Describe( + "Backup resources should follow the specific order in schedule", + Label("NamespaceMapping", "Single", "Snapshot", "SkipVanillaZfs"), + OneNamespaceMappingSnapshotTest, +) +var _ = Describe( + "Backup resources should follow the specific order in schedule", + Label("NamespaceMapping", "Multiple", "Snapshot", "SkipVanillaZfs"), + MultiNamespacesMappingSnapshotTest, +) + +var _ = Describe( + "Backup resources should follow the specific order in schedule", + Label("PVBackup", "OptIn"), + OptInPVBackupTest, +) +var _ = Describe( + "Backup resources should follow the specific order in schedule", + Label("PVBackup", "OptOut"), + OptOutPVBackupTest, +) + +var _ = Describe( + "Velero test on parallel files upload", + Label("UploaderConfig", "ParallelFilesUpload"), + ParallelFilesUploadTest, +) +var _ = Describe( + "Velero test on parallel files download", + Label("UploaderConfig", "ParallelFilesDownload"), + ParallelFilesDownloadTest, +) func GetKubeConfigContext() error { var err error - var tcDefault, tcStandby TestClient - tcDefault, err = NewTestClient(VeleroCfg.DefaultClusterContext) - VeleroCfg.DefaultClient = &tcDefault - VeleroCfg.ClientToInstallVelero = VeleroCfg.DefaultClient - VeleroCfg.ClusterToInstallVelero = VeleroCfg.DefaultClusterName - VeleroCfg.ServiceAccountNameToInstall = VeleroCfg.DefaultCLSServiceAccountName + var tcDefault, tcStandby k8s.TestClient + tcDefault, err = k8s.NewTestClient(test.VeleroCfg.DefaultClusterContext) + test.VeleroCfg.DefaultClient = &tcDefault + test.VeleroCfg.ClientToInstallVelero = test.VeleroCfg.DefaultClient + test.VeleroCfg.ClusterToInstallVelero = test.VeleroCfg.DefaultClusterName + test.VeleroCfg.ServiceAccountNameToInstall = test.VeleroCfg.DefaultCLSServiceAccountName if err != nil { return err } - if VeleroCfg.DefaultClusterContext != "" { - err = KubectlConfigUseContext(context.Background(), VeleroCfg.DefaultClusterContext) + if test.VeleroCfg.DefaultClusterContext != "" { + err = k8s.KubectlConfigUseContext(context.Background(), test.VeleroCfg.DefaultClusterContext) if err != nil { return err } - if VeleroCfg.StandbyClusterContext != "" { - tcStandby, err = NewTestClient(VeleroCfg.StandbyClusterContext) - VeleroCfg.StandbyClient = &tcStandby + if test.VeleroCfg.StandbyClusterContext != "" { + tcStandby, err = k8s.NewTestClient(test.VeleroCfg.StandbyClusterContext) + test.VeleroCfg.StandbyClient = &tcStandby if err != nil { return err } @@ -275,14 +669,14 @@ func TestE2e(t *testing.T) { t.Skip("Skipping E2E tests") } - if !slices.Contains(LocalCloudProviders, VeleroCfg.CloudProvider) { + if !slices.Contains(test.LocalCloudProviders, test.VeleroCfg.CloudProvider) { fmt.Println("For cloud platforms, object store plugin provider will be set as cloud provider") // If ObjectStoreProvider is not provided, then using the value same as CloudProvider - if VeleroCfg.ObjectStoreProvider == "" { - VeleroCfg.ObjectStoreProvider = VeleroCfg.CloudProvider + if test.VeleroCfg.ObjectStoreProvider == "" { + test.VeleroCfg.ObjectStoreProvider = test.VeleroCfg.CloudProvider } } else { - if VeleroCfg.ObjectStoreProvider == "" { + if test.VeleroCfg.ObjectStoreProvider == "" { t.Error(errors.New("No object store provider specified - must be specified when using kind as the cloud provider")) // Must have an object store provider } } @@ -298,19 +692,67 @@ func TestE2e(t *testing.T) { } var _ = BeforeSuite(func() { - if InstallVelero { + By("Install StorageClass for E2E.") + Expect(veleroutil.InstallStorageClasses(test.VeleroCfg.CloudProvider)).To(Succeed()) + + if strings.EqualFold(test.VeleroCfg.Features, test.FeatureCSI) && + test.VeleroCfg.UseVolumeSnapshots { + By("Install VolumeSnapshotClass for E2E.") + Expect( + k8s.KubectlApplyByFile( + context.Background(), + fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", test.VeleroCfg.CloudProvider), + ), + ).To(Succeed()) + } + + if test.InstallVelero { By("Install test resources before testing") - Expect(PrepareVelero(context.Background(), "install resource before testing", VeleroCfg)).To(Succeed()) + Expect( + veleroutil.PrepareVelero( + context.Background(), + "install resource before testing", + test.VeleroCfg, + ), + ).To(Succeed()) } }) var _ = AfterSuite(func() { + ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) + defer ctxCancel() + + By("Delete StorageClasses created by E2E") + Expect( + k8s.DeleteStorageClass( + ctx, + *test.VeleroCfg.ClientToInstallVelero, + test.StorageClassName, + ), + ).To(Succeed()) + Expect( + k8s.DeleteStorageClass( + ctx, + *test.VeleroCfg.ClientToInstallVelero, + test.StorageClassName2, + ), + ).To(Succeed()) + + if strings.EqualFold(test.VeleroCfg.Features, test.FeatureCSI) && + test.VeleroCfg.UseVolumeSnapshots { + By("Delete VolumeSnapshotClass created by E2E") + Expect( + k8s.KubectlDeleteByFile( + ctx, + fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", test.VeleroCfg.CloudProvider), + ), + ).To(Succeed()) + } + // If the Velero is installed during test, and the FailFast is not enabled, // uninstall Velero. If not, either Velero is not installed, or kept it for debug on failure. - if InstallVelero && (testSuitePassed || !VeleroCfg.FailFast) { + if test.InstallVelero && (testSuitePassed || !test.VeleroCfg.FailFast) { By("release test resources after testing") - ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) - defer ctxCancel() - Expect(VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) + Expect(veleroutil.VeleroUninstall(ctx, test.VeleroCfg)).To(Succeed()) } }) diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index ade423b76b..1604806208 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -75,8 +75,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) By("Uninstall Velero", func() { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() - Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace)).To(Succeed()) + Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed()) }) } }) @@ -87,28 +86,72 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.DefaultClusterContext), func() { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() + Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) - Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace)).To(Succeed()) - DeleteNamespace(context.Background(), *veleroCfg.DefaultClient, migrationNamespace, true) + veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient + veleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName + Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed()) + + By(fmt.Sprintf("Delete sample workload namespace %s", migrationNamespace), func() { + Expect( + DeleteNamespace( + context.Background(), + *veleroCfg.DefaultClient, + migrationNamespace, + true), + ).To(Succeed()) + }) }) By(fmt.Sprintf("Uninstall Velero on cluster %s", veleroCfg.StandbyClusterContext), func() { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() Expect(KubectlConfigUseContext(context.Background(), veleroCfg.StandbyClusterContext)).To(Succeed()) - Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace)).To(Succeed()) - DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true) - }) + veleroCfg.ClientToInstallVelero = veleroCfg.StandbyClient + veleroCfg.ClusterToInstallVelero = veleroCfg.StandbyClusterName + + By("Delete StorageClasses created by E2E") + Expect( + DeleteStorageClass( + ctx, + *veleroCfg.ClientToInstallVelero, + StorageClassName, + ), + ).To(Succeed()) + Expect( + DeleteStorageClass( + ctx, + *veleroCfg.ClientToInstallVelero, + StorageClassName2, + ), + ).To(Succeed()) + + if strings.EqualFold(veleroCfg.Features, FeatureCSI) && + veleroCfg.UseVolumeSnapshots { + By("Delete VolumeSnapshotClass created by E2E") + Expect( + KubectlDeleteByFile( + ctx, + fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", veleroCfg.CloudProvider), + ), + ).To(Succeed()) + } + + Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed()) - if InstallVelero { By(fmt.Sprintf("Delete sample workload namespace %s", migrationNamespace), func() { - DeleteNamespace(context.Background(), *veleroCfg.StandbyClient, migrationNamespace, true) + Expect( + DeleteNamespace( + context.Background(), + *veleroCfg.StandbyClient, + migrationNamespace, + true, + ), + ).To(Succeed()) }) - } + }) - By(fmt.Sprintf("Switch to default kubeconfig context %s", veleroCfg.DefaultClusterContext), func() { + By(fmt.Sprintf("Switch to default KubeConfig context %s", veleroCfg.DefaultClusterContext), func() { Expect(KubectlConfigUseContext(context.Background(), veleroCfg.DefaultClusterContext)).To(Succeed()) veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient veleroCfg.ClusterToInstallVelero = veleroCfg.DefaultClusterName @@ -297,6 +340,20 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) veleroCfg.ObjectStoreProvider = veleroCfg.StandbyClusterObjectStoreProvider } + By("Install StorageClass for E2E.") + Expect(InstallStorageClasses(veleroCfg.StandbyClusterCloudProvider)).To(Succeed()) + + if strings.EqualFold(veleroCfg.Features, FeatureCSI) && + veleroCfg.UseVolumeSnapshots { + By("Install VolumeSnapshotClass for E2E.") + Expect( + KubectlApplyByFile( + context.Background(), + fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", veleroCfg.StandbyClusterCloudProvider), + ), + ).To(Succeed()) + } + Expect(VeleroInstall(context.Background(), &veleroCfg, true)).To(Succeed()) }) @@ -307,16 +364,13 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) By(fmt.Sprintf("Restore %s", migrationNamespace), func() { if OriginVeleroCfg.SnapshotMoveData { - By(fmt.Sprintf("Create a storage class %s for restore PV provisioned by storage class %s on different cloud provider", StorageClassName, KibishiiStorageClassName), func() { - Expect(InstallStorageClass(context.Background(), fmt.Sprintf("../testdata/storage-class/%s.yaml", veleroCfg.StandbyClusterCloudProvider))).To(Succeed()) - }) - configmaptName := "datamover-storage-class-config" + cmName := "datamover-storage-class-config" labels := map[string]string{"velero.io/change-storage-class": "RestoreItemAction", "velero.io/plugin-config": ""} data := map[string]string{KibishiiStorageClassName: StorageClassName} - By(fmt.Sprintf("Create ConfigMap %s in namespace %s", configmaptName, veleroCfg.VeleroNamespace), func() { - _, err := CreateConfigMap(veleroCfg.StandbyClient.ClientGo, veleroCfg.VeleroNamespace, configmaptName, labels, data) + By(fmt.Sprintf("Create ConfigMap %s in namespace %s", cmName, veleroCfg.VeleroNamespace), func() { + _, err := CreateConfigMap(veleroCfg.StandbyClient.ClientGo, veleroCfg.VeleroNamespace, cmName, labels, data) Expect(err).To(Succeed(), fmt.Sprintf("failed to create configmap in the namespace %q", veleroCfg.VeleroNamespace)) }) } else { @@ -343,7 +397,7 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) // TODO: delete backup created by case self, not all By("Clean backups after test", func() { veleroCfg.ClientToInstallVelero = veleroCfg.DefaultClient - DeleteBackups(context.Background(), backupNames, &veleroCfg) + Expect(DeleteBackups(context.Background(), backupNames, &veleroCfg)).To(Succeed()) }) }) }) diff --git a/test/e2e/parallelfilesdownload/parallel_files_download.go b/test/e2e/parallelfilesdownload/parallel_files_download.go index bc77dffc8c..a678c6c80e 100644 --- a/test/e2e/parallelfilesdownload/parallel_files_download.go +++ b/test/e2e/parallelfilesdownload/parallel_files_download.go @@ -21,7 +21,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" . "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/e2e/test" @@ -90,11 +89,6 @@ func (p *ParallelFilesDownload) Init() error { } func (p *ParallelFilesDownload) CreateResources() error { - err := InstallStorageClass(p.Ctx, fmt.Sprintf("../testdata/storage-class/%s.yaml", p.VeleroCfg.CloudProvider)) - if err != nil { - return errors.Wrapf(err, "failed to install storage class for pv backup filtering test") - } - By(fmt.Sprintf("Create namespace %s", p.namespace), func() { Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", p.namespace)) diff --git a/test/e2e/parallelfilesupload/parallel_files_upload.go b/test/e2e/parallelfilesupload/parallel_files_upload.go index 6af813f985..5478e35186 100644 --- a/test/e2e/parallelfilesupload/parallel_files_upload.go +++ b/test/e2e/parallelfilesupload/parallel_files_upload.go @@ -21,7 +21,6 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "github.com/pkg/errors" . "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/e2e/test" @@ -81,11 +80,6 @@ func (p *ParallelFilesUpload) Init() error { } func (p *ParallelFilesUpload) CreateResources() error { - err := InstallStorageClass(p.Ctx, fmt.Sprintf("../testdata/storage-class/%s.yaml", p.VeleroCfg.CloudProvider)) - if err != nil { - return errors.Wrapf(err, "failed to install storage class for pv backup filtering test") - } - By(fmt.Sprintf("Create namespace %s", p.namespace), func() { Expect(CreateNamespace(p.Ctx, p.Client, p.namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", p.namespace)) diff --git a/test/e2e/pv-backup/pv-backup-filter.go b/test/e2e/pv-backup/pv-backup-filter.go index d89688c4f5..b58cb8a55b 100644 --- a/test/e2e/pv-backup/pv-backup-filter.go +++ b/test/e2e/pv-backup/pv-backup-filter.go @@ -63,11 +63,6 @@ func (p *PVBackupFiltering) Init() error { } func (p *PVBackupFiltering) CreateResources() error { - err := InstallStorageClass(p.Ctx, fmt.Sprintf("../testdata/storage-class/%s.yaml", p.VeleroCfg.CloudProvider)) - if err != nil { - return errors.Wrapf(err, "failed to install storage class for pv backup filtering test") - } - for _, ns := range *p.NSIncluded { By(fmt.Sprintf("Create namespaces %s for workload\n", ns), func() { Expect(CreateNamespace(p.Ctx, p.Client, ns)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", ns)) diff --git a/test/e2e/resourcemodifiers/resource_modifiers.go b/test/e2e/resourcemodifiers/resource_modifiers.go index 1ab2c63945..e3bd9ea11d 100644 --- a/test/e2e/resourcemodifiers/resource_modifiers.go +++ b/test/e2e/resourcemodifiers/resource_modifiers.go @@ -134,7 +134,7 @@ func (r *ResourceModifiersCase) Clean() error { if CurrentSpecReport().Failed() && r.VeleroCfg.FailFast { fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } else { - if err := DeleteConfigmap(r.Client.ClientGo, r.VeleroCfg.VeleroNamespace, r.cmName); err != nil { + if err := DeleteConfigMap(r.Client.ClientGo, r.VeleroCfg.VeleroNamespace, r.cmName); err != nil { return err } diff --git a/test/e2e/resourcepolicies/resource_policies.go b/test/e2e/resourcepolicies/resource_policies.go index 7239c49fd5..94e238eda3 100644 --- a/test/e2e/resourcepolicies/resource_policies.go +++ b/test/e2e/resourcepolicies/resource_policies.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package filtering +package resourcepolicies import ( "fmt" @@ -29,7 +29,6 @@ import ( . "github.com/vmware-tanzu/velero/test" . "github.com/vmware-tanzu/velero/test/e2e/test" . "github.com/vmware-tanzu/velero/test/util/k8s" - . "github.com/vmware-tanzu/velero/test/util/velero" ) const FileName = "test-data.txt" @@ -101,10 +100,6 @@ func (r *ResourcePoliciesCase) Init() error { } func (r *ResourcePoliciesCase) CreateResources() error { - By(("Installing storage class..."), func() { - Expect(InstallTestStorageClasses(fmt.Sprintf("../testdata/storage-class/%s.yaml", r.VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install storage class") - }) - By(fmt.Sprintf("Create configmap %s in namespaces %s for workload\n", r.cmName, r.VeleroCfg.VeleroNamespace), func() { Expect(CreateConfigMapFromYAMLData(r.Client.ClientGo, r.yamlConfig, r.cmName, r.VeleroCfg.VeleroNamespace)).To(Succeed(), fmt.Sprintf("Failed to create configmap %s in namespaces %s for workload\n", r.cmName, r.VeleroCfg.VeleroNamespace)) }) @@ -181,11 +176,7 @@ func (r *ResourcePoliciesCase) Clean() error { if CurrentSpecReport().Failed() && r.VeleroCfg.FailFast { fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } else { - if err := r.deleteTestStorageClassList([]string{StorageClassName, StorageClassName2}); err != nil { - return err - } - - if err := DeleteConfigmap(r.Client.ClientGo, r.VeleroCfg.VeleroNamespace, r.cmName); err != nil { + if err := DeleteConfigMap(r.Client.ClientGo, r.VeleroCfg.VeleroNamespace, r.cmName); err != nil { return err } @@ -248,12 +239,3 @@ func (r *ResourcePoliciesCase) writeDataIntoPods(namespace, volName string) erro } return nil } - -func (r *ResourcePoliciesCase) deleteTestStorageClassList(scList []string) error { - for _, v := range scList { - if err := DeleteStorageClass(r.Ctx, r.Client, v); err != nil { - return err - } - } - return nil -} diff --git a/test/e2e/schedule/schedule-backup-creation.go b/test/e2e/schedule/schedule-backup-creation.go index 9376f22904..df30b1752f 100644 --- a/test/e2e/schedule/schedule-backup-creation.go +++ b/test/e2e/schedule/schedule-backup-creation.go @@ -10,13 +10,14 @@ import ( . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - . "github.com/vmware-tanzu/velero/test/e2e/test" - . "github.com/vmware-tanzu/velero/test/util/k8s" - . "github.com/vmware-tanzu/velero/test/util/velero" + "github.com/vmware-tanzu/velero/test" + framework "github.com/vmware-tanzu/velero/test/e2e/test" + k8sutil "github.com/vmware-tanzu/velero/test/util/k8s" + veleroutil "github.com/vmware-tanzu/velero/test/util/velero" ) type ScheduleBackupCreation struct { - TestCase + framework.TestCase namespace string ScheduleName string ScheduleArgs []string @@ -30,7 +31,7 @@ type ScheduleBackupCreation struct { podSleepDuration time.Duration } -var ScheduleBackupCreationTest func() = TestFunc(&ScheduleBackupCreation{}) +var ScheduleBackupCreationTest func() = framework.TestFunc(&ScheduleBackupCreation{}) func (s *ScheduleBackupCreation) Init() error { s.TestCase.Init() @@ -41,7 +42,7 @@ func (s *ScheduleBackupCreation) Init() error { s.verifyTimes = 5 // More larger verify times more confidence we have podSleepDurationStr := "300s" s.podSleepDuration, _ = time.ParseDuration(podSleepDurationStr) - s.TestMsg = &TestMSG{ + s.TestMsg = &framework.TestMSG{ Desc: "Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", FailedMSG: "Failed to verify schedule back creation behavior", Text: "Schedule controller wouldn't create a new backup when it still has pending or InProgress backup", @@ -64,14 +65,14 @@ func (s *ScheduleBackupCreation) Init() error { func (s *ScheduleBackupCreation) CreateResources() error { By(fmt.Sprintf("Create namespace %s", s.namespace), func() { - Expect(CreateNamespace(s.Ctx, s.Client, s.namespace)).To(Succeed(), + Expect(k8sutil.CreateNamespace(s.Ctx, s.Client, s.namespace)).To(Succeed(), fmt.Sprintf("Failed to create namespace %s", s.namespace)) }) By(fmt.Sprintf("Create pod %s in namespace %s", s.podName, s.namespace), func() { - _, err := CreatePod(s.Client, s.namespace, s.podName, "default", s.pvcName, []string{s.volume}, nil, s.podAnn) + _, err := k8sutil.CreatePod(s.Client, s.namespace, s.podName, test.StorageClassName, s.pvcName, []string{s.volume}, nil, s.podAnn) Expect(err).To(Succeed()) - err = WaitForPods(s.Ctx, s.Client, s.namespace, []string{s.podName}) + err = k8sutil.WaitForPods(s.Ctx, s.Client, s.namespace, []string{s.podName}) Expect(err).To(Succeed()) }) return nil @@ -87,8 +88,8 @@ func (s *ScheduleBackupCreation) Backup() error { now := time.Now().Minute() triggerNow := now % s.Period if triggerNow == 0 { - Expect(VeleroScheduleCreate(s.Ctx, s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, s.ScheduleName, s.ScheduleArgs)).To(Succeed(), func() string { - RunDebug(context.Background(), s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, "", "") + Expect(veleroutil.VeleroScheduleCreate(s.Ctx, s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, s.ScheduleName, s.ScheduleArgs)).To(Succeed(), func() string { + veleroutil.RunDebug(context.Background(), s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, "", "") return "Fail to create schedule" }) break @@ -106,7 +107,7 @@ func (s *ScheduleBackupCreation) Backup() error { mi, _ := time.ParseDuration("60s") time.Sleep(s.podSleepDuration + mi) bMap := make(map[string]string) - backupsInfo, err := GetScheduledBackupsCreationTime(s.Ctx, s.VeleroCfg.VeleroCLI, "default", s.ScheduleName) + backupsInfo, err := veleroutil.GetScheduledBackupsCreationTime(s.Ctx, s.VeleroCfg.VeleroCLI, "default", s.ScheduleName) Expect(err).To(Succeed()) Expect(backupsInfo).To(HaveLen(i)) for index, bi := range backupsInfo { @@ -129,7 +130,7 @@ func (s *ScheduleBackupCreation) Clean() error { if CurrentSpecReport().Failed() && s.VeleroCfg.FailFast { fmt.Println("Test case failed and fail fast is enabled. Skip resource clean up.") } else { - Expect(VeleroScheduleDelete(s.Ctx, s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, s.ScheduleName)).To(Succeed()) + Expect(veleroutil.VeleroScheduleDelete(s.Ctx, s.VeleroCfg.VeleroCLI, s.VeleroCfg.VeleroNamespace, s.ScheduleName)).To(Succeed()) Expect(s.TestCase.Clean()).To(Succeed()) } diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index 3c371064c3..38a8c8a0ca 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -78,8 +78,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC By("Uninstall Velero", func() { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() - Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace)).To(Succeed()) + Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed()) }) } }) @@ -97,8 +96,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC By("Uninstall Velero", func() { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() - Expect(VeleroUninstall(ctx, veleroCfg.VeleroCLI, - veleroCfg.VeleroNamespace)).To(Succeed()) + Expect(VeleroUninstall(ctx, veleroCfg)).To(Succeed()) }) } } diff --git a/test/perf/e2e_suite_test.go b/test/perf/e2e_suite_test.go index 383e78a78f..cb4522d699 100644 --- a/test/perf/e2e_suite_test.go +++ b/test/perf/e2e_suite_test.go @@ -139,7 +139,7 @@ var _ = AfterSuite(func() { By("release test resources after testing") ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() - Expect(VeleroUninstall(ctx, VeleroCfg.VeleroCLI, VeleroCfg.VeleroNamespace)).To(Succeed()) + Expect(VeleroUninstall(ctx, VeleroCfg)).To(Succeed()) } } }) diff --git a/test/testdata/storage-class/README.md b/test/testdata/storage-class/README.md new file mode 100644 index 0000000000..52f27382fd --- /dev/null +++ b/test/testdata/storage-class/README.md @@ -0,0 +1,12 @@ +The `test/testdata/storage-class` directory contains the StorageClass YAMLs used for E2E. +The public cloud provider (including AWS, Azure and GCP) has two StorageClasses. +* The `provider-name`.yaml contains the default StorageClass for the provider. It uses the CSI provisioner. +* The `provider-name`-legacy.yaml contains the legacy StorageClass for the provider. It uses the in-tree volume plugin as the provisioner. By far, there is no E2E case using them. + +The vSphere environment also has two StorageClass files. +* The vsphere-legacy.yaml is used for the TKGm environment. +* The vsphere.yaml is used for the VKS environment. + +The ZFS StorageClasses only have the default one. There is no in-tree volume plugin used StorageClass used in E2E. + +The kind StorageClass uses the local-path provisioner. Will consider adding the CSI provisioner when there is a need. diff --git a/test/testdata/storage-class/aws-csi.yaml b/test/testdata/storage-class/aws-legecy.yaml similarity index 70% rename from test/testdata/storage-class/aws-csi.yaml rename to test/testdata/storage-class/aws-legecy.yaml index 48c95fdc9c..29e79c8eac 100644 --- a/test/testdata/storage-class/aws-csi.yaml +++ b/test/testdata/storage-class/aws-legecy.yaml @@ -1,8 +1,8 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: e2e-csi-storage-class -provisioner: ebs.csi.aws.com + name: e2e-storage-class +provisioner: kubernetes.io/aws-ebs parameters: type: gp2 reclaimPolicy: Delete diff --git a/test/testdata/storage-class/aws.yaml b/test/testdata/storage-class/aws.yaml index 29e79c8eac..848357aecd 100644 --- a/test/testdata/storage-class/aws.yaml +++ b/test/testdata/storage-class/aws.yaml @@ -2,7 +2,7 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: e2e-storage-class -provisioner: kubernetes.io/aws-ebs +provisioner: ebs.csi.aws.com parameters: type: gp2 reclaimPolicy: Delete diff --git a/test/testdata/storage-class/azure-csi.yaml b/test/testdata/storage-class/azure-legacy.yaml similarity index 76% rename from test/testdata/storage-class/azure-csi.yaml rename to test/testdata/storage-class/azure-legacy.yaml index 5ef573b48b..aa9451bf67 100644 --- a/test/testdata/storage-class/azure-csi.yaml +++ b/test/testdata/storage-class/azure-legacy.yaml @@ -1,8 +1,8 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: e2e-csi-storage-class -provisioner: disk.csi.azure.com + name: e2e-storage-class +provisioner: kubernetes.io/azure-disk parameters: cachingmode: ReadOnly kind: Managed diff --git a/test/testdata/storage-class/azure.yaml b/test/testdata/storage-class/azure.yaml index aa9451bf67..4a153cab3a 100644 --- a/test/testdata/storage-class/azure.yaml +++ b/test/testdata/storage-class/azure.yaml @@ -2,7 +2,7 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: name: e2e-storage-class -provisioner: kubernetes.io/azure-disk +provisioner: disk.csi.azure.com parameters: cachingmode: ReadOnly kind: Managed diff --git a/test/testdata/storage-class/gcp-csi.yaml b/test/testdata/storage-class/gcp-legacy.yaml similarity index 78% rename from test/testdata/storage-class/gcp-csi.yaml rename to test/testdata/storage-class/gcp-legacy.yaml index 399061fa84..397a4b19f3 100644 --- a/test/testdata/storage-class/gcp-csi.yaml +++ b/test/testdata/storage-class/gcp-legacy.yaml @@ -4,10 +4,10 @@ kind: StorageClass metadata: labels: addonmanager.kubernetes.io/mode: EnsureExists - name: e2e-csi-storage-class + name: e2e-storage-class parameters: type: pd-standard -provisioner: pd.csi.storage.gke.io +provisioner: kubernetes.io/gce-pd reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer diff --git a/test/testdata/storage-class/gcp.yaml b/test/testdata/storage-class/gcp.yaml index 397a4b19f3..8a60ed4e5b 100644 --- a/test/testdata/storage-class/gcp.yaml +++ b/test/testdata/storage-class/gcp.yaml @@ -7,7 +7,7 @@ metadata: name: e2e-storage-class parameters: type: pd-standard -provisioner: kubernetes.io/gce-pd +provisioner: pd.csi.storage.gke.io reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer diff --git a/test/testdata/storage-class/vanilla-zfs-csi.yaml b/test/testdata/storage-class/vanilla-zfs-csi.yaml deleted file mode 100644 index 697b80a536..0000000000 --- a/test/testdata/storage-class/vanilla-zfs-csi.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: e2e-csi-storage-class -parameters: - recordsize: "128k" - compression: "off" - dedup: "off" - fstype: "zfs" - poolname: "zfspv-pool" -provisioner: zfs.csi.openebs.io -volumeBindingMode: WaitForFirstConsumer -reclaimPolicy: Retain \ No newline at end of file diff --git a/test/testdata/storage-class/vanilla-zfs.yaml b/test/testdata/storage-class/vanilla-zfs.yaml index c7ada98b83..34c063b1a7 100644 --- a/test/testdata/storage-class/vanilla-zfs.yaml +++ b/test/testdata/storage-class/vanilla-zfs.yaml @@ -9,4 +9,4 @@ parameters: fstype: "zfs" poolname: "zfspv-pool" provisioner: zfs.csi.openebs.io -volumeBindingMode: WaitForFirstConsumer \ No newline at end of file +volumeBindingMode: WaitForFirstConsumer diff --git a/test/testdata/storage-class/vsphere-csi.yaml b/test/testdata/storage-class/vsphere-legacy.yaml similarity index 53% rename from test/testdata/storage-class/vsphere-csi.yaml rename to test/testdata/storage-class/vsphere-legacy.yaml index 911751cfba..9449fa0113 100644 --- a/test/testdata/storage-class/vsphere-csi.yaml +++ b/test/testdata/storage-class/vsphere-legacy.yaml @@ -1,12 +1,11 @@ apiVersion: storage.k8s.io/v1 kind: StorageClass metadata: - name: e2e-csi-storage-class + name: e2e-storage-class annotations: storageclass.kubernetes.io/is-default-class: "false" parameters: - # StoragePolicyName: "vSAN Default Storage Policy" # This is used for the TKGm environment. - svStorageClass: worker-storagepolicy # This is used for TKGs/uTKG environment. + StoragePolicyName: "vSAN Default Storage Policy" # This is used for TKGm environment. provisioner: csi.vsphere.vmware.com reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer \ No newline at end of file diff --git a/test/testdata/storage-class/vsphere.yaml b/test/testdata/storage-class/vsphere.yaml index e09b09a603..f26bdf1f0b 100644 --- a/test/testdata/storage-class/vsphere.yaml +++ b/test/testdata/storage-class/vsphere.yaml @@ -5,8 +5,7 @@ metadata: annotations: storageclass.kubernetes.io/is-default-class: "false" parameters: - #StoragePolicyName: "vSAN Default Storage Policy" # This is used for TKGm environment. - svStorageClass: worker-storagepolicy # This is used for TKGs/uTKG environment. + svStorageClass: worker-storagepolicy provisioner: csi.vsphere.vmware.com reclaimPolicy: Delete volumeBindingMode: WaitForFirstConsumer \ No newline at end of file diff --git a/test/testdata/volume-snapshot-class/vanilla-zfs.yaml b/test/testdata/volume-snapshot-class/vanilla-zfs.yaml index 3649215e84..c5ed37a449 100644 --- a/test/testdata/volume-snapshot-class/vanilla-zfs.yaml +++ b/test/testdata/volume-snapshot-class/vanilla-zfs.yaml @@ -1,7 +1,7 @@ kind: VolumeSnapshotClass apiVersion: snapshot.storage.k8s.io/v1 metadata: - name: zfspv-snapclass + name: e2e-volume-snapshot-class annotations: snapshot.storage.kubernetes.io/is-default-class: "true" labels: diff --git a/test/testdata/volume-snapshot-class/vsphere.yaml b/test/testdata/volume-snapshot-class/vsphere.yaml index 08ea9b2225..6bab9b5a28 100644 --- a/test/testdata/volume-snapshot-class/vsphere.yaml +++ b/test/testdata/volume-snapshot-class/vsphere.yaml @@ -8,6 +8,6 @@ metadata: snapshot.storage.kubernetes.io/is-default-class: "true" labels: velero.io/csi-volumesnapshot-class: "true" - name: volumesnapshotclass-delete + name: e2e-volume-snapshot-class parameters: svVolumeSnapshotClass: volumesnapshotclass-delete diff --git a/test/types.go b/test/types.go index 527068599c..35a1502c2d 100644 --- a/test/types.go +++ b/test/types.go @@ -25,9 +25,12 @@ import ( "github.com/vmware-tanzu/velero/test/util/k8s" ) +// e2e-storage-class is the default StorageClass for E2E. const StorageClassName = "e2e-storage-class" + +// e2e-storage-class-2 is used for the StorageClass mapping test case. const StorageClassName2 = "e2e-storage-class-2" -const CSIStorageClassName = "e2e-csi-storage-class" + const FeatureCSI = "EnableCSI" const VanillaZFS = "vanilla-zfs" const Kind = "kind" diff --git a/test/util/k8s/configmap.go b/test/util/k8s/configmap.go index e4edd0667b..5a49932bae 100644 --- a/test/util/k8s/configmap.go +++ b/test/util/k8s/configmap.go @@ -71,7 +71,7 @@ func GetConfigmap(c clientset.Interface, ns, secretName string) (*v1.ConfigMap, return c.CoreV1().ConfigMaps(ns).Get(context.TODO(), secretName, metav1.GetOptions{}) } -func DeleteConfigmap(c clientset.Interface, ns, name string) error { +func DeleteConfigMap(c clientset.Interface, ns, name string) error { if err := c.CoreV1().ConfigMaps(ns).Delete(context.TODO(), name, metav1.DeleteOptions{}); err != nil { return errors.Wrap(err, fmt.Sprintf("failed to delete configmap in namespace %q", ns)) } @@ -79,7 +79,7 @@ func DeleteConfigmap(c clientset.Interface, ns, name string) error { } func WaitForConfigmapDelete(c clientset.Interface, ns, name string) error { - if err := DeleteConfigmap(c, ns, name); err != nil { + if err := DeleteConfigMap(c, ns, name); err != nil { return err } diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 9da1c4b83a..7d6a6d1c49 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -23,7 +23,6 @@ import ( "fmt" "os" "os/exec" - "strings" "time" "github.com/pkg/errors" @@ -129,7 +128,7 @@ func VeleroInstall(ctx context.Context, veleroCfg *test.VeleroConfig, isStandbyC _, err = k8s.GetNamespace(ctx, *veleroCfg.ClientToInstallVelero, veleroCfg.VeleroNamespace) // We should uninstall Velero for a new service account creation. if !apierrors.IsNotFound(err) { - if err := VeleroUninstall(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace); err != nil { + if err := VeleroUninstall(context.Background(), *veleroCfg); err != nil { return errors.Wrapf(err, "Failed to uninstall velero %s", veleroCfg.VeleroNamespace) } } @@ -150,15 +149,19 @@ func VeleroInstall(ctx context.Context, veleroCfg *test.VeleroConfig, isStandbyC return errors.Wrapf(err, "Failed to create service account %s to %s namespace", veleroInstallOptions.ServiceAccountName, veleroCfg.VeleroNamespace) } } - err = installVeleroServer(ctx, veleroCfg.VeleroCLI, veleroCfg.CloudProvider, &installOptions{ - Options: veleroInstallOptions, - RegistryCredentialFile: veleroCfg.RegistryCredentialFile, - RestoreHelperImage: veleroCfg.RestoreHelperImage, - VeleroServerDebugMode: veleroCfg.VeleroServerDebugMode, - WithoutDisableInformerCacheParam: veleroCfg.WithoutDisableInformerCacheParam, - }) - if err != nil { + if err := installVeleroServer( + ctx, + veleroCfg.VeleroCLI, + veleroCfg.CloudProvider, + &installOptions{ + Options: veleroInstallOptions, + RegistryCredentialFile: veleroCfg.RegistryCredentialFile, + RestoreHelperImage: veleroCfg.RestoreHelperImage, + VeleroServerDebugMode: veleroCfg.VeleroServerDebugMode, + WithoutDisableInformerCacheParam: veleroCfg.WithoutDisableInformerCacheParam, + }, + ); err != nil { time.Sleep(9 * time.Hour) RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, "", "") return errors.WithMessagef(err, "Failed to install Velero in the cluster") @@ -320,14 +323,6 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options if len(options.Features) > 0 { args = append(args, "--features", options.Features) - if strings.EqualFold(options.Features, test.FeatureCSI) && options.UseVolumeSnapshots { - // https://github.com/openebs/zfs-localpv/blob/develop/docs/snapshot.md - fmt.Printf("Start to install %s VolumeSnapshotClass ... \n", cloudProvider) - if err := k8s.KubectlApplyByFile(ctx, fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", cloudProvider)); err != nil { - fmt.Println("Fail to install VolumeSnapshotClass when CSI feature is enabled: ", err) - return err - } - } } if options.GarbageCollectionFrequency > 0 { @@ -374,14 +369,14 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options args = append(args, fmt.Sprintf("--uploader-type=%v", options.UploaderType)) } - if err := createVelereResources(ctx, cli, namespace, args, options); err != nil { + if err := createVeleroResources(ctx, cli, namespace, args, options); err != nil { return err } return waitVeleroReady(ctx, namespace, options.UseNodeAgent) } -func createVelereResources(ctx context.Context, cli, namespace string, args []string, options *installOptions) error { +func createVeleroResources(ctx context.Context, cli, namespace string, args []string, options *installOptions) error { args = append(args, "--dry-run", "--output", "json", "--crds-only") // get the CRD definitions @@ -670,7 +665,7 @@ func PrepareVelero(ctx context.Context, caseName string, veleroCfg test.VeleroCo fmt.Printf("error in checking velero status with %v", err) ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() - VeleroUninstall(ctx, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace) + VeleroUninstall(ctx, veleroCfg) ready = false } if ready { @@ -681,9 +676,15 @@ func PrepareVelero(ctx context.Context, caseName string, veleroCfg test.VeleroCo return VeleroInstall(context.Background(), &veleroCfg, false) } -func VeleroUninstall(ctx context.Context, cli, namespace string) error { - stdout, stderr, err := velerexec.RunCommand(exec.CommandContext(ctx, cli, "uninstall", "--force", "-n", namespace)) - if err != nil { +func VeleroUninstall(ctx context.Context, veleroCfg test.VeleroConfig) error { + if stdout, stderr, err := velerexec.RunCommand(exec.CommandContext( + ctx, + veleroCfg.VeleroCLI, + "uninstall", + "--force", + "-n", + veleroCfg.VeleroNamespace, + )); err != nil { return errors.Wrapf(err, "failed to uninstall velero, stdout=%s, stderr=%s", stdout, stderr) } fmt.Println("Velero uninstalled ⛵") diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index d998246636..639fefb015 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -616,9 +616,7 @@ func RunDebug(ctx context.Context, veleroCLI, veleroNamespace, backup, restore s if len(backup) > 0 { args = append(args, "--backup", backup) } - if len(restore) > 0 { - //args = append(args, "--restore", restore) - } + fmt.Printf("Generating the debug tarball at %s\n", output) if err := VeleroCmdExec(ctx, veleroCLI, args); err != nil { fmt.Println(errors.Wrapf(err, "failed to run the debug command")) @@ -1228,6 +1226,7 @@ func GetBackupsCreationTime(ctx context.Context, veleroCLI, bslName string) ([]s func GetAllBackups(ctx context.Context, veleroCLI string) ([]string, error) { return GetBackupsFromBsl(ctx, veleroCLI, "") } + func DeleteBslResource(ctx context.Context, veleroCLI string, bslName string) error { args := []string{"backup-location", "delete", bslName, "--confirm"} @@ -1431,6 +1430,7 @@ func VeleroUpgrade(ctx context.Context, veleroCfg VeleroConfig) error { } return waitVeleroReady(ctx, veleroCfg.VeleroNamespace, veleroCfg.UseNodeAgent) } + func ApplyCRDs(ctx context.Context, veleroCLI string) ([]string, error) { cmds := []*common.OsCommandLine{} @@ -1629,20 +1629,32 @@ func GetVeleroPodName(ctx context.Context) ([]string, error) { return common.GetListByCmdPipes(ctx, cmds) } -func InstallTestStorageClasses(path string) error { +// InstallStorageClasses create the "e2e-storage-class" and "e2e-storage-class-2" +// StorageClasses for E2E tests. +// +// e2e-storage-class is the default StorageClass for E2E. +// e2e-storage-class-2 is used for the StorageClass mapping test case. +// Kibishii StorageClass is not covered here. +func InstallStorageClasses(provider string) error { ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*5) defer ctxCancel() - err := InstallStorageClass(ctx, path) - if err != nil { + + storageClassFilePath := fmt.Sprintf("../testdata/storage-class/%s.yaml", provider) + + if err := InstallStorageClass(ctx, storageClassFilePath); err != nil { return err } - content, err := os.ReadFile(path) + content, err := os.ReadFile(storageClassFilePath) if err != nil { - return errors.Wrapf(err, "failed to get %s when install storage class", path) + return errors.Wrapf(err, "failed to get %s when install storage class", storageClassFilePath) } - // replace sc to new value - newContent := strings.ReplaceAll(string(content), fmt.Sprintf("name: %s", StorageClassName), fmt.Sprintf("name: %s", StorageClassName2)) + // Replace the name to e2e-storage-class-2 + newContent := strings.ReplaceAll( + string(content), + fmt.Sprintf("name: %s", StorageClassName), + fmt.Sprintf("name: %s", StorageClassName2), + ) tmpFile, err := os.CreateTemp("", "sc-file") if err != nil {