From e62b50b5b64fd23bf7a91d051b8391f9be5d5413 Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Tue, 12 Nov 2024 14:29:57 +0800 Subject: [PATCH] Modify other cases to support VKS environment. Signed-off-by: Xun Jiang --- test/e2e/backups/deletion.go | 67 +++++++--- test/e2e/backups/ttl.go | 8 +- test/e2e/bsl-mgmt/deletion.go | 186 +++++++++++++++------------ test/e2e/migration/migration.go | 9 +- test/e2e/upgrade/upgrade.go | 9 +- test/util/kibishii/kibishii_utils.go | 9 +- test/util/providers/common.go | 82 ++++++++---- 7 files changed, 229 insertions(+), 141 deletions(-) diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index f5bba56c2dc..1f934c08333 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -99,8 +99,6 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc providerName := veleroCfg.CloudProvider veleroNamespace := veleroCfg.VeleroNamespace registryCredentialFile := veleroCfg.RegistryCredentialFile - bslPrefix := veleroCfg.BSLPrefix - bslConfig := veleroCfg.BSLConfig veleroFeatures := veleroCfg.Features for _, ns := range workloadNamespaceList { if err := CreateNamespace(oneHourTimeout, client, ns); err != nil { @@ -153,7 +151,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc } } } - err = ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) + err = ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix) if err != nil { return err } @@ -165,9 +163,12 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc for _, ns := range workloadNamespaceList { snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, DefaultKibishiiWorkerCounts, ns, backupName, KibishiiPVCNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig, - backupName, snapshotCheckPoint) + err = SnapshotsShouldBeCreatedInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + ) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } @@ -179,9 +180,12 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") // Get all snapshots base on backup name, regardless of namespaces - err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig, - backupName, snapshotCheckPoint) + err = SnapshotsShouldBeCreatedInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + ) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } @@ -216,14 +220,28 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc } } - err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5) + err = ObjectsShouldNotBeInBucket( + veleroCfg.ObjectStoreProvider, + veleroCfg.CloudCredentialsFile, + veleroCfg.BSLBucket, + veleroCfg.BSLPrefix, + veleroCfg.BSLConfig, + backupName, + BackupObjectsPrefix, + 5, + ) if err != nil { return err } if useVolumeSnapshots { - if err := SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - bslConfig, backupName, snapshotCheckPoint); err != nil { + if err := SnapshotsShouldNotExistInCloud( + veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, + veleroCfg.BSLBucket, + veleroCfg.BSLConfig, + backupName, + snapshotCheckPoint, + ); err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } } @@ -243,13 +261,28 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc }) }) - err = DeleteObjectsInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) - if err != nil { + if err := DeleteObjectsInBucket( + veleroCfg.ObjectStoreProvider, + veleroCfg.CloudCredentialsFile, + veleroCfg.BSLBucket, + veleroCfg.BSLPrefix, + veleroCfg.BSLConfig, + backupName, + BackupObjectsPrefix, + ); err != nil { return err } - err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1) - if err != nil { + if err := ObjectsShouldNotBeInBucket( + veleroCfg.ObjectStoreProvider, + veleroCfg.CloudCredentialsFile, + veleroCfg.BSLBucket, + veleroCfg.BSLPrefix, + veleroCfg.BSLConfig, + backupName, + BackupObjectsPrefix, + 1, + ); err != nil { return err } diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 342eac03786..6c8d2878b83 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -140,13 +140,11 @@ func TTLTest() { Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") Expect( - SnapshotsShouldBeCreatedInCloud( - veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, - veleroCfg.BSLBucket, - veleroCfg.BSLConfig, + SnapshotsShouldBeCreatedInProvider( + veleroCfg, test.backupName, snapshotCheckPoint, + false, ), ).NotTo(HaveOccurred(), "Fail to verify the created snapshots") } diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 6c3ccee88ec..fdede2af2c6 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -36,8 +36,8 @@ import ( const ( // Please make sure length of this namespace should be shorter, - // otherwise ResticRepositories name verification will be wrong - // when making combination of ResticRepositories name(max length is 63) + // otherwise BackupRepositories name verification will be wrong + // when making combination of BackupRepositories name(max length is 63) bslDeletionTestNs = "bsl-deletion" ) @@ -134,12 +134,12 @@ func BslDeletionTest(useVolumeSnapshots bool) { )).To(Succeed()) }) - backupName_1 := "backup1-" + UUIDgen.String() - backupName_2 := "backup2-" + UUIDgen.String() - backupLocation_1 := "default" - backupLocation_2 := additionalBsl - podName_1 := "kibishii-deployment-0" - podName_2 := "kibishii-deployment-1" + backupName1 := "backup1-" + UUIDgen.String() + backupName2 := "backup2-" + UUIDgen.String() + backupLocation1 := "default" + backupLocation2 := additionalBsl + podName1 := "kibishii-deployment-0" + podName2 := "kibishii-deployment-1" label_1 := "for=1" // TODO remove when issue https://github.com/vmware-tanzu/velero/issues/4724 is fixed @@ -157,17 +157,17 @@ func BslDeletionTest(useVolumeSnapshots bool) { // Restic can not backup PV only, so pod need to be labeled also By("Label all 2 worker-pods of Kibishii", func() { - Expect(AddLabelToPod(context.Background(), podName_1, bslDeletionTestNs, label_1)).To(Succeed()) + Expect(AddLabelToPod(context.Background(), podName1, bslDeletionTestNs, label_1)).To(Succeed()) Expect(AddLabelToPod(context.Background(), "kibishii-deployment-1", bslDeletionTestNs, label_2)).To(Succeed()) }) By("Get all 2 PVCs of Kibishii and label them separately ", func() { - pvc, err := GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName_1) + pvc, err := GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName1) Expect(err).To(Succeed()) fmt.Println(pvc) Expect(pvc).To(HaveLen(1)) pvc1 := pvc[0] - pvc, err = GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName_2) + pvc, err = GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName2) Expect(err).To(Succeed()) fmt.Println(pvc) Expect(pvc).To(HaveLen(1)) @@ -177,13 +177,13 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) var BackupCfg BackupConfig - BackupCfg.BackupName = backupName_1 + BackupCfg.BackupName = backupName1 BackupCfg.Namespace = bslDeletionTestNs - BackupCfg.BackupLocation = backupLocation_1 + BackupCfg.BackupLocation = backupLocation1 BackupCfg.UseVolumeSnapshots = useVolumeSnapshots BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots BackupCfg.Selector = label_1 - By(fmt.Sprintf("Backup one of PV of sample workload by label-1 - Kibishii by the first BSL %s", backupLocation_1), func() { + By(fmt.Sprintf("Backup one of PV of sample workload by label-1 - Kibishii by the first BSL %s", backupLocation1), func() { // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case Expect(VeleroBackupNamespace(oneHourTimeout, veleroCfg.VeleroCLI, @@ -193,10 +193,10 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) }) - BackupCfg.BackupName = backupName_2 - BackupCfg.BackupLocation = backupLocation_2 + BackupCfg.BackupName = backupName2 + BackupCfg.BackupLocation = backupLocation2 BackupCfg.Selector = label_2 - By(fmt.Sprintf("Back up the other one PV of sample workload with label-2 into the additional BSL %s", backupLocation_2), func() { + By(fmt.Sprintf("Back up the other one PV of sample workload with label-2 into the additional BSL %s", backupLocation2), func() { Expect(VeleroBackupNamespace(oneHourTimeout, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupCfg.BackupName, "") @@ -211,82 +211,94 @@ func BslDeletionTest(useVolumeSnapshots bool) { Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, bslDeletionTestNs, 2)).To(Succeed()) }) - By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName_1), func() { + By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName1), func() { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, - backupName_1, 1)).To(Succeed()) + backupName1, 1)).To(Succeed()) }) - By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName_2), func() { + By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName2), func() { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, - backupName_2, 1)).To(Succeed()) + backupName2, 1)).To(Succeed()) }) } if veleroCfg.CloudProvider != VanillaZFS { var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs - By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() { - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) + By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation1), func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint( + *veleroCfg.ClientToInstallVelero, + veleroCfg, + 1, + bslDeletionTestNs, + backupName1, + []string{podName1}, + ) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) + Expect(SnapshotsShouldBeCreatedInProvider( + veleroCfg, + backupName1, + snapshotCheckPoint, + false, + )).To(Succeed()) }) - By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() { - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) + By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation2), func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint( + *veleroCfg.ClientToInstallVelero, + veleroCfg, + 1, + bslDeletionTestNs, + backupName2, + []string{podName2}, + ) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - var BSLCredentials, BSLConfig string - if veleroCfg.CloudProvider == Vsphere { - BSLCredentials = veleroCfg.AdditionalBSLCredentials - BSLConfig = veleroCfg.AdditionalBSLConfig - } else { // Snapshotting with non-vSphere provider has nothing to do with BSL - BSLCredentials = veleroCfg.CloudCredentialsFile - BSLConfig = veleroCfg.BSLConfig - } - - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - BSLCredentials, veleroCfg.AdditionalBSLBucket, - BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed()) + + Expect(SnapshotsShouldBeCreatedInProvider( + veleroCfg, + backupName2, + snapshotCheckPoint, + true, + )).To(Succeed()) }) } - } else { // For Restics - By(fmt.Sprintf("Resticrepositories for BSL %s should be created in Velero namespace", backupLocation_1), func() { + } else { + By(fmt.Sprintf("BackupRepositories for BSL %s should be created in Velero namespace", backupLocation1), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_1, 1)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation1, 1)).To(Succeed()) }) - By(fmt.Sprintf("Resticrepositories for BSL %s should be created in Velero namespace", backupLocation_2), func() { + By(fmt.Sprintf("BackupRepositories for BSL %s should be created in Velero namespace", backupLocation2), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_2, 1)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation2, 1)).To(Succeed()) }) } - By(fmt.Sprintf("Backup 1 %s should be created.", backupName_1), func() { + By(fmt.Sprintf("Backup 1 %s should be created.", backupName1), func() { Expect(WaitForBackupToBeCreated(context.Background(), - backupName_1, 10*time.Minute, &veleroCfg)).To(Succeed()) + backupName1, 10*time.Minute, &veleroCfg)).To(Succeed()) }) - By(fmt.Sprintf("Backup 2 %s should be created.", backupName_2), func() { + By(fmt.Sprintf("Backup 2 %s should be created.", backupName2), func() { Expect(WaitForBackupToBeCreated(context.Background(), - backupName_2, 10*time.Minute, &veleroCfg)).To(Succeed()) + backupName2, 10*time.Minute, &veleroCfg)).To(Succeed()) }) - backupsInBSL1, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation_1) + backupsInBSL1, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation1) Expect(err).To(Succeed()) - backupsInBSL2, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation_2) + backupsInBSL2, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation2) Expect(err).To(Succeed()) backupsInBsl1AndBsl2 := append(backupsInBSL1, backupsInBSL2...) - By(fmt.Sprintf("Get all backups from 2 BSLs %s before deleting one of them", backupLocation_1), func() { + By(fmt.Sprintf("Get all backups from 2 BSLs %s before deleting one of them", backupLocation1), func() { backupsBeforeDel, err := GetAllBackups(context.Background(), veleroCfg.VeleroCLI) Expect(err).To(Succeed()) Expect(cmp.Diff(backupsInBsl1AndBsl2, backupsBeforeDel, cmpopts.SortSlices(less))).Should(BeEmpty()) - By(fmt.Sprintf("Backup1 %s should exist in cloud object store before bsl deletion", backupName_1), func() { + By(fmt.Sprintf("Backup1 %s should exist in cloud object store before bsl deletion", backupName1), func() { Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, - backupName_1, BackupObjectsPrefix)).To(Succeed()) + backupName1, BackupObjectsPrefix)).To(Succeed()) }) - By(fmt.Sprintf("Delete one of backup locations - %s", backupLocation_1), func() { - Expect(DeleteBslResource(context.Background(), veleroCfg.VeleroCLI, backupLocation_1)).To(Succeed()) + By(fmt.Sprintf("Delete one of backup locations - %s", backupLocation1), func() { + Expect(DeleteBslResource(context.Background(), veleroCfg.VeleroCLI, backupLocation1)).To(Succeed()) Expect(WaitForBackupsToBeDeleted(context.Background(), backupsInBSL1, 10*time.Minute, &veleroCfg)).To(Succeed()) }) @@ -298,10 +310,10 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) }) - By(fmt.Sprintf("Backup1 %s should still exist in cloud object store after bsl deletion", backupName_1), func() { + By(fmt.Sprintf("Backup1 %s should still exist in cloud object store after bsl deletion", backupName1), func() { Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, - backupName_1, BackupObjectsPrefix)).To(Succeed()) + backupName1, BackupObjectsPrefix)).To(Succeed()) }) // TODO: Choose additional BSL to be deleted as an new test case @@ -313,45 +325,51 @@ func BslDeletionTest(useVolumeSnapshots bool) { if useVolumeSnapshots { if veleroCfg.CloudProvider == Vsphere { - By(fmt.Sprintf("Snapshot in backup %s should still exist, because snapshot CR will be deleted 24 hours later if the status is a success", backupName_2), func() { + By(fmt.Sprintf("Snapshot in backup %s should still exist, because snapshot CR will be deleted 24 hours later if the status is a success", backupName2), func() { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, - backupName_1, 1)).To(Succeed()) + backupName1, 1)).To(Succeed()) Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, - backupName_2, 1)).To(Succeed()) + backupName2, 1)).To(Succeed()) }) } var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs - By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_1), func() { - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) + By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation1), func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName1, []string{podName1}) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) + Expect(SnapshotsShouldBeCreatedInProvider( + veleroCfg, + backupName1, + snapshotCheckPoint, + false, + )).To(Succeed()) }) - By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_2), func() { - var BSLCredentials, BSLConfig string - if veleroCfg.CloudProvider == Vsphere { - BSLCredentials = veleroCfg.AdditionalBSLCredentials - BSLConfig = veleroCfg.AdditionalBSLConfig - } else { - BSLCredentials = veleroCfg.CloudCredentialsFile - BSLConfig = veleroCfg.BSLConfig - } - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) + By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation2), func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint( + *veleroCfg.ClientToInstallVelero, + veleroCfg, + 1, + bslDeletionTestNs, + backupName2, + []string{podName2}, + ) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - BSLCredentials, veleroCfg.AdditionalBSLBucket, - BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed()) + + Expect(SnapshotsShouldBeCreatedInProvider( + veleroCfg, + backupName2, + snapshotCheckPoint, + true, + )).To(Succeed()) }) - } else { // For Restic - By(fmt.Sprintf("Resticrepositories for BSL %s should be deleted in Velero namespace", backupLocation_1), func() { + } else { + By(fmt.Sprintf("BackupRepositories for BSL %s should be deleted in Velero namespace", backupLocation1), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_1, 0)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation1, 0)).To(Succeed()) }) - By(fmt.Sprintf("Resticrepositories for BSL %s should still exist in Velero namespace", backupLocation_2), func() { + By(fmt.Sprintf("BackupRepositories for BSL %s should still exist in Velero namespace", backupLocation2), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_2, 1)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation2, 1)).To(Succeed()) }) } fmt.Printf("|| EXPECTED || - Backup deletion test completed successfully\n") diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 51edf340721..ef13a5b840e 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -288,9 +288,12 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, kibishiiWorkerCount, migrationNamespace, backupName, GetKibishiiPVCNameList(kibishiiWorkerCount)) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) + Expect(SnapshotsShouldBeCreatedInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + )).To(Succeed()) }) } } diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index ffb8f5f22bf..0938be9b0a4 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -202,9 +202,12 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 2, upgradeNamespace, backupName, KibishiiPVCNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) + Expect(SnapshotsShouldBeCreatedInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + )).To(Succeed()) }) } diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index 4ab07839462..83f741c084c 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -131,9 +131,12 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc if err != nil { return errors.Wrap(err, "Fail to get snapshot checkpoint") } - err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, - backupName, snapshotCheckPoint) + err = SnapshotsShouldBeCreatedInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + ) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } diff --git a/test/util/providers/common.go b/test/util/providers/common.go index 46e42894ef9..ee0bf113454 100644 --- a/test/util/providers/common.go +++ b/test/util/providers/common.go @@ -28,14 +28,14 @@ import ( "github.com/pkg/errors" "github.com/vmware-tanzu/velero/internal/volume" - . "github.com/vmware-tanzu/velero/test" + velerotest "github.com/vmware-tanzu/velero/test" velero "github.com/vmware-tanzu/velero/test/util/velero" ) type ObjectsInStorage interface { IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error - IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error + IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck velerotest.SnapshotCheckPoint) error GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, objectKey string) (io.ReadCloser, error) } @@ -75,13 +75,13 @@ func ObjectsShouldNotBeInBucket(objectStoreProvider, cloudCredentialsFile, bslBu func getProvider(cloudProvider string) (ObjectsInStorage, error) { var s ObjectsInStorage switch cloudProvider { - case AWS, Vsphere: + case velerotest.AWS, velerotest.Vsphere: aws := AWSStorage("") s = &aws - case GCP: + case velerotest.GCP: gcs := GCSStorage("") s = &gcs - case Azure: + case velerotest.Azure: az := AzureStorage("") s = &az default: @@ -102,7 +102,7 @@ func IsObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bsl bslPrefix = getFullPrefix(bslPrefix, subPrefix) s, err := getProvider(objectStoreProvider) if err != nil { - return false, errors.Wrapf(err, fmt.Sprintf("Object store provider %s is not valid", objectStoreProvider)) + return false, errors.Wrapf(err, "Object store provider %s is not valid", objectStoreProvider) } return s.IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName) } @@ -116,19 +116,19 @@ func DeleteObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, } s, err := getProvider(objectStoreProvider) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Object store provider %s is not valid", objectStoreProvider)) + return errors.Wrapf(err, "Object store provider %s is not valid", objectStoreProvider) } err = s.DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Fail to delete %s", bslPrefix)) + return errors.Wrapf(err, "Fail to delete %s", bslPrefix) } return nil } -func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { +func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint velerotest.SnapshotCheckPoint) error { fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName) - if cloudProvider == VanillaZFS { + if cloudProvider == velerotest.VanillaZFS { fmt.Printf("Skip snapshot check for cloud provider %s", cloudProvider) return nil } @@ -137,45 +137,75 @@ func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBuck } snapshotCheckPoint.ExpectCount = 0 - err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) + err := isSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s exist in cloud after backup as expected", backupName)) + return errors.Wrapf(err, "|| UNEXPECTED ||Snapshots %s exist in cloud after backup as expected", backupName) } fmt.Printf("|| EXPECTED || - Snapshots do not exist in cloud, backup %s\n", backupName) return nil } -func SnapshotsShouldBeCreatedInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { +func SnapshotsShouldBeCreatedInProvider( + veleroCfg velerotest.VeleroConfig, + backupName string, + snapshotCheckPoint velerotest.SnapshotCheckPoint, + checkStandbyCluster bool, +) error { fmt.Printf("|| VERIFICATION || - Snapshots should exist in cloud, backup %s\n", backupName) - if cloudProvider == VanillaZFS { - fmt.Printf("Skip snapshot check for cloud provider %s", cloudProvider) + if veleroCfg.CloudProvider == velerotest.VanillaZFS { + fmt.Printf("Skip snapshot check for cloud provider %s", veleroCfg.CloudProvider) return nil } - if cloudCredentialsFile == "" { - return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider)) + if veleroCfg.CloudProvider == velerotest.Vsphere && !veleroCfg.HasVspherePlugin { + fmt.Printf("Skip snapshot check for vSphere environment that doesn't have Velero vSphere plugin.") + return nil } + if veleroCfg.CloudCredentialsFile == "" { + return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", veleroCfg.CloudProvider)) + } + + cloudCredentialsFile := veleroCfg.CloudCredentialsFile + bslBucket := veleroCfg.BSLBucket + bslConfig := veleroCfg.BSLConfig + if checkStandbyCluster { + bslBucket = veleroCfg.AdditionalBSLBucket - err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) + // Only vSphere environment's snapshot on standby cluster is related to BSL. + if veleroCfg.CloudProvider == velerotest.Vsphere { + cloudCredentialsFile = veleroCfg.AdditionalBSLCredentials + bslConfig = veleroCfg.AdditionalBSLConfig + } + } + + err := isSnapshotExisted( + veleroCfg.CloudProvider, + cloudCredentialsFile, + bslBucket, + bslConfig, + backupName, + snapshotCheckPoint, + ) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED || - Snapshots %s do not exist in cloud after backup as expected", backupName)) + return errors.Wrapf(err, "|| UNEXPECTED || - Snapshots %s do not exist in cloud after backup as expected", backupName) } - fmt.Printf("|| EXPECTED || - Snapshots of backup %s exist in cloud %s\n", backupName, cloudProvider) + + fmt.Printf("|| EXPECTED || - Snapshots of backup %s exist in cloud %s\n", backupName, veleroCfg.CloudProvider) return nil } -func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error { +func isSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheck velerotest.SnapshotCheckPoint) error { s, err := getProvider(cloudProvider) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) + return errors.Wrapf(err, "Cloud provider %s is not valid", cloudProvider) } - if cloudProvider == Vsphere { + if cloudProvider == velerotest.Vsphere { var retSnapshotIDs []string ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*2) defer ctxCancel() retSnapshotIDs, err = velero.GetVsphereSnapshotIDs(ctx, time.Hour, snapshotCheck.NamespaceBackedUp, snapshotCheck.PodName) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Fail to get snapshot CRs of backup%s", backupName)) + return errors.Wrapf(err, "Fail to get snapshot CRs of backup%s", backupName) } bslPrefix := "plugins" @@ -201,7 +231,7 @@ func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig } else { err = s.IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName, snapshotCheck) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Fail to get snapshot of backup %s", backupName)) + return errors.Wrapf(err, "Fail to get snapshot of backup %s", backupName) } } return nil @@ -221,7 +251,7 @@ func GetVolumeInfoMetadataContent( fmt.Printf("|| VERIFICATION || - Get backup %s volumeinfo file in storage %s\n", backupName, bslPrefix) s, err := getProvider(objectStoreProvider) if err != nil { - return nil, errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", objectStoreProvider)) + return nil, errors.Wrapf(err, "Cloud provider %s is not valid", objectStoreProvider) } return s.GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, volumeFileName)