From b02fc1da96d81cc029098459d05b3314abadc720 Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Tue, 5 Nov 2024 17:29:36 +0800 Subject: [PATCH 1/3] E2E supports VKS data mover environment. * Add new flag HAS_VSPHERE_PLUGIN for E2E test. * Modify the E2E README for the new parameter. * Add the VolumeSnapshotClass for VKS. * Modify the plugin install logic. * Modify the cases to support data mover case in VKS. Signed-off-by: Xun Jiang --- test/Makefile | 19 ++++++- test/e2e/README.md | 49 +++++++++++++++++-- test/e2e/backups/deletion.go | 5 +- test/e2e/backups/ttl.go | 28 ++++++++--- test/e2e/e2e_suite_test.go | 1 + test/e2e/migration/migration.go | 18 ++++--- test/e2e/upgrade/upgrade.go | 3 +- .../volume-snapshot-class/vsphere.yaml | 13 +++++ test/types.go | 1 + test/util/kibishii/kibishii_utils.go | 3 +- test/util/velero/install.go | 2 +- test/util/velero/velero_utils.go | 18 ++----- 12 files changed, 119 insertions(+), 41 deletions(-) create mode 100644 test/testdata/volume-snapshot-class/vsphere.yaml diff --git a/test/Makefile b/test/Makefile index 276cc76afd..b3cf6eadd5 100644 --- a/test/Makefile +++ b/test/Makefile @@ -41,25 +41,41 @@ help: ## Display this help TOOLS_DIR := $(REPO_ROOT)/hack/tools BIN_DIR := bin + # Try to not modify PATH if possible GOBIN := $(REPO_ROOT)/.go/bin + TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR) + GINKGO := $(GOBIN)/ginkgo + KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize + OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin + # Please reference to this document for Ginkgo label spec format. # https://onsi.github.io/ginkgo/#spec-labels GINKGO_LABELS ?= + # When --fail-fast is set, the entire suite will stop when the first failure occurs. # Enable --fail-fast by default. # https://onsi.github.io/ginkgo/#mental-model-how-ginkgo-handles-failure FAIL_FAST ?= false + VELERO_CLI ?=$$(pwd)/../_output/bin/$(GOOS)/$(GOARCH)/velero + VELERO_IMAGE ?= velero/velero:main + PLUGINS ?= + +# Flag used to tell E2E whether the Velero vSphere plugin is installed. +HAS_VSPHERE_PLUGIN ?= false + RESTORE_HELPER_IMAGE ?= + #Released version only UPGRADE_FROM_VELERO_VERSION ?= v1.13.2,v1.14.1 + # UPGRADE_FROM_VELERO_CLI can has the same format(a list divided by comma) with UPGRADE_FROM_VELERO_VERSION # Upgrade tests will be executed sequently according to the list by UPGRADE_FROM_VELERO_VERSION # So although length of UPGRADE_FROM_VELERO_CLI list is not equal with UPGRADE_FROM_VELERO_VERSION @@ -150,7 +166,8 @@ COMMON_ARGS := --velerocli=$(VELERO_CLI) \ --velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \ --uploader-type=$(UPLOADER_TYPE) \ --debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART) \ - --fail-fast=$(FAIL_FAST) + --fail-fast=$(FAIL_FAST) \ + --has-vsphere-plugin=$(HAS_VSPHERE_PLUGIN) # Make sure ginkgo is in $GOBIN .PHONY:ginkgo diff --git a/test/e2e/README.md b/test/e2e/README.md index 65231e01af..2ca71a7420 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -78,6 +78,7 @@ These configuration parameters are expected as values to the following command l 1. `--standby-cluster-object-store-provider`: Object store provider for standby cluster. 1. `--debug-velero-pod-restart`: A switch for debugging velero pod restart. 1. `--fail-fast`: A switch for for failing fast on meeting error. +1. `--disable-vsphere-plugin`: A switch for not install the Velero vSphere plugin when the provider is set to `vsphere`. These configurations or parameters are used to generate install options for Velero for each test suite. @@ -129,12 +130,13 @@ Below is a mapping between `make` variables to E2E configuration flags. 1. `INSTALL_VELERO `: `-install-velero`. Optional. 1. `DEBUG_VELERO_POD_RESTART`: `-debug-velero-pod-restart`. Optional. 1. `FAIL_FAST`: `--fail-fast`. Optional. +1. `DISABLE_VSPHERE_PLUGIN`: `--diable-vsphere-plugin`. Optional. ### Examples -Basic examples: +#### Basic examples: 1. Run Velero tests in a kind cluster with AWS (or MinIO) as the storage provider: @@ -208,7 +210,7 @@ ADDITIONAL_CREDS_FILE=/path/to/azure-creds \ make test-e2e ``` -Upgrade examples: +#### Upgrade examples: 1. Run Velero upgrade tests with pre-upgrade version: @@ -234,7 +236,7 @@ UPGRADE_FROM_VELERO_VERSION=v1.10.2,v1.11.0 \ make test-e2e ``` -Migration examples: +#### Migration examples: 1. Migration between 2 cluster of the same provider tests: @@ -275,7 +277,7 @@ GINKGO_LABELS="Migration" \ make test-e2e ``` -## 5. Filtering tests +#### Filtering tests In release-1.15, Velero bumps the [Ginkgo](https://onsi.github.io/ginkgo/) version to [v2](https://onsi.github.io/ginkgo/MIGRATING_TO_V2). Velero E2E start to use [labels](https://onsi.github.io/ginkgo/#spec-labels) to filter cases instead of [`-focus` and `-skip`](https://onsi.github.io/ginkgo/#focused-specs) parameters. @@ -285,7 +287,6 @@ Both `make run-e2e` and `make run-perf` CLI support using parameter `GINKGO_LABE `GINKGO_LABELS` is interpreted into `ginkgo run` CLI's parameter [`--label-filter`](https://onsi.github.io/ginkgo/#spec-labels). -### Examples E2E tests can be run with specific cases to be included and/or excluded using the commands below: 1. Run Velero tests with specific cases to be included: @@ -316,6 +317,44 @@ In this example, cases are labelled as * `Migration` and `Restic` will be skipped. +#### VKS environment test +1. Run the CSI data mover test. + +`HAS_VSPHERE_PLUGIN` should be set to `false` to not install the Velero vSphere plugin. +``` bash +CLOUD_PROVIDER=vsphere \ +DEFAULT_CLUSTER=wl-antreav1301 \ +STANDBY_CLUSTER=wl-antreav1311 \ +DEFAULT_CLUSTER_NAME=192.168.0.4 \ +STANDBY_CLUSTER_NAME=192.168.0.3 \ +FEATURES=EnableCSI \ +PLUGINS=gcr.io/velero-gcp/velero-plugin-for-aws:main \ +HAS_VSPHERE_PLUGIN=false \ +OBJECT_STORE_PROVIDER=aws \ +CREDS_FILE=$HOME/aws-credential \ +BSL_CONFIG=region=us-east-1 \ +BSL_BUCKET=nightly-normal-account4-test \ +BSL_PREFIX=nightly \ +ADDITIONAL_BSL_PLUGINS=gcr.io/velero-gcp/velero-plugin-for-aws:main \ +ADDITIONAL_OBJECT_STORE_PROVIDER=aws \ +ADDITIONAL_BSL_CONFIG=region=us-east-1 \ +ADDITIONAL_BSL_BUCKET=nightly-normal-account4-test \ +ADDITIONAL_BSL_PREFIX=addition-bsl \ +ADDITIONAL_CREDS_FILE=$HOME/aws-credential \ +VELERO_IMAGE=gcr.io/velero-gcp/velero:main \ +RESTORE_HELPER_IMAGE=gcr.io/velero-gcp/velero-restore-helper:main \ +VERSION=main \ +SNAPSHOT_MOVE_DATA=true \ +STANDBY_CLUSTER_CLOUD_PROVIDER=vsphere \ +STANDBY_CLUSTER_OBJECT_STORE_PROVIDER=aws \ +STANDBY_CLUSTER_PLUGINS=gcr.io/velero-gcp/velero-plugin-for-aws:main \ +DISABLE_INFORMER_CACHE=true \ +REGISTRY_CREDENTIAL_FILE=$HOME/.docker/config.json \ +GINKGO_LABELS=Migration \ +KIBISHII_DIRECTORY=$HOME/kibishii/kubernetes/yaml/ \ +make test-e2e +``` + ## 6. Full Tests execution As we provided several examples for E2E test execution, if no filter is involved and despite difference of test environment, diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index 0443b74446..f5bba56c2d 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -34,7 +34,7 @@ import ( . "github.com/vmware-tanzu/velero/test/util/velero" ) -// Test backup and restore of Kibishi using restic +// Test backup and restore of Kibishii using restic func BackupDeletionWithSnapshots() { backup_deletion_test(true) @@ -143,7 +143,8 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc }) }) for _, ns := range workloadNamespaceList { - if providerName == Vsphere && useVolumeSnapshots { + if useVolumeSnapshots && + veleroCfg.HasVspherePlugin { // Wait for uploads started by the Velero Plugin for vSphere to complete // TODO - remove after upload progress monitoring is implemented fmt.Println("Waiting for vSphere uploads to complete") diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 1f101e0555..342eac0378 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -122,19 +122,33 @@ func TTLTest() { var snapshotCheckPoint SnapshotCheckPoint if useVolumeSnapshots { - if veleroCfg.CloudProvider == Vsphere { - // TODO - remove after upload progress monitoring is implemented + if veleroCfg.HasVspherePlugin { By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(ctx, time.Hour, test.testNS, 2)).To(Succeed()) }) } - snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, test.testNS, test.backupName, KibishiiPVCNameList) - Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, - test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + snapshotCheckPoint, err = GetSnapshotCheckPoint( + client, + veleroCfg, + 2, + test.testNS, + test.backupName, + KibishiiPVCNameList, + ) + Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") + + Expect( + SnapshotsShouldBeCreatedInCloud( + veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, + veleroCfg.BSLBucket, + veleroCfg.BSLConfig, + test.backupName, + snapshotCheckPoint, + ), + ).NotTo(HaveOccurred(), "Fail to verify the created snapshots") } By(fmt.Sprintf("Simulating a disaster by removing namespace %s\n", BackupCfg.BackupName), func() { diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 1b4cd32dec..36cb0b0e5d 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -103,6 +103,7 @@ func init() { flag.StringVar(&VeleroCfg.DefaultCLSServiceAccountName, "default-cls-service-account-name", "", "default cluster service account name.") flag.StringVar(&VeleroCfg.StandbyCLSServiceAccountName, "standby-cls-service-account-name", "", "standby cluster service account name.") flag.BoolVar(&VeleroCfg.FailFast, "fail-fast", true, "a switch for failing fast on meeting error.") + flag.BoolVar(&VeleroCfg.HasVspherePlugin, "has-vsphere-plugin", false, "a switch for installing vSphere plugin.") } // Add label [SkipVanillaZfs]: diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 8316126e6e..51edf34072 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -158,16 +158,21 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) if OriginVeleroCfg.CloudProvider == AWS { OriginVeleroCfg.Plugins = migrationNeedPlugins[AWS][0] } + // If HasVspherePlugin is false, only install the AWS plugin. + // If do nothing here, the default behavior is + // installing both AWS and vSphere plugins. + if OriginVeleroCfg.CloudProvider == Vsphere && + !OriginVeleroCfg.HasVspherePlugin { + OriginVeleroCfg.Plugins = migrationNeedPlugins[AWS][0] + } // Because Velero CSI plugin is deprecated in v1.14, // only need to install it for version lower than v1.14. if strings.Contains(OriginVeleroCfg.Features, FeatureCSI) && semver.Compare(versionWithoutPatch, "v1.14") < 0 { OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[CSI][0] } - if OriginVeleroCfg.SnapshotMoveData { - if OriginVeleroCfg.CloudProvider == Azure { - OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[AWS][0] - } + if OriginVeleroCfg.SnapshotMoveData && OriginVeleroCfg.CloudProvider == Azure { + OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[AWS][0] } veleroCLI2Version.VeleroCLI, err = InstallVeleroCLI(veleroCLI2Version.VeleroVersion) Expect(err).To(Succeed()) @@ -253,8 +258,9 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) }) if useVolumeSnapshots { - if veleroCfg.CloudProvider == Vsphere { - // TODO - remove after upload progress monitoring is implemented + // Only wait for the snapshots.backupdriver.cnsdp.vmware.com + // when the vSphere plugin is used. + if veleroCfg.HasVspherePlugin { By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(context.Background(), time.Hour, migrationNamespace, kibishiiWorkerCount)).To(Succeed()) diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index cfda01feff..ffb8f5f22b 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -190,8 +190,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC }) if useVolumeSnapshots { - if veleroCfg.CloudProvider == Vsphere { - // TODO - remove after upload progress monitoring is implemented + if veleroCfg.HasVspherePlugin { By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, upgradeNamespace, 2)).To(Succeed()) diff --git a/test/testdata/volume-snapshot-class/vsphere.yaml b/test/testdata/volume-snapshot-class/vsphere.yaml new file mode 100644 index 0000000000..08ea9b2225 --- /dev/null +++ b/test/testdata/volume-snapshot-class/vsphere.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: snapshot.storage.k8s.io/v1 +deletionPolicy: Delete +driver: csi.vsphere.vmware.com +kind: VolumeSnapshotClass +metadata: + annotations: + snapshot.storage.kubernetes.io/is-default-class: "true" + labels: + velero.io/csi-volumesnapshot-class: "true" + name: volumesnapshotclass-delete +parameters: + svVolumeSnapshotClass: volumesnapshotclass-delete diff --git a/test/types.go b/test/types.go index 961eebe5dc..798f721448 100644 --- a/test/types.go +++ b/test/types.go @@ -118,6 +118,7 @@ type VeleroConfig struct { ServiceAccountNameToInstall string EKSPolicyARN string FailFast bool + HasVspherePlugin bool } type VeleroCfgInPerf struct { diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index a749538d72..4ab0783946 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -120,9 +120,8 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc // Checkpoint for a successful backup if useVolumeSnapshots { - if providerName == Vsphere { + if veleroCfg.HasVspherePlugin { // Wait for uploads started by the Velero Plugin for vSphere to complete - // TODO - remove after upload progress monitoring is implemented fmt.Println("Waiting for vSphere uploads to complete") if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace, 2); err != nil { return errors.Wrapf(err, "Error waiting for uploads to complete") diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 81204a9863..9da1c4b83a 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -320,7 +320,7 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options if len(options.Features) > 0 { args = append(args, "--features", options.Features) - if !strings.EqualFold(cloudProvider, test.Vsphere) && strings.EqualFold(options.Features, test.FeatureCSI) && options.UseVolumeSnapshots { + if strings.EqualFold(options.Features, test.FeatureCSI) && options.UseVolumeSnapshots { // https://github.com/openebs/zfs-localpv/blob/develop/docs/snapshot.md fmt.Printf("Start to install %s VolumeSnapshotClass ... \n", cloudProvider) if err := k8s.KubectlApplyByFile(ctx, fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", cloudProvider)); err != nil { diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index cec0d61869..38ada7a9e9 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -107,7 +107,7 @@ var PluginsMatrix = map[string]map[string][]string{ }, } -func getPluginsByVersion(version, cloudProvider, objectStoreProvider string, needDataMoverPlugin bool) ([]string, error) { +func getPluginsByVersion(version string, cloudProvider string, needDataMoverPlugin bool) ([]string, error) { var cloudMap map[string][]string arr := strings.Split(version, ".") if len(arr) >= 3 { @@ -133,18 +133,6 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider string, nee return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, cloudProvider) } } - // TODO: Is this section need? - if objectStoreProvider != cloudProvider { - pluginsForObjectStoreProvider, ok := cloudMap[objectStoreProvider] - if !ok { - return nil, errors.Errorf("fail to get plugins by version: %s and object store provider %s", version, objectStoreProvider) - } - for _, p := range pluginsForObjectStoreProvider { - if !slices.Contains(plugins, p) { - plugins = append(plugins, p) - } - } - } if needDataMoverPlugin { pluginsForDatamover, ok := cloudMap["datamover"] @@ -630,7 +618,7 @@ func getProviderPlugins(ctx context.Context, veleroCLI string, cloudProvider str return nil, errors.WithMessage(err, "failed to get velero version") } - plugins, err := getPluginsByVersion(version, cloudProvider, cloudProvider, false) + plugins, err := getPluginsByVersion(version, cloudProvider, false) if err != nil { return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", cloudProvider, version) } @@ -673,7 +661,7 @@ func getPlugins(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) { if veleroCfg.SnapshotMoveData && veleroCfg.DataMoverPlugin == "" && !veleroCfg.IsUpgradeTest { needDataMoverPlugin = true } - plugins, err = getPluginsByVersion(version, cloudProvider, objectStoreProvider, needDataMoverPlugin) + plugins, err = getPluginsByVersion(version, cloudProvider, needDataMoverPlugin) if err != nil { return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", objectStoreProvider, version) } From bcb60ed783d8d1295e3e6d402037f5ae81e2e476 Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Tue, 12 Nov 2024 14:29:57 +0800 Subject: [PATCH 2/3] Modify other cases to support VKS environment. Signed-off-by: Xun Jiang --- test/e2e/backups/deletion.go | 77 +++++++---- test/e2e/backups/ttl.go | 18 +-- test/e2e/bsl-mgmt/deletion.go | 186 +++++++++++++++------------ test/e2e/migration/migration.go | 9 +- test/e2e/upgrade/upgrade.go | 9 +- test/util/kibishii/kibishii_utils.go | 18 ++- test/util/providers/common.go | 89 +++++++------ 7 files changed, 238 insertions(+), 168 deletions(-) diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index f5bba56c2d..e4bf69702d 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -99,8 +99,6 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc providerName := veleroCfg.CloudProvider veleroNamespace := veleroCfg.VeleroNamespace registryCredentialFile := veleroCfg.RegistryCredentialFile - bslPrefix := veleroCfg.BSLPrefix - bslConfig := veleroCfg.BSLConfig veleroFeatures := veleroCfg.Features for _, ns := range workloadNamespaceList { if err := CreateNamespace(oneHourTimeout, client, ns); err != nil { @@ -153,7 +151,7 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc } } } - err = ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) + err = ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, backupName, BackupObjectsPrefix) if err != nil { return err } @@ -165,9 +163,12 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc for _, ns := range workloadNamespaceList { snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, DefaultKibishiiWorkerCounts, ns, backupName, KibishiiPVCNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig, - backupName, snapshotCheckPoint) + err = CheckSnapshotsInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + ) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } @@ -179,9 +180,12 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") // Get all snapshots base on backup name, regardless of namespaces - err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslConfig, - backupName, snapshotCheckPoint) + err = CheckSnapshotsInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + ) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } @@ -207,26 +211,34 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc return err } + // Verify snapshots are deleted after backup deletion. if useVolumeSnapshots { - err = SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, - backupName, snapshotCheckPoint) + snapshotCheckPoint.ExpectCount = 0 + err = CheckSnapshotsInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + ) if err != nil { - return errors.Wrap(err, "exceed waiting for snapshot created in cloud") + return errors.Wrap(err, "fail to verify snapshots are deleted in provider.") } } - err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 5) + // Verify backup metadata files are deleted in OSS after backup deletion. + err = ObjectsShouldNotBeInBucket( + veleroCfg.ObjectStoreProvider, + veleroCfg.CloudCredentialsFile, + veleroCfg.BSLBucket, + veleroCfg.BSLPrefix, + veleroCfg.BSLConfig, + backupName, + BackupObjectsPrefix, + 5, + ) if err != nil { return err } - if useVolumeSnapshots { - if err := SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - bslConfig, backupName, snapshotCheckPoint); err != nil { - return errors.Wrap(err, "exceed waiting for snapshot created in cloud") - } - } // Hit issue: https://docs.aws.amazon.com/AWSEC2/latest/APIReference/errors-overview.html#:~:text=SnapshotCreationPerVolumeRateExceeded // Sleep for more than 15 seconds to avoid this issue. @@ -243,13 +255,28 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc }) }) - err = DeleteObjectsInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix) - if err != nil { + if err := DeleteObjectsInBucket( + veleroCfg.ObjectStoreProvider, + veleroCfg.CloudCredentialsFile, + veleroCfg.BSLBucket, + veleroCfg.BSLPrefix, + veleroCfg.BSLConfig, + backupName, + BackupObjectsPrefix, + ); err != nil { return err } - err = ObjectsShouldNotBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, bslPrefix, bslConfig, backupName, BackupObjectsPrefix, 1) - if err != nil { + if err := ObjectsShouldNotBeInBucket( + veleroCfg.ObjectStoreProvider, + veleroCfg.CloudCredentialsFile, + veleroCfg.BSLBucket, + veleroCfg.BSLPrefix, + veleroCfg.BSLConfig, + backupName, + BackupObjectsPrefix, + 1, + ); err != nil { return err } diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 342eac0378..5135c4b0b4 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -140,13 +140,11 @@ func TTLTest() { Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") Expect( - SnapshotsShouldBeCreatedInCloud( - veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, - veleroCfg.BSLBucket, - veleroCfg.BSLConfig, + CheckSnapshotsInProvider( + veleroCfg, test.backupName, snapshotCheckPoint, + false, ), ).NotTo(HaveOccurred(), "Fail to verify the created snapshots") } @@ -202,9 +200,13 @@ func TTLTest() { By("PersistentVolume snapshots should be deleted", func() { if useVolumeSnapshots { - Expect(SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, - test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + snapshotCheckPoint.ExpectCount = 0 + Expect(CheckSnapshotsInProvider( + veleroCfg, + test.backupName, + snapshotCheckPoint, + false, + )).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") } }) diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index 6c3ccee88e..ca9d42addb 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -36,8 +36,8 @@ import ( const ( // Please make sure length of this namespace should be shorter, - // otherwise ResticRepositories name verification will be wrong - // when making combination of ResticRepositories name(max length is 63) + // otherwise BackupRepositories name verification will be wrong + // when making combination of BackupRepositories name(max length is 63) bslDeletionTestNs = "bsl-deletion" ) @@ -134,12 +134,12 @@ func BslDeletionTest(useVolumeSnapshots bool) { )).To(Succeed()) }) - backupName_1 := "backup1-" + UUIDgen.String() - backupName_2 := "backup2-" + UUIDgen.String() - backupLocation_1 := "default" - backupLocation_2 := additionalBsl - podName_1 := "kibishii-deployment-0" - podName_2 := "kibishii-deployment-1" + backupName1 := "backup1-" + UUIDgen.String() + backupName2 := "backup2-" + UUIDgen.String() + backupLocation1 := "default" + backupLocation2 := additionalBsl + podName1 := "kibishii-deployment-0" + podName2 := "kibishii-deployment-1" label_1 := "for=1" // TODO remove when issue https://github.com/vmware-tanzu/velero/issues/4724 is fixed @@ -157,17 +157,17 @@ func BslDeletionTest(useVolumeSnapshots bool) { // Restic can not backup PV only, so pod need to be labeled also By("Label all 2 worker-pods of Kibishii", func() { - Expect(AddLabelToPod(context.Background(), podName_1, bslDeletionTestNs, label_1)).To(Succeed()) + Expect(AddLabelToPod(context.Background(), podName1, bslDeletionTestNs, label_1)).To(Succeed()) Expect(AddLabelToPod(context.Background(), "kibishii-deployment-1", bslDeletionTestNs, label_2)).To(Succeed()) }) By("Get all 2 PVCs of Kibishii and label them separately ", func() { - pvc, err := GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName_1) + pvc, err := GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName1) Expect(err).To(Succeed()) fmt.Println(pvc) Expect(pvc).To(HaveLen(1)) pvc1 := pvc[0] - pvc, err = GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName_2) + pvc, err = GetPvcByPVCName(context.Background(), bslDeletionTestNs, podName2) Expect(err).To(Succeed()) fmt.Println(pvc) Expect(pvc).To(HaveLen(1)) @@ -177,13 +177,13 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) var BackupCfg BackupConfig - BackupCfg.BackupName = backupName_1 + BackupCfg.BackupName = backupName1 BackupCfg.Namespace = bslDeletionTestNs - BackupCfg.BackupLocation = backupLocation_1 + BackupCfg.BackupLocation = backupLocation1 BackupCfg.UseVolumeSnapshots = useVolumeSnapshots BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots BackupCfg.Selector = label_1 - By(fmt.Sprintf("Backup one of PV of sample workload by label-1 - Kibishii by the first BSL %s", backupLocation_1), func() { + By(fmt.Sprintf("Backup one of PV of sample workload by label-1 - Kibishii by the first BSL %s", backupLocation1), func() { // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case Expect(VeleroBackupNamespace(oneHourTimeout, veleroCfg.VeleroCLI, @@ -193,10 +193,10 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) }) - BackupCfg.BackupName = backupName_2 - BackupCfg.BackupLocation = backupLocation_2 + BackupCfg.BackupName = backupName2 + BackupCfg.BackupLocation = backupLocation2 BackupCfg.Selector = label_2 - By(fmt.Sprintf("Back up the other one PV of sample workload with label-2 into the additional BSL %s", backupLocation_2), func() { + By(fmt.Sprintf("Back up the other one PV of sample workload with label-2 into the additional BSL %s", backupLocation2), func() { Expect(VeleroBackupNamespace(oneHourTimeout, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { RunDebug(context.Background(), veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupCfg.BackupName, "") @@ -211,82 +211,94 @@ func BslDeletionTest(useVolumeSnapshots bool) { Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, bslDeletionTestNs, 2)).To(Succeed()) }) - By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName_1), func() { + By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName1), func() { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, - backupName_1, 1)).To(Succeed()) + backupName1, 1)).To(Succeed()) }) - By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName_2), func() { + By(fmt.Sprintf("Snapshot CR in backup %s should be created", backupName2), func() { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, - backupName_2, 1)).To(Succeed()) + backupName2, 1)).To(Succeed()) }) } if veleroCfg.CloudProvider != VanillaZFS { var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs - By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_1), func() { - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) + By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation1), func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint( + *veleroCfg.ClientToInstallVelero, + veleroCfg, + 1, + bslDeletionTestNs, + backupName1, + []string{podName1}, + ) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) + Expect(CheckSnapshotsInProvider( + veleroCfg, + backupName1, + snapshotCheckPoint, + false, + )).To(Succeed()) }) - By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation_2), func() { - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) + By(fmt.Sprintf("Snapshot of bsl %s should be created in cloud object store", backupLocation2), func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint( + *veleroCfg.ClientToInstallVelero, + veleroCfg, + 1, + bslDeletionTestNs, + backupName2, + []string{podName2}, + ) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - var BSLCredentials, BSLConfig string - if veleroCfg.CloudProvider == Vsphere { - BSLCredentials = veleroCfg.AdditionalBSLCredentials - BSLConfig = veleroCfg.AdditionalBSLConfig - } else { // Snapshotting with non-vSphere provider has nothing to do with BSL - BSLCredentials = veleroCfg.CloudCredentialsFile - BSLConfig = veleroCfg.BSLConfig - } - - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - BSLCredentials, veleroCfg.AdditionalBSLBucket, - BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed()) + + Expect(CheckSnapshotsInProvider( + veleroCfg, + backupName2, + snapshotCheckPoint, + true, + )).To(Succeed()) }) } - } else { // For Restics - By(fmt.Sprintf("Resticrepositories for BSL %s should be created in Velero namespace", backupLocation_1), func() { + } else { + By(fmt.Sprintf("BackupRepositories for BSL %s should be created in Velero namespace", backupLocation1), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_1, 1)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation1, 1)).To(Succeed()) }) - By(fmt.Sprintf("Resticrepositories for BSL %s should be created in Velero namespace", backupLocation_2), func() { + By(fmt.Sprintf("BackupRepositories for BSL %s should be created in Velero namespace", backupLocation2), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_2, 1)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation2, 1)).To(Succeed()) }) } - By(fmt.Sprintf("Backup 1 %s should be created.", backupName_1), func() { + By(fmt.Sprintf("Backup 1 %s should be created.", backupName1), func() { Expect(WaitForBackupToBeCreated(context.Background(), - backupName_1, 10*time.Minute, &veleroCfg)).To(Succeed()) + backupName1, 10*time.Minute, &veleroCfg)).To(Succeed()) }) - By(fmt.Sprintf("Backup 2 %s should be created.", backupName_2), func() { + By(fmt.Sprintf("Backup 2 %s should be created.", backupName2), func() { Expect(WaitForBackupToBeCreated(context.Background(), - backupName_2, 10*time.Minute, &veleroCfg)).To(Succeed()) + backupName2, 10*time.Minute, &veleroCfg)).To(Succeed()) }) - backupsInBSL1, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation_1) + backupsInBSL1, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation1) Expect(err).To(Succeed()) - backupsInBSL2, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation_2) + backupsInBSL2, err := GetBackupsFromBsl(context.Background(), veleroCfg.VeleroCLI, backupLocation2) Expect(err).To(Succeed()) backupsInBsl1AndBsl2 := append(backupsInBSL1, backupsInBSL2...) - By(fmt.Sprintf("Get all backups from 2 BSLs %s before deleting one of them", backupLocation_1), func() { + By(fmt.Sprintf("Get all backups from 2 BSLs %s before deleting one of them", backupLocation1), func() { backupsBeforeDel, err := GetAllBackups(context.Background(), veleroCfg.VeleroCLI) Expect(err).To(Succeed()) Expect(cmp.Diff(backupsInBsl1AndBsl2, backupsBeforeDel, cmpopts.SortSlices(less))).Should(BeEmpty()) - By(fmt.Sprintf("Backup1 %s should exist in cloud object store before bsl deletion", backupName_1), func() { + By(fmt.Sprintf("Backup1 %s should exist in cloud object store before bsl deletion", backupName1), func() { Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, - backupName_1, BackupObjectsPrefix)).To(Succeed()) + backupName1, BackupObjectsPrefix)).To(Succeed()) }) - By(fmt.Sprintf("Delete one of backup locations - %s", backupLocation_1), func() { - Expect(DeleteBslResource(context.Background(), veleroCfg.VeleroCLI, backupLocation_1)).To(Succeed()) + By(fmt.Sprintf("Delete one of backup locations - %s", backupLocation1), func() { + Expect(DeleteBslResource(context.Background(), veleroCfg.VeleroCLI, backupLocation1)).To(Succeed()) Expect(WaitForBackupsToBeDeleted(context.Background(), backupsInBSL1, 10*time.Minute, &veleroCfg)).To(Succeed()) }) @@ -298,10 +310,10 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) }) - By(fmt.Sprintf("Backup1 %s should still exist in cloud object store after bsl deletion", backupName_1), func() { + By(fmt.Sprintf("Backup1 %s should still exist in cloud object store after bsl deletion", backupName1), func() { Expect(ObjectsShouldBeInBucket(veleroCfg.ObjectStoreProvider, veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLPrefix, veleroCfg.BSLConfig, - backupName_1, BackupObjectsPrefix)).To(Succeed()) + backupName1, BackupObjectsPrefix)).To(Succeed()) }) // TODO: Choose additional BSL to be deleted as an new test case @@ -313,45 +325,51 @@ func BslDeletionTest(useVolumeSnapshots bool) { if useVolumeSnapshots { if veleroCfg.CloudProvider == Vsphere { - By(fmt.Sprintf("Snapshot in backup %s should still exist, because snapshot CR will be deleted 24 hours later if the status is a success", backupName_2), func() { + By(fmt.Sprintf("Snapshot in backup %s should still exist, because snapshot CR will be deleted 24 hours later if the status is a success", backupName2), func() { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, - backupName_1, 1)).To(Succeed()) + backupName1, 1)).To(Succeed()) Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, - backupName_2, 1)).To(Succeed()) + backupName2, 1)).To(Succeed()) }) } var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs - By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_1), func() { - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_1, []string{podName_1}) + By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation1), func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName1, []string{podName1}) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName_1, snapshotCheckPoint)).To(Succeed()) + Expect(CheckSnapshotsInProvider( + veleroCfg, + backupName1, + snapshotCheckPoint, + false, + )).To(Succeed()) }) - By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation_2), func() { - var BSLCredentials, BSLConfig string - if veleroCfg.CloudProvider == Vsphere { - BSLCredentials = veleroCfg.AdditionalBSLCredentials - BSLConfig = veleroCfg.AdditionalBSLConfig - } else { - BSLCredentials = veleroCfg.CloudCredentialsFile - BSLConfig = veleroCfg.BSLConfig - } - snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 1, bslDeletionTestNs, backupName_2, []string{podName_2}) + By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation2), func() { + snapshotCheckPoint, err = GetSnapshotCheckPoint( + *veleroCfg.ClientToInstallVelero, + veleroCfg, + 1, + bslDeletionTestNs, + backupName2, + []string{podName2}, + ) Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - BSLCredentials, veleroCfg.AdditionalBSLBucket, - BSLConfig, backupName_2, snapshotCheckPoint)).To(Succeed()) + + Expect(CheckSnapshotsInProvider( + veleroCfg, + backupName2, + snapshotCheckPoint, + true, + )).To(Succeed()) }) - } else { // For Restic - By(fmt.Sprintf("Resticrepositories for BSL %s should be deleted in Velero namespace", backupLocation_1), func() { + } else { + By(fmt.Sprintf("BackupRepositories for BSL %s should be deleted in Velero namespace", backupLocation1), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_1, 0)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation1, 0)).To(Succeed()) }) - By(fmt.Sprintf("Resticrepositories for BSL %s should still exist in Velero namespace", backupLocation_2), func() { + By(fmt.Sprintf("BackupRepositories for BSL %s should still exist in Velero namespace", backupLocation2), func() { Expect(BackupRepositoriesCountShouldBe(context.Background(), - veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation_2, 1)).To(Succeed()) + veleroCfg.VeleroNamespace, bslDeletionTestNs+"-"+backupLocation2, 1)).To(Succeed()) }) } fmt.Printf("|| EXPECTED || - Backup deletion test completed successfully\n") diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 51edf34072..915ef8c697 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -288,9 +288,12 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) snapshotCheckPoint, err = GetSnapshotCheckPoint(*veleroCfg.DefaultClient, veleroCfg, kibishiiWorkerCount, migrationNamespace, backupName, GetKibishiiPVCNameList(kibishiiWorkerCount)) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) + Expect(CheckSnapshotsInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + )).To(Succeed()) }) } } diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index ffb8f5f22b..f3f63fc54b 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -202,9 +202,12 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC snapshotCheckPoint, err := GetSnapshotCheckPoint(*veleroCfg.ClientToInstallVelero, veleroCfg, 2, upgradeNamespace, backupName, KibishiiPVCNameList) Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, - veleroCfg.BSLConfig, backupName, snapshotCheckPoint)).To(Succeed()) + Expect(CheckSnapshotsInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + )).To(Succeed()) }) } diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index 4ab0783946..29652fce38 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -131,9 +131,12 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc if err != nil { return errors.Wrap(err, "Fail to get snapshot checkpoint") } - err = SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, - backupName, snapshotCheckPoint) + err = CheckSnapshotsInProvider( + veleroCfg, + backupName, + snapshotCheckPoint, + false, + ) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } @@ -166,9 +169,12 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc return errors.Wrap(err, "failed to get snapshot checkPoint") } } else { - err = SnapshotsShouldNotExistInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, - backupName, SnapshotCheckPoint{}) + err = CheckSnapshotsInProvider( + veleroCfg, + backupName, + SnapshotCheckPoint{}, + false, + ) if err != nil { return errors.Wrap(err, "exceed waiting for snapshot created in cloud") } diff --git a/test/util/providers/common.go b/test/util/providers/common.go index 46e42894ef..a7c68dc37e 100644 --- a/test/util/providers/common.go +++ b/test/util/providers/common.go @@ -28,14 +28,14 @@ import ( "github.com/pkg/errors" "github.com/vmware-tanzu/velero/internal/volume" - . "github.com/vmware-tanzu/velero/test" + velerotest "github.com/vmware-tanzu/velero/test" velero "github.com/vmware-tanzu/velero/test/util/velero" ) type ObjectsInStorage interface { IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error - IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error + IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck velerotest.SnapshotCheckPoint) error GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, objectKey string) (io.ReadCloser, error) } @@ -75,13 +75,13 @@ func ObjectsShouldNotBeInBucket(objectStoreProvider, cloudCredentialsFile, bslBu func getProvider(cloudProvider string) (ObjectsInStorage, error) { var s ObjectsInStorage switch cloudProvider { - case AWS, Vsphere: + case velerotest.AWS, velerotest.Vsphere: aws := AWSStorage("") s = &aws - case GCP: + case velerotest.GCP: gcs := GCSStorage("") s = &gcs - case Azure: + case velerotest.Azure: az := AzureStorage("") s = &az default: @@ -102,7 +102,7 @@ func IsObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bsl bslPrefix = getFullPrefix(bslPrefix, subPrefix) s, err := getProvider(objectStoreProvider) if err != nil { - return false, errors.Wrapf(err, fmt.Sprintf("Object store provider %s is not valid", objectStoreProvider)) + return false, errors.Wrapf(err, "Object store provider %s is not valid", objectStoreProvider) } return s.IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName) } @@ -116,66 +116,77 @@ func DeleteObjectsInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, } s, err := getProvider(objectStoreProvider) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Object store provider %s is not valid", objectStoreProvider)) + return errors.Wrapf(err, "Object store provider %s is not valid", objectStoreProvider) } err = s.DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Fail to delete %s", bslPrefix)) + return errors.Wrapf(err, "Fail to delete %s", bslPrefix) } return nil } -func SnapshotsShouldNotExistInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { - fmt.Printf("|| VERIFICATION || - Snapshots should not exist in cloud, backup %s\n", backupName) +func CheckSnapshotsInProvider( + veleroCfg velerotest.VeleroConfig, + backupName string, + snapshotCheckPoint velerotest.SnapshotCheckPoint, + checkStandbyCluster bool, +) error { + fmt.Printf("|| VERIFICATION || - there should %d snapshots in provider, backup %s\n", + snapshotCheckPoint.ExpectCount, backupName) - if cloudProvider == VanillaZFS { - fmt.Printf("Skip snapshot check for cloud provider %s", cloudProvider) + if veleroCfg.CloudProvider == velerotest.VanillaZFS { + fmt.Printf("Skip snapshot check for cloud provider %s", veleroCfg.CloudProvider) return nil } - if cloudCredentialsFile == "" { - return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider)) + if veleroCfg.CloudProvider == velerotest.Vsphere && !veleroCfg.HasVspherePlugin { + fmt.Printf("Skip snapshot check for vSphere environment that doesn't have Velero vSphere plugin.") + return nil } - - snapshotCheckPoint.ExpectCount = 0 - err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) - if err != nil { - return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED ||Snapshots %s exist in cloud after backup as expected", backupName)) + if veleroCfg.CloudCredentialsFile == "" { + return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", veleroCfg.CloudProvider)) } - fmt.Printf("|| EXPECTED || - Snapshots do not exist in cloud, backup %s\n", backupName) - return nil -} -func SnapshotsShouldBeCreatedInCloud(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheckPoint SnapshotCheckPoint) error { - fmt.Printf("|| VERIFICATION || - Snapshots should exist in cloud, backup %s\n", backupName) + cloudCredentialsFile := veleroCfg.CloudCredentialsFile + bslBucket := veleroCfg.BSLBucket + bslConfig := veleroCfg.BSLConfig + if checkStandbyCluster { + bslBucket = veleroCfg.AdditionalBSLBucket - if cloudProvider == VanillaZFS { - fmt.Printf("Skip snapshot check for cloud provider %s", cloudProvider) - return nil - } - if cloudCredentialsFile == "" { - return errors.New(fmt.Sprintf("|| ERROR || - Please provide credential file of cloud %s \n", cloudProvider)) + // Only vSphere environment's snapshot on standby cluster is related to BSL. + if veleroCfg.CloudProvider == velerotest.Vsphere { + cloudCredentialsFile = veleroCfg.AdditionalBSLCredentials + bslConfig = veleroCfg.AdditionalBSLConfig + } } - err := IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName, snapshotCheckPoint) + err := isSnapshotExisted( + veleroCfg.CloudProvider, + cloudCredentialsFile, + bslBucket, + bslConfig, + backupName, + snapshotCheckPoint, + ) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("|| UNEXPECTED || - Snapshots %s do not exist in cloud after backup as expected", backupName)) + return errors.Wrapf(err, "|| UNEXPECTED || - Snapshots count is not as expected after backup %s", backupName) } - fmt.Printf("|| EXPECTED || - Snapshots of backup %s exist in cloud %s\n", backupName, cloudProvider) + + fmt.Printf("|| EXPECTED || - Snapshots of backup %s exist in provider %s\n", backupName, veleroCfg.CloudProvider) return nil } -func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error { +func isSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig, backupName string, snapshotCheck velerotest.SnapshotCheckPoint) error { s, err := getProvider(cloudProvider) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) + return errors.Wrapf(err, "Cloud provider %s is not valid", cloudProvider) } - if cloudProvider == Vsphere { + if cloudProvider == velerotest.Vsphere { var retSnapshotIDs []string ctx, ctxCancel := context.WithTimeout(context.Background(), time.Minute*2) defer ctxCancel() retSnapshotIDs, err = velero.GetVsphereSnapshotIDs(ctx, time.Hour, snapshotCheck.NamespaceBackedUp, snapshotCheck.PodName) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Fail to get snapshot CRs of backup%s", backupName)) + return errors.Wrapf(err, "Fail to get snapshot CRs of backup%s", backupName) } bslPrefix := "plugins" @@ -201,7 +212,7 @@ func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig } else { err = s.IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName, snapshotCheck) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("Fail to get snapshot of backup %s", backupName)) + return errors.Wrapf(err, "Fail to get snapshot of backup %s", backupName) } } return nil @@ -221,7 +232,7 @@ func GetVolumeInfoMetadataContent( fmt.Printf("|| VERIFICATION || - Get backup %s volumeinfo file in storage %s\n", backupName, bslPrefix) s, err := getProvider(objectStoreProvider) if err != nil { - return nil, errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", objectStoreProvider)) + return nil, errors.Wrapf(err, "Cloud provider %s is not valid", objectStoreProvider) } return s.GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, volumeFileName) From bebea4d278d9c55be58399b1ea4ee604ebd325b5 Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Tue, 12 Nov 2024 23:24:28 +0800 Subject: [PATCH 3/3] Modify upgrade and migration cases. Signed-off-by: Xun Jiang --- test/e2e/README.md | 8 +- test/e2e/backup/backup.go | 8 +- test/e2e/bsl-mgmt/deletion.go | 26 ++--- test/e2e/migration/migration.go | 39 ++------ test/e2e/upgrade/upgrade.go | 15 ++- test/types.go | 2 + test/util/kibishii/kibishii_utils.go | 11 ++- test/util/velero/velero_utils.go | 138 +++++++++++++++++++-------- 8 files changed, 144 insertions(+), 103 deletions(-) diff --git a/test/e2e/README.md b/test/e2e/README.md index 2ca71a7420..331c2920a7 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -78,7 +78,7 @@ These configuration parameters are expected as values to the following command l 1. `--standby-cluster-object-store-provider`: Object store provider for standby cluster. 1. `--debug-velero-pod-restart`: A switch for debugging velero pod restart. 1. `--fail-fast`: A switch for for failing fast on meeting error. -1. `--disable-vsphere-plugin`: A switch for not install the Velero vSphere plugin when the provider is set to `vsphere`. +1. `--has-vsphere-plugin`: A switch to indicate whether the Velero vSphere plugin is installed for vSphere environment. These configurations or parameters are used to generate install options for Velero for each test suite. @@ -130,7 +130,7 @@ Below is a mapping between `make` variables to E2E configuration flags. 1. `INSTALL_VELERO `: `-install-velero`. Optional. 1. `DEBUG_VELERO_POD_RESTART`: `-debug-velero-pod-restart`. Optional. 1. `FAIL_FAST`: `--fail-fast`. Optional. -1. `DISABLE_VSPHERE_PLUGIN`: `--diable-vsphere-plugin`. Optional. +1. `HAS_VSPHERE_PLUGIN`: `--has-vsphere-plugin`. Optional. @@ -338,8 +338,8 @@ BSL_PREFIX=nightly \ ADDITIONAL_BSL_PLUGINS=gcr.io/velero-gcp/velero-plugin-for-aws:main \ ADDITIONAL_OBJECT_STORE_PROVIDER=aws \ ADDITIONAL_BSL_CONFIG=region=us-east-1 \ -ADDITIONAL_BSL_BUCKET=nightly-normal-account4-test \ -ADDITIONAL_BSL_PREFIX=addition-bsl \ +ADDITIONAL_BSL_BUCKET=nightly-restrict-account-test \ +ADDITIONAL_BSL_PREFIX=nightly \ ADDITIONAL_CREDS_FILE=$HOME/aws-credential \ VELERO_IMAGE=gcr.io/velero-gcp/velero:main \ RESTORE_HELPER_IMAGE=gcr.io/velero-gcp/velero-restore-helper:main \ diff --git a/test/e2e/backup/backup.go b/test/e2e/backup/backup.go index 6f47ed2d9e..cfdc897244 100644 --- a/test/e2e/backup/backup.go +++ b/test/e2e/backup/backup.go @@ -197,9 +197,9 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) { secretKey, )).To(Succeed()) - bsls := []string{"default", additionalBsl} + BSLs := []string{"default", additionalBsl} - for _, bsl := range bsls { + for _, bsl := range BSLs { backupName = fmt.Sprintf("backup-%s", bsl) restoreName = fmt.Sprintf("restore-%s", bsl) // We limit the length of backup name here to avoid the issue of vsphere plugin https://github.com/vmware-tanzu/velero-plugin-for-vsphere/issues/370 @@ -209,8 +209,8 @@ func BackupRestoreTest(backupRestoreTestConfig BackupRestoreTestConfig) { restoreName = fmt.Sprintf("%s-%s", restoreName, UUIDgen) } veleroCfg.ProvideSnapshotsVolumeParam = !provideSnapshotVolumesParmInBackup - workloadNmespace := kibishiiNamespace + bsl - Expect(RunKibishiiTests(veleroCfg, backupName, restoreName, bsl, workloadNmespace, useVolumeSnapshots, !useVolumeSnapshots)).To(Succeed(), + workloadNS := kibishiiNamespace + bsl + Expect(RunKibishiiTests(veleroCfg, backupName, restoreName, bsl, workloadNS, useVolumeSnapshots, !useVolumeSnapshots)).To(Succeed(), "Failed to successfully backup and restore Kibishii namespace using BSL %s", bsl) } }) diff --git a/test/e2e/bsl-mgmt/deletion.go b/test/e2e/bsl-mgmt/deletion.go index ca9d42addb..51584d85ba 100644 --- a/test/e2e/bsl-mgmt/deletion.go +++ b/test/e2e/bsl-mgmt/deletion.go @@ -41,7 +41,7 @@ const ( bslDeletionTestNs = "bsl-deletion" ) -// Test backup and restore of Kibishi using restic +// Test backup and restore of Kibishii using restic func BslDeletionWithSnapshots() { BslDeletionTest(true) @@ -141,10 +141,10 @@ func BslDeletionTest(useVolumeSnapshots bool) { podName1 := "kibishii-deployment-0" podName2 := "kibishii-deployment-1" - label_1 := "for=1" + label1 := "for=1" // TODO remove when issue https://github.com/vmware-tanzu/velero/issues/4724 is fixed - //label_2 := "for!=1" - label_2 := "for=2" + //label2 := "for!=1" + label2 := "for=2" By("Create namespace for sample workload", func() { Expect(CreateNamespace(oneHourTimeout, *veleroCfg.ClientToInstallVelero, bslDeletionTestNs)).To(Succeed()) }) @@ -157,8 +157,8 @@ func BslDeletionTest(useVolumeSnapshots bool) { // Restic can not backup PV only, so pod need to be labeled also By("Label all 2 worker-pods of Kibishii", func() { - Expect(AddLabelToPod(context.Background(), podName1, bslDeletionTestNs, label_1)).To(Succeed()) - Expect(AddLabelToPod(context.Background(), "kibishii-deployment-1", bslDeletionTestNs, label_2)).To(Succeed()) + Expect(AddLabelToPod(context.Background(), podName1, bslDeletionTestNs, label1)).To(Succeed()) + Expect(AddLabelToPod(context.Background(), "kibishii-deployment-1", bslDeletionTestNs, label2)).To(Succeed()) }) By("Get all 2 PVCs of Kibishii and label them separately ", func() { @@ -172,8 +172,8 @@ func BslDeletionTest(useVolumeSnapshots bool) { fmt.Println(pvc) Expect(pvc).To(HaveLen(1)) pvc2 := pvc[0] - Expect(AddLabelToPvc(context.Background(), pvc1, bslDeletionTestNs, label_1)).To(Succeed()) - Expect(AddLabelToPvc(context.Background(), pvc2, bslDeletionTestNs, label_2)).To(Succeed()) + Expect(AddLabelToPvc(context.Background(), pvc1, bslDeletionTestNs, label1)).To(Succeed()) + Expect(AddLabelToPvc(context.Background(), pvc2, bslDeletionTestNs, label2)).To(Succeed()) }) var BackupCfg BackupConfig @@ -182,7 +182,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { BackupCfg.BackupLocation = backupLocation1 BackupCfg.UseVolumeSnapshots = useVolumeSnapshots BackupCfg.DefaultVolumesToFsBackup = !useVolumeSnapshots - BackupCfg.Selector = label_1 + BackupCfg.Selector = label1 By(fmt.Sprintf("Backup one of PV of sample workload by label-1 - Kibishii by the first BSL %s", backupLocation1), func() { // TODO currently, the upgrade case covers the upgrade path from 1.6 to main and the velero v1.6 doesn't support "debug" command // TODO move to "runDebug" after we bump up to 1.7 in the upgrade case @@ -195,7 +195,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { BackupCfg.BackupName = backupName2 BackupCfg.BackupLocation = backupLocation2 - BackupCfg.Selector = label_2 + BackupCfg.Selector = label2 By(fmt.Sprintf("Back up the other one PV of sample workload with label-2 into the additional BSL %s", backupLocation2), func() { Expect(VeleroBackupNamespace(oneHourTimeout, veleroCfg.VeleroCLI, veleroCfg.VeleroNamespace, BackupCfg)).To(Succeed(), func() string { @@ -205,8 +205,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { }) if useVolumeSnapshots { - if veleroCfg.CloudProvider == Vsphere { - // TODO - remove after upload progress monitoring is implemented + if veleroCfg.HasVspherePlugin { By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, bslDeletionTestNs, 2)).To(Succeed()) @@ -324,7 +323,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { // }) if useVolumeSnapshots { - if veleroCfg.CloudProvider == Vsphere { + if veleroCfg.HasVspherePlugin { By(fmt.Sprintf("Snapshot in backup %s should still exist, because snapshot CR will be deleted 24 hours later if the status is a success", backupName2), func() { Expect(SnapshotCRsCountShouldBe(context.Background(), bslDeletionTestNs, backupName1, 1)).To(Succeed()) @@ -332,6 +331,7 @@ func BslDeletionTest(useVolumeSnapshots bool) { backupName2, 1)).To(Succeed()) }) } + var snapshotCheckPoint SnapshotCheckPoint snapshotCheckPoint.NamespaceBackedUp = bslDeletionTestNs By(fmt.Sprintf("Snapshot should not be deleted in cloud object store after deleting bsl %s", backupLocation1), func() { diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 915ef8c697..ade423b76b 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -25,7 +25,6 @@ import ( "github.com/google/uuid" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" - "golang.org/x/mod/semver" . "github.com/vmware-tanzu/velero/test" util "github.com/vmware-tanzu/velero/test/util/csi" @@ -142,38 +141,12 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) if veleroCLI2Version.VeleroVersion == "self" { veleroCLI2Version.VeleroCLI = veleroCfg.VeleroCLI } else { - fmt.Printf("Using default images address of Velero CLI %s\n", veleroCLI2Version.VeleroVersion) - OriginVeleroCfg.VeleroImage = "" - OriginVeleroCfg.RestoreHelperImage = "" - OriginVeleroCfg.Plugins = "" - - versionWithoutPatch := semver.MajorMinor(veleroCLI2Version.VeleroVersion) - // Read migration case needs plugins from the PluginsMatrix map. - migrationNeedPlugins, ok := PluginsMatrix[versionWithoutPatch] - Expect(ok).To(BeTrue()) - - if OriginVeleroCfg.CloudProvider == Azure { - OriginVeleroCfg.Plugins = migrationNeedPlugins[Azure][0] - } - if OriginVeleroCfg.CloudProvider == AWS { - OriginVeleroCfg.Plugins = migrationNeedPlugins[AWS][0] - } - // If HasVspherePlugin is false, only install the AWS plugin. - // If do nothing here, the default behavior is - // installing both AWS and vSphere plugins. - if OriginVeleroCfg.CloudProvider == Vsphere && - !OriginVeleroCfg.HasVspherePlugin { - OriginVeleroCfg.Plugins = migrationNeedPlugins[AWS][0] - } - // Because Velero CSI plugin is deprecated in v1.14, - // only need to install it for version lower than v1.14. - if strings.Contains(OriginVeleroCfg.Features, FeatureCSI) && - semver.Compare(versionWithoutPatch, "v1.14") < 0 { - OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[CSI][0] - } - if OriginVeleroCfg.SnapshotMoveData && OriginVeleroCfg.CloudProvider == Azure { - OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[AWS][0] - } + OriginVeleroCfg, err = SetImagesToDefaultValues( + OriginVeleroCfg, + veleroCLI2Version.VeleroVersion, + ) + Expect(err).To(Succeed(), "Fail to set images for the migrate-from Velero installation.") + veleroCLI2Version.VeleroCLI, err = InstallVeleroCLI(veleroCLI2Version.VeleroVersion) Expect(err).To(Succeed()) } diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index f3f63fc54b..3c371064c3 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -124,14 +124,16 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC veleroCfg.GCFrequency = "" By(fmt.Sprintf("Install the expected old version Velero (%s) for upgrade", veleroCLI2Version.VeleroVersion), func() { - //Set VeleroImage and RestoreHelperImage to blank - //VeleroImage and RestoreHelperImage should be the default value in originalCli tmpCfgForOldVeleroInstall := veleroCfg tmpCfgForOldVeleroInstall.UpgradeFromVeleroVersion = veleroCLI2Version.VeleroVersion tmpCfgForOldVeleroInstall.VeleroCLI = veleroCLI2Version.VeleroCLI - tmpCfgForOldVeleroInstall.VeleroImage = "" - tmpCfgForOldVeleroInstall.RestoreHelperImage = "" - tmpCfgForOldVeleroInstall.Plugins = "" + + tmpCfgForOldVeleroInstall, err = SetImagesToDefaultValues( + tmpCfgForOldVeleroInstall, + veleroCLI2Version.VeleroVersion, + ) + Expect(err).To(Succeed(), "Fail to set the images for upgrade-from Velero installation.") + tmpCfgForOldVeleroInstall.UploaderType = "" version, err := GetVeleroVersion(oneHourTimeout, tmpCfgForOldVeleroInstall.VeleroCLI, true) Expect(err).To(Succeed(), "Fail to get Velero version") @@ -145,9 +147,6 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC tmpCfgForOldVeleroInstall.UseRestic = !useVolumeSnapshots tmpCfgForOldVeleroInstall.UseNodeAgent = false } - //TODO: Remove this setting when upgrade path is from 1.13 to higher - //TODO: version, or self version 1.12 and older versions have no this parameter. - tmpCfgForOldVeleroInstall.WithoutDisableInformerCacheParam = true Expect(VeleroInstall(context.Background(), &tmpCfgForOldVeleroInstall, false)).To(Succeed()) Expect(CheckVeleroVersion(context.Background(), tmpCfgForOldVeleroInstall.VeleroCLI, diff --git a/test/types.go b/test/types.go index 798f721448..527068599c 100644 --- a/test/types.go +++ b/test/types.go @@ -38,6 +38,8 @@ const AWS = "aws" const GCP = "gcp" const Vsphere = "vsphere" const CSI = "csi" +const Velero = "velero" +const VeleroRestoreHelper = "velero-restore-helper" const UploaderTypeRestic = "restic" diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index 29652fce38..be503ff0b1 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -65,8 +65,15 @@ func GetKibishiiPVCNameList(workerCount int) []string { } // RunKibishiiTests runs kibishii tests on the provider. -func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLocation, kibishiiNamespace string, - useVolumeSnapshots, defaultVolumesToFsBackup bool) error { +func RunKibishiiTests( + veleroCfg VeleroConfig, + backupName string, + restoreName string, + backupLocation string, + kibishiiNamespace string, + useVolumeSnapshots bool, + defaultVolumesToFsBackup bool, +) error { pvCount := len(KibishiiPVCNameList) client := *veleroCfg.ClientToInstallVelero oneHourTimeout, ctxCancel := context.WithTimeout(context.Background(), time.Minute*60) diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index 38ada7a9e9..d998246636 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -36,6 +36,7 @@ import ( "github.com/pkg/errors" "golang.org/x/exp/slices" + "golang.org/x/mod/semver" ver "k8s.io/apimachinery/pkg/util/version" "k8s.io/apimachinery/pkg/util/wait" kbclient "sigs.k8s.io/controller-runtime/pkg/client" @@ -54,67 +55,126 @@ const BackupObjectsPrefix = "backups" const RestoreObjectsPrefix = "restores" const PluginsObjectsPrefix = "plugins" -var PluginsMatrix = map[string]map[string][]string{ +var ImagesMatrix = map[string]map[string][]string{ "v1.10": { - "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.6.0"}, - "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.6.0"}, - "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.1"}, - "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.6.0"}, - "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.4.0"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.6.0"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.6.0"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.1"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.6.0"}, + "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.4.0"}, + "velero": {"gcr.io/velero-gcp/velero:v1.10.2"}, + "velero-restore-helper": {"gcr.io/velero-gcp/velero-restore-helper:v1.10.2"}, }, "v1.11": { - "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.7.0"}, - "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.7.0"}, - "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.1"}, - "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.7.0"}, - "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.5.0"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.7.0"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.7.0"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.1"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.7.0"}, + "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.5.0"}, + "velero": {"gcr.io/velero-gcp/velero:v1.11.1"}, + "velero-restore-helper": {"gcr.io/velero-gcp/velero-restore-helper:v1.11.1"}, }, "v1.12": { - "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.8.0"}, - "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.8.0"}, - "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.1"}, - "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.8.0"}, - "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.6.0"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.8.0"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.8.0"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.1"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.8.0"}, + "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.6.0"}, + "velero": {"gcr.io/velero-gcp/velero:v1.12.4"}, + "velero-restore-helper": {"gcr.io/velero-gcp/velero-restore-helper:v1.12.4"}, }, "v1.13": { - "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.9.2"}, - "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.9.2"}, - "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, - "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.9.2"}, - "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.7.1"}, - "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.9.2"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.9.2"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.9.2"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.9.2"}, + "csi": {"gcr.io/velero-gcp/velero-plugin-for-csi:v0.7.1"}, + "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.9.2"}, + "velero": {"gcr.io/velero-gcp/velero:v1.13.2"}, + "velero-restore-helper": {"gcr.io/velero-gcp/velero-restore-helper:v1.13.2"}, }, "v1.14": { - "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.10.1"}, - "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.10.1"}, - "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, - "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.10.1"}, - "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.10.1"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.10.1"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.10.1"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.10.1"}, + "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.10.1"}, + "velero": {"gcr.io/velero-gcp/velero:v1.14.1"}, + "velero-restore-helper": {"gcr.io/velero-gcp/velero-restore-helper:v1.14.1"}, }, "v1.15": { - "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.11.0"}, - "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.11.0"}, - "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, - "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.11.0"}, - "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.11.0"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.11.0"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:v1.11.0"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:v1.11.0"}, + "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:v1.11.0"}, + "velero": {"gcr.io/velero-gcp/velero:v1.15.0"}, + "velero-restore-helper": {"gcr.io/velero-gcp/velero-restore-helper:v1.15.0"}, }, "main": { - "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:main"}, - "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:main"}, - "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, - "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:main"}, - "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:main"}, + "aws": {"gcr.io/velero-gcp/velero-plugin-for-aws:main"}, + "azure": {"gcr.io/velero-gcp/velero-plugin-for-microsoft-azure:main"}, + "vsphere": {"gcr.io/velero-gcp/velero-plugin-for-vsphere:v1.5.2"}, + "gcp": {"gcr.io/velero-gcp/velero-plugin-for-gcp:main"}, + "datamover": {"gcr.io/velero-gcp/velero-plugin-for-aws:main"}, + "velero": {"gcr.io/velero-gcp/velero:main"}, + "velero-restore-helper": {"gcr.io/velero-gcp/velero-restore-helper:main"}, }, } +func SetImagesToDefaultValues(config VeleroConfig, version string) (VeleroConfig, error) { + fmt.Printf("Get the images for version %s\n", version) + + ret := config + + ret.Plugins = "" + + versionWithoutPatch := semver.MajorMinor(version) + // Read migration case needs images from the PluginsMatrix map. + images, ok := ImagesMatrix[versionWithoutPatch] + if !ok { + return config, fmt.Errorf("fail to read the images for version %s from the ImagesMatrix", + versionWithoutPatch) + } + + ret.VeleroImage = images[Velero][0] + ret.RestoreHelperImage = images[VeleroRestoreHelper][0] + + if ret.CloudProvider == Azure { + ret.Plugins = images[Azure][0] + } + if ret.CloudProvider == AWS { + ret.Plugins = images[AWS][0] + } + // If HasVspherePlugin is false, only install the AWS plugin. + // If do nothing here, the default behavior is + // installing both AWS and vSphere plugins. + if ret.CloudProvider == Vsphere && + !ret.HasVspherePlugin { + ret.Plugins = images[AWS][0] + } + + // Because Velero CSI plugin is deprecated in v1.14, + // only need to install it for version lower than v1.14. + if strings.Contains(ret.Features, FeatureCSI) && + semver.Compare(versionWithoutPatch, "v1.14") < 0 { + ret.Plugins = ret.Plugins + "," + images[CSI][0] + } + if ret.SnapshotMoveData && ret.CloudProvider == Azure { + ret.Plugins = ret.Plugins + "," + images[AWS][0] + } + + return ret, nil +} + func getPluginsByVersion(version string, cloudProvider string, needDataMoverPlugin bool) ([]string, error) { var cloudMap map[string][]string arr := strings.Split(version, ".") if len(arr) >= 3 { - cloudMap = PluginsMatrix[arr[0]+"."+arr[1]] + cloudMap = ImagesMatrix[arr[0]+"."+arr[1]] } if len(cloudMap) == 0 { - cloudMap = PluginsMatrix["main"] + cloudMap = ImagesMatrix["main"] if len(cloudMap) == 0 { return nil, errors.Errorf("fail to get plugins by version: main") }