E2E supports VKS data mover environment.

* Add new flag HAS_VSPHERE_PLUGIN for E2E test.
* Modify the E2E README for the new parameter.
* Add the VolumeSnapshotClass for VKS.
* Modify the plugin install logic.
* Modify the cases to support data mover case in VKS.

Signed-off-by: Xun Jiang <[email protected]>

test/Makefile

@@ -41,25 +41,41 @@ help:  ## Display this help
 
 TOOLS_DIR := $(REPO_ROOT)/hack/tools
 BIN_DIR := bin
+
 # Try to not modify PATH if possible
 GOBIN := $(REPO_ROOT)/.go/bin
+
 TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR)
+
 GINKGO := $(GOBIN)/ginkgo
+
 KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize
+
 OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin
+
 # Please reference to this document for Ginkgo label spec format.
 # https://onsi.github.io/ginkgo/#spec-labels
 GINKGO_LABELS ?=
+
 # When --fail-fast is set, the entire suite will stop when the first failure occurs.
 # Enable --fail-fast by default.
 # https://onsi.github.io/ginkgo/#mental-model-how-ginkgo-handles-failure
 FAIL_FAST ?= false
+
 VELERO_CLI ?=$$(pwd)/../_output/bin/$(GOOS)/$(GOARCH)/velero
+
 VELERO_IMAGE ?= velero/velero:main
+
 PLUGINS ?=
+
+# Flag used to tell E2E whether the Velero vSphere plugin is installed.
+HAS_VSPHERE_PLUGIN ?= false
+
 RESTORE_HELPER_IMAGE ?=
+
 #Released version only
 UPGRADE_FROM_VELERO_VERSION ?= v1.13.2,v1.14.1
+
 # UPGRADE_FROM_VELERO_CLI can has the same format(a list divided by comma) with UPGRADE_FROM_VELERO_VERSION
 # Upgrade tests will be executed sequently according to the list by UPGRADE_FROM_VELERO_VERSION
 # So although length of UPGRADE_FROM_VELERO_CLI list is not equal with UPGRADE_FROM_VELERO_VERSION
@@ -150,7 +166,8 @@ COMMON_ARGS := --velerocli=$(VELERO_CLI) \
 	--velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \
 	--uploader-type=$(UPLOADER_TYPE) \
 	--debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART) \
-	--fail-fast=$(FAIL_FAST)
+	--fail-fast=$(FAIL_FAST) \
+	--has-vsphere-plugin=$(HAS_VSPHERE_PLUGIN)
 
 # Make sure ginkgo is in $GOBIN
 .PHONY:ginkgo

test/e2e/README.md

@@ -78,6 +78,7 @@ These configuration parameters are expected as values to the following command l
 1. `--standby-cluster-object-store-provider`: Object store provider for standby cluster.
 1. `--debug-velero-pod-restart`: A switch for debugging velero pod restart.
 1. `--fail-fast`: A switch for for failing fast on meeting error.
+1. `--disable-vsphere-plugin`: A switch for not install the Velero vSphere plugin when the provider is set to `vsphere`.
 
 These configurations or parameters are used to generate install options for Velero for each test suite.
 
@@ -129,12 +130,13 @@ Below is a mapping between `make` variables to E2E configuration flags.
 1. `INSTALL_VELERO `: `-install-velero`. Optional.
 1. `DEBUG_VELERO_POD_RESTART`: `-debug-velero-pod-restart`. Optional.
 1. `FAIL_FAST`: `--fail-fast`. Optional.
+1. `DISABLE_VSPHERE_PLUGIN`: `--diable-vsphere-plugin`. Optional.
 
 
 
 ### Examples
 
-Basic examples:
+#### Basic examples:
 
 1. Run Velero tests in a kind cluster with AWS (or MinIO) as the storage provider:
 
@@ -208,7 +210,7 @@ ADDITIONAL_CREDS_FILE=/path/to/azure-creds \
 make test-e2e
 ```
 
-Upgrade examples:
+#### Upgrade examples:
 
 1. Run Velero upgrade tests with pre-upgrade version:
 
@@ -234,7 +236,7 @@ UPGRADE_FROM_VELERO_VERSION=v1.10.2,v1.11.0 \
 make test-e2e
 ```
 
-Migration examples:
+#### Migration examples:
 
 1. Migration between 2 cluster of the same provider tests:
 
@@ -275,7 +277,7 @@ GINKGO_LABELS="Migration" \
 make test-e2e
 ```
 
-## 5. Filtering tests
+#### Filtering tests
 
 In release-1.15, Velero bumps the [Ginkgo](https://onsi.github.io/ginkgo/) version to [v2](https://onsi.github.io/ginkgo/MIGRATING_TO_V2).
 Velero E2E start to use [labels](https://onsi.github.io/ginkgo/#spec-labels) to filter cases instead of [`-focus` and `-skip`](https://onsi.github.io/ginkgo/#focused-specs) parameters.
@@ -285,7 +287,6 @@ Both `make run-e2e` and `make run-perf` CLI support using parameter `GINKGO_LABE
 `GINKGO_LABELS` is interpreted into `ginkgo run` CLI's parameter [`--label-filter`](https://onsi.github.io/ginkgo/#spec-labels).
 
 
-### Examples
 E2E tests can be run with specific cases to be included and/or excluded using the commands below:
 
 1. Run Velero tests with specific cases to be included:
@@ -316,6 +317,44 @@ In this example, cases are labelled as
 * `Migration` and `Restic` 
 will be skipped.
 
+#### VKS environment test
+1. Run the CSI data mover test. 
+
+`HAS_VSPHERE_PLUGIN` should be set to `false` to not install the Velero vSphere plugin.
+``` bash
+CLOUD_PROVIDER=vsphere \
+DEFAULT_CLUSTER=wl-antreav1301 \
+STANDBY_CLUSTER=wl-antreav1311 \
+DEFAULT_CLUSTER_NAME=192.168.0.4 \
+STANDBY_CLUSTER_NAME=192.168.0.3 \
+FEATURES=EnableCSI \
+PLUGINS=gcr.io/velero-gcp/velero-plugin-for-aws:main \
+HAS_VSPHERE_PLUGIN=false \
+OBJECT_STORE_PROVIDER=aws \
+CREDS_FILE=$HOME/aws-credential \
+BSL_CONFIG=region=us-east-1 \
+BSL_BUCKET=nightly-normal-account4-test \
+BSL_PREFIX=nightly \
+ADDITIONAL_BSL_PLUGINS=gcr.io/velero-gcp/velero-plugin-for-aws:main \
+ADDITIONAL_OBJECT_STORE_PROVIDER=aws \
+ADDITIONAL_BSL_CONFIG=region=us-east-1 \
+ADDITIONAL_BSL_BUCKET=nightly-normal-account4-test \
+ADDITIONAL_BSL_PREFIX=addition-bsl \
+ADDITIONAL_CREDS_FILE=$HOME/aws-credential \
+VELERO_IMAGE=gcr.io/velero-gcp/velero:main \
+RESTORE_HELPER_IMAGE=gcr.io/velero-gcp/velero-restore-helper:main \
+VERSION=main \
+SNAPSHOT_MOVE_DATA=true \
+STANDBY_CLUSTER_CLOUD_PROVIDER=vsphere \
+STANDBY_CLUSTER_OBJECT_STORE_PROVIDER=aws \
+STANDBY_CLUSTER_PLUGINS=gcr.io/velero-gcp/velero-plugin-for-aws:main \
+DISABLE_INFORMER_CACHE=true \
+REGISTRY_CREDENTIAL_FILE=$HOME/.docker/config.json \
+GINKGO_LABELS=Migration \
+KIBISHII_DIRECTORY=$HOME/kibishii/kubernetes/yaml/ \
+make test-e2e
+```
+
 ## 6. Full Tests execution
 
 As we provided several examples for E2E test execution, if no filter is involved and despite difference of test environment, 

test/e2e/backups/deletion.go

@@ -34,7 +34,7 @@ import (
 	. "github.com/vmware-tanzu/velero/test/util/velero"
 )
 
-// Test backup and restore of Kibishi using restic
+// Test backup and restore of Kibishii using restic
 
 func BackupDeletionWithSnapshots() {
 	backup_deletion_test(true)
@@ -143,7 +143,8 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc
 		})
 	})
 	for _, ns := range workloadNamespaceList {
-		if providerName == Vsphere && useVolumeSnapshots {
+		if useVolumeSnapshots &&
+			veleroCfg.HasVspherePlugin {
 			// Wait for uploads started by the Velero Plugin for vSphere to complete
 			// TODO - remove after upload progress monitoring is implemented
 			fmt.Println("Waiting for vSphere uploads to complete")

test/e2e/backups/ttl.go

@@ -122,19 +122,33 @@ func TTLTest() {
 
 		var snapshotCheckPoint SnapshotCheckPoint
 		if useVolumeSnapshots {
-			if veleroCfg.CloudProvider == Vsphere {
-				// TODO - remove after upload progress monitoring is implemented
+			if veleroCfg.HasVspherePlugin {
 				By("Waiting for vSphere uploads to complete", func() {
 					Expect(WaitForVSphereUploadCompletion(ctx, time.Hour,
 						test.testNS, 2)).To(Succeed())
 				})
 			}
-			snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, test.testNS, test.backupName, KibishiiPVCNameList)
-			Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
 
-			Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider,
-				veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig,
-				test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint")
+			snapshotCheckPoint, err = GetSnapshotCheckPoint(
+				client,
+				veleroCfg,
+				2,
+				test.testNS,
+				test.backupName,
+				KibishiiPVCNameList,
+			)
+			Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint")
+
+			Expect(
+				SnapshotsShouldBeCreatedInCloud(
+					veleroCfg.CloudProvider,
+					veleroCfg.CloudCredentialsFile,
+					veleroCfg.BSLBucket,
+					veleroCfg.BSLConfig,
+					test.backupName,
+					snapshotCheckPoint,
+				),
+			).NotTo(HaveOccurred(), "Fail to verify the created snapshots")
 		}
 
 		By(fmt.Sprintf("Simulating a disaster by removing namespace %s\n", BackupCfg.BackupName), func() {

test/e2e/e2e_suite_test.go

@@ -103,6 +103,7 @@ func init() {
 	flag.StringVar(&VeleroCfg.DefaultCLSServiceAccountName, "default-cls-service-account-name", "", "default cluster service account name.")
 	flag.StringVar(&VeleroCfg.StandbyCLSServiceAccountName, "standby-cls-service-account-name", "", "standby cluster service account name.")
 	flag.BoolVar(&VeleroCfg.FailFast, "fail-fast", true, "a switch for failing fast on meeting error.")
+	flag.BoolVar(&VeleroCfg.HasVspherePlugin, "has-vsphere-plugin", false, "a switch for installing vSphere plugin.")
 }
 
 // Add label [SkipVanillaZfs]:

test/e2e/migration/migration.go

@@ -158,16 +158,21 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 						if OriginVeleroCfg.CloudProvider == AWS {
 							OriginVeleroCfg.Plugins = migrationNeedPlugins[AWS][0]
 						}
+						// If HasVspherePlugin is false, only install the AWS plugin.
+						// If do nothing here, the default behavior is
+						// installing both AWS and vSphere plugins.
+						if OriginVeleroCfg.CloudProvider == Vsphere &&
+							!OriginVeleroCfg.HasVspherePlugin {
+							OriginVeleroCfg.Plugins = migrationNeedPlugins[AWS][0]
+						}
 						// Because Velero CSI plugin is deprecated in v1.14,
 						// only need to install it for version lower than v1.14.
 						if strings.Contains(OriginVeleroCfg.Features, FeatureCSI) &&
 							semver.Compare(versionWithoutPatch, "v1.14") < 0 {
 							OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[CSI][0]
 						}
-						if OriginVeleroCfg.SnapshotMoveData {
-							if OriginVeleroCfg.CloudProvider == Azure {
-								OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[AWS][0]
-							}
+						if OriginVeleroCfg.SnapshotMoveData && OriginVeleroCfg.CloudProvider == Azure {
+							OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[AWS][0]
 						}
 						veleroCLI2Version.VeleroCLI, err = InstallVeleroCLI(veleroCLI2Version.VeleroVersion)
 						Expect(err).To(Succeed())
@@ -253,8 +258,9 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version)
 			})
 
 			if useVolumeSnapshots {
-				if veleroCfg.CloudProvider == Vsphere {
-					// TODO - remove after upload progress monitoring is implemented
+				// Only wait for the snapshots.backupdriver.cnsdp.vmware.com
+				// when the vSphere plugin is used.
+				if veleroCfg.HasVspherePlugin {
 					By("Waiting for vSphere uploads to complete", func() {
 						Expect(WaitForVSphereUploadCompletion(context.Background(), time.Hour,
 							migrationNamespace, kibishiiWorkerCount)).To(Succeed())

test/e2e/upgrade/upgrade.go

@@ -190,8 +190,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC
 			})
 
 			if useVolumeSnapshots {
-				if veleroCfg.CloudProvider == Vsphere {
-					// TODO - remove after upload progress monitoring is implemented
+				if veleroCfg.HasVspherePlugin {
 					By("Waiting for vSphere uploads to complete", func() {
 						Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour,
 							upgradeNamespace, 2)).To(Succeed())

test/testdata/volume-snapshot-class/vsphere.yaml

@@ -0,0 +1,13 @@
+---
+apiVersion: snapshot.storage.k8s.io/v1
+deletionPolicy: Delete
+driver: csi.vsphere.vmware.com
+kind: VolumeSnapshotClass
+metadata:
+  annotations:
+    snapshot.storage.kubernetes.io/is-default-class: "true"
+  labels:
+    velero.io/csi-volumesnapshot-class: "true"
+  name: volumesnapshotclass-delete
+parameters:
+  svVolumeSnapshotClass: volumesnapshotclass-delete

test/types.go

@@ -118,6 +118,7 @@ type VeleroConfig struct {
 	ServiceAccountNameToInstall       string
 	EKSPolicyARN                      string
 	FailFast                          bool
+	HasVspherePlugin                  bool
 }
 
 type VeleroCfgInPerf struct {

test/util/kibishii/kibishii_utils.go

@@ -120,9 +120,8 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc
 
 	// Checkpoint for a successful backup
 	if useVolumeSnapshots {
-		if providerName == Vsphere {
+		if veleroCfg.HasVspherePlugin {
 			// Wait for uploads started by the Velero Plugin for vSphere to complete
-			// TODO - remove after upload progress monitoring is implemented
 			fmt.Println("Waiting for vSphere uploads to complete")
 			if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace, 2); err != nil {
 				return errors.Wrapf(err, "Error waiting for uploads to complete")

test/util/velero/install.go

@@ -320,7 +320,7 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options
 
 	if len(options.Features) > 0 {
 		args = append(args, "--features", options.Features)
-		if !strings.EqualFold(cloudProvider, test.Vsphere) && strings.EqualFold(options.Features, test.FeatureCSI) && options.UseVolumeSnapshots {
+		if strings.EqualFold(options.Features, test.FeatureCSI) && options.UseVolumeSnapshots {
 			// https://github.com/openebs/zfs-localpv/blob/develop/docs/snapshot.md
 			fmt.Printf("Start to install %s VolumeSnapshotClass ... \n", cloudProvider)
 			if err := k8s.KubectlApplyByFile(ctx, fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", cloudProvider)); err != nil {

test/util/velero/velero_utils.go

@@ -107,7 +107,7 @@ var PluginsMatrix = map[string]map[string][]string{
 	},
 }
 
-func getPluginsByVersion(version, cloudProvider, objectStoreProvider string, needDataMoverPlugin bool) ([]string, error) {
+func getPluginsByVersion(version string, cloudProvider string, needDataMoverPlugin bool) ([]string, error) {
 	var cloudMap map[string][]string
 	arr := strings.Split(version, ".")
 	if len(arr) >= 3 {
@@ -133,18 +133,6 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider string, nee
 			return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, cloudProvider)
 		}
 	}
-	// TODO: Is this section need?
-	if objectStoreProvider != cloudProvider {
-		pluginsForObjectStoreProvider, ok := cloudMap[objectStoreProvider]
-		if !ok {
-			return nil, errors.Errorf("fail to get plugins by version: %s and object store provider %s", version, objectStoreProvider)
-		}
-		for _, p := range pluginsForObjectStoreProvider {
-			if !slices.Contains(plugins, p) {
-				plugins = append(plugins, p)
-			}
-		}
-	}
 
 	if needDataMoverPlugin {
 		pluginsForDatamover, ok := cloudMap["datamover"]
@@ -630,7 +618,7 @@ func getProviderPlugins(ctx context.Context, veleroCLI string, cloudProvider str
 		return nil, errors.WithMessage(err, "failed to get velero version")
 	}
 
-	plugins, err := getPluginsByVersion(version, cloudProvider, cloudProvider, false)
+	plugins, err := getPluginsByVersion(version, cloudProvider, false)
 	if err != nil {
 		return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", cloudProvider, version)
 	}
@@ -673,7 +661,7 @@ func getPlugins(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) {
 		if veleroCfg.SnapshotMoveData && veleroCfg.DataMoverPlugin == "" && !veleroCfg.IsUpgradeTest {
 			needDataMoverPlugin = true
 		}
-		plugins, err = getPluginsByVersion(version, cloudProvider, objectStoreProvider, needDataMoverPlugin)
+		plugins, err = getPluginsByVersion(version, cloudProvider, needDataMoverPlugin)
 		if err != nil {
 			return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", objectStoreProvider, version)
 		}