From b02fc1da96d81cc029098459d05b3314abadc720 Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Tue, 5 Nov 2024 17:29:36 +0800 Subject: [PATCH] E2E supports VKS data mover environment. * Add new flag HAS_VSPHERE_PLUGIN for E2E test. * Modify the E2E README for the new parameter. * Add the VolumeSnapshotClass for VKS. * Modify the plugin install logic. * Modify the cases to support data mover case in VKS. Signed-off-by: Xun Jiang --- test/Makefile | 19 ++++++- test/e2e/README.md | 49 +++++++++++++++++-- test/e2e/backups/deletion.go | 5 +- test/e2e/backups/ttl.go | 28 ++++++++--- test/e2e/e2e_suite_test.go | 1 + test/e2e/migration/migration.go | 18 ++++--- test/e2e/upgrade/upgrade.go | 3 +- .../volume-snapshot-class/vsphere.yaml | 13 +++++ test/types.go | 1 + test/util/kibishii/kibishii_utils.go | 3 +- test/util/velero/install.go | 2 +- test/util/velero/velero_utils.go | 18 ++----- 12 files changed, 119 insertions(+), 41 deletions(-) create mode 100644 test/testdata/volume-snapshot-class/vsphere.yaml diff --git a/test/Makefile b/test/Makefile index 276cc76afd..b3cf6eadd5 100644 --- a/test/Makefile +++ b/test/Makefile @@ -41,25 +41,41 @@ help: ## Display this help TOOLS_DIR := $(REPO_ROOT)/hack/tools BIN_DIR := bin + # Try to not modify PATH if possible GOBIN := $(REPO_ROOT)/.go/bin + TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR) + GINKGO := $(GOBIN)/ginkgo + KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize + OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin + # Please reference to this document for Ginkgo label spec format. # https://onsi.github.io/ginkgo/#spec-labels GINKGO_LABELS ?= + # When --fail-fast is set, the entire suite will stop when the first failure occurs. # Enable --fail-fast by default. # https://onsi.github.io/ginkgo/#mental-model-how-ginkgo-handles-failure FAIL_FAST ?= false + VELERO_CLI ?=$$(pwd)/../_output/bin/$(GOOS)/$(GOARCH)/velero + VELERO_IMAGE ?= velero/velero:main + PLUGINS ?= + +# Flag used to tell E2E whether the Velero vSphere plugin is installed. +HAS_VSPHERE_PLUGIN ?= false + RESTORE_HELPER_IMAGE ?= + #Released version only UPGRADE_FROM_VELERO_VERSION ?= v1.13.2,v1.14.1 + # UPGRADE_FROM_VELERO_CLI can has the same format(a list divided by comma) with UPGRADE_FROM_VELERO_VERSION # Upgrade tests will be executed sequently according to the list by UPGRADE_FROM_VELERO_VERSION # So although length of UPGRADE_FROM_VELERO_CLI list is not equal with UPGRADE_FROM_VELERO_VERSION @@ -150,7 +166,8 @@ COMMON_ARGS := --velerocli=$(VELERO_CLI) \ --velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \ --uploader-type=$(UPLOADER_TYPE) \ --debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART) \ - --fail-fast=$(FAIL_FAST) + --fail-fast=$(FAIL_FAST) \ + --has-vsphere-plugin=$(HAS_VSPHERE_PLUGIN) # Make sure ginkgo is in $GOBIN .PHONY:ginkgo diff --git a/test/e2e/README.md b/test/e2e/README.md index 65231e01af..2ca71a7420 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -78,6 +78,7 @@ These configuration parameters are expected as values to the following command l 1. `--standby-cluster-object-store-provider`: Object store provider for standby cluster. 1. `--debug-velero-pod-restart`: A switch for debugging velero pod restart. 1. `--fail-fast`: A switch for for failing fast on meeting error. +1. `--disable-vsphere-plugin`: A switch for not install the Velero vSphere plugin when the provider is set to `vsphere`. These configurations or parameters are used to generate install options for Velero for each test suite. @@ -129,12 +130,13 @@ Below is a mapping between `make` variables to E2E configuration flags. 1. `INSTALL_VELERO `: `-install-velero`. Optional. 1. `DEBUG_VELERO_POD_RESTART`: `-debug-velero-pod-restart`. Optional. 1. `FAIL_FAST`: `--fail-fast`. Optional. +1. `DISABLE_VSPHERE_PLUGIN`: `--diable-vsphere-plugin`. Optional. ### Examples -Basic examples: +#### Basic examples: 1. Run Velero tests in a kind cluster with AWS (or MinIO) as the storage provider: @@ -208,7 +210,7 @@ ADDITIONAL_CREDS_FILE=/path/to/azure-creds \ make test-e2e ``` -Upgrade examples: +#### Upgrade examples: 1. Run Velero upgrade tests with pre-upgrade version: @@ -234,7 +236,7 @@ UPGRADE_FROM_VELERO_VERSION=v1.10.2,v1.11.0 \ make test-e2e ``` -Migration examples: +#### Migration examples: 1. Migration between 2 cluster of the same provider tests: @@ -275,7 +277,7 @@ GINKGO_LABELS="Migration" \ make test-e2e ``` -## 5. Filtering tests +#### Filtering tests In release-1.15, Velero bumps the [Ginkgo](https://onsi.github.io/ginkgo/) version to [v2](https://onsi.github.io/ginkgo/MIGRATING_TO_V2). Velero E2E start to use [labels](https://onsi.github.io/ginkgo/#spec-labels) to filter cases instead of [`-focus` and `-skip`](https://onsi.github.io/ginkgo/#focused-specs) parameters. @@ -285,7 +287,6 @@ Both `make run-e2e` and `make run-perf` CLI support using parameter `GINKGO_LABE `GINKGO_LABELS` is interpreted into `ginkgo run` CLI's parameter [`--label-filter`](https://onsi.github.io/ginkgo/#spec-labels). -### Examples E2E tests can be run with specific cases to be included and/or excluded using the commands below: 1. Run Velero tests with specific cases to be included: @@ -316,6 +317,44 @@ In this example, cases are labelled as * `Migration` and `Restic` will be skipped. +#### VKS environment test +1. Run the CSI data mover test. + +`HAS_VSPHERE_PLUGIN` should be set to `false` to not install the Velero vSphere plugin. +``` bash +CLOUD_PROVIDER=vsphere \ +DEFAULT_CLUSTER=wl-antreav1301 \ +STANDBY_CLUSTER=wl-antreav1311 \ +DEFAULT_CLUSTER_NAME=192.168.0.4 \ +STANDBY_CLUSTER_NAME=192.168.0.3 \ +FEATURES=EnableCSI \ +PLUGINS=gcr.io/velero-gcp/velero-plugin-for-aws:main \ +HAS_VSPHERE_PLUGIN=false \ +OBJECT_STORE_PROVIDER=aws \ +CREDS_FILE=$HOME/aws-credential \ +BSL_CONFIG=region=us-east-1 \ +BSL_BUCKET=nightly-normal-account4-test \ +BSL_PREFIX=nightly \ +ADDITIONAL_BSL_PLUGINS=gcr.io/velero-gcp/velero-plugin-for-aws:main \ +ADDITIONAL_OBJECT_STORE_PROVIDER=aws \ +ADDITIONAL_BSL_CONFIG=region=us-east-1 \ +ADDITIONAL_BSL_BUCKET=nightly-normal-account4-test \ +ADDITIONAL_BSL_PREFIX=addition-bsl \ +ADDITIONAL_CREDS_FILE=$HOME/aws-credential \ +VELERO_IMAGE=gcr.io/velero-gcp/velero:main \ +RESTORE_HELPER_IMAGE=gcr.io/velero-gcp/velero-restore-helper:main \ +VERSION=main \ +SNAPSHOT_MOVE_DATA=true \ +STANDBY_CLUSTER_CLOUD_PROVIDER=vsphere \ +STANDBY_CLUSTER_OBJECT_STORE_PROVIDER=aws \ +STANDBY_CLUSTER_PLUGINS=gcr.io/velero-gcp/velero-plugin-for-aws:main \ +DISABLE_INFORMER_CACHE=true \ +REGISTRY_CREDENTIAL_FILE=$HOME/.docker/config.json \ +GINKGO_LABELS=Migration \ +KIBISHII_DIRECTORY=$HOME/kibishii/kubernetes/yaml/ \ +make test-e2e +``` + ## 6. Full Tests execution As we provided several examples for E2E test execution, if no filter is involved and despite difference of test environment, diff --git a/test/e2e/backups/deletion.go b/test/e2e/backups/deletion.go index 0443b74446..f5bba56c2d 100644 --- a/test/e2e/backups/deletion.go +++ b/test/e2e/backups/deletion.go @@ -34,7 +34,7 @@ import ( . "github.com/vmware-tanzu/velero/test/util/velero" ) -// Test backup and restore of Kibishi using restic +// Test backup and restore of Kibishii using restic func BackupDeletionWithSnapshots() { backup_deletion_test(true) @@ -143,7 +143,8 @@ func runBackupDeletionTests(client TestClient, veleroCfg VeleroConfig, backupLoc }) }) for _, ns := range workloadNamespaceList { - if providerName == Vsphere && useVolumeSnapshots { + if useVolumeSnapshots && + veleroCfg.HasVspherePlugin { // Wait for uploads started by the Velero Plugin for vSphere to complete // TODO - remove after upload progress monitoring is implemented fmt.Println("Waiting for vSphere uploads to complete") diff --git a/test/e2e/backups/ttl.go b/test/e2e/backups/ttl.go index 1f101e0555..342eac0378 100644 --- a/test/e2e/backups/ttl.go +++ b/test/e2e/backups/ttl.go @@ -122,19 +122,33 @@ func TTLTest() { var snapshotCheckPoint SnapshotCheckPoint if useVolumeSnapshots { - if veleroCfg.CloudProvider == Vsphere { - // TODO - remove after upload progress monitoring is implemented + if veleroCfg.HasVspherePlugin { By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(ctx, time.Hour, test.testNS, 2)).To(Succeed()) }) } - snapshotCheckPoint, err = GetSnapshotCheckPoint(client, veleroCfg, 2, test.testNS, test.backupName, KibishiiPVCNameList) - Expect(err).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") - Expect(SnapshotsShouldBeCreatedInCloud(veleroCfg.CloudProvider, - veleroCfg.CloudCredentialsFile, veleroCfg.BSLBucket, veleroCfg.BSLConfig, - test.backupName, snapshotCheckPoint)).NotTo(HaveOccurred(), "Fail to get Azure CSI snapshot checkpoint") + snapshotCheckPoint, err = GetSnapshotCheckPoint( + client, + veleroCfg, + 2, + test.testNS, + test.backupName, + KibishiiPVCNameList, + ) + Expect(err).NotTo(HaveOccurred(), "Fail to get snapshot checkpoint") + + Expect( + SnapshotsShouldBeCreatedInCloud( + veleroCfg.CloudProvider, + veleroCfg.CloudCredentialsFile, + veleroCfg.BSLBucket, + veleroCfg.BSLConfig, + test.backupName, + snapshotCheckPoint, + ), + ).NotTo(HaveOccurred(), "Fail to verify the created snapshots") } By(fmt.Sprintf("Simulating a disaster by removing namespace %s\n", BackupCfg.BackupName), func() { diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 1b4cd32dec..36cb0b0e5d 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -103,6 +103,7 @@ func init() { flag.StringVar(&VeleroCfg.DefaultCLSServiceAccountName, "default-cls-service-account-name", "", "default cluster service account name.") flag.StringVar(&VeleroCfg.StandbyCLSServiceAccountName, "standby-cls-service-account-name", "", "standby cluster service account name.") flag.BoolVar(&VeleroCfg.FailFast, "fail-fast", true, "a switch for failing fast on meeting error.") + flag.BoolVar(&VeleroCfg.HasVspherePlugin, "has-vsphere-plugin", false, "a switch for installing vSphere plugin.") } // Add label [SkipVanillaZfs]: diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 8316126e6e..51edf34072 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -158,16 +158,21 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) if OriginVeleroCfg.CloudProvider == AWS { OriginVeleroCfg.Plugins = migrationNeedPlugins[AWS][0] } + // If HasVspherePlugin is false, only install the AWS plugin. + // If do nothing here, the default behavior is + // installing both AWS and vSphere plugins. + if OriginVeleroCfg.CloudProvider == Vsphere && + !OriginVeleroCfg.HasVspherePlugin { + OriginVeleroCfg.Plugins = migrationNeedPlugins[AWS][0] + } // Because Velero CSI plugin is deprecated in v1.14, // only need to install it for version lower than v1.14. if strings.Contains(OriginVeleroCfg.Features, FeatureCSI) && semver.Compare(versionWithoutPatch, "v1.14") < 0 { OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[CSI][0] } - if OriginVeleroCfg.SnapshotMoveData { - if OriginVeleroCfg.CloudProvider == Azure { - OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[AWS][0] - } + if OriginVeleroCfg.SnapshotMoveData && OriginVeleroCfg.CloudProvider == Azure { + OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[AWS][0] } veleroCLI2Version.VeleroCLI, err = InstallVeleroCLI(veleroCLI2Version.VeleroVersion) Expect(err).To(Succeed()) @@ -253,8 +258,9 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) }) if useVolumeSnapshots { - if veleroCfg.CloudProvider == Vsphere { - // TODO - remove after upload progress monitoring is implemented + // Only wait for the snapshots.backupdriver.cnsdp.vmware.com + // when the vSphere plugin is used. + if veleroCfg.HasVspherePlugin { By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(context.Background(), time.Hour, migrationNamespace, kibishiiWorkerCount)).To(Succeed()) diff --git a/test/e2e/upgrade/upgrade.go b/test/e2e/upgrade/upgrade.go index cfda01feff..ffb8f5f22b 100644 --- a/test/e2e/upgrade/upgrade.go +++ b/test/e2e/upgrade/upgrade.go @@ -190,8 +190,7 @@ func BackupUpgradeRestoreTest(useVolumeSnapshots bool, veleroCLI2Version VeleroC }) if useVolumeSnapshots { - if veleroCfg.CloudProvider == Vsphere { - // TODO - remove after upload progress monitoring is implemented + if veleroCfg.HasVspherePlugin { By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, upgradeNamespace, 2)).To(Succeed()) diff --git a/test/testdata/volume-snapshot-class/vsphere.yaml b/test/testdata/volume-snapshot-class/vsphere.yaml new file mode 100644 index 0000000000..08ea9b2225 --- /dev/null +++ b/test/testdata/volume-snapshot-class/vsphere.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: snapshot.storage.k8s.io/v1 +deletionPolicy: Delete +driver: csi.vsphere.vmware.com +kind: VolumeSnapshotClass +metadata: + annotations: + snapshot.storage.kubernetes.io/is-default-class: "true" + labels: + velero.io/csi-volumesnapshot-class: "true" + name: volumesnapshotclass-delete +parameters: + svVolumeSnapshotClass: volumesnapshotclass-delete diff --git a/test/types.go b/test/types.go index 961eebe5dc..798f721448 100644 --- a/test/types.go +++ b/test/types.go @@ -118,6 +118,7 @@ type VeleroConfig struct { ServiceAccountNameToInstall string EKSPolicyARN string FailFast bool + HasVspherePlugin bool } type VeleroCfgInPerf struct { diff --git a/test/util/kibishii/kibishii_utils.go b/test/util/kibishii/kibishii_utils.go index a749538d72..4ab0783946 100644 --- a/test/util/kibishii/kibishii_utils.go +++ b/test/util/kibishii/kibishii_utils.go @@ -120,9 +120,8 @@ func RunKibishiiTests(veleroCfg VeleroConfig, backupName, restoreName, backupLoc // Checkpoint for a successful backup if useVolumeSnapshots { - if providerName == Vsphere { + if veleroCfg.HasVspherePlugin { // Wait for uploads started by the Velero Plugin for vSphere to complete - // TODO - remove after upload progress monitoring is implemented fmt.Println("Waiting for vSphere uploads to complete") if err := WaitForVSphereUploadCompletion(oneHourTimeout, time.Hour, kibishiiNamespace, 2); err != nil { return errors.Wrapf(err, "Error waiting for uploads to complete") diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 81204a9863..9da1c4b83a 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -320,7 +320,7 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options if len(options.Features) > 0 { args = append(args, "--features", options.Features) - if !strings.EqualFold(cloudProvider, test.Vsphere) && strings.EqualFold(options.Features, test.FeatureCSI) && options.UseVolumeSnapshots { + if strings.EqualFold(options.Features, test.FeatureCSI) && options.UseVolumeSnapshots { // https://github.com/openebs/zfs-localpv/blob/develop/docs/snapshot.md fmt.Printf("Start to install %s VolumeSnapshotClass ... \n", cloudProvider) if err := k8s.KubectlApplyByFile(ctx, fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", cloudProvider)); err != nil { diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index cec0d61869..38ada7a9e9 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -107,7 +107,7 @@ var PluginsMatrix = map[string]map[string][]string{ }, } -func getPluginsByVersion(version, cloudProvider, objectStoreProvider string, needDataMoverPlugin bool) ([]string, error) { +func getPluginsByVersion(version string, cloudProvider string, needDataMoverPlugin bool) ([]string, error) { var cloudMap map[string][]string arr := strings.Split(version, ".") if len(arr) >= 3 { @@ -133,18 +133,6 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider string, nee return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, cloudProvider) } } - // TODO: Is this section need? - if objectStoreProvider != cloudProvider { - pluginsForObjectStoreProvider, ok := cloudMap[objectStoreProvider] - if !ok { - return nil, errors.Errorf("fail to get plugins by version: %s and object store provider %s", version, objectStoreProvider) - } - for _, p := range pluginsForObjectStoreProvider { - if !slices.Contains(plugins, p) { - plugins = append(plugins, p) - } - } - } if needDataMoverPlugin { pluginsForDatamover, ok := cloudMap["datamover"] @@ -630,7 +618,7 @@ func getProviderPlugins(ctx context.Context, veleroCLI string, cloudProvider str return nil, errors.WithMessage(err, "failed to get velero version") } - plugins, err := getPluginsByVersion(version, cloudProvider, cloudProvider, false) + plugins, err := getPluginsByVersion(version, cloudProvider, false) if err != nil { return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", cloudProvider, version) } @@ -673,7 +661,7 @@ func getPlugins(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) { if veleroCfg.SnapshotMoveData && veleroCfg.DataMoverPlugin == "" && !veleroCfg.IsUpgradeTest { needDataMoverPlugin = true } - plugins, err = getPluginsByVersion(version, cloudProvider, objectStoreProvider, needDataMoverPlugin) + plugins, err = getPluginsByVersion(version, cloudProvider, needDataMoverPlugin) if err != nil { return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", objectStoreProvider, version) }