diff --git a/test/Makefile b/test/Makefile index 276cc76afd..6c646f5b9f 100644 --- a/test/Makefile +++ b/test/Makefile @@ -41,25 +41,42 @@ help: ## Display this help TOOLS_DIR := $(REPO_ROOT)/hack/tools BIN_DIR := bin + # Try to not modify PATH if possible GOBIN := $(REPO_ROOT)/.go/bin + TOOLS_BIN_DIR := $(TOOLS_DIR)/$(BIN_DIR) + GINKGO := $(GOBIN)/ginkgo + KUSTOMIZE := $(TOOLS_BIN_DIR)/kustomize + OUTPUT_DIR := _output/$(GOOS)/$(GOARCH)/bin + # Please reference to this document for Ginkgo label spec format. # https://onsi.github.io/ginkgo/#spec-labels GINKGO_LABELS ?= + # When --fail-fast is set, the entire suite will stop when the first failure occurs. # Enable --fail-fast by default. # https://onsi.github.io/ginkgo/#mental-model-how-ginkgo-handles-failure FAIL_FAST ?= false + VELERO_CLI ?=$$(pwd)/../_output/bin/$(GOOS)/$(GOARCH)/velero + VELERO_IMAGE ?= velero/velero:main + PLUGINS ?= + +# Flag used to tell the E2E not install the Velero vSphere plugin +# when the provider is set to `vsphere`. +DISABLE_VSPHERE_PLUGIN ?= false + RESTORE_HELPER_IMAGE ?= + #Released version only UPGRADE_FROM_VELERO_VERSION ?= v1.13.2,v1.14.1 + # UPGRADE_FROM_VELERO_CLI can has the same format(a list divided by comma) with UPGRADE_FROM_VELERO_VERSION # Upgrade tests will be executed sequently according to the list by UPGRADE_FROM_VELERO_VERSION # So although length of UPGRADE_FROM_VELERO_CLI list is not equal with UPGRADE_FROM_VELERO_VERSION @@ -150,7 +167,8 @@ COMMON_ARGS := --velerocli=$(VELERO_CLI) \ --velero-server-debug-mode=$(VELERO_SERVER_DEBUG_MODE) \ --uploader-type=$(UPLOADER_TYPE) \ --debug-velero-pod-restart=$(DEBUG_VELERO_POD_RESTART) \ - --fail-fast=$(FAIL_FAST) + --fail-fast=$(FAIL_FAST) \ + --disable-vsphere-plugin=$(DISABLE_VSPHERE_PLUGIN) # Make sure ginkgo is in $GOBIN .PHONY:ginkgo diff --git a/test/e2e/README.md b/test/e2e/README.md index 65231e01af..0d158dbf55 100644 --- a/test/e2e/README.md +++ b/test/e2e/README.md @@ -78,6 +78,7 @@ These configuration parameters are expected as values to the following command l 1. `--standby-cluster-object-store-provider`: Object store provider for standby cluster. 1. `--debug-velero-pod-restart`: A switch for debugging velero pod restart. 1. `--fail-fast`: A switch for for failing fast on meeting error. +1. `--disable-vsphere-plugin`: A switch for not install the Velero vSphere plugin when the provider is set to `vsphere`. These configurations or parameters are used to generate install options for Velero for each test suite. @@ -129,12 +130,13 @@ Below is a mapping between `make` variables to E2E configuration flags. 1. `INSTALL_VELERO `: `-install-velero`. Optional. 1. `DEBUG_VELERO_POD_RESTART`: `-debug-velero-pod-restart`. Optional. 1. `FAIL_FAST`: `--fail-fast`. Optional. +1. `DISABLE_VSPHERE_PLUGIN`: `--diable-vsphere-plugin`. Optional. ### Examples -Basic examples: +#### Basic examples: 1. Run Velero tests in a kind cluster with AWS (or MinIO) as the storage provider: @@ -208,7 +210,7 @@ ADDITIONAL_CREDS_FILE=/path/to/azure-creds \ make test-e2e ``` -Upgrade examples: +#### Upgrade examples: 1. Run Velero upgrade tests with pre-upgrade version: @@ -234,7 +236,7 @@ UPGRADE_FROM_VELERO_VERSION=v1.10.2,v1.11.0 \ make test-e2e ``` -Migration examples: +#### Migration examples: 1. Migration between 2 cluster of the same provider tests: @@ -275,7 +277,7 @@ GINKGO_LABELS="Migration" \ make test-e2e ``` -## 5. Filtering tests +#### Filtering tests In release-1.15, Velero bumps the [Ginkgo](https://onsi.github.io/ginkgo/) version to [v2](https://onsi.github.io/ginkgo/MIGRATING_TO_V2). Velero E2E start to use [labels](https://onsi.github.io/ginkgo/#spec-labels) to filter cases instead of [`-focus` and `-skip`](https://onsi.github.io/ginkgo/#focused-specs) parameters. @@ -285,7 +287,6 @@ Both `make run-e2e` and `make run-perf` CLI support using parameter `GINKGO_LABE `GINKGO_LABELS` is interpreted into `ginkgo run` CLI's parameter [`--label-filter`](https://onsi.github.io/ginkgo/#spec-labels). -### Examples E2E tests can be run with specific cases to be included and/or excluded using the commands below: 1. Run Velero tests with specific cases to be included: @@ -316,6 +317,44 @@ In this example, cases are labelled as * `Migration` and `Restic` will be skipped. +#### VKS environment test +1. Run the CSI data mover test. + +`DISABLE_VSPHERE_PLUGIN` should be set to `true` to not install the Velero vSphere plugin. +``` bash +CLOUD_PROVIDER=vsphere \ +DEFAULT_CLUSTER=wl-antreav1301 \ +STANDBY_CLUSTER=wl-antreav1311 \ +DEFAULT_CLUSTER_NAME=192.168.0.4 \ +STANDBY_CLUSTER_NAME=192.168.0.3 \ +FEATURES=EnableCSI \ +PLUGINS=gcr.io/velero-gcp/velero-plugin-for-aws:main \ +DISABLE_VSPHERE_PLUGIN=true \ +OBJECT_STORE_PROVIDER=aws \ +CREDS_FILE=$HOME/aws-credential \ +BSL_CONFIG=region=us-east-1 \ +BSL_BUCKET=nightly-normal-account4-test \ +BSL_PREFIX=nightly \ +ADDITIONAL_BSL_PLUGINS=gcr.io/velero-gcp/velero-plugin-for-aws:main \ +ADDITIONAL_OBJECT_STORE_PROVIDER=aws \ +ADDITIONAL_BSL_CONFIG=region=us-east-1 \ +ADDITIONAL_BSL_BUCKET=nightly-normal-account4-test \ +ADDITIONAL_BSL_PREFIX=addition-bsl \ +ADDITIONAL_CREDS_FILE=$HOME/aws-credential \ +VELERO_IMAGE=gcr.io/velero-gcp/velero:main \ +RESTORE_HELPER_IMAGE=gcr.io/velero-gcp/velero-restore-helper:main \ +VERSION=main \ +SNAPSHOT_MOVE_DATA=true \ +STANDBY_CLUSTER_CLOUD_PROVIDER=vsphere \ +STANDBY_CLUSTER_OBJECT_STORE_PROVIDER=aws \ +STANDBY_CLUSTER_PLUGINS=gcr.io/velero-gcp/velero-plugin-for-aws:main \ +DISABLE_INFORMER_CACHE=true \ +REGISTRY_CREDENTIAL_FILE=$HOME/.docker/config.json \ +GINKGO_LABELS=Migration \ +KIBISHII_DIRECTORY=$HOME/kibishii/kubernetes/yaml/ \ +make test-e2e +``` + ## 6. Full Tests execution As we provided several examples for E2E test execution, if no filter is involved and despite difference of test environment, diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 1b4cd32dec..c0864ee035 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -103,6 +103,7 @@ func init() { flag.StringVar(&VeleroCfg.DefaultCLSServiceAccountName, "default-cls-service-account-name", "", "default cluster service account name.") flag.StringVar(&VeleroCfg.StandbyCLSServiceAccountName, "standby-cls-service-account-name", "", "standby cluster service account name.") flag.BoolVar(&VeleroCfg.FailFast, "fail-fast", true, "a switch for failing fast on meeting error.") + flag.BoolVar(&VeleroCfg.DisableVspherePlugin, "disable-vsphere-plugin", false, "a switch for not installing vSphere plugin.") } // Add label [SkipVanillaZfs]: diff --git a/test/e2e/migration/migration.go b/test/e2e/migration/migration.go index 8316126e6e..180eac8d3b 100644 --- a/test/e2e/migration/migration.go +++ b/test/e2e/migration/migration.go @@ -158,16 +158,22 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) if OriginVeleroCfg.CloudProvider == AWS { OriginVeleroCfg.Plugins = migrationNeedPlugins[AWS][0] } + // If the CloudProvider is vSphere and don't need vSphere plugin, + // Only install the AWS plugin. + // If do nothing here, the default behavior is + // installing both AWS and vSphere plugins. + if OriginVeleroCfg.CloudProvider == Vsphere && + OriginVeleroCfg.DisableVspherePlugin { + OriginVeleroCfg.Plugins = migrationNeedPlugins[AWS][0] + } // Because Velero CSI plugin is deprecated in v1.14, // only need to install it for version lower than v1.14. if strings.Contains(OriginVeleroCfg.Features, FeatureCSI) && semver.Compare(versionWithoutPatch, "v1.14") < 0 { OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[CSI][0] } - if OriginVeleroCfg.SnapshotMoveData { - if OriginVeleroCfg.CloudProvider == Azure { - OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[AWS][0] - } + if OriginVeleroCfg.SnapshotMoveData && OriginVeleroCfg.CloudProvider == Azure { + OriginVeleroCfg.Plugins = OriginVeleroCfg.Plugins + "," + migrationNeedPlugins[AWS][0] } veleroCLI2Version.VeleroCLI, err = InstallVeleroCLI(veleroCLI2Version.VeleroVersion) Expect(err).To(Succeed()) @@ -253,8 +259,10 @@ func MigrationTest(useVolumeSnapshots bool, veleroCLI2Version VeleroCLI2Version) }) if useVolumeSnapshots { - if veleroCfg.CloudProvider == Vsphere { - // TODO - remove after upload progress monitoring is implemented + // Only wait for the snapshots.backupdriver.cnsdp.vmware.com + // when the provider is vSphere and vSphere plugin is used. + if veleroCfg.CloudProvider == Vsphere && + !veleroCfg.DisableVspherePlugin { By("Waiting for vSphere uploads to complete", func() { Expect(WaitForVSphereUploadCompletion(context.Background(), time.Hour, migrationNamespace, kibishiiWorkerCount)).To(Succeed()) diff --git a/test/testdata/volume-snapshot-class/vsphere.yaml b/test/testdata/volume-snapshot-class/vsphere.yaml new file mode 100644 index 0000000000..08ea9b2225 --- /dev/null +++ b/test/testdata/volume-snapshot-class/vsphere.yaml @@ -0,0 +1,13 @@ +--- +apiVersion: snapshot.storage.k8s.io/v1 +deletionPolicy: Delete +driver: csi.vsphere.vmware.com +kind: VolumeSnapshotClass +metadata: + annotations: + snapshot.storage.kubernetes.io/is-default-class: "true" + labels: + velero.io/csi-volumesnapshot-class: "true" + name: volumesnapshotclass-delete +parameters: + svVolumeSnapshotClass: volumesnapshotclass-delete diff --git a/test/types.go b/test/types.go index 961eebe5dc..792ba7f010 100644 --- a/test/types.go +++ b/test/types.go @@ -118,6 +118,7 @@ type VeleroConfig struct { ServiceAccountNameToInstall string EKSPolicyARN string FailFast bool + DisableVspherePlugin bool } type VeleroCfgInPerf struct { diff --git a/test/util/velero/install.go b/test/util/velero/install.go index 81204a9863..9da1c4b83a 100644 --- a/test/util/velero/install.go +++ b/test/util/velero/install.go @@ -320,7 +320,7 @@ func installVeleroServer(ctx context.Context, cli, cloudProvider string, options if len(options.Features) > 0 { args = append(args, "--features", options.Features) - if !strings.EqualFold(cloudProvider, test.Vsphere) && strings.EqualFold(options.Features, test.FeatureCSI) && options.UseVolumeSnapshots { + if strings.EqualFold(options.Features, test.FeatureCSI) && options.UseVolumeSnapshots { // https://github.com/openebs/zfs-localpv/blob/develop/docs/snapshot.md fmt.Printf("Start to install %s VolumeSnapshotClass ... \n", cloudProvider) if err := k8s.KubectlApplyByFile(ctx, fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", cloudProvider)); err != nil { diff --git a/test/util/velero/velero_utils.go b/test/util/velero/velero_utils.go index cec0d61869..38ada7a9e9 100644 --- a/test/util/velero/velero_utils.go +++ b/test/util/velero/velero_utils.go @@ -107,7 +107,7 @@ var PluginsMatrix = map[string]map[string][]string{ }, } -func getPluginsByVersion(version, cloudProvider, objectStoreProvider string, needDataMoverPlugin bool) ([]string, error) { +func getPluginsByVersion(version string, cloudProvider string, needDataMoverPlugin bool) ([]string, error) { var cloudMap map[string][]string arr := strings.Split(version, ".") if len(arr) >= 3 { @@ -133,18 +133,6 @@ func getPluginsByVersion(version, cloudProvider, objectStoreProvider string, nee return nil, errors.Errorf("fail to get plugins by version: %s and provider %s", version, cloudProvider) } } - // TODO: Is this section need? - if objectStoreProvider != cloudProvider { - pluginsForObjectStoreProvider, ok := cloudMap[objectStoreProvider] - if !ok { - return nil, errors.Errorf("fail to get plugins by version: %s and object store provider %s", version, objectStoreProvider) - } - for _, p := range pluginsForObjectStoreProvider { - if !slices.Contains(plugins, p) { - plugins = append(plugins, p) - } - } - } if needDataMoverPlugin { pluginsForDatamover, ok := cloudMap["datamover"] @@ -630,7 +618,7 @@ func getProviderPlugins(ctx context.Context, veleroCLI string, cloudProvider str return nil, errors.WithMessage(err, "failed to get velero version") } - plugins, err := getPluginsByVersion(version, cloudProvider, cloudProvider, false) + plugins, err := getPluginsByVersion(version, cloudProvider, false) if err != nil { return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", cloudProvider, version) } @@ -673,7 +661,7 @@ func getPlugins(ctx context.Context, veleroCfg VeleroConfig) ([]string, error) { if veleroCfg.SnapshotMoveData && veleroCfg.DataMoverPlugin == "" && !veleroCfg.IsUpgradeTest { needDataMoverPlugin = true } - plugins, err = getPluginsByVersion(version, cloudProvider, objectStoreProvider, needDataMoverPlugin) + plugins, err = getPluginsByVersion(version, cloudProvider, needDataMoverPlugin) if err != nil { return nil, errors.WithMessagef(err, "Fail to get plugin by provider %s and version %s", objectStoreProvider, version) }