From effbcba52190e101cc01860dbb9bd9670352f712 Mon Sep 17 00:00:00 2001 From: Xun Jiang Date: Tue, 12 Dec 2023 09:50:55 +0800 Subject: [PATCH] Add E2E test cases for backup VolumeInfo feature. Signed-off-by: Xun Jiang --- .../velero-restore-helper.go | 2 +- test/e2e/basic/backup-volume-info/base.go | 148 ++++++++++++++++++ .../backup-volume-info/csi_data_mover.go | 60 +++++++ .../basic/backup-volume-info/csi_snapshot.go | 59 +++++++ .../backup-volume-info/filesystem_upload.go | 59 +++++++ .../backup-volume-info/native_snapshot.go | 59 +++++++ .../backup-volume-info/skipped_volumes.go | 59 +++++++ test/e2e/e2e_suite_test.go | 8 + test/testdata/storage-class/aws-csi.yaml | 9 ++ test/testdata/storage-class/azure-csi.yaml | 11 ++ test/testdata/storage-class/gcp-csi.yaml | 13 ++ test/testdata/volume-snapshot-class/aws.yaml | 9 ++ .../testdata/volume-snapshot-class/azure.yaml | 9 ++ test/testdata/volume-snapshot-class/gcp.yaml | 9 ++ test/types.go | 1 + test/util/k8s/common.go | 6 + test/util/providers/aws_utils.go | 54 +++++++ test/util/providers/azure_utils.go | 30 ++++ test/util/providers/common.go | 61 ++++++++ test/util/providers/gcloud_utils.go | 12 ++ 20 files changed, 677 insertions(+), 1 deletion(-) create mode 100644 test/e2e/basic/backup-volume-info/base.go create mode 100644 test/e2e/basic/backup-volume-info/csi_data_mover.go create mode 100644 test/e2e/basic/backup-volume-info/csi_snapshot.go create mode 100644 test/e2e/basic/backup-volume-info/filesystem_upload.go create mode 100644 test/e2e/basic/backup-volume-info/native_snapshot.go create mode 100644 test/e2e/basic/backup-volume-info/skipped_volumes.go create mode 100644 test/testdata/storage-class/aws-csi.yaml create mode 100644 test/testdata/storage-class/azure-csi.yaml create mode 100644 test/testdata/storage-class/gcp-csi.yaml create mode 100644 test/testdata/volume-snapshot-class/aws.yaml create mode 100644 test/testdata/volume-snapshot-class/azure.yaml create mode 100644 test/testdata/volume-snapshot-class/gcp.yaml diff --git a/cmd/velero-restore-helper/velero-restore-helper.go b/cmd/velero-restore-helper/velero-restore-helper.go index 748f1e6c88..d310f807e9 100644 --- a/cmd/velero-restore-helper/velero-restore-helper.go +++ b/cmd/velero-restore-helper/velero-restore-helper.go @@ -73,7 +73,7 @@ func done() bool { return false } - fmt.Printf("Found %s", doneFile) + fmt.Printf("Found the done file %s\n", doneFile) } return true diff --git a/test/e2e/basic/backup-volume-info/base.go b/test/e2e/basic/backup-volume-info/base.go new file mode 100644 index 0000000000..8b60e3b03a --- /dev/null +++ b/test/e2e/basic/backup-volume-info/base.go @@ -0,0 +1,148 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package basic + +import ( + "context" + "fmt" + "strconv" + "strings" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/pkg/errors" + + . "github.com/vmware-tanzu/velero/test" + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/util/k8s" + . "github.com/vmware-tanzu/velero/test/util/velero" +) + +type BackupVolumeInfo struct { + TestCase + SnapshotVolumes bool + DefaultVolumesToFSBackup bool + SnapshotMoveData bool + TimeoutDuration time.Duration +} + +func (v *BackupVolumeInfo) Init() error { + v.TestCase.Init() + + BeforeEach(func() { + if strings.Contains(v.VeleroCfg.Features, "EnableCSI") { + if strings.Contains(v.CaseBaseName, "native-snapshot") { + fmt.Printf("Skip native snapshot case %s when the CSI feature is enabled.\n", v.CaseBaseName) + Skip("Skip due to vSphere CSI driver long time issue of Static provisioning") + } + } else { + if strings.Contains(v.CaseBaseName, "csi") { + fmt.Printf("Skip CSI related case %s when the CSI feature is not enabled.\n", v.CaseBaseName) + Skip("Skip due to vSphere CSI driver long time issue of Static provisioning") + } + } + }) + + v.CaseBaseName = v.CaseBaseName + v.UUIDgen + v.BackupName = "backup-" + v.CaseBaseName + v.RestoreName = "restore-" + v.CaseBaseName + v.TimeoutDuration = 10 * time.Minute + v.NamespacesTotal = 1 + + v.VeleroCfg = VeleroCfg + v.Client = *v.VeleroCfg.ClientToInstallVelero + v.NSIncluded = &[]string{v.CaseBaseName} + + v.TestMsg = &TestMSG{ + Desc: "Test backup's VolumeInfo metadata content", + Text: "The VolumeInfo should be generated based on the backup type", + FailedMSG: "Failed to verify the backup VolumeInfo's content", + } + + v.BackupArgs = []string{ + "backup", "create", v.BackupName, + "--namespace", v.VeleroCfg.VeleroNamespace, + "--include-namespaces", v.CaseBaseName, + "--snapshot-volumes" + "=" + strconv.FormatBool(v.SnapshotVolumes), + "--default-volumes-to-fs-backup" + "=" + strconv.FormatBool(v.DefaultVolumesToFSBackup), + "--snapshot-move-data" + "=" + strconv.FormatBool(v.SnapshotMoveData), + "--wait", + } + + v.RestoreArgs = []string{ + "create", "--namespace", v.VeleroCfg.VeleroNamespace, "restore", v.RestoreName, + "--from-backup", v.BackupName, "--wait", + } + return nil +} + +func (v *BackupVolumeInfo) CreateResources() error { + v.Ctx, v.CtxCancel = context.WithTimeout(context.Background(), v.TimeoutDuration) + labels := map[string]string{ + "volume-info": "true", + } + for nsNum := 0; nsNum < v.NamespacesTotal; nsNum++ { + fmt.Printf("Creating namespaces ...\n") + createNSName := v.CaseBaseName + if err := CreateNamespaceWithLabel(v.Ctx, v.Client, createNSName, labels); err != nil { + return errors.Wrapf(err, "Failed to create namespace %s", createNSName) + } + + Expect(InstallTestStorageClasses(fmt.Sprintf("../testdata/storage-class/%s-csi.yaml", v.VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install StorageClass") + + // Install VolumeSnapshotClass + Expect(KubectlApplyByFile(v.Ctx, fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", v.VeleroCfg.CloudProvider))).To(Succeed(), "Failed to install VolumeSnapshotClass") + + pvc, err := CreatePVC(v.Client, createNSName, "volume-info", CSIStorageClassName, nil) + Expect(err).To(Succeed()) + vols := CreateVolumes(pvc.Name, []string{"volume-info"}) + + //Create deployment + fmt.Printf("Creating deployment in namespaces ...%s\n", createNSName) + deployment := NewDeployment(v.CaseBaseName, createNSName, 1, labels, nil).WithVolume(vols).Result() + deployment, err = CreateDeployment(v.Client.ClientGo, createNSName, deployment) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to delete the namespace %q", createNSName)) + } + err = WaitForReadyDeployment(v.Client.ClientGo, createNSName, deployment.Name) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to ensure job completion in namespace: %q", createNSName)) + } + } + return nil +} + +func (v *BackupVolumeInfo) Destroy() error { + err := CleanupNamespaces(v.Ctx, v.Client, v.CaseBaseName) + if err != nil { + return errors.Wrap(err, "Could cleanup retrieve namespaces") + } + + return WaitAllSelectedNSDeleted(v.Ctx, v.Client, "ns-test=true") +} + +func (v *BackupVolumeInfo) cleanResource() error { + if err := DeleteStorageClass(v.Ctx, v.Client, CSIStorageClassName); err != nil { + return errors.Wrap(err, "fail to delete the StorageClass") + } + + if err := KubectlDeleteByFile(v.Ctx, fmt.Sprintf("../testdata/volume-snapshot-class/%s.yaml", v.VeleroCfg.CloudProvider)); err != nil { + return errors.Wrap(err, "fail to delete the VolumeSnapshotClass") + } + return nil +} diff --git a/test/e2e/basic/backup-volume-info/csi_data_mover.go b/test/e2e/basic/backup-volume-info/csi_data_mover.go new file mode 100644 index 0000000000..b18412ef73 --- /dev/null +++ b/test/e2e/basic/backup-volume-info/csi_data_mover.go @@ -0,0 +1,60 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package basic + +import ( + "fmt" + + . "github.com/onsi/gomega" + + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/util/providers" + . "github.com/vmware-tanzu/velero/test/util/velero" +) + +var CSIDataMoverVolumeInfoTest func() = TestFunc(&CSIDataMoverVolumeInfo{ + BackupVolumeInfo{ + SnapshotMoveData: true, + SnapshotVolumes: true, + TestCase: TestCase{ + CaseBaseName: "csi-data-mover-volumeinfo", + }, + }, +}) + +type CSIDataMoverVolumeInfo struct { + BackupVolumeInfo +} + +func (c *CSIDataMoverVolumeInfo) Verify() error { + volumeInfo, err := GetVolumeInfo( + c.VeleroCfg.CloudProvider, + c.VeleroCfg.CloudCredentialsFile, + c.VeleroCfg.BSLBucket, + c.VeleroCfg.BSLPrefix, + c.VeleroCfg.BSLConfig, + c.BackupName, + BackupObjectsPrefix+"/"+c.BackupName, + ) + + Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Fail to get VolumeInfo metadata in the Backup Repository.")) + Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true)) + Expect(volumeInfo[0].SnapshotDataMovementInfo).NotTo(BeNil()) + + // Clean SC and VSC + return c.cleanResource() +} diff --git a/test/e2e/basic/backup-volume-info/csi_snapshot.go b/test/e2e/basic/backup-volume-info/csi_snapshot.go new file mode 100644 index 0000000000..9fa8bb3afe --- /dev/null +++ b/test/e2e/basic/backup-volume-info/csi_snapshot.go @@ -0,0 +1,59 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package basic + +import ( + "fmt" + + . "github.com/onsi/gomega" + + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/util/providers" + . "github.com/vmware-tanzu/velero/test/util/velero" +) + +var CSISnapshotVolumeInfoTest func() = TestFunc(&CSISnapshotVolumeInfo{ + BackupVolumeInfo{ + SnapshotVolumes: true, + TestCase: TestCase{ + CaseBaseName: "csi-snapshot-volumeinfo", + }, + }, +}) + +type CSISnapshotVolumeInfo struct { + BackupVolumeInfo +} + +func (c *CSISnapshotVolumeInfo) Verify() error { + volumeInfo, err := GetVolumeInfo( + c.VeleroCfg.CloudProvider, + c.VeleroCfg.CloudCredentialsFile, + c.VeleroCfg.BSLBucket, + c.VeleroCfg.BSLPrefix, + c.VeleroCfg.BSLConfig, + c.BackupName, + BackupObjectsPrefix+"/"+c.BackupName, + ) + + Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Fail to get VolumeInfo metadata in the Backup Repository.")) + Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true)) + Expect(volumeInfo[0].CSISnapshotInfo).NotTo(BeNil()) + + // Clean SC and VSC + return c.cleanResource() +} diff --git a/test/e2e/basic/backup-volume-info/filesystem_upload.go b/test/e2e/basic/backup-volume-info/filesystem_upload.go new file mode 100644 index 0000000000..a6ff9dc547 --- /dev/null +++ b/test/e2e/basic/backup-volume-info/filesystem_upload.go @@ -0,0 +1,59 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package basic + +import ( + "fmt" + + . "github.com/onsi/gomega" + + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/util/providers" + . "github.com/vmware-tanzu/velero/test/util/velero" +) + +var FilesystemUploadVolumeInfoTest func() = TestFunc(&FilesystemUploadVolumeInfo{ + BackupVolumeInfo{ + DefaultVolumesToFSBackup: true, + TestCase: TestCase{ + CaseBaseName: "fs-upload-volumeinfo", + }, + }, +}) + +type FilesystemUploadVolumeInfo struct { + BackupVolumeInfo +} + +func (f *FilesystemUploadVolumeInfo) Verify() error { + volumeInfo, err := GetVolumeInfo( + f.VeleroCfg.CloudProvider, + f.VeleroCfg.CloudCredentialsFile, + f.VeleroCfg.BSLBucket, + f.VeleroCfg.BSLPrefix, + f.VeleroCfg.BSLConfig, + f.BackupName, + BackupObjectsPrefix+"/"+f.BackupName, + ) + + Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Fail to get VolumeInfo metadata in the Backup Repository.")) + Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true)) + Expect(volumeInfo[0].PVBInfo).NotTo(BeNil()) + + // Clean SC and VSC + return f.cleanResource() +} diff --git a/test/e2e/basic/backup-volume-info/native_snapshot.go b/test/e2e/basic/backup-volume-info/native_snapshot.go new file mode 100644 index 0000000000..ef3d37b1c6 --- /dev/null +++ b/test/e2e/basic/backup-volume-info/native_snapshot.go @@ -0,0 +1,59 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package basic + +import ( + "fmt" + + . "github.com/onsi/gomega" + + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/util/providers" + . "github.com/vmware-tanzu/velero/test/util/velero" +) + +var NativeSnapshotVolumeInfoTest func() = TestFunc(&NativeSnapshotVolumeInfo{ + BackupVolumeInfo{ + SnapshotVolumes: true, + TestCase: TestCase{ + CaseBaseName: "native-snapshot-volumeinfo", + }, + }, +}) + +type NativeSnapshotVolumeInfo struct { + BackupVolumeInfo +} + +func (n *NativeSnapshotVolumeInfo) Verify() error { + volumeInfo, err := GetVolumeInfo( + n.VeleroCfg.CloudProvider, + n.VeleroCfg.CloudCredentialsFile, + n.VeleroCfg.BSLBucket, + n.VeleroCfg.BSLPrefix, + n.VeleroCfg.BSLConfig, + n.BackupName, + BackupObjectsPrefix+"/"+n.BackupName, + ) + + Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Fail to get VolumeInfo metadata in the Backup Repository.")) + Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true)) + Expect(volumeInfo[0].NativeSnapshotInfo).NotTo(BeNil()) + + // Clean SC and VSC + return n.cleanResource() +} diff --git a/test/e2e/basic/backup-volume-info/skipped_volumes.go b/test/e2e/basic/backup-volume-info/skipped_volumes.go new file mode 100644 index 0000000000..73a1c9be9a --- /dev/null +++ b/test/e2e/basic/backup-volume-info/skipped_volumes.go @@ -0,0 +1,59 @@ +/* +Copyright the Velero contributors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package basic + +import ( + "fmt" + + . "github.com/onsi/gomega" + + . "github.com/vmware-tanzu/velero/test/e2e/test" + . "github.com/vmware-tanzu/velero/test/util/providers" + . "github.com/vmware-tanzu/velero/test/util/velero" +) + +var SkippedVolumeInfoTest func() = TestFunc(&SkippedVolumeInfo{ + BackupVolumeInfo{ + SnapshotVolumes: false, + TestCase: TestCase{ + CaseBaseName: "skipped-volumes-volumeinfo", + }, + }, +}) + +type SkippedVolumeInfo struct { + BackupVolumeInfo +} + +func (s *SkippedVolumeInfo) Verify() error { + volumeInfo, err := GetVolumeInfo( + s.VeleroCfg.CloudProvider, + s.VeleroCfg.CloudCredentialsFile, + s.VeleroCfg.BSLBucket, + s.VeleroCfg.BSLPrefix, + s.VeleroCfg.BSLConfig, + s.BackupName, + BackupObjectsPrefix+"/"+s.BackupName, + ) + + Expect(err).ShouldNot(HaveOccurred(), fmt.Sprintf("Fail to get VolumeInfo metadata in the Backup Repository.")) + Expect(len(volumeInfo) > 0).To(BeIdenticalTo(true)) + Expect(volumeInfo[0].Skipped == true).To(BeIdenticalTo(true)) + + // Clean SC and VSC + return s.cleanResource() +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index 7c7da5c5c6..aa205c39ca 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -34,6 +34,7 @@ import ( . "github.com/vmware-tanzu/velero/test/e2e/backups" . "github.com/vmware-tanzu/velero/test/e2e/basic" . "github.com/vmware-tanzu/velero/test/e2e/basic/api-group" + . "github.com/vmware-tanzu/velero/test/e2e/basic/backup-volume-info" . "github.com/vmware-tanzu/velero/test/e2e/basic/resources-check" . "github.com/vmware-tanzu/velero/test/e2e/bsl-mgmt" . "github.com/vmware-tanzu/velero/test/e2e/migration" @@ -136,6 +137,13 @@ var _ = Describe("[ResourceFiltering][IncludeResources][Restore] Velero test on var _ = Describe("[ResourceFiltering][LabelSelector] Velero test on backup include resources matching the label selector", BackupWithLabelSelector) var _ = Describe("[ResourceFiltering][ResourcePolicies][Restic] Velero test on skip backup of volume by resource policies", ResourcePoliciesTest) +// backup VolumeInfo test +var _ = Describe("[BackupVolumeInfo][SkippedVolume]", SkippedVolumeInfoTest) +var _ = Describe("[BackupVolumeInfo][FilesystemUpload]", FilesystemUploadVolumeInfoTest) +var _ = Describe("[BackupVolumeInfo][CSIDataMover]", CSIDataMoverVolumeInfoTest) +var _ = Describe("[BackupVolumeInfo][CSISnapshot]", CSISnapshotVolumeInfoTest) +var _ = Describe("[BackupVolumeInfo][NativeSnapshot]", NativeSnapshotVolumeInfoTest) + var _ = Describe("[ResourceModifier][Restore] Velero test on resource modifiers from the cluster restore", ResourceModifiersTest) var _ = Describe("[Backups][Deletion][Restic] Velero tests of Restic backup deletion", BackupDeletionWithRestic) diff --git a/test/testdata/storage-class/aws-csi.yaml b/test/testdata/storage-class/aws-csi.yaml new file mode 100644 index 0000000000..48c95fdc9c --- /dev/null +++ b/test/testdata/storage-class/aws-csi.yaml @@ -0,0 +1,9 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: e2e-csi-storage-class +provisioner: ebs.csi.aws.com +parameters: + type: gp2 +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer diff --git a/test/testdata/storage-class/azure-csi.yaml b/test/testdata/storage-class/azure-csi.yaml new file mode 100644 index 0000000000..5ef573b48b --- /dev/null +++ b/test/testdata/storage-class/azure-csi.yaml @@ -0,0 +1,11 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: e2e-csi-storage-class +provisioner: disk.csi.azure.com +parameters: + cachingmode: ReadOnly + kind: Managed + storageaccounttype: StandardSSD_LRS +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer diff --git a/test/testdata/storage-class/gcp-csi.yaml b/test/testdata/storage-class/gcp-csi.yaml new file mode 100644 index 0000000000..399061fa84 --- /dev/null +++ b/test/testdata/storage-class/gcp-csi.yaml @@ -0,0 +1,13 @@ +allowVolumeExpansion: true +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + labels: + addonmanager.kubernetes.io/mode: EnsureExists + name: e2e-csi-storage-class +parameters: + type: pd-standard +provisioner: pd.csi.storage.gke.io +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer + diff --git a/test/testdata/volume-snapshot-class/aws.yaml b/test/testdata/volume-snapshot-class/aws.yaml new file mode 100644 index 0000000000..005b48f509 --- /dev/null +++ b/test/testdata/volume-snapshot-class/aws.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: snapshot.storage.k8s.io/v1 +deletionPolicy: Delete +driver: ebs.csi.aws.com +kind: VolumeSnapshotClass +metadata: + labels: + velero.io/csi-volumesnapshot-class: "true" + name: e2e-volume-snapshot-class \ No newline at end of file diff --git a/test/testdata/volume-snapshot-class/azure.yaml b/test/testdata/volume-snapshot-class/azure.yaml new file mode 100644 index 0000000000..f04c7c9371 --- /dev/null +++ b/test/testdata/volume-snapshot-class/azure.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: snapshot.storage.k8s.io/v1 +deletionPolicy: Delete +driver: disk.csi.azure.com +kind: VolumeSnapshotClass +metadata: + labels: + velero.io/csi-volumesnapshot-class: "true" + name: e2e-volume-snapshot-class \ No newline at end of file diff --git a/test/testdata/volume-snapshot-class/gcp.yaml b/test/testdata/volume-snapshot-class/gcp.yaml new file mode 100644 index 0000000000..f8e958d5ae --- /dev/null +++ b/test/testdata/volume-snapshot-class/gcp.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: snapshot.storage.k8s.io/v1 +deletionPolicy: Delete +driver: pd.csi.storage.gke.io +kind: VolumeSnapshotClass +metadata: + labels: + velero.io/csi-volumesnapshot-class: "true" + name: e2e-volume-snapshot-class \ No newline at end of file diff --git a/test/types.go b/test/types.go index 4b1d78485e..2a4e06f291 100644 --- a/test/types.go +++ b/test/types.go @@ -27,6 +27,7 @@ import ( const StorageClassName = "e2e-storage-class" const StorageClassName2 = "e2e-storage-class-2" +const CSIStorageClassName = "e2e-csi-storage-class" const FeatureCSI = "EnableCSI" var InstallVelero bool diff --git a/test/util/k8s/common.go b/test/util/k8s/common.go index da439f24c7..4f98f26340 100644 --- a/test/util/k8s/common.go +++ b/test/util/k8s/common.go @@ -204,6 +204,12 @@ func KubectlApplyByFile(ctx context.Context, file string) error { return exec.CommandContext(ctx, "kubectl", args...).Run() } +func KubectlDeleteByFile(ctx context.Context, file string) error { + args := []string{"delete", "-f", file, "--force=true"} + fmt.Println(args) + return exec.CommandContext(ctx, "kubectl", args...).Run() +} + func KubectlConfigUseContext(ctx context.Context, kubectlContext string) error { cmd := exec.CommandContext(ctx, "kubectl", "config", "use-context", kubectlContext) diff --git a/test/util/providers/aws_utils.go b/test/util/providers/aws_utils.go index 10ce13e678..116c6b351c 100644 --- a/test/util/providers/aws_utils.go +++ b/test/util/providers/aws_utils.go @@ -20,6 +20,7 @@ import ( "crypto/tls" "crypto/x509" "fmt" + "io" "net/http" "net/url" "os" @@ -415,3 +416,56 @@ func (s AWSStorage) GetMinioBucketSize(cloudCredentialsFile, bslBucket, bslPrefi } return totalSize, nil } + +func (s AWSStorage) GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, objectKey string) (io.ReadCloser, error) { + config := flag.NewMap() + config.Set(bslConfig) + objectsInput := s3.ListObjectsV2Input{} + objectsInput.Bucket = aws.String(bslBucket) + objectsInput.Delimiter = aws.String("/") + + if bslPrefix != "" { + objectsInput.Prefix = aws.String(bslPrefix) + } + + var err error + var s3Config aws.Config + var s3Client *s3.Client + region := config.Data()["region"] + s3url := "" + if region == "" { + region, err = GetBucketRegion(bslBucket) + if err != nil { + return nil, errors.Wrapf(err, "failed to get region for bucket %s", bslBucket) + } + } + if region == "minio" { + s3url = config.Data()["s3Url"] + s3Config, err = newAWSConfig(region, "", cloudCredentialsFile, true, "") + if err != nil { + return nil, errors.Wrapf(err, "Failed to create AWS config of region %s", region) + } + s3Client, err = newS3Client(s3Config, s3url, true) + } else { + s3Config, err = newAWSConfig(region, "", cloudCredentialsFile, false, "") + if err != nil { + return nil, errors.Wrapf(err, "Failed to create AWS config of region %s", region) + } + s3Client, err = newS3Client(s3Config, s3url, true) + } + if err != nil { + return nil, errors.Wrapf(err, "failed to create S3 client of region %s", region) + } + + fullObjectKey := strings.Trim(bslPrefix, "/") + "/" + strings.Trim(objectKey, "/") + + result, err := s3Client.GetObject(context.TODO(), &s3.GetObjectInput{ + Bucket: aws.String(bslBucket), + Key: aws.String(fullObjectKey), + }) + if err != nil { + return nil, errors.Wrapf(err, "failed to get object %s", fullObjectKey) + } + + return result.Body, nil +} diff --git a/test/util/providers/azure_utils.go b/test/util/providers/azure_utils.go index af7339905f..06063dcfa4 100644 --- a/test/util/providers/azure_utils.go +++ b/test/util/providers/azure_utils.go @@ -18,6 +18,7 @@ package providers import ( "fmt" + "io" "log" "net/url" "os" @@ -364,3 +365,32 @@ func (s AzureStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupN return nil } } + +func (s AzureStorage) GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, objectKey string) (io.ReadCloser, error) { + ctx := context.Background() + accountName, accountKey, err := getStorageCredential(cloudCredentialsFile, bslConfig) + if err != nil { + return nil, errors.Wrapf(err, "Fail to get storage account name and key of bucket %s", bslBucket) + } + + credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) + if err != nil { + log.Fatal("Invalid credentials with error: " + err.Error()) + } + p := azblob.NewPipeline(credential, azblob.PipelineOptions{}) + + URL, _ := url.Parse( + fmt.Sprintf("https://%s.blob.core.windows.net/%s", accountName, bslBucket)) + + containerURL := azblob.NewContainerURL(*URL, p) + _, err = containerURL.Create(ctx, azblob.Metadata{}, azblob.PublicAccessNone) + handleErrors(err) + + blobURL := containerURL.NewBlockBlobURL(strings.Join([]string{bslPrefix, objectKey}, "/")) + downloadResponse, err := blobURL.Download(ctx, 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{}) + if err != nil { + handleErrors(err) + } + + return downloadResponse.Body(azblob.RetryReaderOptions{}), nil +} diff --git a/test/util/providers/common.go b/test/util/providers/common.go index 13b1cf8eef..6bd37dcb3e 100644 --- a/test/util/providers/common.go +++ b/test/util/providers/common.go @@ -17,13 +17,17 @@ limitations under the License. package providers import ( + "compress/gzip" "context" + "encoding/json" "fmt" + "io" "strings" "time" "github.com/pkg/errors" + "github.com/vmware-tanzu/velero/internal/volume" . "github.com/vmware-tanzu/velero/test" velero "github.com/vmware-tanzu/velero/test/util/velero" ) @@ -32,6 +36,7 @@ type ObjectsInStorage interface { IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) (bool, error) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupName string, snapshotCheck SnapshotCheckPoint) error + GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, objectKey string) (io.ReadCloser, error) } func ObjectsShouldBeInBucket(objectStoreProvider, cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupName, subPrefix string) error { @@ -189,3 +194,59 @@ func IsSnapshotExisted(cloudProvider, cloudCredentialsFile, bslBucket, bslConfig } return nil } + +func GetVolumeInfoMetadataContent( + cloudProvider, + cloudCredentialsFile, + bslBucket, + bslPrefix, + bslConfig, + backupName, + subPrefix string, +) (io.Reader, error) { + bslPrefix = strings.Trim(getFullPrefix(bslPrefix, subPrefix), "/") + volumeFileName := backupName + "-volumeinfo.json.gz" + fmt.Printf("|| VERIFICATION || - Get backup %s volumeinfo file in storage %s\n", backupName, bslPrefix) + s, err := getProvider(cloudProvider) + if err != nil { + return nil, errors.Wrapf(err, fmt.Sprintf("Cloud provider %s is not valid", cloudProvider)) + } + + return s.GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, volumeFileName) +} + +func GetVolumeInfo( + cloudProvider, + cloudCredentialsFile, + bslBucket, + bslPrefix, + bslConfig, + backupName, + subPrefix string, +) ([]*volume.VolumeInfo, error) { + readCloser, err := GetVolumeInfoMetadataContent(cloudProvider, + cloudCredentialsFile, + bslBucket, + bslPrefix, + bslConfig, + backupName, + subPrefix, + ) + if err != nil { + return nil, err + } + + gzr, err := gzip.NewReader(readCloser) + if err != nil { + return nil, errors.WithStack(err) + } + defer gzr.Close() + + volumeInfos := make([]*volume.VolumeInfo, 0) + + if err := json.NewDecoder(gzr).Decode(&volumeInfos); err != nil { + return nil, errors.Wrap(err, "error decoding object data") + } + + return volumeInfos, nil +} diff --git a/test/util/providers/gcloud_utils.go b/test/util/providers/gcloud_utils.go index 6702f37ba4..f653aa02af 100644 --- a/test/util/providers/gcloud_utils.go +++ b/test/util/providers/gcloud_utils.go @@ -19,6 +19,7 @@ package providers import ( "encoding/json" "fmt" + "io" "os" "strings" @@ -65,6 +66,7 @@ func (s GCSStorage) IsObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix } } } + func (s GCSStorage) DeleteObjectsInBucket(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, backupObject string) error { q := &storage.Query{ Prefix: bslPrefix, @@ -141,3 +143,13 @@ func (s GCSStorage) IsSnapshotExisted(cloudCredentialsFile, bslConfig, backupObj return nil } } + +func (s GCSStorage) GetObject(cloudCredentialsFile, bslBucket, bslPrefix, bslConfig, objectKey string) (io.ReadCloser, error) { + ctx := context.Background() + client, err := storage.NewClient(ctx, option.WithCredentialsFile(cloudCredentialsFile)) + if err != nil { + return nil, errors.Wrapf(err, "fail to create GCloud client") + } + + return client.Bucket(bslBucket).Object(strings.Join([]string{bslPrefix, objectKey}, "/")).NewReader(ctx) +}