From b52144b051fd3fcee19dc6fd09fffde7976db46c Mon Sep 17 00:00:00 2001 From: Drew Sirenko <68304519+AndrewSirenko@users.noreply.github.com> Date: Wed, 4 Oct 2023 09:05:50 -0400 Subject: [PATCH] Add e2e tests for format options --- tests/e2e/README.md | 7 +- tests/e2e/driver/ebs_csi_driver.go | 6 +- tests/e2e/format_options.go | 89 ++++++++++++++++++ ...ically_provisioned_resize_volume_tester.go | 45 +++++---- tests/e2e/testsuites/format_options_tester.go | 93 +++++++++++++++++++ tests/e2e/testsuites/specs.go | 11 +-- 6 files changed, 224 insertions(+), 27 deletions(-) create mode 100644 tests/e2e/format_options.go create mode 100644 tests/e2e/testsuites/format_options_tester.go diff --git a/tests/e2e/README.md b/tests/e2e/README.md index c61904d618..b4b819a3f7 100644 --- a/tests/e2e/README.md +++ b/tests/e2e/README.md @@ -1,5 +1,5 @@ ## E2E Testing -E2E test verifies the funcitonality of EBS CSI driver in the context of Kubernetes. It exercises driver feature e2e including static provisioning, dynamic provisioning, volume scheduling, mount options, etc. +E2E test verifies the functionality of EBS CSI driver in the context of Kubernetes. It exercises driver feature e2e including static provisioning, dynamic provisioning, volume scheduling, mount options, etc. ### Requirements 1. AWS credential is [configured](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html) @@ -10,6 +10,11 @@ E2E test verifies the funcitonality of EBS CSI driver in the context of Kubernet 1. sed 1. Golang 1.11+ +### Running a specific test +1. Make sure you have a cluster up with ebs csi driver deployed +2. Current directory is `/aws-ebs-csi-driver/tests/e2e` +3. Run `'ginkgo run --focus='should create multiple PV objects, bind to PVCs and attach all to a single pod'` + ### Notes Some tests marked with `[env]` require specific environmental variables to be set, if not set these tests will be skipped. diff --git a/tests/e2e/driver/ebs_csi_driver.go b/tests/e2e/driver/ebs_csi_driver.go index 265034dd8b..9bdca76bcd 100644 --- a/tests/e2e/driver/ebs_csi_driver.go +++ b/tests/e2e/driver/ebs_csi_driver.go @@ -103,7 +103,7 @@ func (d *ebsCSIDriver) GetPersistentVolume(volumeID string, fsType string, size } // GetParameters returns the parameters specific for this driver -func GetParameters(volumeType string, fsType string, encrypted bool) map[string]string { +func GetParameters(volumeType string, fsType string, encrypted bool, additionalParameters map[string]string) map[string]string { parameters := map[string]string{ "type": volumeType, "csi.storage.k8s.io/fstype": fsType, @@ -120,6 +120,10 @@ func GetParameters(volumeType string, fsType string, encrypted bool) map[string] if encrypted { parameters[ebscsidriver.EncryptedKey] = True } + for k, v := range additionalParameters { + parameters[k] = v + } + return parameters } diff --git a/tests/e2e/format_options.go b/tests/e2e/format_options.go new file mode 100644 index 0000000000..d2e44b2b6b --- /dev/null +++ b/tests/e2e/format_options.go @@ -0,0 +1,89 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + ebscsidriver "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/testsuites" + . "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + admissionapi "k8s.io/pod-security-admission/api" +) + +var ( + testedFsTypes = []string{ebscsidriver.FSTypeExt4} + + formatOptionTests = []testsuites.FormatOptionTest{ + { + CreateVolumeParameterKey: ebscsidriver.BlockSizeKey, + CreateVolumeParameterValue: "1024", + }, + { + CreateVolumeParameterKey: ebscsidriver.INodeSizeKey, + CreateVolumeParameterValue: "512", + }, + { + CreateVolumeParameterKey: ebscsidriver.BytesPerINodeKey, + CreateVolumeParameterValue: "8192", + }, + { + CreateVolumeParameterKey: ebscsidriver.NumberOfINodesKey, + CreateVolumeParameterValue: "200192", + }, + } +) + +var _ = Describe("[ebs-csi-e2e] [single-az] [format-options] Formatting a volume", func() { + f := framework.NewDefaultFramework("ebs") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + + var ( + cs clientset.Interface + ns *v1.Namespace + ebsDriver driver.PVTestDriver + ) + + BeforeEach(func() { + cs = f.ClientSet + ns = f.Namespace + ebsDriver = driver.InitEbsCSIDriver() + }) + + for _, fsType := range testedFsTypes { + Context(fmt.Sprintf("with an %s filesystem", fsType), func() { + for _, formatOptionTestCase := range formatOptionTests { + formatOptionTestCase := formatOptionTestCase // Go trap + if fsTypeDoesNotSupportFormatOptionParameter(fsType, formatOptionTestCase.CreateVolumeParameterKey) { + continue + } + + Context(fmt.Sprintf("with a custom %s parameter", formatOptionTestCase.CreateVolumeParameterKey), func() { + It("successfully mounts and is resizable", func() { + formatOptionTestCase.Run(cs, ns, ebsDriver, fsType) + }) + }) + } + }) + } +}) + +func fsTypeDoesNotSupportFormatOptionParameter(fsType string, createVolumeParameterKey string) bool { + _, paramNotSupported := ebscsidriver.FileSystemConfigs[fsType].NotSupportedParams[createVolumeParameterKey] + return paramNotSupported +} diff --git a/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go b/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go index c264467104..daf1af49fc 100644 --- a/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go +++ b/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go @@ -45,25 +45,7 @@ func (t *DynamicallyProvisionedResizeVolumeTest) Run(client clientset.Interface, tpvc, _ := volume.SetupDynamicPersistentVolumeClaim(client, namespace, t.CSIDriver) defer tpvc.Cleanup() - pvcName := tpvc.persistentVolumeClaim.Name - pvc, _ := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(context.TODO(), pvcName, metav1.GetOptions{}) - By(fmt.Sprintf("Get pvc name: %v", pvc.Name)) - originalSize := pvc.Spec.Resources.Requests["storage"] - delta := resource.Quantity{} - delta.Set(util.GiBToBytes(1)) - originalSize.Add(delta) - pvc.Spec.Resources.Requests["storage"] = originalSize - - By("resizing the pvc") - updatedPvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Update(context.TODO(), pvc, metav1.UpdateOptions{}) - if err != nil { - framework.ExpectNoError(err, fmt.Sprintf("fail to resize pvc(%s): %v", pvcName, err)) - } - updatedSize := updatedPvc.Spec.Resources.Requests["storage"] - - By("checking the resizing PV result") - error := WaitForPvToResize(client, namespace, updatedPvc.Spec.VolumeName, updatedSize, 1*time.Minute, 5*time.Second) - framework.ExpectNoError(error) + ResizeTestPvc(client, namespace, tpvc, 1) By("Validate volume can be attached") tpod := NewTestPod(client, namespace, t.Pod.Cmd) @@ -92,3 +74,28 @@ func WaitForPvToResize(c clientset.Interface, ns *v1.Namespace, pvName string, d } return fmt.Errorf("Gave up after waiting %v for pv %q to complete resizing", timeout, pvName) } + +// ResizeTestPvc increases size of given persistent volume claim by `sizeIncreaseGi` Gigabytes +func ResizeTestPvc(client clientset.Interface, namespace *v1.Namespace, testPvc *TestPersistentVolumeClaim, sizeIncreaseGi int64) (updatedPvc *v1.PersistentVolumeClaim, updatedSize resource.Quantity) { + By(fmt.Sprintf("getting pvc name: %v", testPvc.persistentVolumeClaim.Name)) + pvc, _ := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(context.TODO(), testPvc.persistentVolumeClaim.Name, metav1.GetOptions{}) + + originalSize := pvc.Spec.Resources.Requests["storage"] + delta := resource.Quantity{} + delta.Set(util.GiBToBytes(sizeIncreaseGi)) + originalSize.Add(delta) + pvc.Spec.Resources.Requests["storage"] = originalSize + + By("resizing the pvc") + updatedPvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Update(context.TODO(), pvc, metav1.UpdateOptions{}) + if err != nil { + framework.ExpectNoError(err, fmt.Sprintf("fail to resize pvc(%s): %v", pvc.Name, err)) + } + updatedSize = updatedPvc.Spec.Resources.Requests["storage"] + + By("checking the resizing PV result") + err = WaitForPvToResize(client, namespace, updatedPvc.Spec.VolumeName, updatedSize, 1*time.Minute, 5*time.Second) + framework.ExpectNoError(err) + + return +} diff --git a/tests/e2e/testsuites/format_options_tester.go b/tests/e2e/testsuites/format_options_tester.go new file mode 100644 index 0000000000..a7aa7ace38 --- /dev/null +++ b/tests/e2e/testsuites/format_options_tester.go @@ -0,0 +1,93 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testsuites + +import ( + "fmt" + awscloud "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + . "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" + clientset "k8s.io/client-go/kubernetes" +) + +// FormatOptionTest will provision required StorageClass(es), PVC(s) and Pod(s) in order to test that volumes with +// a specified custom format options will mount and are able to be later resized. +type FormatOptionTest struct { + CreateVolumeParameterKey string + CreateVolumeParameterValue string +} + +const ( + volumeSizeIncreaseAmtGi = 1 + volumeMountPath = "/mnt/test-format-option" // TODO should I keep this as mnt/test-1, and refactor to be `DefaultMountPath` globally in testsuites? +) + +var ( + podCmdWriteToVolume = fmt.Sprintf("echo 'hello world' >> %s/data && grep 'hello world' %s/data && sync", volumeMountPath, volumeMountPath) // TODO Debt: All the dynamic provisioning tests use this same cmd. Should we refactor out into exported constant? +) + +func (t *FormatOptionTest) Run(client clientset.Interface, namespace *v1.Namespace, ebsDriver driver.PVTestDriver, fsType string) { + By("setting up pvc with custom format option") + volumeDetails := createFormatOptionVolumeDetails(fsType, volumeMountPath, t) + testPvc, _ := volumeDetails.SetupDynamicPersistentVolumeClaim(client, namespace, ebsDriver) + defer testPvc.Cleanup() + + By("deploying pod with custom format option") + getFsInfoTestPod := createPodWithVolume(client, namespace, podCmdWriteToVolume, testPvc, volumeDetails) + defer getFsInfoTestPod.Cleanup() + getFsInfoTestPod.WaitForSuccess() + + By("testing that pvc is able to be resized") + ResizeTestPvc(client, namespace, testPvc, volumeSizeIncreaseAmtGi) + + By("validating resized pvc by deploying new pod") + resizeTestPod := createPodWithVolume(client, namespace, podCmdWriteToVolume, testPvc, volumeDetails) + defer resizeTestPod.Cleanup() + + By("confirming new pod can write to resized volume") + resizeTestPod.WaitForSuccess() +} + +// TODO should we improve this across e2e tests via builder design pattern? Or is that not go-like? +func createFormatOptionVolumeDetails(fsType string, volumeMountPath string, t *FormatOptionTest) *VolumeDetails { + allowVolumeExpansion := true + + volume := VolumeDetails{ + VolumeType: awscloud.VolumeTypeGP2, + FSType: fsType, + MountOptions: []string{"rw"}, + ClaimSize: driver.MinimumSizeForVolumeType(awscloud.VolumeTypeGP2), + VolumeMount: VolumeMountDetails{ + NameGenerate: "test-volume-format-option", + MountPathGenerate: volumeMountPath, + }, + AllowVolumeExpansion: &allowVolumeExpansion, + AdditionalParameters: map[string]string{ + t.CreateVolumeParameterKey: t.CreateVolumeParameterValue, + }, + } + + return &volume +} + +// TODO putting this in function may be overkill? In an ideal world we refactor out TestEverything objects so testPod.SetupVolume isn't gross. +func createPodWithVolume(client clientset.Interface, namespace *v1.Namespace, cmd string, testPvc *TestPersistentVolumeClaim, volumeDetails *VolumeDetails) *TestPod { + testPod := NewTestPod(client, namespace, cmd) + testPod.SetupVolume(testPvc.persistentVolumeClaim, volumeDetails.VolumeMount.NameGenerate, volumeDetails.VolumeMount.MountPathGenerate, volumeDetails.VolumeMount.ReadOnly) + testPod.Create() + + return testPod +} diff --git a/tests/e2e/testsuites/specs.go b/tests/e2e/testsuites/specs.go index e2f24c9743..ff068283ca 100644 --- a/tests/e2e/testsuites/specs.go +++ b/tests/e2e/testsuites/specs.go @@ -45,10 +45,9 @@ type VolumeDetails struct { VolumeMode VolumeMode VolumeMount VolumeMountDetails VolumeDevice VolumeDeviceDetails - // Optional, used with pre-provisioned volumes - VolumeID string - // Optional, used with PVCs created from snapshots - DataSource *DataSource + VolumeID string // Optional, used with pre-provisioned volumes + DataSource *DataSource // Optional, used with PVCs created from snapshots + AdditionalParameters map[string]string // Optional, used when testing formatting options } type VolumeMode int @@ -121,7 +120,7 @@ func (pod *PodDetails) SetupDeployment(client clientset.Interface, namespace *v1 volume := pod.Volumes[0] By("setting up the StorageClass") - storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name) + storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted, volume.AdditionalParameters), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name) tsc := NewTestStorageClass(client, namespace, storageClass) createdStorageClass := tsc.Create() cleanupFuncs = append(cleanupFuncs, tsc.Cleanup) @@ -141,7 +140,7 @@ func (pod *PodDetails) SetupDeployment(client clientset.Interface, namespace *v1 func (volume *VolumeDetails) SetupDynamicPersistentVolumeClaim(client clientset.Interface, namespace *v1.Namespace, csiDriver driver.DynamicPVTestDriver) (*TestPersistentVolumeClaim, []func()) { cleanupFuncs := make([]func(), 0) By("setting up the StorageClass") - storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name) + storageClass := csiDriver.GetDynamicProvisionStorageClass(driver.GetParameters(volume.VolumeType, volume.FSType, volume.Encrypted, volume.AdditionalParameters), volume.MountOptions, volume.ReclaimPolicy, volume.AllowVolumeExpansion, volume.VolumeBindingMode, volume.AllowedTopologyValues, namespace.Name) tsc := NewTestStorageClass(client, namespace, storageClass) createdStorageClass := tsc.Create() cleanupFuncs = append(cleanupFuncs, tsc.Cleanup)