From de85e64e9d82adb045ca536b9cd4df77390d3d65 Mon Sep 17 00:00:00 2001 From: Drew Sirenko <68304519+AndrewSirenko@users.noreply.github.com> Date: Fri, 1 Dec 2023 15:01:02 -0600 Subject: [PATCH] Add E2E tests for modifying volumes via annotations --- Makefile | 2 +- tests/e2e/modify_volume.go | 131 ++++++++++++++++++ ...ically_provisioned_resize_volume_tester.go | 56 +------- tests/e2e/testsuites/e2e_utils.go | 104 +++++++++++++- tests/e2e/testsuites/format_options_tester.go | 6 +- tests/e2e/testsuites/modify_volume_tester.go | 94 +++++++++++++ 6 files changed, 334 insertions(+), 59 deletions(-) create mode 100644 tests/e2e/modify_volume.go create mode 100644 tests/e2e/testsuites/modify_volume_tester.go diff --git a/Makefile b/Makefile index 32b52c84dd..cae3921f20 100644 --- a/Makefile +++ b/Makefile @@ -157,7 +157,7 @@ test-sanity: test-e2e-single-az: AWS_REGION=us-west-2 \ AWS_AVAILABILITY_ZONES=us-west-2a \ - HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME' \ + HELM_EXTRA_FLAGS='--set=controller.k8sTagClusterId=$$CLUSTER_NAME,controller.volumeModificationFeature.enabled=true' \ EBS_INSTALL_SNAPSHOT="true" \ TEST_PATH=./tests/e2e/... \ GINKGO_FOCUS="\[ebs-csi-e2e\] \[single-az\]" \ diff --git a/tests/e2e/modify_volume.go b/tests/e2e/modify_volume.go new file mode 100644 index 0000000000..38567d9408 --- /dev/null +++ b/tests/e2e/modify_volume.go @@ -0,0 +1,131 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + awscloud "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" + ebscsidriver "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/testsuites" + . "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + admissionapi "k8s.io/pod-security-admission/api" +) + +var ( + defaultModifyVolumeTestGp3CreateVolumeParameters = map[string]string{ + ebscsidriver.VolumeTypeKey: awscloud.VolumeTypeGP3, + ebscsidriver.FSTypeKey: ebscsidriver.FSTypeExt4, + } +) + +var ( + modifyVolumeTests = map[string]testsuites.ModifyVolumeTest{ + "with a new iops annotation": { + CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters, + ModifyVolumeAnnotations: map[string]string{ + testsuites.AnnotationIops: "4000", + }, + ShouldResizeVolume: false, + ShouldTestInvalidModificationRecovery: false, + }, + "with a new io2 volumeType annotation": { + CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters, + ModifyVolumeAnnotations: map[string]string{ + testsuites.AnnotationVolumeType: awscloud.VolumeTypeIO2, + testsuites.AnnotationIops: testsuites.DefaultIopsIoVolumes, // As of aws-ebs-csi-driver v1.25.0, parameter iops must be re-specified when modifying volumeType io2 volumes. + }, + ShouldResizeVolume: false, + ShouldTestInvalidModificationRecovery: false, + }, + "with a new throughput annotation": { + CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters, + ModifyVolumeAnnotations: map[string]string{ + testsuites.AnnotationThroughput: "150", + }, + ShouldResizeVolume: false, + ShouldTestInvalidModificationRecovery: false, + }, + "with new throughput and iops annotations": { + CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters, + ModifyVolumeAnnotations: map[string]string{ + testsuites.AnnotationIops: "4000", + testsuites.AnnotationThroughput: "150", + }, + ShouldResizeVolume: false, + ShouldTestInvalidModificationRecovery: false, + }, + "with a larger size and new throughput and iops annotations": { + CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters, + ModifyVolumeAnnotations: map[string]string{ + testsuites.AnnotationIops: "4000", + testsuites.AnnotationThroughput: "150", + }, + ShouldResizeVolume: true, + ShouldTestInvalidModificationRecovery: false, + }, + "with a larger size and new throughput and iops annotations after providing an invalid annotation": { + CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters, + ModifyVolumeAnnotations: map[string]string{ + testsuites.AnnotationIops: "4000", + testsuites.AnnotationThroughput: "150", + }, + ShouldResizeVolume: true, + ShouldTestInvalidModificationRecovery: true, + }, + "from io2 to gp3 with larger size and new iops and throughput annotations": { + CreateVolumeParameters: map[string]string{ + ebscsidriver.VolumeTypeKey: awscloud.VolumeTypeIO2, + ebscsidriver.FSTypeKey: ebscsidriver.FSTypeExt4, + ebscsidriver.IopsKey: testsuites.DefaultIopsIoVolumes, + }, + ModifyVolumeAnnotations: map[string]string{ + testsuites.AnnotationVolumeType: awscloud.VolumeTypeGP3, + testsuites.AnnotationIops: "4000", + testsuites.AnnotationThroughput: "150", + }, + ShouldResizeVolume: true, + ShouldTestInvalidModificationRecovery: false, + }, + } +) + +var _ = Describe("[ebs-csi-e2e] [single-az] [modify-volume] Modifying a PVC", func() { + f := framework.NewDefaultFramework("ebs") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + + var ( + cs clientset.Interface + ns *v1.Namespace + ebsDriver driver.PVTestDriver + ) + + BeforeEach(func() { + cs = f.ClientSet + ns = f.Namespace + ebsDriver = driver.InitEbsCSIDriver() + }) + + for testName, modifyVolumeTest := range modifyVolumeTests { + modifyVolumeTest := modifyVolumeTest + Context(testName, func() { + It("will modify associated PV and EBS Volume", func() { + modifyVolumeTest.Run(cs, ns, ebsDriver) + }) + }) + } +}) diff --git a/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go b/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go index daf1af49fc..c6b0a513bc 100644 --- a/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go +++ b/tests/e2e/testsuites/dynamically_provisioned_resize_volume_tester.go @@ -15,18 +15,9 @@ limitations under the License. package testsuites import ( - "context" - "fmt" - "time" - - "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util" "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" - v1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/kubernetes/test/e2e/framework" - . "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" clientset "k8s.io/client-go/kubernetes" ) @@ -45,9 +36,10 @@ func (t *DynamicallyProvisionedResizeVolumeTest) Run(client clientset.Interface, tpvc, _ := volume.SetupDynamicPersistentVolumeClaim(client, namespace, t.CSIDriver) defer tpvc.Cleanup() - ResizeTestPvc(client, namespace, tpvc, 1) + By("resizing the volume") + ResizeTestPvc(client, namespace, tpvc, DefaultSizeIncreaseGi) - By("Validate volume can be attached") + By("validate volume can be attached") tpod := NewTestPod(client, namespace, t.Pod.Cmd) tpod.SetupVolume(tpvc.persistentVolumeClaim, volume.VolumeMount.NameGenerate+"1", volume.VolumeMount.MountPathGenerate+"1", volume.VolumeMount.ReadOnly) @@ -58,44 +50,4 @@ func (t *DynamicallyProvisionedResizeVolumeTest) Run(client clientset.Interface, tpod.WaitForSuccess() defer tpod.Cleanup() - -} - -// WaitForPvToResize waiting for pvc size to be resized to desired size -func WaitForPvToResize(c clientset.Interface, ns *v1.Namespace, pvName string, desiredSize resource.Quantity, timeout time.Duration, interval time.Duration) error { - By(fmt.Sprintf("Waiting up to %v for pv in namespace %q to be complete", timeout, ns.Name)) - for start := time.Now(); time.Since(start) < timeout; time.Sleep(interval) { - newPv, _ := c.CoreV1().PersistentVolumes().Get(context.TODO(), pvName, metav1.GetOptions{}) - newPvSize := newPv.Spec.Capacity["storage"] - if desiredSize.Equal(newPvSize) { - By(fmt.Sprintf("Pv size is updated to %v", newPvSize.String())) - return nil - } - } - return fmt.Errorf("Gave up after waiting %v for pv %q to complete resizing", timeout, pvName) -} - -// ResizeTestPvc increases size of given persistent volume claim by `sizeIncreaseGi` Gigabytes -func ResizeTestPvc(client clientset.Interface, namespace *v1.Namespace, testPvc *TestPersistentVolumeClaim, sizeIncreaseGi int64) (updatedPvc *v1.PersistentVolumeClaim, updatedSize resource.Quantity) { - By(fmt.Sprintf("getting pvc name: %v", testPvc.persistentVolumeClaim.Name)) - pvc, _ := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(context.TODO(), testPvc.persistentVolumeClaim.Name, metav1.GetOptions{}) - - originalSize := pvc.Spec.Resources.Requests["storage"] - delta := resource.Quantity{} - delta.Set(util.GiBToBytes(sizeIncreaseGi)) - originalSize.Add(delta) - pvc.Spec.Resources.Requests["storage"] = originalSize - - By("resizing the pvc") - updatedPvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Update(context.TODO(), pvc, metav1.UpdateOptions{}) - if err != nil { - framework.ExpectNoError(err, fmt.Sprintf("fail to resize pvc(%s): %v", pvc.Name, err)) - } - updatedSize = updatedPvc.Spec.Resources.Requests["storage"] - - By("checking the resizing PV result") - err = WaitForPvToResize(client, namespace, updatedPvc.Spec.VolumeName, updatedSize, 1*time.Minute, 5*time.Second) - framework.ExpectNoError(err) - - return } diff --git a/tests/e2e/testsuites/e2e_utils.go b/tests/e2e/testsuites/e2e_utils.go index 33a8b0b472..4fbf7792f9 100644 --- a/tests/e2e/testsuites/e2e_utils.go +++ b/tests/e2e/testsuites/e2e_utils.go @@ -14,19 +14,121 @@ limitations under the License. package testsuites -import "fmt" +import ( + "context" + "fmt" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/util" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + "time" +) const ( DefaultVolumeName = "test-volume-1" DefaultMountPath = "/mnt/default-mount" DefaultIopsIoVolumes = "100" + + DefaultSizeIncreaseGi = 1 + + DefaultModificationTimeout = 3 * time.Minute + DefaultResizeTimout = 1 * time.Minute + DefaultK8sApiPollingInterval = 5 * time.Second + + AnnotationIops = "ebs.csi.aws.com/iops" + AnnotationThroughput = "ebs.csi.aws.com/throughput" + AnnotationVolumeType = "ebs.csi.aws.com/volumeType" ) +// PodCmdWriteToVolume returns pod command that would write to mounted volume func PodCmdWriteToVolume(volumeMountPath string) string { return fmt.Sprintf("echo 'hello world' >> %s/data && grep 'hello world' %s/data && sync", volumeMountPath, volumeMountPath) } +// PodCmdContinuousWrite returns pod command that would continuously write to mounted volume +func PodCmdContinuousWrite(volumeMountPath string) string { + return fmt.Sprintf("while true; do echo \"$(date -u)\" >> /%s/out.txt; sleep 5; done", volumeMountPath) +} + +// IncreasePvcObjectStorage increases `storage` of a K8s PVC object by specified Gigabytes +func IncreasePvcObjectStorage(pvc *v1.PersistentVolumeClaim, sizeIncreaseGi int64) resource.Quantity { + pvcSize := pvc.Spec.Resources.Requests["storage"] + delta := resource.Quantity{} + delta.Set(util.GiBToBytes(sizeIncreaseGi)) + pvcSize.Add(delta) + pvc.Spec.Resources.Requests["storage"] = pvcSize + return pvcSize +} + +// WaitForPvToResize waiting for pvc size to be resized to desired size +func WaitForPvToResize(c clientset.Interface, ns *v1.Namespace, pvName string, desiredSize resource.Quantity, timeout time.Duration, interval time.Duration) error { + framework.Logf("waiting up to %v for pv resize in namespace %q to be complete", timeout, ns.Name) + for start := time.Now(); time.Since(start) < timeout; time.Sleep(interval) { + newPv, _ := c.CoreV1().PersistentVolumes().Get(context.TODO(), pvName, metav1.GetOptions{}) + newPvSize := newPv.Spec.Capacity["storage"] + if desiredSize.Equal(newPvSize) { + framework.Logf("pv size is updated to %v", newPvSize.String()) + return nil + } + } + return fmt.Errorf("gave up after waiting %v for pv %q to complete resizing", timeout, pvName) +} + +// ResizeTestPvc increases size of given `TestPersistentVolumeClaim` by specified Gigabytes +func ResizeTestPvc(client clientset.Interface, namespace *v1.Namespace, testPvc *TestPersistentVolumeClaim, sizeIncreaseGi int64) (updatedSize resource.Quantity) { + framework.Logf("getting pvc name: %v", testPvc.persistentVolumeClaim.Name) + pvc, _ := client.CoreV1().PersistentVolumeClaims(namespace.Name).Get(context.TODO(), testPvc.persistentVolumeClaim.Name, metav1.GetOptions{}) + + IncreasePvcObjectStorage(pvc, sizeIncreaseGi) + + framework.Logf("updating the pvc object") + updatedPvc, err := client.CoreV1().PersistentVolumeClaims(namespace.Name).Update(context.TODO(), pvc, metav1.UpdateOptions{}) + if err != nil { + framework.ExpectNoError(err, fmt.Sprintf("fail to resize pvc(%s): %v", pvc.Name, err)) + } + updatedSize = updatedPvc.Spec.Resources.Requests["storage"] + + framework.Logf("checking the resizing PV result") + err = WaitForPvToResize(client, namespace, updatedPvc.Spec.VolumeName, updatedSize, DefaultResizeTimout, DefaultK8sApiPollingInterval) + framework.ExpectNoError(err) + return updatedSize +} + +// AnnotatePvc annotates supplied k8s pvc object with supplied annotations +func AnnotatePvc(pvc *v1.PersistentVolumeClaim, annotations map[string]string) { + for annotation, value := range annotations { + pvc.Annotations[annotation] = value + } +} + +// CheckPvAnnotations checks whether supplied k8s pv object contains supplied annotations +func CheckPvAnnotations(pv *v1.PersistentVolume, annotations map[string]string) bool { + for annotation, value := range annotations { + if pv.Annotations[annotation] != value { + return false + } + } + return true +} + +// WaitForPvToModify waiting for PV to be modified +func WaitForPvToModify(c clientset.Interface, ns *v1.Namespace, pvName string, expectedAnnotations map[string]string, timeout time.Duration, interval time.Duration) error { + framework.Logf("waiting up to %v for pv in namespace %q to be modified", timeout, ns.Name) + + for start := time.Now(); time.Since(start) < timeout; time.Sleep(interval) { + modifyingPv, _ := c.CoreV1().PersistentVolumes().Get(context.TODO(), pvName, metav1.GetOptions{}) + + if CheckPvAnnotations(modifyingPv, expectedAnnotations) { + framework.Logf("pv annotations are updated to %v", modifyingPv.Annotations) + return nil + } + } + return fmt.Errorf("gave up after waiting %v for pv %q to complete modifying", timeout, pvName) +} + func CreateVolumeDetails(createVolumeParameters map[string]string, volumeSize string) *VolumeDetails { allowVolumeExpansion := true diff --git a/tests/e2e/testsuites/format_options_tester.go b/tests/e2e/testsuites/format_options_tester.go index 80a4eb28a3..96c827a5d3 100644 --- a/tests/e2e/testsuites/format_options_tester.go +++ b/tests/e2e/testsuites/format_options_tester.go @@ -28,10 +28,6 @@ type FormatOptionTest struct { CreateVolumeParameters map[string]string } -const ( - volumeSizeIncreaseAmtGi = 1 -) - func (t *FormatOptionTest) Run(client clientset.Interface, namespace *v1.Namespace, ebsDriver driver.PVTestDriver) { By("setting up pvc with custom format option") volumeDetails := CreateVolumeDetails(t.CreateVolumeParameters, driver.MinimumSizeForVolumeType(t.CreateVolumeParameters[ebscsidriver.VolumeTypeKey])) @@ -44,7 +40,7 @@ func (t *FormatOptionTest) Run(client clientset.Interface, namespace *v1.Namespa formatOptionMountPod.WaitForSuccess() By("testing that pvc is able to be resized") - ResizeTestPvc(client, namespace, testPvc, volumeSizeIncreaseAmtGi) + ResizeTestPvc(client, namespace, testPvc, DefaultSizeIncreaseGi) By("validating resized pvc by deploying new pod") resizeTestPod := createPodWithVolume(client, namespace, PodCmdWriteToVolume(DefaultMountPath), testPvc, volumeDetails) diff --git a/tests/e2e/testsuites/modify_volume_tester.go b/tests/e2e/testsuites/modify_volume_tester.go new file mode 100644 index 0000000000..fb784ec3a6 --- /dev/null +++ b/tests/e2e/testsuites/modify_volume_tester.go @@ -0,0 +1,94 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testsuites + +import ( + "context" + "fmt" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + . "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" +) + +// ModifyVolumeTest will provision pod with attached volume, and test that modifying its pvc will modify the associated pv. +type ModifyVolumeTest struct { + CreateVolumeParameters map[string]string + ModifyVolumeAnnotations map[string]string + ShouldResizeVolume bool + ShouldTestInvalidModificationRecovery bool +} + +var ( + invalidAnnotations = map[string]string{ + AnnotationIops: "1", + } + volumeSize = "10Gi" // Different from driver.MinimumSizeForVolumeType to simplify iops, throughput, volumeType modification +) + +func (modifyVolumeTest *ModifyVolumeTest) Run(c clientset.Interface, ns *v1.Namespace, ebsDriver driver.PVTestDriver) { + By("setting up pvc") + volumeDetails := CreateVolumeDetails(modifyVolumeTest.CreateVolumeParameters, volumeSize) + testVolume, _ := volumeDetails.SetupDynamicPersistentVolumeClaim(c, ns, ebsDriver) + defer testVolume.Cleanup() + + By("deploying pod continuously writing to volume") + formatOptionMountPod := createPodWithVolume(c, ns, PodCmdContinuousWrite(DefaultMountPath), testVolume, volumeDetails) + defer formatOptionMountPod.Cleanup() + formatOptionMountPod.WaitForRunning() + + if modifyVolumeTest.ShouldTestInvalidModificationRecovery { + By("modifying the pvc with invalid annotations") + attemptInvalidModification(c, ns, testVolume) + } + + By("modifying the pvc") + modifyingPvc, _ := c.CoreV1().PersistentVolumeClaims(ns.Name).Get(context.TODO(), testVolume.persistentVolumeClaim.Name, metav1.GetOptions{}) + AnnotatePvc(modifyingPvc, modifyVolumeTest.ModifyVolumeAnnotations) + + var updatedPvcSize resource.Quantity + if modifyVolumeTest.ShouldResizeVolume { + By("resizing the pvc") + updatedPvcSize = IncreasePvcObjectStorage(modifyingPvc, DefaultSizeIncreaseGi) + } + + modifiedPvc, err := c.CoreV1().PersistentVolumeClaims(ns.Name).Update(context.TODO(), modifyingPvc, metav1.UpdateOptions{}) + if err != nil { + framework.ExpectNoError(err, fmt.Sprintf("fail to modify pvc(%s): %v", modifyingPvc.Name, err)) + } + framework.Logf("updated pvc: %s\n", modifiedPvc.Annotations) + + // Confirm Volume Modified + By("wait for and confirm pv modification") + err = WaitForPvToModify(c, ns, testVolume.persistentVolume.Name, modifyVolumeTest.ModifyVolumeAnnotations, DefaultModificationTimeout, DefaultK8sApiPollingInterval) + framework.ExpectNoError(err, fmt.Sprintf("fail to modify pv(%s): %v", modifyingPvc.Name, err)) + if modifyVolumeTest.ShouldResizeVolume { + err = WaitForPvToResize(c, ns, testVolume.persistentVolume.Name, updatedPvcSize, DefaultResizeTimout, DefaultK8sApiPollingInterval) + framework.ExpectNoError(err, fmt.Sprintf("fail to resize pv(%s): %v", modifyingPvc.Name, err)) + } +} + +func attemptInvalidModification(c clientset.Interface, ns *v1.Namespace, testVolume *TestPersistentVolumeClaim) { + modifyingPvc, _ := c.CoreV1().PersistentVolumeClaims(ns.Name).Get(context.TODO(), testVolume.persistentVolumeClaim.Name, metav1.GetOptions{}) + AnnotatePvc(modifyingPvc, invalidAnnotations) + modifiedPvc, err := c.CoreV1().PersistentVolumeClaims(ns.Name).Update(context.TODO(), modifyingPvc, metav1.UpdateOptions{}) + if err != nil { + framework.ExpectNoError(err, fmt.Sprintf("fail to modify pvc(%s): %v", modifyingPvc.Name, err)) + } + framework.Logf("pvc %q/%q has been modified with invalid annotations: %s", ns.Name, modifiedPvc.Name, modifiedPvc.Annotations) +}