From 417e416a43b1773a65c152744e35c9c71f7882ee Mon Sep 17 00:00:00 2001 From: Drew Sirenko <68304519+AndrewSirenko@users.noreply.github.com> Date: Fri, 24 Nov 2023 10:51:32 -0800 Subject: [PATCH] Add E2E tests for modifying volumes via annotations --- tests/e2e/modify_volume.go | 132 +++++++++++++++++++ tests/e2e/testsuites/e2e_utils.go | 44 +++++++ tests/e2e/testsuites/modify_volume_tester.go | 104 +++++++++++++++ 3 files changed, 280 insertions(+) create mode 100644 tests/e2e/modify_volume.go create mode 100644 tests/e2e/testsuites/modify_volume_tester.go diff --git a/tests/e2e/modify_volume.go b/tests/e2e/modify_volume.go new file mode 100644 index 0000000000..b70fb5f21a --- /dev/null +++ b/tests/e2e/modify_volume.go @@ -0,0 +1,132 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + awscloud "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud" + ebscsidriver "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/testsuites" + . "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" + admissionapi "k8s.io/pod-security-admission/api" +) + +var ( + defaultModifyVolumeTestGp3CreateVolumeParameters = map[string]string{ + ebscsidriver.VolumeTypeKey: awscloud.VolumeTypeGP3, + ebscsidriver.FSTypeKey: ebscsidriver.FSTypeExt4, + } +) + +var ( + modifyVolumeTests = map[string]testsuites.ModifyVolumeTest{ + "with a new iops annotation": { + CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters, + ModifyVolumeAnnotations: map[string]string{ + testsuites.AnnotationIops: "4000", + }, + ShouldResizeVolume: false, + ShouldTestInvalidModificationRecovery: false, + }, + "with a new io2 volumeType annotation": { + CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters, + ModifyVolumeAnnotations: map[string]string{ + testsuites.AnnotationVolumeType: awscloud.VolumeTypeIO2, + testsuites.AnnotationIops: testsuites.DefaultE2EIopsIoVolumes, // As of aws-ebs-csi-driver v1.25.0, parameter iops must be re-specified when modifying volumeType io2 volumes. + }, + ShouldResizeVolume: false, + ShouldTestInvalidModificationRecovery: false, + }, + "with a new throughput annotation": { + CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters, + ModifyVolumeAnnotations: map[string]string{ + testsuites.AnnotationThroughput: "150", + }, + ShouldResizeVolume: false, + ShouldTestInvalidModificationRecovery: false, + }, + "with new throughput and iops annotations": { + CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters, + ModifyVolumeAnnotations: map[string]string{ + testsuites.AnnotationIops: "4000", + testsuites.AnnotationThroughput: "150", + }, + ShouldResizeVolume: false, + ShouldTestInvalidModificationRecovery: false, + }, + "with a larger size and new throughput and iops annotations": { + CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters, + ModifyVolumeAnnotations: map[string]string{ + testsuites.AnnotationIops: "4000", + testsuites.AnnotationThroughput: "150", + }, + ShouldResizeVolume: true, + ShouldTestInvalidModificationRecovery: false, + }, + "with a larger size and new throughput and iops annotations after providing an invalid annotation": { + CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters, + ModifyVolumeAnnotations: map[string]string{ + testsuites.AnnotationIops: "4000", + testsuites.AnnotationThroughput: "150", + }, + ShouldResizeVolume: true, + ShouldTestInvalidModificationRecovery: true, + }, + "from io2 to gp3 with larger size and new iops and throughput annotations": { + CreateVolumeParameters: map[string]string{ + ebscsidriver.VolumeTypeKey: awscloud.VolumeTypeIO2, + ebscsidriver.FSTypeKey: ebscsidriver.FSTypeExt4, + ebscsidriver.IopsKey: testsuites.DefaultE2EIopsIoVolumes, + }, + ModifyVolumeAnnotations: map[string]string{ + testsuites.AnnotationVolumeType: awscloud.VolumeTypeGP3, + testsuites.AnnotationIops: "4000", + testsuites.AnnotationThroughput: "150", + }, + ShouldResizeVolume: true, + ShouldTestInvalidModificationRecovery: false, + }, + } +) + +var _ = Describe("[ebs-csi-e2e] [single-az] [modify-volume] Modifying a PVC", func() { + f := framework.NewDefaultFramework("ebs") + f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged + + var ( + cs clientset.Interface + ns *v1.Namespace + ebsDriver driver.PVTestDriver + ) + + BeforeEach(func() { + cs = f.ClientSet + ns = f.Namespace + ebsDriver = driver.InitEbsCSIDriver() + }) + + for testName, modifyVolumeTest := range modifyVolumeTests { + modifyVolumeTest := modifyVolumeTest + Context(fmt.Sprintf("%s", testName), func() { + It(fmt.Sprintf("will modify associated PV and EBS Volume"), func() { + modifyVolumeTest.Run(cs, ns, ebsDriver) + }) + }) + } +}) diff --git a/tests/e2e/testsuites/e2e_utils.go b/tests/e2e/testsuites/e2e_utils.go index ba339f12ea..83a6f9379f 100644 --- a/tests/e2e/testsuites/e2e_utils.go +++ b/tests/e2e/testsuites/e2e_utils.go @@ -34,14 +34,25 @@ const ( DefaultSizeIncreaseGi = 1 + DefaultModificationTimeout = 3 * time.Minute DefaultResizeTimout = 1 * time.Minute DefaultK8sApiPollingInterval = 5 * time.Second + + AnnotationIops = "ebs.csi.aws.com/iops" + AnnotationThroughput = "ebs.csi.aws.com/throughput" + AnnotationVolumeType = "ebs.csi.aws.com/volumeType" ) +// PodCmdWriteToVolume returns pod command that would write to mounted volume func PodCmdWriteToVolume(volumeMountPath string) string { return fmt.Sprintf("echo 'hello world' >> %s/data && grep 'hello world' %s/data && sync", volumeMountPath, volumeMountPath) } +// PodCmdContinuousWrite returns pod command that would continuously write to mounted volume +func PodCmdContinuousWrite(volumeMountPath string) string { + return fmt.Sprintf("while true; do echo \"$(date -u)\" >> /%s/out.txt; sleep 5; done", volumeMountPath) +} + // IncreasePvcObjectStorage increases `storage` of a K8s PVC object by `sizeIncreaseGi` func IncreasePvcObjectStorage(pvc *v1.PersistentVolumeClaim, sizeIncreaseGi int64) resource.Quantity { pvcSize := pvc.Spec.Resources.Requests["storage"] @@ -86,6 +97,39 @@ func ResizeTestPvc(client clientset.Interface, namespace *v1.Namespace, testPvc return updatedSize } +// AnnotatePvc annotates supplied k8s pvc object with supplied annotations +func AnnotatePvc(pvc *v1.PersistentVolumeClaim, annotations map[string]string) { + for annotation, value := range annotations { + pvc.Annotations[annotation] = value + } +} + +// CheckPvAnnotations checks whether supplied k8s pv object contains supplied annotations +func CheckPvAnnotations(pv *v1.PersistentVolume, annotations map[string]string) bool { + for annotation, value := range annotations { + if pv.Annotations[annotation] != value { + return false + } + } + return true +} + +// WaitForPvToModify waiting for PV to be modified +// TODO Consider replacing this and WaitForPvToResize with [Kubernetes Client Go Watch](https://pkg.go.dev/k8s.io/client-go/tools/watch) +func WaitForPvToModify(c clientset.Interface, ns *v1.Namespace, pvName string, expectedAnnotations map[string]string, timeout time.Duration, interval time.Duration) error { + framework.Logf("waiting up to %v for pv in namespace %q to be modified", timeout, ns.Name) + + for start := time.Now(); time.Since(start) < timeout; time.Sleep(interval) { + modifyingPv, _ := c.CoreV1().PersistentVolumes().Get(context.TODO(), pvName, metav1.GetOptions{}) + + if CheckPvAnnotations(modifyingPv, expectedAnnotations) { + framework.Logf("pv annotations are updated to %v", modifyingPv.Annotations) + return nil + } + } + return fmt.Errorf("gave up after waiting %v for pv %q to complete modifying", timeout, pvName) +} + func CreateVolumeDetails(createVolumeParameters map[string]string, volumeSize string) *VolumeDetails { allowVolumeExpansion := true diff --git a/tests/e2e/testsuites/modify_volume_tester.go b/tests/e2e/testsuites/modify_volume_tester.go new file mode 100644 index 0000000000..8c886e8d92 --- /dev/null +++ b/tests/e2e/testsuites/modify_volume_tester.go @@ -0,0 +1,104 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package testsuites + +import ( + "context" + "fmt" + "github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver" + . "github.com/onsi/ginkgo/v2" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientset "k8s.io/client-go/kubernetes" + "k8s.io/kubernetes/test/e2e/framework" +) + +// ModifyVolumeTest will provision pod with attached volume, and test that modifying its pvc will modify the associated pv. +type ModifyVolumeTest struct { + CreateVolumeParameters map[string]string + ModifyVolumeAnnotations map[string]string + ShouldResizeVolume bool + ShouldTestInvalidModificationRecovery bool +} + +const ( + // TODO Q: This `modifyVolumeTest` prefix is a code smell. Does it signify that `testsuites` package should be broken up into separate testsuite packages importing a shared testsuite-utils package? + modifyVolumeTestDefaultVolumeSize = "10Gi" // Different from driver.MinimumSizeForVolumeType to simplify iops, throughput, volumeType modification +) + +var ( + modifyVolumeTestInvalidAnnotations = map[string]string{ + AnnotationIops: "1", + } +) + +func (modifyVolumeTest *ModifyVolumeTest) Run(c clientset.Interface, ns *v1.Namespace, ebsDriver driver.PVTestDriver) { + By("setting up pvc") + volumeDetails := CreateVolumeDetails(modifyVolumeTest.CreateVolumeParameters, modifyVolumeTestDefaultVolumeSize) + testVolume, _ := volumeDetails.SetupDynamicPersistentVolumeClaim(c, ns, ebsDriver) + defer testVolume.Cleanup() + + By("deploying pod continously writing to volume") + // TODO Q: fix continuous write command + formatOptionMountPod := createPodWithVolume(c, ns, PodCmdContinuousWrite(DefaultMountPath), testVolume, volumeDetails) + defer formatOptionMountPod.Cleanup() + formatOptionMountPod.WaitForRunning() + + if modifyVolumeTest.ShouldTestInvalidModificationRecovery { + By("modifying the pvc with invalid annotations") + attemptInvalidModification(c, ns, testVolume) + } + + By("modifying the pvc") + modifyingPvc, _ := c.CoreV1().PersistentVolumeClaims(ns.Name).Get(context.TODO(), testVolume.persistentVolumeClaim.Name, metav1.GetOptions{}) + AnnotatePvc(modifyingPvc, modifyVolumeTest.ModifyVolumeAnnotations) + + var updatedPvcSize resource.Quantity + if modifyVolumeTest.ShouldResizeVolume { + By("resizing the pvc") + updatedPvcSize = IncreasePvcObjectStorage(modifyingPvc, DefaultSizeIncreaseGi) + } + + modifiedPvc, err := c.CoreV1().PersistentVolumeClaims(ns.Name).Update(context.TODO(), modifyingPvc, metav1.UpdateOptions{}) + if err != nil { + framework.ExpectNoError(err, fmt.Sprintf("fail to modify pvc(%s): %v", modifyingPvc.Name, err)) + } + framework.Logf("updated pvc: %s\n", modifiedPvc.Annotations) + + // Confirm Volume Modified + By("wait for and confirm pv modification") + err = WaitForPvToModify(c, ns, testVolume.persistentVolume.Name, modifyVolumeTest.ModifyVolumeAnnotations, DefaultModificationTimeout, DefaultK8sApiPollingInterval) + framework.ExpectNoError(err, fmt.Sprintf("fail to modify pv(%s): %v", modifyingPvc.Name, err)) + //_, err = fmt.Fprintf(GinkgoWriter, "updated PV: %s", pv.Annotations) // TODO add what expected annotations are here and earlier in test + if modifyVolumeTest.ShouldResizeVolume { + err = WaitForPvToResize(c, ns, testVolume.persistentVolume.Name, updatedPvcSize, DefaultResizeTimout, DefaultK8sApiPollingInterval) + framework.ExpectNoError(err, fmt.Sprintf("fail to resize pv(%s): %v", modifyingPvc.Name, err)) + } +} + +func attemptInvalidModification(c clientset.Interface, ns *v1.Namespace, testVolume *TestPersistentVolumeClaim) bool { + modifyingPvc, _ := c.CoreV1().PersistentVolumeClaims(ns.Name).Get(context.TODO(), testVolume.persistentVolumeClaim.Name, metav1.GetOptions{}) + AnnotatePvc(modifyingPvc, modifyVolumeTestInvalidAnnotations) + modifiedPvc, err := c.CoreV1().PersistentVolumeClaims(ns.Name).Update(context.TODO(), modifyingPvc, metav1.UpdateOptions{}) + if err != nil { + framework.ExpectNoError(err, fmt.Sprintf("fail to modify pvc(%s): %v", modifyingPvc.Name, err)) + } + framework.Logf("pvc %q/%q has been modified with invalid annotations: %s", ns.Name, modifiedPvc.Name, modifiedPvc.Annotations) + if err != nil { + return true + } + return false +}