Add E2E tests for modifying volumes via annotations

tests/e2e/modify_volume.go

@@ -0,0 +1,132 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+   http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"fmt"
+	awscloud "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/cloud"
+	ebscsidriver "github.com/kubernetes-sigs/aws-ebs-csi-driver/pkg/driver"
+	"github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver"
+	"github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/testsuites"
+	. "github.com/onsi/ginkgo/v2"
+	v1 "k8s.io/api/core/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+	admissionapi "k8s.io/pod-security-admission/api"
+)
+
+var (
+	defaultModifyVolumeTestGp3CreateVolumeParameters = map[string]string{
+		ebscsidriver.VolumeTypeKey: awscloud.VolumeTypeGP3,
+		ebscsidriver.FSTypeKey:     ebscsidriver.FSTypeExt4,
+	}
+)
+
+var (
+	modifyVolumeTests = map[string]testsuites.ModifyVolumeTest{
+		"with a new iops annotation": {
+			CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters,
+			ModifyVolumeAnnotations: map[string]string{
+				testsuites.AnnotationIops: "4000",
+			},
+			ShouldResizeVolume:                    false,
+			ShouldTestInvalidModificationRecovery: false,
+		},
+		"with a new io2 volumeType annotation": {
+			CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters,
+			ModifyVolumeAnnotations: map[string]string{
+				testsuites.AnnotationVolumeType: awscloud.VolumeTypeIO2,
+				testsuites.AnnotationIops:       testsuites.DefaultE2EIopsIoVolumes, // As of aws-ebs-csi-driver v1.25.0, parameter iops must be re-specified when modifying volumeType io2 volumes.
+			},
+			ShouldResizeVolume:                    false,
+			ShouldTestInvalidModificationRecovery: false,
+		},
+		"with a new throughput annotation": {
+			CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters,
+			ModifyVolumeAnnotations: map[string]string{
+				testsuites.AnnotationThroughput: "150",
+			},
+			ShouldResizeVolume:                    false,
+			ShouldTestInvalidModificationRecovery: false,
+		},
+		"with new throughput and iops annotations": {
+			CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters,
+			ModifyVolumeAnnotations: map[string]string{
+				testsuites.AnnotationIops:       "4000",
+				testsuites.AnnotationThroughput: "150",
+			},
+			ShouldResizeVolume:                    false,
+			ShouldTestInvalidModificationRecovery: false,
+		},
+		"with a larger size and new throughput and iops annotations": {
+			CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters,
+			ModifyVolumeAnnotations: map[string]string{
+				testsuites.AnnotationIops:       "4000",
+				testsuites.AnnotationThroughput: "150",
+			},
+			ShouldResizeVolume:                    true,
+			ShouldTestInvalidModificationRecovery: false,
+		},
+		"with a larger size and new throughput and iops annotations after providing an invalid annotation": {
+			CreateVolumeParameters: defaultModifyVolumeTestGp3CreateVolumeParameters,
+			ModifyVolumeAnnotations: map[string]string{
+				testsuites.AnnotationIops:       "4000",
+				testsuites.AnnotationThroughput: "150",
+			},
+			ShouldResizeVolume:                    true,
+			ShouldTestInvalidModificationRecovery: true,
+		},
+		"from io2 to gp3 with larger size and new iops and throughput annotations": {
+			CreateVolumeParameters: map[string]string{
+				ebscsidriver.VolumeTypeKey: awscloud.VolumeTypeIO2,
+				ebscsidriver.FSTypeKey:     ebscsidriver.FSTypeExt4,
+				ebscsidriver.IopsKey:       testsuites.DefaultE2EIopsIoVolumes,
+			},
+			ModifyVolumeAnnotations: map[string]string{
+				testsuites.AnnotationVolumeType: awscloud.VolumeTypeGP3,
+				testsuites.AnnotationIops:       "4000",
+				testsuites.AnnotationThroughput: "150",
+			},
+			ShouldResizeVolume:                    true,
+			ShouldTestInvalidModificationRecovery: false,
+		},
+	}
+)
+
+var _ = Describe("[ebs-csi-e2e] [single-az] [modify-volume] Modifying a PVC", func() {
+	f := framework.NewDefaultFramework("ebs")
+	f.NamespacePodSecurityEnforceLevel = admissionapi.LevelPrivileged
+
+	var (
+		cs        clientset.Interface
+		ns        *v1.Namespace
+		ebsDriver driver.PVTestDriver
+	)
+
+	BeforeEach(func() {
+		cs = f.ClientSet
+		ns = f.Namespace
+		ebsDriver = driver.InitEbsCSIDriver()
+	})
+
+	for testName, modifyVolumeTest := range modifyVolumeTests {
+		modifyVolumeTest := modifyVolumeTest
+		Context(fmt.Sprintf("%s", testName), func() {
+			It(fmt.Sprintf("will modify associated PV and EBS Volume"), func() {
+				modifyVolumeTest.Run(cs, ns, ebsDriver)
+			})
+		})
+	}
+})

tests/e2e/testsuites/e2e_utils.go

@@ -34,14 +34,25 @@ const (
 
 	DefaultSizeIncreaseGi = 1
 
+	DefaultModificationTimeout   = 3 * time.Minute
 	DefaultResizeTimout          = 1 * time.Minute
 	DefaultK8sApiPollingInterval = 5 * time.Second
+
+	AnnotationIops       = "ebs.csi.aws.com/iops"
+	AnnotationThroughput = "ebs.csi.aws.com/throughput"
+	AnnotationVolumeType = "ebs.csi.aws.com/volumeType"
 )
 
+// PodCmdWriteToVolume returns pod command that would write to mounted volume
 func PodCmdWriteToVolume(volumeMountPath string) string {
 	return fmt.Sprintf("echo 'hello world' >> %s/data && grep 'hello world' %s/data && sync", volumeMountPath, volumeMountPath)
 }
 
+// PodCmdContinuousWrite returns pod command that would continuously write to mounted volume
+func PodCmdContinuousWrite(volumeMountPath string) string {
+	return fmt.Sprintf("while true; do echo \"$(date -u)\" >> /%s/out.txt; sleep 5; done", volumeMountPath)
+}
+
 // IncreasePvcObjectStorage increases `storage` of a K8s PVC object by `sizeIncreaseGi`
 func IncreasePvcObjectStorage(pvc *v1.PersistentVolumeClaim, sizeIncreaseGi int64) resource.Quantity {
 	pvcSize := pvc.Spec.Resources.Requests["storage"]
@@ -86,6 +97,39 @@ func ResizeTestPvc(client clientset.Interface, namespace *v1.Namespace, testPvc
 	return updatedSize
 }
 
+// AnnotatePvc annotates supplied k8s pvc object with supplied annotations
+func AnnotatePvc(pvc *v1.PersistentVolumeClaim, annotations map[string]string) {
+	for annotation, value := range annotations {
+		pvc.Annotations[annotation] = value
+	}
+}
+
+// CheckPvAnnotations checks whether supplied k8s pv object contains supplied annotations
+func CheckPvAnnotations(pv *v1.PersistentVolume, annotations map[string]string) bool {
+	for annotation, value := range annotations {
+		if pv.Annotations[annotation] != value {
+			return false
+		}
+	}
+	return true
+}
+
+// WaitForPvToModify waiting for PV to be modified
+// TODO Consider replacing this and WaitForPvToResize with [Kubernetes Client Go Watch](https://pkg.go.dev/k8s.io/client-go/tools/watch)
+func WaitForPvToModify(c clientset.Interface, ns *v1.Namespace, pvName string, expectedAnnotations map[string]string, timeout time.Duration, interval time.Duration) error {
+	framework.Logf("waiting up to %v for pv in namespace %q to be modified", timeout, ns.Name)
+
+	for start := time.Now(); time.Since(start) < timeout; time.Sleep(interval) {
+		modifyingPv, _ := c.CoreV1().PersistentVolumes().Get(context.TODO(), pvName, metav1.GetOptions{})
+
+		if CheckPvAnnotations(modifyingPv, expectedAnnotations) {
+			framework.Logf("pv annotations are updated to %v", modifyingPv.Annotations)
+			return nil
+		}
+	}
+	return fmt.Errorf("gave up after waiting %v for pv %q to complete modifying", timeout, pvName)
+}
+
 func CreateVolumeDetails(createVolumeParameters map[string]string, volumeSize string) *VolumeDetails {
 	allowVolumeExpansion := true
 

tests/e2e/testsuites/modify_volume_tester.go

@@ -0,0 +1,104 @@
+/*
+Copyright 2023 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+   http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package testsuites
+
+import (
+	"context"
+	"fmt"
+	"github.com/kubernetes-sigs/aws-ebs-csi-driver/tests/e2e/driver"
+	. "github.com/onsi/ginkgo/v2"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	clientset "k8s.io/client-go/kubernetes"
+	"k8s.io/kubernetes/test/e2e/framework"
+)
+
+// ModifyVolumeTest will provision pod with attached volume, and test that modifying its pvc will modify the associated pv.
+type ModifyVolumeTest struct {
+	CreateVolumeParameters                map[string]string
+	ModifyVolumeAnnotations               map[string]string
+	ShouldResizeVolume                    bool
+	ShouldTestInvalidModificationRecovery bool
+}
+
+const (
+	// TODO Q: This `modifyVolumeTest` prefix is a code smell. Does it signify that `testsuites` package should be broken up into separate testsuite packages importing a shared testsuite-utils package?
+	modifyVolumeTestDefaultVolumeSize = "10Gi" // Different from driver.MinimumSizeForVolumeType to simplify iops, throughput, volumeType modification
+)
+
+var (
+	modifyVolumeTestInvalidAnnotations = map[string]string{
+		AnnotationIops: "1",
+	}
+)
+
+func (modifyVolumeTest *ModifyVolumeTest) Run(c clientset.Interface, ns *v1.Namespace, ebsDriver driver.PVTestDriver) {
+	By("setting up pvc")
+	volumeDetails := CreateVolumeDetails(modifyVolumeTest.CreateVolumeParameters, modifyVolumeTestDefaultVolumeSize)
+	testVolume, _ := volumeDetails.SetupDynamicPersistentVolumeClaim(c, ns, ebsDriver)
+	defer testVolume.Cleanup()
+
+	By("deploying pod continously writing to volume")
+	// TODO Q: fix continuous write command
+	formatOptionMountPod := createPodWithVolume(c, ns, PodCmdContinuousWrite(DefaultMountPath), testVolume, volumeDetails)
+	defer formatOptionMountPod.Cleanup()
+	formatOptionMountPod.WaitForRunning()
+
+	if modifyVolumeTest.ShouldTestInvalidModificationRecovery {
+		By("modifying the pvc with invalid annotations")
+		attemptInvalidModification(c, ns, testVolume)
+	}
+
+	By("modifying the pvc")
+	modifyingPvc, _ := c.CoreV1().PersistentVolumeClaims(ns.Name).Get(context.TODO(), testVolume.persistentVolumeClaim.Name, metav1.GetOptions{})
+	AnnotatePvc(modifyingPvc, modifyVolumeTest.ModifyVolumeAnnotations)
+
+	var updatedPvcSize resource.Quantity
+	if modifyVolumeTest.ShouldResizeVolume {
+		By("resizing the pvc")
+		updatedPvcSize = IncreasePvcObjectStorage(modifyingPvc, DefaultSizeIncreaseGi)
+	}
+
+	modifiedPvc, err := c.CoreV1().PersistentVolumeClaims(ns.Name).Update(context.TODO(), modifyingPvc, metav1.UpdateOptions{})
+	if err != nil {
+		framework.ExpectNoError(err, fmt.Sprintf("fail to modify pvc(%s): %v", modifyingPvc.Name, err))
+	}
+	framework.Logf("updated pvc: %s\n", modifiedPvc.Annotations)
+
+	// Confirm Volume Modified
+	By("wait for and confirm pv modification")
+	err = WaitForPvToModify(c, ns, testVolume.persistentVolume.Name, modifyVolumeTest.ModifyVolumeAnnotations, DefaultModificationTimeout, DefaultK8sApiPollingInterval)
+	framework.ExpectNoError(err, fmt.Sprintf("fail to modify pv(%s): %v", modifyingPvc.Name, err))
+	//_, err = fmt.Fprintf(GinkgoWriter, "updated PV: %s", pv.Annotations) // TODO add what expected annotations are here and earlier in test
+	if modifyVolumeTest.ShouldResizeVolume {
+		err = WaitForPvToResize(c, ns, testVolume.persistentVolume.Name, updatedPvcSize, DefaultResizeTimout, DefaultK8sApiPollingInterval)
+		framework.ExpectNoError(err, fmt.Sprintf("fail to resize pv(%s): %v", modifyingPvc.Name, err))
+	}
+}
+
+func attemptInvalidModification(c clientset.Interface, ns *v1.Namespace, testVolume *TestPersistentVolumeClaim) bool {
+	modifyingPvc, _ := c.CoreV1().PersistentVolumeClaims(ns.Name).Get(context.TODO(), testVolume.persistentVolumeClaim.Name, metav1.GetOptions{})
+	AnnotatePvc(modifyingPvc, modifyVolumeTestInvalidAnnotations)
+	modifiedPvc, err := c.CoreV1().PersistentVolumeClaims(ns.Name).Update(context.TODO(), modifyingPvc, metav1.UpdateOptions{})
+	if err != nil {
+		framework.ExpectNoError(err, fmt.Sprintf("fail to modify pvc(%s): %v", modifyingPvc.Name, err))
+	}
+	framework.Logf("pvc %q/%q has been modified with invalid annotations: %s", ns.Name, modifiedPvc.Name, modifiedPvc.Annotations)
+	if err != nil {
+		return true
+	}
+	return false
+}