support PVC capacity expand (#679)

* add controller expand

.github/scripts/e2e-test.py

@@ -15,7 +15,7 @@
 
 from kubernetes import config
 
-from config import GLOBAL_MOUNTPOINT, LOG, IS_CE
+from config import GLOBAL_MOUNTPOINT, LOG
 from test_case import (
     test_dynamic_mount_image_with_webhook,
     test_static_mount_image_with_webhook,
@@ -45,6 +45,7 @@
     test_dynamic_pvc_delete_with_path_pattern,
     test_dynamic_pvc_delete_not_last_with_path_pattern,
     test_webhook_two_volume,
+    test_dynamic_expand,
 )
 from util import die, mount_on_host, umount, clean_juicefs_volume, deploy_secret_and_sc, check_do_test
 
@@ -78,6 +79,7 @@
                 test_static_mount_image()
                 test_pod_resource_err()
                 test_quota_using_storage_rw()
+                test_dynamic_expand()
 
             elif test_mode == "pod-mount-share":
                 test_share_mount()
@@ -95,6 +97,8 @@
                 test_delete_all()
                 test_dynamic_delete_pod()
                 test_static_delete_pod()
+                test_quota_using_storage_rw()
+                test_dynamic_expand()
 
             elif test_mode == "webhook":
                 test_webhook_two_volume()
@@ -107,6 +111,7 @@
                 test_deployment_dynamic_patch_pv_with_webhook()
                 test_deployment_static_patch_pv_with_webhook()
                 test_quota_using_storage_rw()
+                test_dynamic_expand()
 
             elif test_mode == "webhook-provisioner":
                 test_webhook_two_volume()
@@ -122,6 +127,8 @@
                 test_path_pattern_in_storage_class()
                 test_dynamic_pvc_delete_with_path_pattern()
                 test_job_complete_using_storage()
+                test_quota_using_storage_rw()
+                test_dynamic_expand()
 
             elif test_mode == "process":
                 test_static_cache_clean_upon_umount()
@@ -132,6 +139,7 @@
                 test_deployment_use_pv_ro()
                 test_delete_pvc()
                 test_quota_using_storage_rw()
+                test_dynamic_expand()
             else:
                 raise Exception("unknown test mode: %s" % test_mode)
         except Exception as e:

.github/scripts/model.py

@@ -89,11 +89,14 @@ def create(self):
                 "csi.storage.k8s.io/node-publish-secret-namespace": self.secret_namespace,
                 "csi.storage.k8s.io/provisioner-secret-name": self.secret_name,
                 "csi.storage.k8s.io/provisioner-secret-namespace": self.secret_namespace,
+                "csi.storage.k8s.io/controller-expand-secret-name": self.secret_name,
+                "csi.storage.k8s.io/controller-expand-secret-namespace": self.secret_namespace,
                 "juicefs/mount-cpu-limit": "5",
                 "juicefs/mount-memory-limit": "5Gi",
                 "juicefs/mount-cpu-request": "100m",
                 "juicefs/mount-memory-request": "500Mi",
-            }
+            },
+            allow_volume_expansion=True,
         )
         if self.parameters:
             for k, v in self.parameters.items():
@@ -117,12 +120,13 @@ def __init__(self, *, name, access_mode, storage_name, pv, labels=None, annotati
         self.pv = pv
         self.labels = labels
         self.annotations = annotations
+        self.capacity = "1Gi"
 
     def create(self):
         spec = client.V1PersistentVolumeClaimSpec(
             access_modes=[self.access_mode],
             resources=client.V1ResourceRequirements(
-                requests={"storage": "1Gi"}
+                requests={"storage": self.capacity}
             )
         )
         if self.pv != "":
@@ -137,6 +141,15 @@ def create(self):
         client.CoreV1Api().create_namespaced_persistent_volume_claim(namespace=self.namespace, body=pvc)
         PVCs.append(self)
 
+    def update_capacity(self, capacity):
+        pvc = client.CoreV1Api().read_namespaced_persistent_volume_claim(name=self.name, namespace=self.namespace)
+        pvc.spec.resources = client.V1ResourceRequirements(
+            requests={"storage": capacity}
+        )
+        client.CoreV1Api().replace_namespaced_persistent_volume_claim(
+            name=self.name, namespace=self.namespace, body=pvc
+        )
+
     def delete(self):
         client.CoreV1Api().delete_namespaced_persistent_volume_claim(name=self.name, namespace=self.namespace)
         PVCs.remove(self)

.github/scripts/test_case.py

@@ -2274,3 +2274,76 @@ def test_webhook_two_volume():
     LOG.info("Remove pvc 2 {}".format(pvc2.name))
     pvc2.delete()
     return
+
+
+def test_dynamic_expand():
+    LOG.info("[test case] Dynamic PVC capacity expand begin..")
+    # deploy pvc
+    pvc = PVC(name="pvc-cap-expand", access_mode="ReadWriteMany", storage_name=STORAGECLASS_NAME, pv="")
+    LOG.info("Deploy pvc {}".format(pvc.name))
+    pvc.create()
+
+    # wait for pvc bound
+    for i in range(0, 60):
+        if pvc.check_is_bound():
+            break
+        time.sleep(1)
+
+    # deploy pod
+    deployment = Deployment(name="app-dynamic-cap-expand", pvc=pvc.name, replicas=1)
+    LOG.info("Deploy deployment {}".format(deployment.name))
+    deployment.create()
+    pod = Pod(name="", deployment_name=deployment.name, replicas=deployment.replicas)
+    LOG.info("Watch for pods of {} for success.".format(deployment.name))
+    result = pod.watch_for_success()
+    if not result:
+        if MOUNT_MODE == "webhook":
+            pods = client.CoreV1Api().list_namespaced_pod(
+                namespace="default",
+                label_selector="deployment={}".format(deployment.name)
+            )
+            for po in pods.items:
+                pod_name = po.metadata.name
+                if not check_pod_ready(po):
+                    subprocess.check_call(["kubectl", "get", "po", pod_name, "-o", "yaml", "-n", "default"])
+        raise Exception("Pods of deployment {} are not ready within 10 min.".format(deployment.name))
+
+    # check mount point
+    LOG.info("Check mount point..")
+    volume_id = pvc.get_volume_id()
+    LOG.info("Get volume_id {}".format(volume_id))
+    check_path = volume_id + "/out.txt"
+    result = check_mount_point(check_path)
+    if not result:
+        if MOUNT_MODE == "webhook":
+            pods = client.CoreV1Api().list_namespaced_pod(
+                namespace="default",
+                label_selector="deployment={}".format(deployment.name)
+            )
+            for po in pods.items:
+                pod_name = po.metadata.name
+                subprocess.check_call(["kubectl", "logs", pod_name, "-c", "jfs-mount", "-n", "default"])
+                subprocess.check_call(["kubectl", "logs", pod_name, "-c", "app", "-n", "default"])
+        raise Exception("mount Point of /jfs/{}/out.txt are not ready within 5 min.".format(volume_id))
+    LOG.info("Test pass.")
+
+    # expand pvc
+    LOG.info("Expand pvc {} to 2Gi".format(pvc.name))
+    pvc.update_capacity("2Gi")
+    pods = client.CoreV1Api().list_namespaced_pod(
+        namespace="default",
+        label_selector="deployment={}".format(deployment.name)
+    )
+    check_quota(pods.items[0].metadata.name, "2.0G")
+
+    # delete test resources
+    LOG.info("Remove deployment {}".format(deployment.name))
+    deployment.delete()
+    pod = Pod(name="", deployment_name=deployment.name, replicas=deployment.replicas)
+    LOG.info("Watch for pods of deployment {} for delete.".format(deployment.name))
+    result = pod.watch_for_delete(deployment.replicas)
+    if not result:
+        raise Exception("Pods of deployment {} are not delete within 5 min.".format(deployment.name))
+    LOG.info("Remove pvc {}".format(pvc.name))
+    pvc.delete()
+    return

.github/scripts/util.py

@@ -103,6 +103,7 @@ def check_mount_point(check_path):
             raise e
     return False
 
+
 def check_quota(name, expected):
     output = ""
     for i in range(0, 10):
@@ -127,6 +128,7 @@ def check_quota(name, expected):
         return
     raise Exception("quota is not set:\n{}".format(output))
 
+
 def wait_dir_empty(check_path):
     LOG.info(f"check path {check_path} empty")
     for i in range(0, 60):

deploy/kubernetes/base/resources.yaml

@@ -14,10 +14,10 @@ metadata:
 rules:
   - apiGroups: [""]
     resources: ["persistentvolumes"]
-    verbs: ["get", "list", "watch", "create", "delete"]
+    verbs: ["get", "list", "watch", "create", "delete", "patch"]
   - apiGroups: [""]
-    resources: ["persistentvolumeclaims"]
-    verbs: ["get", "list", "watch", "update"]
+    resources: ["persistentvolumeclaims", "persistentvolumeclaims/status"]
+    verbs: ["get", "list", "watch", "update", "patch"]
   - apiGroups: ["storage.k8s.io"]
     resources: ["storageclasses"]
     verbs: ["get", "list", "watch"]
@@ -298,6 +298,19 @@ spec:
           volumeMounts:
             - name: socket-dir
               mountPath: /var/lib/csi/sockets/pluginproxy/
+        - name: csi-resizer
+          image: quay.io/k8scsi/csi-resizer:v1.0.1
+          imagePullPolicy: Always
+          args:
+            - --csi-address=$(ADDRESS)
+            - --leader-election
+            - --v=2
+          env:
+            - name: ADDRESS
+              value: /var/lib/csi/sockets/pluginproxy/csi.sock
+          volumeMounts:
+            - name: socket-dir
+              mountPath: /var/lib/csi/sockets/pluginproxy/
         - name: liveness-probe
           image: quay.io/k8scsi/livenessprobe:v1.1.0
           args:

go.mod

@@ -5,13 +5,12 @@ require (
 	github.com/container-storage-interface/spec v1.5.0
 	github.com/go-logr/logr v1.2.0
 	github.com/golang/mock v1.6.0
-	github.com/kubernetes-csi/csi-test v1.1.0
 	github.com/onsi/ginkgo v1.16.5
 	github.com/onsi/gomega v1.17.0
 	github.com/pkg/errors v0.9.1
 	github.com/smartystreets/goconvey v1.6.4
 	github.com/spf13/cobra v1.1.3
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4
 	google.golang.org/grpc v1.53.0
 	k8s.io/api v0.23.0
 	k8s.io/apimachinery v0.23.0
@@ -35,7 +34,6 @@ require (
 	github.com/google/gofuzz v1.1.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
-	github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/inconshreveable/mousetrap v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
@@ -46,11 +44,9 @@ require (
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
-	github.com/prometheus/client_golang v1.11.1 // indirect
 	github.com/prometheus/client_model v0.2.0 // indirect
 	github.com/prometheus/common v0.28.0 // indirect
 	github.com/prometheus/procfs v0.6.0 // indirect
-	github.com/smartystreets/assertions v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	golang.org/x/crypto v0.1.0 // indirect
 	golang.org/x/net v0.5.0 // indirect
@@ -66,16 +62,23 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.23.0 // indirect
 	k8s.io/component-base v0.23.0 // indirect
-	k8s.io/klog/v2 v2.30.0 // indirect
+	k8s.io/klog/v2 v2.70.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.0 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
-go 1.17
+require (
+	github.com/gopherjs/gopherjs v0.0.0-20200217142428-fce0ec30dd00 // indirect
+	github.com/kubernetes-csi/csi-test v1.1.1
+	github.com/prometheus/client_golang v1.11.1 // indirect
+	github.com/smartystreets/assertions v1.1.0 // indirect
+)
+
+go 1.19
 
 replace k8s.io/api => k8s.io/api v0.22.0