support vci serverless (#753)

* support vci serverless * del initContainer & return error if readonly in dynamic provisoner * put quota cmd in poststart script --------- Signed-off-by: zwwhdls <[email protected]>
juicedata · Oct 13, 2023 · 4106e83 · 4106e83
1 parent eb067a3
commit 4106e83
Show file tree

Hide file tree

Showing 27 changed files with 1,218 additions and 576 deletions.
diff --git a/.github/scripts/e2e-test.py b/.github/scripts/e2e-test.py
@@ -25,7 +25,6 @@
     test_static_delete_policy,
     test_deployment_using_storage_rw,
     test_quota_using_storage_rw,
-    test_deployment_using_storage_ro,
     test_deployment_use_pv_rw,
     test_deployment_use_pv_ro,
     test_delete_all,
@@ -67,7 +66,6 @@
                 test_dynamic_cache_clean_upon_umount()
                 test_static_delete_policy()
                 test_deployment_using_storage_rw()
-                test_deployment_using_storage_ro()
                 test_deployment_use_pv_rw()
                 test_deployment_use_pv_ro()
                 test_delete_one()
@@ -109,7 +107,6 @@
                 test_job_complete_using_storage()
                 test_static_delete_policy()
                 test_deployment_using_storage_rw()
-                test_deployment_using_storage_ro()
                 test_dynamic_mount_image_with_webhook()
                 test_static_mount_image_with_webhook()
                 test_deployment_dynamic_patch_pv_with_webhook()
@@ -121,7 +118,6 @@
                 test_webhook_two_volume()
                 test_static_delete_policy()
                 test_deployment_using_storage_rw()
-                test_deployment_using_storage_ro()
                 test_deployment_use_pv_rw()
                 test_deployment_use_pv_ro()
                 test_deployment_dynamic_patch_pv_with_webhook()
@@ -138,7 +134,6 @@
                 test_static_cache_clean_upon_umount()
                 test_dynamic_cache_clean_upon_umount()
                 test_deployment_using_storage_rw()
-                test_deployment_using_storage_ro()
                 test_deployment_use_pv_rw()
                 test_deployment_use_pv_ro()
                 test_delete_pvc()

diff --git a/.github/scripts/test_case.py b/.github/scripts/test_case.py
@@ -164,6 +164,7 @@ def test_quota_using_storage_rw():
     return
 
 
+# this case is not valid.
 def test_deployment_using_storage_ro():
     LOG.info("[test case] Deployment using storageClass with rom begin..")
     # deploy pvc

diff --git a/deploy/k8s.yaml b/deploy/k8s.yaml
@@ -66,6 +66,13 @@ rules:
   - update
   - delete
   - patch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
 - apiGroups:
   - ""
   resources:

diff --git a/deploy/k8s_before_v1_18.yaml b/deploy/k8s_before_v1_18.yaml
@@ -66,6 +66,13 @@ rules:
   - update
   - delete
   - patch
+- apiGroups:
+  - ""
+  resources:
+  - nodes
+  verbs:
+  - get
+  - list
 - apiGroups:
   - ""
   resources:

diff --git a/deploy/kubernetes/base/resources.yaml b/deploy/kubernetes/base/resources.yaml
@@ -517,6 +517,31 @@ webhooks:
       matchLabels:
         juicefs.com/enable-injection: "true"
 ---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
+metadata:
+  name: juicefs-admission-serverless-webhook
+webhooks:
+  - name: sidecar.inject.serverless.juicefs.com
+    rules:
+      - apiGroups:   [""]
+        apiVersions: ["v1"]
+        operations:  ["CREATE"]
+        resources:   ["pods"]
+    clientConfig:
+      service:
+        namespace: kube-system
+        name: juicefs-admission-webhook
+        path: "/juicefs/serverless/inject-v1-pod"
+      caBundle: CA_BUNDLE
+    timeoutSeconds: 20
+    failurePolicy: Fail
+    sideEffects: None
+    admissionReviewVersions: ["v1","v1beta1"]
+    namespaceSelector:
+      matchLabels:
+        juicefs.com/enable-serverless-injection: "true"
+---
 apiVersion: v1
 kind: Service
 metadata:

diff --git a/deploy/kubernetes/release/kustomization.yaml b/deploy/kubernetes/release/kustomization.yaml
@@ -42,3 +42,14 @@ patches:
     version: v1
     kind: MutatingWebhookConfiguration
     name: juicefs-admission-webhook
+- patch: |-
+    $patch: delete
+    apiVersion: admissionregistration.k8s.io/v1
+    kind: MutatingWebhookConfiguration
+    metadata:
+      name: juicefs-admission-serverless-webhook
+  target:
+    group: admissionregistration.k8s.io
+    version: v1
+    kind: MutatingWebhookConfiguration
+    name: juicefs-admission-serverless-webhook
diff --git a/deploy/webhook-with-certmanager.yaml b/deploy/webhook-with-certmanager.yaml
@@ -381,6 +381,41 @@ spec:
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
+metadata:
+  labels:
+    app.kubernetes.io/instance: juicefs-csi-driver
+    app.kubernetes.io/name: juicefs-csi-driver
+    app.kubernetes.io/version: master
+  name: juicefs-admission-serverless-webhook
+webhooks:
+- admissionReviewVersions:
+  - v1
+  - v1beta1
+  clientConfig:
+    caBundle: CA_BUNDLE
+    service:
+      name: juicefs-admission-webhook
+      namespace: kube-system
+      path: /juicefs/serverless/inject-v1-pod
+  failurePolicy: Fail
+  name: sidecar.inject.serverless.juicefs.com
+  namespaceSelector:
+    matchLabels:
+      juicefs.com/enable-serverless-injection: "true"
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    resources:
+    - pods
+  sideEffects: None
+  timeoutSeconds: 20
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
 metadata:
   annotations:
     cert-manager.io/inject-ca-from: kube-system/juicefs-cert

diff --git a/deploy/webhook.yaml b/deploy/webhook.yaml
@@ -386,6 +386,41 @@ spec:
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
+metadata:
+  labels:
+    app.kubernetes.io/instance: juicefs-csi-driver
+    app.kubernetes.io/name: juicefs-csi-driver
+    app.kubernetes.io/version: master
+  name: juicefs-admission-serverless-webhook
+webhooks:
+- admissionReviewVersions:
+  - v1
+  - v1beta1
+  clientConfig:
+    caBundle: CA_BUNDLE
+    service:
+      name: juicefs-admission-webhook
+      namespace: kube-system
+      path: /juicefs/serverless/inject-v1-pod
+  failurePolicy: Fail
+  name: sidecar.inject.serverless.juicefs.com
+  namespaceSelector:
+    matchLabels:
+      juicefs.com/enable-serverless-injection: "true"
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    resources:
+    - pods
+  sideEffects: None
+  timeoutSeconds: 20
+---
+apiVersion: admissionregistration.k8s.io/v1
+kind: MutatingWebhookConfiguration
 metadata:
   labels:
     app.kubernetes.io/instance: juicefs-csi-driver

diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go
@@ -84,6 +84,12 @@ func (d *controllerService) CreateVolume(ctx context.Context, req *csi.CreateVol
 	for k, v := range req.Parameters {
 		volCtx[k] = v
 	}
+	// return error if set readonly in dynamic provisioner
+	for _, vc := range req.VolumeCapabilities {
+		if vc.AccessMode.GetMode() == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY {
+			return nil, status.Errorf(codes.InvalidArgument, "Dynamic mounting uses the sub-path named pv name as data isolation, so read-only mode cannot be used.")
+		}
+	}
 	// create volume
 	//err := d.juicefs.JfsCreateVol(ctx, volumeId, subPath, secrets, volCtx)
 	//if err != nil {

diff --git a/pkg/driver/provisioner.go b/pkg/driver/provisioner.go
@@ -24,6 +24,8 @@ import (
 	"strings"
 	"time"
 
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/klog"
@@ -88,6 +90,16 @@ func (j *provisionerService) Provision(ctx context.Context, options provisioncon
 	if options.StorageClass.Parameters["pathPattern"] != "" {
 		subPath = pvMeta.StringParser(options.StorageClass.Parameters["pathPattern"])
 	}
+	// return error if set readonly in dynamic provisioner
+	for _, am := range options.PVC.Spec.AccessModes {
+		if am == corev1.ReadOnlyMany {
+			if options.StorageClass.Parameters["pathPattern"] == "" {
+				return nil, provisioncontroller.ProvisioningFinished, status.Errorf(codes.InvalidArgument, "Dynamic mounting uses the sub-path named pv name as data isolation, so read-only mode cannot be used.")
+			} else {
+				klog.Warningf("Volume is set readonly, please make sure the subpath %s exists.", subPath)
+			}
+		}
+	}
 
 	mountOptions := make([]string, 0)
 	for _, mo := range options.StorageClass.MountOptions {