diff --git a/.github/scripts/e2e-test.py b/.github/scripts/e2e-test.py index 89f8525008..f9ea63da80 100644 --- a/.github/scripts/e2e-test.py +++ b/.github/scripts/e2e-test.py @@ -25,7 +25,6 @@ test_static_delete_policy, test_deployment_using_storage_rw, test_quota_using_storage_rw, - test_deployment_using_storage_ro, test_deployment_use_pv_rw, test_deployment_use_pv_ro, test_delete_all, @@ -67,7 +66,6 @@ test_dynamic_cache_clean_upon_umount() test_static_delete_policy() test_deployment_using_storage_rw() - test_deployment_using_storage_ro() test_deployment_use_pv_rw() test_deployment_use_pv_ro() test_delete_one() @@ -109,7 +107,6 @@ test_job_complete_using_storage() test_static_delete_policy() test_deployment_using_storage_rw() - test_deployment_using_storage_ro() test_dynamic_mount_image_with_webhook() test_static_mount_image_with_webhook() test_deployment_dynamic_patch_pv_with_webhook() @@ -121,7 +118,6 @@ test_webhook_two_volume() test_static_delete_policy() test_deployment_using_storage_rw() - test_deployment_using_storage_ro() test_deployment_use_pv_rw() test_deployment_use_pv_ro() test_deployment_dynamic_patch_pv_with_webhook() @@ -138,7 +134,6 @@ test_static_cache_clean_upon_umount() test_dynamic_cache_clean_upon_umount() test_deployment_using_storage_rw() - test_deployment_using_storage_ro() test_deployment_use_pv_rw() test_deployment_use_pv_ro() test_delete_pvc() diff --git a/.github/scripts/test_case.py b/.github/scripts/test_case.py index 2e8247bb99..d20a8ae732 100644 --- a/.github/scripts/test_case.py +++ b/.github/scripts/test_case.py @@ -164,6 +164,7 @@ def test_quota_using_storage_rw(): return +# this case is not valid. def test_deployment_using_storage_ro(): LOG.info("[test case] Deployment using storageClass with rom begin..") # deploy pvc diff --git a/deploy/k8s.yaml b/deploy/k8s.yaml index 5c48700c84..42e2dd6cd3 100644 --- a/deploy/k8s.yaml +++ b/deploy/k8s.yaml @@ -66,6 +66,13 @@ rules: - update - delete - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list - apiGroups: - "" resources: diff --git a/deploy/k8s_before_v1_18.yaml b/deploy/k8s_before_v1_18.yaml index 5608988361..2a176f8261 100644 --- a/deploy/k8s_before_v1_18.yaml +++ b/deploy/k8s_before_v1_18.yaml @@ -66,6 +66,13 @@ rules: - update - delete - patch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list - apiGroups: - "" resources: diff --git a/deploy/kubernetes/base/resources.yaml b/deploy/kubernetes/base/resources.yaml index 3592f1006d..f2dfb14f10 100644 --- a/deploy/kubernetes/base/resources.yaml +++ b/deploy/kubernetes/base/resources.yaml @@ -517,6 +517,31 @@ webhooks: matchLabels: juicefs.com/enable-injection: "true" --- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + name: juicefs-admission-serverless-webhook +webhooks: + - name: sidecar.inject.serverless.juicefs.com + rules: + - apiGroups: [""] + apiVersions: ["v1"] + operations: ["CREATE"] + resources: ["pods"] + clientConfig: + service: + namespace: kube-system + name: juicefs-admission-webhook + path: "/juicefs/serverless/inject-v1-pod" + caBundle: CA_BUNDLE + timeoutSeconds: 20 + failurePolicy: Fail + sideEffects: None + admissionReviewVersions: ["v1","v1beta1"] + namespaceSelector: + matchLabels: + juicefs.com/enable-serverless-injection: "true" +--- apiVersion: v1 kind: Service metadata: diff --git a/deploy/kubernetes/release/kustomization.yaml b/deploy/kubernetes/release/kustomization.yaml index f73d95ab46..ff5e07005d 100644 --- a/deploy/kubernetes/release/kustomization.yaml +++ b/deploy/kubernetes/release/kustomization.yaml @@ -42,3 +42,14 @@ patches: version: v1 kind: MutatingWebhookConfiguration name: juicefs-admission-webhook +- patch: |- + $patch: delete + apiVersion: admissionregistration.k8s.io/v1 + kind: MutatingWebhookConfiguration + metadata: + name: juicefs-admission-serverless-webhook + target: + group: admissionregistration.k8s.io + version: v1 + kind: MutatingWebhookConfiguration + name: juicefs-admission-serverless-webhook diff --git a/deploy/webhook-with-certmanager.yaml b/deploy/webhook-with-certmanager.yaml index e00bfee819..9703361f4a 100644 --- a/deploy/webhook-with-certmanager.yaml +++ b/deploy/webhook-with-certmanager.yaml @@ -381,6 +381,41 @@ spec: --- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/instance: juicefs-csi-driver + app.kubernetes.io/name: juicefs-csi-driver + app.kubernetes.io/version: master + name: juicefs-admission-serverless-webhook +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: CA_BUNDLE + service: + name: juicefs-admission-webhook + namespace: kube-system + path: /juicefs/serverless/inject-v1-pod + failurePolicy: Fail + name: sidecar.inject.serverless.juicefs.com + namespaceSelector: + matchLabels: + juicefs.com/enable-serverless-injection: "true" + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None + timeoutSeconds: 20 +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration metadata: annotations: cert-manager.io/inject-ca-from: kube-system/juicefs-cert diff --git a/deploy/webhook.yaml b/deploy/webhook.yaml index ab67f11a20..31b98441a6 100644 --- a/deploy/webhook.yaml +++ b/deploy/webhook.yaml @@ -386,6 +386,41 @@ spec: --- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/instance: juicefs-csi-driver + app.kubernetes.io/name: juicefs-csi-driver + app.kubernetes.io/version: master + name: juicefs-admission-serverless-webhook +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: CA_BUNDLE + service: + name: juicefs-admission-webhook + namespace: kube-system + path: /juicefs/serverless/inject-v1-pod + failurePolicy: Fail + name: sidecar.inject.serverless.juicefs.com + namespaceSelector: + matchLabels: + juicefs.com/enable-serverless-injection: "true" + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None + timeoutSeconds: 20 +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration metadata: labels: app.kubernetes.io/instance: juicefs-csi-driver diff --git a/pkg/driver/controller.go b/pkg/driver/controller.go index a9fe45cea4..dcce300a60 100644 --- a/pkg/driver/controller.go +++ b/pkg/driver/controller.go @@ -84,6 +84,12 @@ func (d *controllerService) CreateVolume(ctx context.Context, req *csi.CreateVol for k, v := range req.Parameters { volCtx[k] = v } + // return error if set readonly in dynamic provisioner + for _, vc := range req.VolumeCapabilities { + if vc.AccessMode.GetMode() == csi.VolumeCapability_AccessMode_MULTI_NODE_READER_ONLY { + return nil, status.Errorf(codes.InvalidArgument, "Dynamic mounting uses the sub-path named pv name as data isolation, so read-only mode cannot be used.") + } + } // create volume //err := d.juicefs.JfsCreateVol(ctx, volumeId, subPath, secrets, volCtx) //if err != nil { diff --git a/pkg/driver/provisioner.go b/pkg/driver/provisioner.go index 417d776ed3..78c4aabfb3 100644 --- a/pkg/driver/provisioner.go +++ b/pkg/driver/provisioner.go @@ -24,6 +24,8 @@ import ( "strings" "time" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog" @@ -88,6 +90,16 @@ func (j *provisionerService) Provision(ctx context.Context, options provisioncon if options.StorageClass.Parameters["pathPattern"] != "" { subPath = pvMeta.StringParser(options.StorageClass.Parameters["pathPattern"]) } + // return error if set readonly in dynamic provisioner + for _, am := range options.PVC.Spec.AccessModes { + if am == corev1.ReadOnlyMany { + if options.StorageClass.Parameters["pathPattern"] == "" { + return nil, provisioncontroller.ProvisioningFinished, status.Errorf(codes.InvalidArgument, "Dynamic mounting uses the sub-path named pv name as data isolation, so read-only mode cannot be used.") + } else { + klog.Warningf("Volume is set readonly, please make sure the subpath %s exists.", subPath) + } + } + } mountOptions := make([]string, 0) for _, mo := range options.StorageClass.MountOptions { diff --git a/pkg/juicefs/mount/builder/common.go b/pkg/juicefs/mount/builder/common.go index afeffb7d9b..74bbf0f81a 100644 --- a/pkg/juicefs/mount/builder/common.go +++ b/pkg/juicefs/mount/builder/common.go @@ -18,53 +18,72 @@ package builder import ( "fmt" - "path/filepath" + "path" + "regexp" + "strconv" + "strings" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - utilpointer "k8s.io/utils/pointer" + "k8s.io/klog" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/juicedata/juicefs-csi-driver/pkg/config" + "github.com/juicedata/juicefs-csi-driver/pkg/util" ) const ( JfsDirName = "jfs-dir" - JfsRootDirName = "jfs-root-dir" UpdateDBDirName = "updatedb" UpdateDBCfgFile = "/etc/updatedb.conf" ) -type Builder struct { +type BaseBuilder struct { jfsSetting *config.JfsSetting capacity int64 } -func NewBuilder(setting *config.JfsSetting, capacity int64) *Builder { - return &Builder{setting, capacity} -} - -func (r *Builder) generateJuicePod() *corev1.Pod { - pod := r.generatePodTemplate() - - volumes := r.getVolumes() - volumeMounts := r.getVolumeMounts() - i := 1 - for k, v := range r.jfsSetting.Configs { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: fmt.Sprintf("config-%v", i), - MountPath: v, - }) - volumes = append(volumes, corev1.Volume{ - Name: fmt.Sprintf("config-%v", i), - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: k, - }, +// genPodTemplate generates a pod template from csi pod +func (r *BaseBuilder) genPodTemplate(baseCnGen func() corev1.Container) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: r.jfsSetting.Attr.Namespace, + Labels: map[string]string{ + config.PodTypeKey: config.PodTypeValue, + config.PodUniqueIdLabelKey: r.jfsSetting.UniqueId, }, - }) - i++ + Annotations: make(map[string]string), + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{baseCnGen()}, + NodeName: config.NodeName, + HostNetwork: r.jfsSetting.Attr.HostNetwork, + HostAliases: r.jfsSetting.Attr.HostAliases, + HostPID: r.jfsSetting.Attr.HostPID, + HostIPC: r.jfsSetting.Attr.HostIPC, + DNSConfig: r.jfsSetting.Attr.DNSConfig, + DNSPolicy: r.jfsSetting.Attr.DNSPolicy, + ServiceAccountName: r.jfsSetting.ServiceAccountName, + ImagePullSecrets: r.jfsSetting.Attr.ImagePullSecrets, + PreemptionPolicy: r.jfsSetting.Attr.PreemptionPolicy, + Tolerations: r.jfsSetting.Attr.Tolerations, + }, } +} +// genCommonJuicePod generates a pod with common settings +func (r *BaseBuilder) genCommonJuicePod(cnGen func() corev1.Container) *corev1.Pod { + pod := r.genPodTemplate(cnGen) + // labels & annotations + pod.ObjectMeta.Labels, pod.ObjectMeta.Annotations = r._genMetadata() + pod.Spec.ServiceAccountName = r.jfsSetting.ServiceAccountName + pod.Spec.PriorityClassName = config.JFSMountPriorityName + pod.Spec.RestartPolicy = corev1.RestartPolicyAlways + gracePeriod := int64(10) + pod.Spec.TerminationGracePeriodSeconds = &gracePeriod + controllerutil.AddFinalizer(pod, config.Finalizer) + + volumes, volumeMounts := r._genJuiceVolumes() pod.Spec.Volumes = volumes pod.Spec.Containers[0].VolumeMounts = volumeMounts pod.Spec.Containers[0].EnvFrom = []corev1.EnvFromSource{{ @@ -74,48 +93,160 @@ func (r *Builder) generateJuicePod() *corev1.Pod { }, }, }} - if r.jfsSetting.FormatCmd != "" { - initContainer := r.getInitContainer() - initContainer.VolumeMounts = append(initContainer.VolumeMounts, volumeMounts...) - pod.Spec.InitContainers = []corev1.Container{initContainer} + pod.Spec.Containers[0].Resources = r.jfsSetting.Resources + pod.Spec.Containers[0].Lifecycle = &corev1.Lifecycle{ + PreStop: &corev1.Handler{ + Exec: &corev1.ExecAction{Command: []string{"sh", "-c", fmt.Sprintf( + "umount %s && rmdir %s", r.jfsSetting.MountPath, r.jfsSetting.MountPath)}}, + }, + } + + if r.jfsSetting.Attr.HostNetwork { + // When using hostNetwork, the MountPod will use a random port for metrics. + // Before inducing any auxiliary method to detect that random port, the + // best way is to avoid announcing any port about that. + pod.Spec.Containers[0].Ports = []corev1.ContainerPort{} + } else { + pod.Spec.Containers[0].Ports = []corev1.ContainerPort{ + {Name: "metrics", ContainerPort: r.genMetricsPort()}, + } } return pod } -func (r *Builder) getVolumes() []corev1.Volume { - dir := corev1.HostPathDirectoryOrCreate - file := corev1.HostPathFileOrCreate - secretName := r.jfsSetting.SecretName - volumes := []corev1.Volume{{ - Name: JfsDirName, - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: config.MountPointPath, - Type: &dir, - }, - }, - }} +// genMountCommand generates mount command +func (r *BaseBuilder) genMountCommand() string { + cmd := "" + options := r.jfsSetting.Options + if r.jfsSetting.IsCe { + klog.V(5).Infof("ceMount: mount %v at %v", util.StripPasswd(r.jfsSetting.Source), r.jfsSetting.MountPath) + mountArgs := []string{config.CeMountPath, "${metaurl}", r.jfsSetting.MountPath} + if !util.ContainsPrefix(options, "metrics=") { + if r.jfsSetting.Attr.HostNetwork { + // Pick up a random (useable) port for hostNetwork MountPods. + options = append(options, "metrics=0.0.0.0:0") + } else { + options = append(options, "metrics=0.0.0.0:9567") + } + } + mountArgs = append(mountArgs, "-o", strings.Join(options, ",")) + cmd = strings.Join(mountArgs, " ") + } else { + klog.V(5).Infof("Mount: mount %v at %v", util.StripPasswd(r.jfsSetting.Source), r.jfsSetting.MountPath) + mountArgs := []string{config.JfsMountPath, r.jfsSetting.Source, r.jfsSetting.MountPath} + mountOptions := []string{"foreground", "no-update"} + if r.jfsSetting.EncryptRsaKey != "" { + mountOptions = append(mountOptions, "rsa-key=/root/.rsa/rsa-key.pem") + } + mountOptions = append(mountOptions, options...) + mountArgs = append(mountArgs, "-o", strings.Join(mountOptions, ",")) + cmd = strings.Join(mountArgs, " ") + } + return util.QuoteForShell(cmd) +} - if !config.Immutable { - volumes = append(volumes, corev1.Volume{ - Name: UpdateDBDirName, - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: UpdateDBCfgFile, - Type: &file, - }, - }}) +// genInitCommand generates init command +func (r *BaseBuilder) genInitCommand() string { + formatCmd := r.jfsSetting.FormatCmd + if r.jfsSetting.EncryptRsaKey != "" { + if r.jfsSetting.IsCe { + formatCmd = formatCmd + " --encrypt-rsa-key=/root/.rsa/rsa-key.pem" + } } - if r.jfsSetting.FormatCmd != "" { - // initContainer will generate xx.conf to share with mount container - volumes = append(volumes, corev1.Volume{ - Name: JfsRootDirName, - VolumeSource: corev1.VolumeSource{ - EmptyDir: nil, - }, - }) + return formatCmd +} + +func (r *BaseBuilder) getQuotaPath() string { + quotaPath := r.jfsSetting.SubPath + var subdir string + for _, o := range r.jfsSetting.Options { + pair := strings.Split(o, "=") + if len(pair) != 2 { + continue + } + if pair[0] == "subdir" { + subdir = path.Join("/", pair[1]) + } + } + targetPath := path.Join(subdir, quotaPath) + return targetPath +} + +// genJobCommand generates job command +func (r *BaseBuilder) getJobCommand() string { + var cmd string + options := util.StripReadonlyOption(r.jfsSetting.Options) + if r.jfsSetting.IsCe { + args := []string{config.CeMountPath, "${metaurl}", "/mnt/jfs"} + if len(options) != 0 { + args = append(args, "-o", strings.Join(options, ",")) + } + cmd = strings.Join(args, " ") + } else { + args := []string{config.JfsMountPath, r.jfsSetting.Source, "/mnt/jfs"} + if r.jfsSetting.EncryptRsaKey != "" { + options = append(options, "rsa-key=/root/.rsa/rsa-key.pem") + } + options = append(options, "background") + args = append(args, "-o", strings.Join(options, ",")) + cmd = strings.Join(args, " ") + } + return util.QuoteForShell(cmd) +} + +// genMetricsPort generates metrics port +func (r *BaseBuilder) genMetricsPort() int32 { + port := int64(9567) + options := r.jfsSetting.Options + + for _, option := range options { + if strings.HasPrefix(option, "metrics=") { + re := regexp.MustCompile(`metrics=.*:([0-9]{1,6})`) + match := re.FindStringSubmatch(option) + if len(match) > 0 { + port, _ = strconv.ParseInt(match[1], 10, 32) + } + } + } + + return int32(port) +} + +// _genMetadata generates labels & annotations +func (r *BaseBuilder) _genMetadata() (labels map[string]string, annotations map[string]string) { + labels = map[string]string{ + config.PodTypeKey: config.PodTypeValue, + config.PodUniqueIdLabelKey: r.jfsSetting.UniqueId, + } + annotations = map[string]string{} + + for k, v := range r.jfsSetting.MountPodLabels { + labels[k] = v + } + for k, v := range r.jfsSetting.MountPodAnnotations { + annotations[k] = v + } + if r.jfsSetting.DeletedDelay != "" { + annotations[config.DeleteDelayTimeKey] = r.jfsSetting.DeletedDelay } + annotations[config.JuiceFSUUID] = r.jfsSetting.UUID + annotations[config.UniqueId] = r.jfsSetting.UniqueId + if r.jfsSetting.CleanCache { + annotations[config.CleanCache] = "true" + } + return +} + +// _genJuiceVolumes generates volumes & volumeMounts +// 1. if encrypt_rsa_key is set, mount secret to /root/.rsa +// 2. if init_config is set, mount secret to /root/.config +// 3. configs in secret +func (r *BaseBuilder) _genJuiceVolumes() ([]corev1.Volume, []corev1.VolumeMount) { + volumes := []corev1.Volume{} + volumeMounts := []corev1.VolumeMount{} + secretName := r.jfsSetting.SecretName + if r.jfsSetting.EncryptRsaKey != "" { volumes = append(volumes, corev1.Volume{ Name: "rsa-key", @@ -129,18 +260,12 @@ func (r *Builder) getVolumes() []corev1.Volume { }, }, }) - } - if config.Webhook { - var mode int32 = 0755 - volumes = append(volumes, corev1.Volume{ - Name: "jfs-check-mount", - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: secretName, - DefaultMode: utilpointer.Int32Ptr(mode), - }, + volumeMounts = append(volumeMounts, + corev1.VolumeMount{ + Name: "rsa-key", + MountPath: "/root/.rsa", }, - }) + ) } if r.jfsSetting.InitConfig != "" { volumes = append(volumes, corev1.Volume{ @@ -153,45 +278,6 @@ func (r *Builder) getVolumes() []corev1.Volume { }}, }}, }) - } - return volumes -} - -func (r *Builder) getVolumeMounts() []corev1.VolumeMount { - mp := corev1.MountPropagationBidirectional - volumeMounts := []corev1.VolumeMount{{ - Name: JfsDirName, - MountPath: config.PodMountBase, - MountPropagation: &mp, - }} - - if !config.Immutable { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: UpdateDBDirName, - MountPath: UpdateDBCfgFile, - MountPropagation: &mp, - }) - } - - if r.jfsSetting.FormatCmd != "" { - // initContainer will generate xx.conf to share with mount container - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: JfsRootDirName, - MountPath: "/root/.juicefs", - MountPropagation: &mp, - }) - } - if r.jfsSetting.EncryptRsaKey != "" { - if !r.jfsSetting.IsCe { - volumeMounts = append(volumeMounts, - corev1.VolumeMount{ - Name: "rsa-key", - MountPath: "/root/.rsa", - }, - ) - } - } - if r.jfsSetting.InitConfig != "" { volumeMounts = append(volumeMounts, corev1.VolumeMount{ Name: "init-config", @@ -199,100 +285,21 @@ func (r *Builder) getVolumeMounts() []corev1.VolumeMount { }, ) } - if config.Webhook { + i := 1 + for k, v := range r.jfsSetting.Configs { volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: "jfs-check-mount", - MountPath: checkMountScriptPath, - SubPath: checkMountScriptName, + Name: fmt.Sprintf("config-%v", i), + MountPath: v, }) - } - return volumeMounts -} - -func (r *Builder) generateCleanCachePod() *corev1.Pod { - volumeMountPrefix := "/var/jfsCache" - cacheVolumes := []corev1.Volume{} - cacheVolumeMounts := []corev1.VolumeMount{} - - hostPathType := corev1.HostPathDirectory - - for idx, cacheDir := range r.jfsSetting.CacheDirs { - name := fmt.Sprintf("cachedir-%d", idx) - - hostPathVolume := corev1.Volume{ - Name: name, - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ - Path: filepath.Join(cacheDir, r.jfsSetting.UUID, "raw"), - Type: &hostPathType, - }}, - } - cacheVolumes = append(cacheVolumes, hostPathVolume) - - volumeMount := corev1.VolumeMount{ - Name: name, - MountPath: filepath.Join(volumeMountPrefix, name), - } - cacheVolumeMounts = append(cacheVolumeMounts, volumeMount) - } - - pod := &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: r.jfsSetting.Attr.Namespace, - Labels: map[string]string{ - config.PodTypeKey: config.PodTypeValue, - }, - Annotations: make(map[string]string), - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{{ - Name: "jfs-cache-clean", - Image: r.jfsSetting.Attr.Image, - Command: []string{"sh", "-c", "rm -rf /var/jfsCache/*/chunks"}, - VolumeMounts: cacheVolumeMounts, - }}, - Volumes: cacheVolumes, - }, - } - return pod -} - -func (r *Builder) generatePodTemplate() *corev1.Pod { - return &corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: r.jfsSetting.Attr.Namespace, - Labels: map[string]string{ - config.PodTypeKey: config.PodTypeValue, - config.PodUniqueIdLabelKey: r.jfsSetting.UniqueId, + volumes = append(volumes, corev1.Volume{ + Name: fmt.Sprintf("config-%v", i), + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: k, + }, }, - Annotations: make(map[string]string), - }, - Spec: corev1.PodSpec{ - Containers: []corev1.Container{r.genCommonContainer()}, - NodeName: config.NodeName, - HostNetwork: r.jfsSetting.Attr.HostNetwork, - HostAliases: r.jfsSetting.Attr.HostAliases, - HostPID: r.jfsSetting.Attr.HostPID, - HostIPC: r.jfsSetting.Attr.HostIPC, - DNSConfig: r.jfsSetting.Attr.DNSConfig, - DNSPolicy: r.jfsSetting.Attr.DNSPolicy, - ServiceAccountName: r.jfsSetting.ServiceAccountName, - ImagePullSecrets: r.jfsSetting.Attr.ImagePullSecrets, - PreemptionPolicy: r.jfsSetting.Attr.PreemptionPolicy, - Tolerations: r.jfsSetting.Attr.Tolerations, - }, - } -} - -func (r *Builder) genCommonContainer() corev1.Container { - isPrivileged := true - rootUser := int64(0) - return corev1.Container{ - Name: config.MountContainerName, - Image: r.jfsSetting.Attr.Image, - SecurityContext: &corev1.SecurityContext{ - Privileged: &isPrivileged, - RunAsUser: &rootUser, - }, - Env: []corev1.EnvVar{}, + }) + i++ } + return volumes, volumeMounts } diff --git a/pkg/juicefs/mount/builder/container.go b/pkg/juicefs/mount/builder/container.go index 9c03f4d2d7..f514850a56 100644 --- a/pkg/juicefs/mount/builder/container.go +++ b/pkg/juicefs/mount/builder/container.go @@ -1,5 +1,5 @@ /* - Copyright 2022 Juicedata Inc + Copyright 2023 Juicedata Inc Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -17,21 +17,100 @@ package builder import ( + "fmt" + "path/filepath" + "strconv" + corev1 "k8s.io/api/core/v1" + utilpointer "k8s.io/utils/pointer" + + "github.com/juicedata/juicefs-csi-driver/pkg/config" ) -func (r *Builder) NewMountSidecar() *corev1.Pod { +type ContainerBuilder struct { + PodBuilder +} + +var _ SidecarInterface = &ContainerBuilder{} + +func NewContainerBuilder(setting *config.JfsSetting, capacity int64) SidecarInterface { + return &ContainerBuilder{ + PodBuilder{BaseBuilder{ + jfsSetting: setting, + capacity: capacity, + }}, + } +} + +// NewMountSidecar generates a pod with a juicefs sidecar +// exactly the same spec as Mount Pod +func (r *ContainerBuilder) NewMountSidecar() *corev1.Pod { pod := r.NewMountPod("") - for i, v := range pod.Spec.Volumes { - if v.Name == JfsRootDirName { - v = corev1.Volume{ - Name: v.Name, - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - } - pod.Spec.Volumes[i] = v - } + // no annotation and label for sidecar + pod.Annotations = map[string]string{} + pod.Labels = map[string]string{} + + volumes, volumeMounts := r.genSidecarVolumes() + pod.Spec.Volumes = append(pod.Spec.Volumes, volumes...) + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, volumeMounts...) + + // check mount & create subpath & set quota + capacity := strconv.FormatInt(r.capacity, 10) + subpath := r.jfsSetting.SubPath + community := "ce" + if !r.jfsSetting.IsCe { + community = "ee" + } + quotaPath := r.getQuotaPath() + name := r.jfsSetting.Name + pod.Spec.Containers[0].Lifecycle.PostStart = &corev1.Handler{ + Exec: &corev1.ExecAction{Command: []string{"bash", "-c", + fmt.Sprintf("time subpath=%s name=%s capacity=%s community=%s quotaPath=%s %s '%s' >> /proc/1/fd/1", + subpath, + name, + capacity, + community, + quotaPath, + checkMountScriptPath, + r.jfsSetting.MountPath, + )}}, } return pod } + +func (r *ContainerBuilder) OverwriteVolumeMounts(mount *corev1.VolumeMount) { + // do not overwrite volumeMounts + return +} + +func (r *ContainerBuilder) OverwriteVolumes(volume *corev1.Volume, mountPath string) { + // overwrite original volume and use juicefs volume mountpoint instead + hostMount := filepath.Join(config.MountPointPath, mountPath, r.jfsSetting.SubPath) + volume.VolumeSource = corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: hostMount, + }, + } +} + +// genSidecarVolumes generates volumes and volumeMounts for sidecar container +// extra volumes and volumeMounts are used to check mount status +func (r *ContainerBuilder) genSidecarVolumes() (volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) { + var mode int32 = 0755 + secretName := r.jfsSetting.SecretName + volumes = []corev1.Volume{{ + Name: "jfs-check-mount", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secretName, + DefaultMode: utilpointer.Int32Ptr(mode), + }, + }, + }} + volumeMounts = []corev1.VolumeMount{{ + Name: "jfs-check-mount", + MountPath: checkMountScriptPath, + SubPath: checkMountScriptName, + }} + return +} diff --git a/pkg/juicefs/mount/builder/interface.go b/pkg/juicefs/mount/builder/interface.go new file mode 100644 index 0000000000..d058923409 --- /dev/null +++ b/pkg/juicefs/mount/builder/interface.go @@ -0,0 +1,26 @@ +/* + Copyright 2023 Juicedata Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package builder + +import corev1 "k8s.io/api/core/v1" + +type SidecarInterface interface { + NewMountSidecar() *corev1.Pod + NewSecret() corev1.Secret + OverwriteVolumes(volume *corev1.Volume, mountPath string) + OverwriteVolumeMounts(mount *corev1.VolumeMount) +} diff --git a/pkg/juicefs/mount/builder/job.go b/pkg/juicefs/mount/builder/job.go index 67c90c9b83..60d6b6dd62 100644 --- a/pkg/juicefs/mount/builder/job.go +++ b/pkg/juicefs/mount/builder/job.go @@ -32,25 +32,43 @@ import ( const DefaultJobTTLSecond = int32(5) -func (r *Builder) NewJobForCreateVolume() *batchv1.Job { +type JobBuilder struct { + PodBuilder +} + +func NewJobBuilder(setting *config.JfsSetting, capacity int64) *JobBuilder { + return &JobBuilder{ + PodBuilder{BaseBuilder{ + jfsSetting: setting, + capacity: capacity, + }}, + } +} + +func (r *JobBuilder) NewJobForCreateVolume() *batchv1.Job { jobName := GenJobNameByVolumeId(r.jfsSetting.VolumeId) + "-createvol" job := r.newJob(jobName) jobCmd := r.getCreateVolumeCmd() - job.Spec.Template.Spec.Containers[0].Command = []string{"sh", "-c", jobCmd} + initCmd := r.genInitCommand() + cmd := strings.Join([]string{initCmd, jobCmd}, "\n") + job.Spec.Template.Spec.Containers[0].Command = []string{"sh", "-c", cmd} + klog.Infof("create volume job cmd: %s", jobCmd) return job } -func (r *Builder) NewJobForDeleteVolume() *batchv1.Job { +func (r *JobBuilder) NewJobForDeleteVolume() *batchv1.Job { jobName := GenJobNameByVolumeId(r.jfsSetting.VolumeId) + "-delvol" job := r.newJob(jobName) jobCmd := r.getDeleteVolumeCmd() - job.Spec.Template.Spec.Containers[0].Command = []string{"sh", "-c", jobCmd} + initCmd := r.genInitCommand() + cmd := strings.Join([]string{initCmd, jobCmd}, "\n") + job.Spec.Template.Spec.Containers[0].Command = []string{"sh", "-c", cmd} klog.Infof("delete volume job cmd: %s", jobCmd) return job } -func (r *Builder) NewJobForCleanCache() *batchv1.Job { +func (r *JobBuilder) NewJobForCleanCache() *batchv1.Job { jobName := GenJobNameByVolumeId(r.jfsSetting.VolumeId) + "-cleancache-" + util.RandStringRunes(6) job := r.newCleanJob(jobName) return job @@ -62,10 +80,10 @@ func GenJobNameByVolumeId(volumeId string) string { return fmt.Sprintf("juicefs-%x", h.Sum(nil))[:16] } -func (r *Builder) newJob(jobName string) *batchv1.Job { +func (r *JobBuilder) newJob(jobName string) *batchv1.Job { secretName := jobName + "-secret" r.jfsSetting.SecretName = secretName - podTemplate := r.generateJuicePod() + podTemplate := r.genCommonJuicePod(r.genCommonContainer) ttlSecond := DefaultJobTTLSecond podTemplate.Spec.Containers[0].Lifecycle = &corev1.Lifecycle{ PreStop: &corev1.Handler{ @@ -95,8 +113,8 @@ func (r *Builder) newJob(jobName string) *batchv1.Job { return &job } -func (r *Builder) newCleanJob(jobName string) *batchv1.Job { - podTemplate := r.generateCleanCachePod() +func (r *JobBuilder) newCleanJob(jobName string) *batchv1.Job { + podTemplate := r.genCleanCachePod() ttlSecond := DefaultJobTTLSecond podTemplate.Spec.RestartPolicy = corev1.RestartPolicyNever podTemplate.Spec.NodeName = config.NodeName @@ -122,12 +140,12 @@ func (r *Builder) newCleanJob(jobName string) *batchv1.Job { return &job } -func (r *Builder) getCreateVolumeCmd() string { +func (r *JobBuilder) getCreateVolumeCmd() string { cmd := r.getJobCommand() return fmt.Sprintf("%s && if [ ! -d /mnt/jfs/%s ]; then mkdir -m 777 /mnt/jfs/%s; fi;", cmd, r.jfsSetting.SubPath, r.jfsSetting.SubPath) } -func (r *Builder) getDeleteVolumeCmd() string { +func (r *JobBuilder) getDeleteVolumeCmd() string { cmd := r.getJobCommand() var jfsPath string if r.jfsSetting.IsCe { @@ -137,24 +155,3 @@ func (r *Builder) getDeleteVolumeCmd() string { } return fmt.Sprintf("%s && if [ -d /mnt/jfs/%s ]; then %s rmr /mnt/jfs/%s; fi;", cmd, r.jfsSetting.SubPath, jfsPath, r.jfsSetting.SubPath) } - -func (r *Builder) getJobCommand() string { - var cmd string - options := util.StripReadonlyOption(r.jfsSetting.Options) - if r.jfsSetting.IsCe { - args := []string{config.CeMountPath, "${metaurl}", "/mnt/jfs"} - if len(options) != 0 { - args = append(args, "-o", strings.Join(options, ",")) - } - cmd = strings.Join(args, " ") - } else { - args := []string{config.JfsMountPath, r.jfsSetting.Source, "/mnt/jfs"} - if r.jfsSetting.EncryptRsaKey != "" { - options = append(options, "rsa-key=/root/.rsa/rsa-key.pem") - } - options = append(options, "background") - args = append(args, "-o", strings.Join(options, ",")) - cmd = strings.Join(args, " ") - } - return util.QuoteForShell(cmd) -} diff --git a/pkg/juicefs/mount/builder/pod.go b/pkg/juicefs/mount/builder/pod.go index f742f7a1e5..1653a2899d 100644 --- a/pkg/juicefs/mount/builder/pod.go +++ b/pkg/juicefs/mount/builder/pod.go @@ -18,30 +18,52 @@ package builder import ( "fmt" - "path" - "regexp" - "strconv" + "path/filepath" "strings" corev1 "k8s.io/api/core/v1" - "k8s.io/klog" - "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/juicedata/juicefs-csi-driver/pkg/config" - "github.com/juicedata/juicefs-csi-driver/pkg/util" ) -func (r *Builder) NewMountPod(podName string) *corev1.Pod { - resourceRequirements := r.jfsSetting.Resources +type PodBuilder struct { + BaseBuilder +} - cmd := r.getCommand() +func NewPodBuilder(setting *config.JfsSetting, capacity int64) *PodBuilder { + return &PodBuilder{ + BaseBuilder: BaseBuilder{ + jfsSetting: setting, + capacity: capacity, + }, + } +} - pod := r.generateJuicePod() +// NewMountPod generates a pod with juicefs client +func (r *PodBuilder) NewMountPod(podName string) *corev1.Pod { + pod := r.genCommonJuicePod(r.genCommonContainer) - metricsPort := r.getMetricsPort() + pod.Name = podName + mountCmd := r.genMountCommand() + cmd := mountCmd + initCmd := r.genInitCommand() + if initCmd != "" { + cmd = strings.Join([]string{initCmd, mountCmd}, "\n") + } + pod.Spec.Containers[0].Command = []string{"sh", "-c", cmd} + pod.Spec.Containers[0].Env = []corev1.EnvVar{{ + Name: "JFS_FOREGROUND", + Value: "1", + }} - // add cache-dir host path volume - cacheVolumes, cacheVolumeMounts := r.getCacheDirVolumes(corev1.MountPropagationBidirectional) + // generate volumes and volumeMounts only used in mount pod + volumes, volumeMounts := r.genPodVolumes() + pod.Spec.Volumes = append(pod.Spec.Volumes, volumes...) + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, volumeMounts...) + + // add cache-dir hostpath & PVC volume + cacheVolumes, cacheVolumeMounts := r.genCacheDirVolumes() pod.Spec.Volumes = append(pod.Spec.Volumes, cacheVolumes...) pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, cacheVolumeMounts...) @@ -50,63 +72,26 @@ func (r *Builder) NewMountPod(podName string) *corev1.Pod { pod.Spec.Volumes = append(pod.Spec.Volumes, mountVolumes...) pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, mountVolumeMounts...) - pod.Name = podName - pod.Spec.ServiceAccountName = r.jfsSetting.ServiceAccountName - controllerutil.AddFinalizer(pod, config.Finalizer) - pod.Spec.PriorityClassName = config.JFSMountPriorityName - pod.Spec.RestartPolicy = corev1.RestartPolicyAlways - pod.Spec.Containers[0].Env = []corev1.EnvVar{{ - Name: "JFS_FOREGROUND", - Value: "1", - }} - pod.Spec.Containers[0].Resources = resourceRequirements - pod.Spec.Containers[0].Command = []string{"sh", "-c", cmd} - pod.Spec.Containers[0].Lifecycle = &corev1.Lifecycle{ - PreStop: &corev1.Handler{ - Exec: &corev1.ExecAction{Command: []string{"sh", "-c", fmt.Sprintf( - "umount %s && rmdir %s", r.jfsSetting.MountPath, r.jfsSetting.MountPath)}}, - }, - } - - if r.jfsSetting.Attr.HostNetwork { - // When using hostNetwork, the MountPod will use a random port for metrics. - // Before inducing any auxiliary method to detect that random port, the - // best way is to avoid announcing any port about that. - pod.Spec.Containers[0].Ports = []corev1.ContainerPort{} - } else { - pod.Spec.Containers[0].Ports = []corev1.ContainerPort{ - {Name: "metrics", ContainerPort: metricsPort}, - } - } - - if config.Webhook { - pod.Spec.Containers[0].Lifecycle.PostStart = &corev1.Handler{ - Exec: &corev1.ExecAction{Command: []string{"bash", "-c", - fmt.Sprintf("time %s %s >> /proc/1/fd/1", checkMountScriptPath, r.jfsSetting.MountPath)}}, - } - } - - gracePeriod := int64(10) - pod.Spec.TerminationGracePeriodSeconds = &gracePeriod + return pod +} - for k, v := range r.jfsSetting.MountPodLabels { - pod.Labels[k] = v - } - for k, v := range r.jfsSetting.MountPodAnnotations { - pod.Annotations[k] = v - } - if r.jfsSetting.DeletedDelay != "" { - pod.Annotations[config.DeleteDelayTimeKey] = r.jfsSetting.DeletedDelay - } - pod.Annotations[config.JuiceFSUUID] = r.jfsSetting.UUID - pod.Annotations[config.UniqueId] = r.jfsSetting.UniqueId - if r.jfsSetting.CleanCache { - pod.Annotations[config.CleanCache] = "true" +// genCommonContainer: generate common privileged container +func (r *PodBuilder) genCommonContainer() corev1.Container { + isPrivileged := true + rootUser := int64(0) + return corev1.Container{ + Name: config.MountContainerName, + Image: r.BaseBuilder.jfsSetting.Attr.Image, + SecurityContext: &corev1.SecurityContext{ + Privileged: &isPrivileged, + RunAsUser: &rootUser, + }, + Env: []corev1.EnvVar{}, } - return pod } -func (r *Builder) getCacheDirVolumes(mountPropagation corev1.MountPropagationMode) ([]corev1.Volume, []corev1.VolumeMount) { +// genCacheDirVolumes: generate cache-dir hostpath & PVC volume +func (r *PodBuilder) genCacheDirVolumes() ([]corev1.Volume, []corev1.VolumeMount) { cacheVolumes := []corev1.Volume{} cacheVolumeMounts := []corev1.VolumeMount{} @@ -128,9 +113,8 @@ func (r *Builder) getCacheDirVolumes(mountPropagation corev1.MountPropagationMod cacheVolumes = append(cacheVolumes, hostPathVolume) volumeMount := corev1.VolumeMount{ - Name: name, - MountPath: cacheDir, - MountPropagation: &mountPropagation, + Name: name, + MountPath: cacheDir, } cacheVolumeMounts = append(cacheVolumeMounts, volumeMount) } @@ -158,13 +142,13 @@ func (r *Builder) getCacheDirVolumes(mountPropagation corev1.MountPropagationMod return cacheVolumes, cacheVolumeMounts } -func (r *Builder) genHostPathVolumes() (volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) { +// genHostPathVolumes: generate host path volumes +func (r *PodBuilder) genHostPathVolumes() (volumes []corev1.Volume, volumeMounts []corev1.VolumeMount) { volumes = []corev1.Volume{} volumeMounts = []corev1.VolumeMount{} if len(r.jfsSetting.HostPath) == 0 { return } - mountPropagation := corev1.MountPropagationBidirectional for idx, hostPath := range r.jfsSetting.HostPath { name := fmt.Sprintf("hostpath-%d", idx) volumes = append(volumes, corev1.Volume{ @@ -176,140 +160,98 @@ func (r *Builder) genHostPathVolumes() (volumes []corev1.Volume, volumeMounts [] }, }) volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: name, - MountPath: hostPath, - MountPropagation: &mountPropagation, + Name: name, + MountPath: hostPath, }) } return } -func (r *Builder) getCommand() string { - cmd := "" - options := r.jfsSetting.Options - if r.jfsSetting.IsCe { - klog.V(5).Infof("ceMount: mount %v at %v", util.StripPasswd(r.jfsSetting.Source), r.jfsSetting.MountPath) - mountArgs := []string{config.CeMountPath, "${metaurl}", r.jfsSetting.MountPath} - if !util.ContainsPrefix(options, "metrics=") { - if r.jfsSetting.Attr.HostNetwork { - // Pick up a random (useable) port for hostNetwork MountPods. - options = append(options, "metrics=0.0.0.0:0") - } else { - options = append(options, "metrics=0.0.0.0:9567") - } - } - mountArgs = append(mountArgs, "-o", strings.Join(options, ",")) - cmd = strings.Join(mountArgs, " ") - } else { - klog.V(5).Infof("Mount: mount %v at %v", util.StripPasswd(r.jfsSetting.Source), r.jfsSetting.MountPath) - mountArgs := []string{config.JfsMountPath, r.jfsSetting.Source, r.jfsSetting.MountPath} - mountOptions := []string{"foreground", "no-update"} - if r.jfsSetting.EncryptRsaKey != "" { - mountOptions = append(mountOptions, "rsa-key=/root/.rsa/rsa-key.pem") - } - mountOptions = append(mountOptions, options...) - mountArgs = append(mountArgs, "-o", strings.Join(mountOptions, ",")) - cmd = strings.Join(mountArgs, " ") - } - return util.QuoteForShell(cmd) -} - -func (r *Builder) getInitContainer() corev1.Container { - isPrivileged := true - rootUser := int64(0) - secretName := r.jfsSetting.SecretName - formatCmd := r.jfsSetting.FormatCmd - container := corev1.Container{ - Name: "jfs-format", - Image: r.jfsSetting.Attr.Image, - SecurityContext: &corev1.SecurityContext{ - Privileged: &isPrivileged, - RunAsUser: &rootUser, +// genPodVolumes: generate volumes for mount pod +// 1. jfs dir: mount point used to propagate the mount point in the mount container to host +// 2. update db dir: mount updatedb.conf from host to mount pod +func (r *PodBuilder) genPodVolumes() ([]corev1.Volume, []corev1.VolumeMount) { + dir := corev1.HostPathDirectoryOrCreate + file := corev1.HostPathFileOrCreate + mp := corev1.MountPropagationBidirectional + volumes := []corev1.Volume{{ + Name: JfsDirName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: config.MountPointPath, + Type: &dir, + }, }, - } - if r.jfsSetting.EncryptRsaKey != "" { - if r.jfsSetting.IsCe { - container.VolumeMounts = append(container.VolumeMounts, - corev1.VolumeMount{ - Name: "rsa-key", - MountPath: "/root/.rsa", + }} + volumeMounts := []corev1.VolumeMount{{ + Name: JfsDirName, + MountPath: config.PodMountBase, + MountPropagation: &mp, + }} + + if !config.Immutable { + volumes = append(volumes, corev1.Volume{ + Name: UpdateDBDirName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: UpdateDBCfgFile, + Type: &file, }, - ) - formatCmd = formatCmd + " --encrypt-rsa-key=/root/.rsa/rsa-key.pem" - } + }}, + ) + volumeMounts = append(volumeMounts, corev1.VolumeMount{ + Name: UpdateDBDirName, + MountPath: UpdateDBCfgFile, + }) } - // create subpath if readonly mount or in webhook mode - if r.jfsSetting.SubPath != "" { - if util.ContainsString(r.jfsSetting.Options, "read-only") || util.ContainsString(r.jfsSetting.Options, "ro") || config.Webhook { - // generate mount command - cmd := r.getJobCommand() - initCmd := fmt.Sprintf("%s && if [ ! -d /mnt/jfs/%s ]; then mkdir -m 777 /mnt/jfs/%s; fi; umount /mnt/jfs", cmd, r.jfsSetting.SubPath, r.jfsSetting.SubPath) - formatCmd = fmt.Sprintf("%s && %s", formatCmd, initCmd) - if config.Webhook && r.capacity > 0 { - quotaPath := r.jfsSetting.SubPath - var subdir string - for _, o := range r.jfsSetting.Options { - pair := strings.Split(o, "=") - if len(pair) != 2 { - continue - } - if pair[0] == "subdir" { - subdir = path.Join("/", pair[1]) - } - } - var setQuotaCmd string - targetPath := path.Join(subdir, quotaPath) - capacity := strconv.FormatInt(r.capacity, 10) - if r.jfsSetting.IsCe { - // juicefs quota; if [ $? -eq 0 ]; then juicefs quota set ${metaurl} --path ${path} --capacity ${capacity}; fi - cmdArgs := []string{ - config.CeCliPath, "quota; if [ $? -eq 0 ]; then", - config.CeCliPath, - "quota", "set", "${metaurl}", - "--path", targetPath, - "--capacity", capacity, - "; fi", - } - setQuotaCmd = strings.Join(cmdArgs, " ") - } else { - cmdArgs := []string{ - config.CliPath, "quota; if [ $? -eq 0 ]; then", - config.CliPath, - "quota", "set", r.jfsSetting.Name, - "--path", targetPath, - "--capacity", capacity, - "; fi", - } - setQuotaCmd = strings.Join(cmdArgs, " ") - } - formatCmd = fmt.Sprintf("%s && %s", formatCmd, setQuotaCmd) - } - } - } - container.Command = []string{"sh", "-c", formatCmd} - - container.EnvFrom = append(container.EnvFrom, corev1.EnvFromSource{ - SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{ - Name: secretName, - }}, - }) - return container + return volumes, volumeMounts } -func (r *Builder) getMetricsPort() int32 { - port := int64(9567) - options := r.jfsSetting.Options - - for _, option := range options { - if strings.HasPrefix(option, "metrics=") { - re := regexp.MustCompile(`metrics=.*:([0-9]{1,6})`) - match := re.FindStringSubmatch(option) - if len(match) > 0 { - port, _ = strconv.ParseInt(match[1], 10, 32) - } +// genCleanCachePod: generate pod to clean cache in host +func (r *PodBuilder) genCleanCachePod() *corev1.Pod { + volumeMountPrefix := "/var/jfsCache" + cacheVolumes := []corev1.Volume{} + cacheVolumeMounts := []corev1.VolumeMount{} + + hostPathType := corev1.HostPathDirectory + + for idx, cacheDir := range r.jfsSetting.CacheDirs { + name := fmt.Sprintf("cachedir-%d", idx) + + hostPathVolume := corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ + Path: filepath.Join(cacheDir, r.jfsSetting.UUID, "raw"), + Type: &hostPathType, + }}, } + cacheVolumes = append(cacheVolumes, hostPathVolume) + + volumeMount := corev1.VolumeMount{ + Name: name, + MountPath: filepath.Join(volumeMountPrefix, name), + } + cacheVolumeMounts = append(cacheVolumeMounts, volumeMount) } - return int32(port) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: r.jfsSetting.Attr.Namespace, + Labels: map[string]string{ + config.PodTypeKey: config.PodTypeValue, + }, + Annotations: make(map[string]string), + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{{ + Name: "jfs-cache-clean", + Image: r.jfsSetting.Attr.Image, + Command: []string{"sh", "-c", "rm -rf /var/jfsCache/*/chunks"}, + VolumeMounts: cacheVolumeMounts, + }}, + Volumes: cacheVolumes, + }, + } + return pod } diff --git a/pkg/juicefs/mount/builder/pod_test.go b/pkg/juicefs/mount/builder/pod_test.go index afd844853a..320995cc4b 100644 --- a/pkg/juicefs/mount/builder/pod_test.go +++ b/pkg/juicefs/mount/builder/pod_test.go @@ -107,9 +107,8 @@ var ( MountPropagation: &mp, }, { - Name: UpdateDBDirName, - MountPath: UpdateDBCfgFile, - MountPropagation: &mp, + Name: UpdateDBDirName, + MountPath: UpdateDBCfgFile, }, }, SecurityContext: &corev1.SecurityContext{ @@ -156,9 +155,8 @@ func putDefaultCacheDir(pod *corev1.Pod) { }, } volumeMount := corev1.VolumeMount{ - Name: "jfs-default-cache", - MountPath: "/var/jfsCache", - MountPropagation: &mp, + Name: "jfs-default-cache", + MountPath: "/var/jfsCache", } pod.Spec.Volumes = append(pod.Spec.Volumes, volume) pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, volumeMount) @@ -170,18 +168,12 @@ func Test_getCacheDirVolumes(t *testing.T) { optionWithCacheDir2 := []string{"cache-dir=/dev/shm/imagenet-0:/dev/shm/imagenet-1"} optionWithCacheDir3 := []string{"cache-dir"} - r := Builder{nil, 0} + r := PodBuilder{BaseBuilder{nil, 0}} - mp := corev1.MountPropagationBidirectional dir := corev1.HostPathDirectory volumeMounts := []corev1.VolumeMount{{ - Name: JfsDirName, - MountPath: config.PodMountBase, - MountPropagation: &mp, - }, { - Name: JfsRootDirName, - MountPath: "/root/.juicefs", - MountPropagation: &mp, + Name: JfsDirName, + MountPath: config.PodMountBase, }} volumes := []corev1.Volume{{ @@ -189,47 +181,41 @@ func Test_getCacheDirVolumes(t *testing.T) { VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ Path: config.MountPointPath, Type: &dir, - }}}, { - Name: JfsRootDirName, - VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{ - Path: config.JFSConfigPath, - Type: &dir, - }}, - }} + }}}} s, _ := config.ParseSetting(map[string]string{"name": "test"}, nil, optionWithoutCacheDir, true) r.jfsSetting = s - cacheVolumes, cacheVolumeMounts := r.getCacheDirVolumes(corev1.MountPropagationBidirectional) + cacheVolumes, cacheVolumeMounts := r.genCacheDirVolumes() volumes = append(volumes, cacheVolumes...) volumeMounts = append(volumeMounts, cacheVolumeMounts...) - if len(volumes) != 3 || len(volumeMounts) != 3 { + if len(volumes) != 2 || len(volumeMounts) != 2 { t.Error("getCacheDirVolumes can't work properly") } s, _ = config.ParseSetting(map[string]string{"name": "test"}, nil, optionWithCacheDir, true) r.jfsSetting = s - cacheVolumes, cacheVolumeMounts = r.getCacheDirVolumes(corev1.MountPropagationBidirectional) + cacheVolumes, cacheVolumeMounts = r.genCacheDirVolumes() volumes = append(volumes, cacheVolumes...) volumeMounts = append(volumeMounts, cacheVolumeMounts...) - if len(volumes) != 4 || len(volumeMounts) != 4 { + if len(volumes) != 3 || len(volumeMounts) != 3 { t.Error("getCacheDirVolumes can't work properly") } s, _ = config.ParseSetting(map[string]string{"name": "test"}, nil, optionWithCacheDir2, true) r.jfsSetting = s - cacheVolumes, cacheVolumeMounts = r.getCacheDirVolumes(corev1.MountPropagationBidirectional) + cacheVolumes, cacheVolumeMounts = r.genCacheDirVolumes() volumes = append(volumes, cacheVolumes...) volumeMounts = append(volumeMounts, cacheVolumeMounts...) - if len(volumes) != 6 || len(volumeMounts) != 6 { + if len(volumes) != 5 || len(volumeMounts) != 5 { t.Error("getCacheDirVolumes can't work properly") } s, _ = config.ParseSetting(map[string]string{"name": "test"}, nil, optionWithCacheDir3, true) r.jfsSetting = s - cacheVolumes, cacheVolumeMounts = r.getCacheDirVolumes(corev1.MountPropagationBidirectional) + cacheVolumes, cacheVolumeMounts = r.genCacheDirVolumes() volumes = append(volumes, cacheVolumes...) volumeMounts = append(volumeMounts, cacheVolumeMounts...) - if len(volumes) != 7 || len(volumeMounts) != 7 { + if len(volumes) != 6 || len(volumeMounts) != 6 { t.Error("getCacheDirVolumes can't work properly") } } @@ -255,19 +241,19 @@ func TestNewMountPod(t *testing.T) { podConfigTest := corev1.Pod{} deepcopyPodFromDefault(&podConfigTest) - podConfigTest.Spec.Volumes = append(podConfigTest.Spec.Volumes, corev1.Volume{ + podConfigTest.Spec.Volumes = append([]corev1.Volume{{ Name: "config-1", VolumeSource: corev1.VolumeSource{Secret: &corev1.SecretVolumeSource{SecretName: "secret-test"}}, - }) - podConfigTest.Spec.Containers[0].VolumeMounts = append(podConfigTest.Spec.Containers[0].VolumeMounts, corev1.VolumeMount{ + }}, podConfigTest.Spec.Volumes...) + podConfigTest.Spec.Containers[0].VolumeMounts = append([]corev1.VolumeMount{{ Name: "config-1", MountPath: "/test", - }) + }}, podConfigTest.Spec.Containers[0].VolumeMounts...) s, _ := config.ParseSetting(map[string]string{"name": "test"}, nil, []string{"cache-dir=/dev/shm/imagenet-0:/dev/shm/imagenet-1", "cache-size=10240", "metrics=0.0.0.0:9567"}, true) - r := Builder{s, 0} + r := PodBuilder{BaseBuilder{s, 0}} cmdWithCacheDir := `/bin/mount.juicefs ${metaurl} /jfs/default-imagenet -o cache-dir=/dev/shm/imagenet-0:/dev/shm/imagenet-1,cache-size=10240,metrics=0.0.0.0:9567` - cacheVolumes, cacheVolumeMounts := r.getCacheDirVolumes(corev1.MountPropagationBidirectional) + cacheVolumes, cacheVolumeMounts := r.genCacheDirVolumes() podCacheTest := corev1.Pod{} deepcopyPodFromDefault(&podCacheTest) podCacheTest.Spec.Containers[0].Command = []string{"sh", "-c", cmdWithCacheDir} @@ -395,7 +381,7 @@ func TestNewMountPod(t *testing.T) { Image: config.CEMountImage, }, } - r := Builder{jfsSetting, 0} + r := PodBuilder{BaseBuilder{jfsSetting, 0}} got := r.NewMountPod(podName) gotStr, _ := json.Marshal(got) wantStr, _ := json.Marshal(tt.want) @@ -448,8 +434,8 @@ func TestPodMount_getCommand(t *testing.T) { MountPath: tt.args.mountPath, Options: tt.args.options, } - r := Builder{jfsSetting, 0} - if got := r.getCommand(); got != tt.want { + r := PodBuilder{BaseBuilder{jfsSetting, 0}} + if got := r.genMountCommand(); got != tt.want { t.Errorf("getCommand() = %v, want %v", got, tt.want) } }) @@ -491,8 +477,8 @@ func TestPodMount_getMetricsPort(t *testing.T) { Name: tt.name, Options: tt.args.options, } - r := Builder{jfsSetting, 0} - if got := r.getMetricsPort(); got != tt.want { + r := PodBuilder{BaseBuilder{jfsSetting, 0}} + if got := r.genMetricsPort(); got != tt.want { t.Errorf("getMetricsPort() = %v, want %v", got, tt.want) } }) @@ -500,7 +486,6 @@ func TestPodMount_getMetricsPort(t *testing.T) { } func TestBuilder_genHostPathVolumes(t *testing.T) { - mountPropagation := corev1.MountPropagationBidirectional type fields struct { jfsSetting *config.JfsSetting } @@ -526,17 +511,16 @@ func TestBuilder_genHostPathVolumes(t *testing.T) { }, }}, wantVolumeMounts: []corev1.VolumeMount{{ - Name: "hostpath-0", - MountPath: "/tmp", - MountPropagation: &mountPropagation, + Name: "hostpath-0", + MountPath: "/tmp", }}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - r := &Builder{ + r := &PodBuilder{BaseBuilder{ jfsSetting: tt.fields.jfsSetting, - } + }} gotVolumes, gotVolumeMounts := r.genHostPathVolumes() if !reflect.DeepEqual(gotVolumes, tt.wantVolumes) { t.Errorf("genHostPathVolumes() gotVolumes = %v, want %v", gotVolumes, tt.wantVolumes) diff --git a/pkg/juicefs/mount/builder/secret.go b/pkg/juicefs/mount/builder/secret.go index 269e5ad6a4..a8a79c81da 100644 --- a/pkg/juicefs/mount/builder/secret.go +++ b/pkg/juicefs/mount/builder/secret.go @@ -44,10 +44,24 @@ do done echo "$(date "+%Y-%m-%d %H:%M:%S")" echo "succeed in checking mount point $ConditionPathIsMountPoint" +if [ -n "${subpath}" ]; then + echo "create subpath ${subpath}" + mkdir -p 777 $ConditionPathIsMountPoint/${subpath} + if [ -n "${capacity}" ]; then + if [ "${community}" == ce ]; then + echo "set quota in ${subpath}" + /usr/local/bin/juicefs quota > /dev/null; if [ $? -eq 0 ]; then /usr/local/bin/juicefs quota set ${metaurl} --path ${quotaPath} --capacity ${capacity}; fi; + fi; + if [ "${community}" == ee ]; then + echo "set quota in ${subpath}" + /usr/bin/juicefs quota > /dev/null; if [ $? -eq 0 ]; then /usr/bin/juicefs quota set ${name} --path ${quotaPath} --capacity ${capacity}; fi; + fi; + fi; +fi; ` ) -func (r *Builder) NewSecret() corev1.Secret { +func (r *BaseBuilder) NewSecret() corev1.Secret { data := make(map[string]string) if r.jfsSetting.MetaUrl != "" { data["metaurl"] = r.jfsSetting.MetaUrl diff --git a/pkg/juicefs/mount/builder/serverless.go b/pkg/juicefs/mount/builder/serverless.go new file mode 100644 index 0000000000..a95fd0de0e --- /dev/null +++ b/pkg/juicefs/mount/builder/serverless.go @@ -0,0 +1,188 @@ +/* + Copyright 2023 Juicedata Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package builder + +import ( + "fmt" + "path/filepath" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + utilpointer "k8s.io/utils/pointer" + + "github.com/juicedata/juicefs-csi-driver/pkg/config" +) + +type ServerlessBuilder struct { + PodBuilder +} + +var _ SidecarInterface = &ServerlessBuilder{} + +func NewServerlessBuilder(setting *config.JfsSetting, capacity int64) SidecarInterface { + return &ServerlessBuilder{ + PodBuilder{BaseBuilder{ + jfsSetting: setting, + capacity: capacity, + }}, + } +} + +// NewMountSidecar generates a pod with a juicefs sidecar in serverless mode +// 1. no hostpath except mount point (the serverless cluster must have this permission.) +// 2. with privileged container (the serverless cluster must have this permission.) +// 3. no initContainer +func (r *ServerlessBuilder) NewMountSidecar() *corev1.Pod { + pod := r.genCommonJuicePod(r.genCommonContainer) + + // no annotation and label for sidecar + pod.Annotations = map[string]string{} + pod.Labels = map[string]string{} + + // check mount & create subpath & set quota + capacity := strconv.FormatInt(r.capacity, 10) + subpath := r.jfsSetting.SubPath + community := "ce" + if !r.jfsSetting.IsCe { + community = "ee" + } + quotaPath := r.getQuotaPath() + name := r.jfsSetting.Name + pod.Spec.Containers[0].Lifecycle.PostStart = &corev1.Handler{ + Exec: &corev1.ExecAction{Command: []string{"bash", "-c", + fmt.Sprintf("time subpath=%s name=%s capacity=%s community=%s quotaPath=%s %s '%s' >> /proc/1/fd/1", + subpath, + name, + capacity, + community, + quotaPath, + checkMountScriptPath, + r.jfsSetting.MountPath, + )}}, + } + pod.Spec.Containers[0].Env = []corev1.EnvVar{{ + Name: "JFS_FOREGROUND", + Value: "1", + }} + + // generate volumes and volumeMounts only used in serverless sidecar + volumes, volumeMounts := r.genServerlessVolumes() + pod.Spec.Volumes = append(pod.Spec.Volumes, volumes...) + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, volumeMounts...) + + // add cache-dir PVC volume + cacheVolumes, cacheVolumeMounts := r.genCacheDirVolumes() + pod.Spec.Volumes = append(pod.Spec.Volumes, cacheVolumes...) + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, cacheVolumeMounts...) + + // command + mountCmd := r.genMountCommand() + initCmd := r.genInitCommand() + cmd := strings.Join([]string{initCmd, mountCmd}, "\n") + pod.Spec.Containers[0].Command = []string{"sh", "-c", cmd} + + return pod +} + +func (r *ServerlessBuilder) OverwriteVolumes(volume *corev1.Volume, mountPath string) { + // overwrite original volume and use juicefs volume mountpoint instead + hostMount := filepath.Join(config.MountPointPath, mountPath, r.jfsSetting.SubPath) + volume.VolumeSource = corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: hostMount, + }, + } +} + +func (r *ServerlessBuilder) OverwriteVolumeMounts(mount *corev1.VolumeMount) { + // do not overwrite volume mount + return +} + +// genServerlessVolumes generates volumes and volumeMounts for serverless sidecar +// 1. jfs dir: mount point as hostPath, used to propagate the mount point in the mount container to the business container +// 2. jfs-check-mount: secret volume, used to check if the mount point is mounted +func (r *ServerlessBuilder) genServerlessVolumes() ([]corev1.Volume, []corev1.VolumeMount) { + dir := corev1.HostPathDirectoryOrCreate + mp := corev1.MountPropagationBidirectional + + var mode int32 = 0755 + secretName := r.jfsSetting.SecretName + volumes := []corev1.Volume{ + { + Name: JfsDirName, + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: config.MountPointPath, + Type: &dir, + }, + }, + }, + { + Name: "jfs-check-mount", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secretName, + DefaultMode: utilpointer.Int32Ptr(mode), + }, + }, + }, + } + volumeMounts := []corev1.VolumeMount{ + { + Name: JfsDirName, + MountPath: config.PodMountBase, + MountPropagation: &mp, + }, + { + Name: "jfs-check-mount", + MountPath: checkMountScriptPath, + SubPath: checkMountScriptName, + }, + } + + return volumes, volumeMounts +} + +// genCacheDirVolumes: in serverless, only support PVC cache, do not support hostpath cache +func (r *ServerlessBuilder) genCacheDirVolumes() ([]corev1.Volume, []corev1.VolumeMount) { + cacheVolumes := []corev1.Volume{} + cacheVolumeMounts := []corev1.VolumeMount{} + + for i, cache := range r.jfsSetting.CachePVCs { + name := fmt.Sprintf("cachedir-pvc-%d", i) + pvcVolume := corev1.Volume{ + Name: name, + VolumeSource: corev1.VolumeSource{ + PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ + ClaimName: cache.PVCName, + ReadOnly: false, + }, + }, + } + cacheVolumes = append(cacheVolumes, pvcVolume) + volumeMount := corev1.VolumeMount{ + Name: name, + ReadOnly: false, + MountPath: cache.Path, + } + cacheVolumeMounts = append(cacheVolumeMounts, volumeMount) + } + + return cacheVolumes, cacheVolumeMounts +} diff --git a/pkg/juicefs/mount/builder/vci-serverless.go b/pkg/juicefs/mount/builder/vci-serverless.go new file mode 100644 index 0000000000..9bfa34f957 --- /dev/null +++ b/pkg/juicefs/mount/builder/vci-serverless.go @@ -0,0 +1,178 @@ +/* + Copyright 2023 Juicedata Inc + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package builder + +import ( + "fmt" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + utilpointer "k8s.io/utils/pointer" + + "github.com/juicedata/juicefs-csi-driver/pkg/config" +) + +const ( + VCIANNOKey = "vke.volcengine.com/burst-to-vci" + VCIANNOValue = "enforce" + VCIPropagation = "vci.vke.volcengine.com/config-bidirectional-mount-propagation" + VCIPropagationConfig = "vke.al.vci-enable-bidirectional-mount-propagation" + VCIPropagationConfigValue = "vke.al.vci-enable-bidirectional-mount-propagation" +) + +type VCIBuilder struct { + ServerlessBuilder + pvc corev1.PersistentVolumeClaim + app corev1.Pod +} + +var _ SidecarInterface = &VCIBuilder{} + +func NewVCIBuilder(setting *config.JfsSetting, capacity int64, app corev1.Pod, pvc corev1.PersistentVolumeClaim) SidecarInterface { + return &VCIBuilder{ + ServerlessBuilder: ServerlessBuilder{PodBuilder{BaseBuilder{ + jfsSetting: setting, + capacity: capacity, + }}}, + pvc: pvc, + app: app, + } +} + +// NewMountSidecar generates a pod with a juicefs sidecar in serverless mode +// 1. no hostpath +// 2. without privileged container +// 3. no propagationBidirectional +// 4. with env JFS_NO_UMOUNT=1 +// 5. annotations for VCI +func (r *VCIBuilder) NewMountSidecar() *corev1.Pod { + pod := r.genCommonJuicePod(r.genNonPrivilegedContainer) + // overwrite annotation + pod.Annotations = map[string]string{ + VCIPropagationConfig: VCIPropagationConfigValue, + VCIPropagation: fmt.Sprintf(`[{ + "container": "jfs-mount", + "mountPath" : "%s" + }]`, r.jfsSetting.MountPath), + } + + // check mount & create subpath & set quota + capacity := strconv.FormatInt(r.capacity, 10) + subpath := r.jfsSetting.SubPath + community := "ce" + if !r.jfsSetting.IsCe { + community = "ee" + } + quotaPath := r.getQuotaPath() + name := r.jfsSetting.Name + pod.Spec.Containers[0].Lifecycle.PostStart = &corev1.Handler{ + Exec: &corev1.ExecAction{Command: []string{"bash", "-c", + fmt.Sprintf("time subpath=%s name=%s capacity=%s community=%s quotaPath=%s %s '%s' >> /proc/1/fd/1", + subpath, + name, + capacity, + community, + quotaPath, + checkMountScriptPath, + r.jfsSetting.MountPath, + )}}, + } + pod.Spec.Containers[0].Env = append(pod.Spec.Containers[0].Env, []corev1.EnvVar{{Name: "JFS_NO_UMOUNT", Value: "1"}, {Name: "JFS_FOREGROUND", Value: "1"}}...) + + // generate volumes and volumeMounts only used in VCI serverless sidecar + volumes, volumeMounts := r.genVCIServerlessVolumes() + pod.Spec.Volumes = append(pod.Spec.Volumes, volumes...) + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, volumeMounts...) + + // add cache-dir PVC volume + cacheVolumes, cacheVolumeMounts := r.genCacheDirVolumes() + pod.Spec.Volumes = append(pod.Spec.Volumes, cacheVolumes...) + pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, cacheVolumeMounts...) + + // command + mountCmd := r.genMountCommand() + initCmd := r.genInitCommand() + cmd := strings.Join([]string{initCmd, mountCmd}, "\n") + pod.Spec.Containers[0].Command = []string{"sh", "-c", cmd} + + return pod +} + +func (r *VCIBuilder) OverwriteVolumes(volume *corev1.Volume, mountPath string) { + volume.VolumeSource = corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + } +} + +func (r *VCIBuilder) OverwriteVolumeMounts(mount *corev1.VolumeMount) { + hostToContainer := corev1.MountPropagationHostToContainer + mount.MountPropagation = &hostToContainer +} + +// genVCIServerlessVolumes generates volumes and volumeMounts for serverless sidecar +// 1. jfs dir: mount point as emptyDir, used to propagate the mount point in the mount container to the business container +// 2. jfs-check-mount: secret volume, used to check if the mount point is mounted +func (r *VCIBuilder) genVCIServerlessVolumes() ([]corev1.Volume, []corev1.VolumeMount) { + var mode int32 = 0755 + var sharedVolumeName string + secretName := r.jfsSetting.SecretName + + // get shared volume name + for _, volume := range r.app.Spec.Volumes { + if volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == r.pvc.Name { + sharedVolumeName = volume.Name + } + } + + volumes := []corev1.Volume{ + { + Name: "jfs-check-mount", + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: secretName, + DefaultMode: utilpointer.Int32Ptr(mode), + }, + }, + }, + } + volumeMounts := []corev1.VolumeMount{ + { + Name: sharedVolumeName, + MountPath: r.jfsSetting.MountPath, + }, + { + Name: "jfs-check-mount", + MountPath: checkMountScriptPath, + SubPath: checkMountScriptName, + }, + } + + return volumes, volumeMounts +} + +func (r *PodBuilder) genNonPrivilegedContainer() corev1.Container { + rootUser := int64(0) + return corev1.Container{ + Name: config.MountContainerName, + Image: r.BaseBuilder.jfsSetting.Attr.Image, + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &rootUser, + }, + Env: []corev1.EnvVar{}, + } +} diff --git a/pkg/juicefs/mount/pod_mount.go b/pkg/juicefs/mount/pod_mount.go index 6708aa5159..e47670e423 100644 --- a/pkg/juicefs/mount/pod_mount.go +++ b/pkg/juicefs/mount/pod_mount.go @@ -220,7 +220,7 @@ func (p *PodMount) JUmount(ctx context.Context, target, podName string) error { func (p *PodMount) JCreateVolume(ctx context.Context, jfsSetting *jfsConfig.JfsSetting) error { var exist *batchv1.Job - r := builder.NewBuilder(jfsSetting, 0) + r := builder.NewJobBuilder(jfsSetting, 0) job := r.NewJobForCreateVolume() exist, err := p.K8sClient.GetJob(ctx, job.Name, job.Namespace) if err != nil && k8serrors.IsNotFound(err) { @@ -252,7 +252,7 @@ func (p *PodMount) JCreateVolume(ctx context.Context, jfsSetting *jfsConfig.JfsS func (p *PodMount) JDeleteVolume(ctx context.Context, jfsSetting *jfsConfig.JfsSetting) error { var exist *batchv1.Job - r := builder.NewBuilder(jfsSetting, 0) + r := builder.NewJobBuilder(jfsSetting, 0) job := r.NewJobForDeleteVolume() exist, err := p.K8sClient.GetJob(ctx, job.Name, job.Namespace) if err != nil && k8serrors.IsNotFound(err) { @@ -321,7 +321,7 @@ func (p *PodMount) createOrAddRef(ctx context.Context, podName string, jfsSettin defer lock.Unlock() jfsSetting.SecretName = podName + "-secret" - r := builder.NewBuilder(jfsSetting, 0) + r := builder.NewPodBuilder(jfsSetting, 0) secret := r.NewSecret() key := util.GetReferenceKey(jfsSetting.TargetPath) @@ -553,7 +553,7 @@ func (p *PodMount) CleanCache(ctx context.Context, image string, id string, volu jfsSetting.VolumeId = volumeId jfsSetting.CacheDirs = cacheDirs jfsSetting.UUID = id - r := builder.NewBuilder(jfsSetting, 0) + r := builder.NewJobBuilder(jfsSetting, 0) job := r.NewJobForCleanCache() klog.V(6).Infof("Clean cache job: %v", job) _, err = p.K8sClient.GetJob(ctx, job.Name, job.Namespace) diff --git a/pkg/util/util.go b/pkg/util/util.go index 0615508262..078627bc7a 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -275,7 +275,9 @@ func GetReferenceKey(target string) string { // ParseMntPath return mntPath, volumeId (/jfs/volumeId, volumeId err) func ParseMntPath(cmd string) (string, string, error) { - args := strings.Fields(cmd) + cmds := strings.Split(cmd, "\n") + mountCmd := cmds[len(cmds)-1] + args := strings.Fields(mountCmd) if len(args) < 3 || !strings.HasPrefix(args[2], config.PodMountBase) { return "", "", fmt.Errorf("err cmd:%s", cmd) } diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index bc9abac60e..c8b30377d3 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -158,28 +158,46 @@ func TestParseMntPath(t *testing.T) { }{ { name: "get sourcePath from pod cmd success", + args: args{cmd: "/usr/local/bin/juicefs format --storage=s3 --bucket=http://juicefs-bucket.minio.default.svc.cluster.local:9000 --access-key=minioadmin --secret-key=${secretkey} ${metaurl} ce-secret\n/bin/mount.juicefs redis://127.0.0.1/6379 /jfs/pvc-xxx"}, + want: "/jfs/pvc-xxx", + want1: "pvc-xxx", + wantErr: false, + }, + { + name: "without init cmd", args: args{cmd: "/bin/mount.juicefs redis://127.0.0.1/6379 /jfs/pvc-xxx"}, want: "/jfs/pvc-xxx", want1: "pvc-xxx", wantErr: false, }, + { + name: "with create subpath", + args: args{cmd: "/usr/local/bin/juicefs format --storage=s3 --bucket=http://juicefs-bucket.minio.default.svc.cluster.local:9000 --access-key=minioadmin --secret-key=${secretkey} ${metaurl} ce-secret\n" + + "/bin/mount.juicefs ${metaurl} /mnt/jfs -o buffer-size=300,cache-size=100,enable-xattr\n" + + "if [ ! -d /mnt/jfs/pvc-fb2ec20c-474f-4804-9504-966da4af9b73 ]; then mkdir -m 777 /mnt/jfs/pvc-fb2ec20c-474f-4804-9504-966da4af9b73; fi;\n" + + "umount /mnt/jfs\n" + + "/bin/mount.juicefs redis://127.0.0.1/6379 /jfs/pvc-xxx"}, + want: "/jfs/pvc-xxx", + want1: "pvc-xxx", + wantErr: false, + }, { name: "err-pod cmd args <3", - args: args{cmd: "/bin/mount.juicefs redis://127.0.0.1/6379"}, + args: args{cmd: "/usr/local/bin/juicefs format --storage=s3 --bucket=http://juicefs-bucket.minio.default.svc.cluster.local:9000 --access-key=minioadmin --secret-key=${secretkey} ${metaurl} ce-secret\n/bin/mount.juicefs redis://127.0.0.1/6379"}, want: "", want1: "", wantErr: true, }, { name: "err-cmd sourcePath no MountBase prefix", - args: args{cmd: "/bin/mount.juicefs redis://127.0.0.1/6379 /err-jfs/pvc-xxx"}, + args: args{cmd: "/usr/local/bin/juicefs format --storage=s3 --bucket=http://juicefs-bucket.minio.default.svc.cluster.local:9000 --access-key=minioadmin --secret-key=${secretkey} ${metaurl} ce-secret\n/bin/mount.juicefs redis://127.0.0.1/6379 /err-jfs/pvc-xxx"}, want: "", want1: "", wantErr: true, }, { name: "err-cmd sourcePath length err", - args: args{cmd: "/bin/mount.juicefs redis://127.0.0.1/6379 /jfs"}, + args: args{cmd: "/usr/local/bin/juicefs format --storage=s3 --bucket=http://juicefs-bucket.minio.default.svc.cluster.local:9000 --access-key=minioadmin --secret-key=${secretkey} ${metaurl} ce-secret\n/bin/mount.juicefs redis://127.0.0.1/6379 /jfs"}, want: "", want1: "", wantErr: true, diff --git a/pkg/webhook/handler/handler.go b/pkg/webhook/handler/handler.go index e65689da20..2fbcc83d89 100644 --- a/pkg/webhook/handler/handler.go +++ b/pkg/webhook/handler/handler.go @@ -36,6 +36,15 @@ type SidecarHandler struct { Client *k8sclient.K8sClient // A decoder will be automatically injected decoder *admission.Decoder + // is in serverless environment + serverless bool +} + +func NewSidecarHandler(client *k8sclient.K8sClient, serverless bool) *SidecarHandler { + return &SidecarHandler{ + Client: client, + serverless: serverless, + } } func (s *SidecarHandler) Handle(ctx context.Context, request admission.Request) admission.Response { @@ -71,7 +80,7 @@ func (s *SidecarHandler) Handle(ctx context.Context, request admission.Request) } jfs := juicefs.NewJfsProvider(nil, s.Client) - sidecarMutate := mutate.NewSidecarMutate(s.Client, jfs, pair) + sidecarMutate := mutate.NewSidecarMutate(s.Client, jfs, s.serverless, pair) klog.Infof("[SidecarHandler] start injecting juicefs client as sidecar in pod [%s] namespace [%s].", pod.Name, pod.Namespace) out, err := sidecarMutate.Mutate(ctx, pod) if err != nil { diff --git a/pkg/webhook/handler/mutate/sidecar.go b/pkg/webhook/handler/mutate/sidecar.go index bc7eed58e0..e7429ef82a 100644 --- a/pkg/webhook/handler/mutate/sidecar.go +++ b/pkg/webhook/handler/mutate/sidecar.go @@ -36,8 +36,9 @@ import ( ) type SidecarMutate struct { - Client *k8sclient.K8sClient - juicefs juicefs.Interface + Client *k8sclient.K8sClient + juicefs juicefs.Interface + Serverless bool Pair []util.PVPair jfsSetting *config.JfsSetting @@ -45,11 +46,12 @@ type SidecarMutate struct { var _ Mutate = &SidecarMutate{} -func NewSidecarMutate(client *k8sclient.K8sClient, jfs juicefs.Interface, pair []util.PVPair) Mutate { +func NewSidecarMutate(client *k8sclient.K8sClient, jfs juicefs.Interface, serverless bool, pair []util.PVPair) Mutate { return &SidecarMutate{ - Client: client, - juicefs: jfs, - Pair: pair, + Client: client, + juicefs: jfs, + Serverless: serverless, + Pair: pair, } } @@ -89,7 +91,15 @@ func (s *SidecarMutate) mutate(ctx context.Context, pod *corev1.Pod, pair util.P if cap <= 0 { return nil, fmt.Errorf("capacity %d is too small, at least 1GiB for quota", capacity) } - r := builder.NewBuilder(jfsSetting, cap) + + var r builder.SidecarInterface + if !s.Serverless { + r = builder.NewContainerBuilder(jfsSetting, cap) + } else if pod.Annotations != nil && pod.Annotations[builder.VCIANNOKey] == builder.VCIANNOValue { + r = builder.NewVCIBuilder(jfsSetting, cap, *pod, *pair.PVC) + } else { + r = builder.NewServerlessBuilder(jfsSetting, cap) + } // create secret per PVC secret := r.NewSecret() @@ -106,14 +116,14 @@ func (s *SidecarMutate) mutate(ctx context.Context, pod *corev1.Pod, pair util.P // deduplicate container name and volume name in pod when multiple volumes are mounted s.Deduplicate(pod, mountPod, index) - // inject container - s.injectContainer(out, mountPod.Spec.Containers[0]) - // inject initContainer - s.injectInitContainer(out, mountPod.Spec.InitContainers[0]) // inject volume - s.injectVolume(out, mountPod.Spec.Volumes, mountPath, pair) + s.injectVolume(out, r, mountPod.Spec.Volumes, mountPath, pair) // inject label s.injectLabel(out) + // inject annotation + s.injectAnnotation(out, mountPod.Annotations) + // inject container + s.injectContainer(out, mountPod.Spec.Containers[0]) return } @@ -132,16 +142,9 @@ func (s *SidecarMutate) Deduplicate(pod, mountPod *corev1.Pod, index int) { return } - // deduplicate initContainer name - for _, c := range pod.Spec.InitContainers { - if c.Name == mountPod.Spec.InitContainers[0].Name { - mountPod.Spec.InitContainers[0].Name = fmt.Sprintf("%s-%d", c.Name, index) - } - } - // deduplicate volume name for i, mv := range mountPod.Spec.Volumes { - if mv.Name == builder.UpdateDBDirName || mv.Name == builder.JfsDirName || mv.Name == builder.JfsRootDirName { + if mv.Name == builder.UpdateDBDirName || mv.Name == builder.JfsDirName { continue } mountIndex := 0 @@ -201,19 +204,14 @@ func (s *SidecarMutate) injectContainer(pod *corev1.Pod, container corev1.Contai pod.Spec.Containers = append([]corev1.Container{container}, pod.Spec.Containers...) } -func (s *SidecarMutate) injectInitContainer(pod *corev1.Pod, container corev1.Container) { - pod.Spec.InitContainers = append([]corev1.Container{container}, pod.Spec.InitContainers...) -} - -func (s *SidecarMutate) injectVolume(pod *corev1.Pod, volumes []corev1.Volume, mountPath string, pair util.PVPair) { - hostMount := filepath.Join(config.MountPointPath, mountPath, s.jfsSetting.SubPath) +func (s *SidecarMutate) injectVolume(pod *corev1.Pod, build builder.SidecarInterface, volumes []corev1.Volume, mountPath string, pair util.PVPair) { mountedVolume := []corev1.Volume{} podVolumes := make(map[string]bool) for _, volume := range pod.Spec.Volumes { podVolumes[volume.Name] = true } for _, v := range volumes { - if v.Name == builder.UpdateDBDirName || v.Name == builder.JfsDirName || v.Name == builder.JfsRootDirName { + if v.Name == builder.UpdateDBDirName || v.Name == builder.JfsDirName { if _, ok := podVolumes[v.Name]; ok { continue } @@ -222,14 +220,17 @@ func (s *SidecarMutate) injectVolume(pod *corev1.Pod, volumes []corev1.Volume, m } for i, volume := range pod.Spec.Volumes { if volume.PersistentVolumeClaim != nil && volume.PersistentVolumeClaim.ClaimName == pair.PVC.Name { - // overwrite original volume and use juicefs volume mountpoint instead - pod.Spec.Volumes[i] = corev1.Volume{ - Name: volume.Name, - VolumeSource: corev1.VolumeSource{ - HostPath: &corev1.HostPathVolumeSource{ - Path: hostMount, - }, - }} + // overwrite volume + build.OverwriteVolumes(&volume, mountPath) + pod.Spec.Volumes[i] = volume + + for j, vm := range pod.Spec.Containers[0].VolumeMounts { + // overwrite volumeMount + if vm.Name == volume.Name { + build.OverwriteVolumeMounts(&vm) + pod.Spec.Containers[0].VolumeMounts[j] = vm + } + } } } // inject volume @@ -247,6 +248,19 @@ func (s *SidecarMutate) injectLabel(pod *corev1.Pod) { metaObj.DeepCopyInto(&pod.ObjectMeta) } +func (s *SidecarMutate) injectAnnotation(pod *corev1.Pod, annotations map[string]string) { + metaObj := pod.ObjectMeta + + if metaObj.Annotations == nil { + metaObj.Annotations = map[string]string{} + } + + for k, v := range annotations { + metaObj.Annotations[k] = v + } + metaObj.DeepCopyInto(&pod.ObjectMeta) +} + func (s *SidecarMutate) createOrUpdateSecret(ctx context.Context, secret *corev1.Secret) error { klog.V(5).Infof("createOrUpdateSecret: %s, %s", secret.Name, secret.Namespace) err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { diff --git a/pkg/webhook/handler/mutate/sidecar_test.go b/pkg/webhook/handler/mutate/sidecar_test.go index d4c53fa3d5..f3fbeeac02 100644 --- a/pkg/webhook/handler/mutate/sidecar_test.go +++ b/pkg/webhook/handler/mutate/sidecar_test.go @@ -25,6 +25,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/juicedata/juicefs-csi-driver/pkg/config" + "github.com/juicedata/juicefs-csi-driver/pkg/juicefs/mount/builder" volconf "github.com/juicedata/juicefs-csi-driver/pkg/util" ) @@ -65,6 +66,10 @@ func TestSidecarMutate_injectVolume(t *testing.T) { }, }, }, + Containers: []corev1.Container{{ + Name: "test", + VolumeMounts: []corev1.VolumeMount{{Name: "app-volume", MountPath: "data"}}, + }}, }, }, volumes: []corev1.Volume{{ @@ -128,6 +133,10 @@ func TestSidecarMutate_injectVolume(t *testing.T) { }, }, }, + Containers: []corev1.Container{{ + Name: "test", + VolumeMounts: []corev1.VolumeMount{{Name: "app-volume", MountPath: "data"}}, + }}, }, }, volumes: []corev1.Volume{{ @@ -155,7 +164,8 @@ func TestSidecarMutate_injectVolume(t *testing.T) { s := &SidecarMutate{ jfsSetting: tt.fields.jfsSetting, } - s.injectVolume(tt.args.pod, tt.args.volumes, tt.args.mountPath, tt.fields.pair) + r := builder.NewContainerBuilder(tt.fields.jfsSetting, 0) + s.injectVolume(tt.args.pod, r, tt.args.volumes, tt.args.mountPath, tt.fields.pair) if len(tt.args.pod.Spec.Volumes) != len(tt.wantPodVolume) { t.Errorf("injectVolume() = %v, want %v", tt.args.pod.Spec.Volumes, tt.wantPodVolume) } @@ -180,39 +190,6 @@ func TestSidecarMutate_injectVolume(t *testing.T) { } } -func TestSidecarMutate_injectInitContainer(t *testing.T) { - type args struct { - pod *corev1.Pod - container corev1.Container - } - tests := []struct { - name string - args args - wantInitContainerLen int - }{ - { - name: "test inject init container", - args: args{ - pod: &corev1.Pod{}, - container: corev1.Container{ - Name: "format", - Image: "juicedata/mount:latest", - }, - }, - wantInitContainerLen: 1, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := &SidecarMutate{} - s.injectInitContainer(tt.args.pod, tt.args.container) - if len(tt.args.pod.Spec.InitContainers) != tt.wantInitContainerLen { - t.Errorf("injectInitContainer() = %v, want %v", tt.args.pod.Spec.InitContainers, tt.wantInitContainerLen) - } - }) - } -} - func TestSidecarMutate_injectContainer(t *testing.T) { type args struct { pod *corev1.Pod diff --git a/pkg/webhook/handler/register.go b/pkg/webhook/handler/register.go index ec77a5439b..a697772fb8 100644 --- a/pkg/webhook/handler/register.go +++ b/pkg/webhook/handler/register.go @@ -24,13 +24,16 @@ import ( "github.com/juicedata/juicefs-csi-driver/pkg/k8sclient" ) -const HandlerPath = "/juicefs/inject-v1-pod" +const ( + SidecarPath = "/juicefs/inject-v1-pod" + ServerlessPath = "/juicefs/serverless/inject-v1-pod" +) // Register registers the handlers to the manager func Register(mgr manager.Manager, client *k8sclient.K8sClient) { server := mgr.GetWebhookServer() - server.Register(HandlerPath, &webhook.Admission{Handler: &SidecarHandler{ - Client: client, - }}) - klog.Infof("Registered webhook handler path %s", HandlerPath) + server.Register(SidecarPath, &webhook.Admission{Handler: NewSidecarHandler(client, false)}) + klog.Infof("Registered webhook handler path %s for sidecar", SidecarPath) + server.Register(ServerlessPath, &webhook.Admission{Handler: NewSidecarHandler(client, true)}) + klog.Infof("Registered webhook handler path %s for serverless", ServerlessPath) } diff --git a/scripts/juicefs-csi-webhook-install.sh b/scripts/juicefs-csi-webhook-install.sh index 05234c6c31..24e2cdefb4 100755 --- a/scripts/juicefs-csi-webhook-install.sh +++ b/scripts/juicefs-csi-webhook-install.sh @@ -457,6 +457,41 @@ spec: --- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/instance: juicefs-csi-driver + app.kubernetes.io/name: juicefs-csi-driver + app.kubernetes.io/version: master + name: juicefs-admission-serverless-webhook +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: CA_BUNDLE + service: + name: juicefs-admission-webhook + namespace: kube-system + path: /juicefs/serverless/inject-v1-pod + failurePolicy: Fail + name: sidecar.inject.serverless.juicefs.com + namespaceSelector: + matchLabels: + juicefs.com/enable-serverless-injection: "true" + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None + timeoutSeconds: 20 +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration metadata: labels: app.kubernetes.io/instance: juicefs-csi-driver @@ -882,6 +917,41 @@ spec: --- apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration +metadata: + labels: + app.kubernetes.io/instance: juicefs-csi-driver + app.kubernetes.io/name: juicefs-csi-driver + app.kubernetes.io/version: master + name: juicefs-admission-serverless-webhook +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + caBundle: CA_BUNDLE + service: + name: juicefs-admission-webhook + namespace: kube-system + path: /juicefs/serverless/inject-v1-pod + failurePolicy: Fail + name: sidecar.inject.serverless.juicefs.com + namespaceSelector: + matchLabels: + juicefs.com/enable-serverless-injection: "true" + rules: + - apiGroups: + - "" + apiVersions: + - v1 + operations: + - CREATE + resources: + - pods + sideEffects: None + timeoutSeconds: 20 +--- +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration metadata: annotations: cert-manager.io/inject-ca-from: kube-system/juicefs-cert