Skip to content

Commit

Permalink
delete initContainer and put init cmd in mount container && keep init…
Browse files Browse the repository at this point in the history 
…Container only if need to create subpath

Signed-off-by: zwwhdls <[email protected]>
  • Loading branch information
@zwwhdls
zwwhdls committed Oct 12, 2023
1 parent a7510b6 commit c1ad4c3
Show file tree
Hide file tree
Showing 9 changed files with 131 additions and 188 deletions.
122 changes: 70 additions & 52 deletions pkg/juicefs/mount/builder/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@ import (

const (
JfsDirName = "jfs-dir"
JfsRootDirName = "jfs-root-dir"
UpdateDBDirName = "updatedb"
UpdateDBCfgFile = "/etc/updatedb.conf"
)
Expand Down Expand Up @@ -112,6 +111,11 @@ func (r *BaseBuilder) genCommonJuicePod(cnGen func() corev1.Container) *corev1.P
{Name: "metrics", ContainerPort: r.genMetricsPort()},
}
}
if initContainer := r.genInitContainer(cnGen); initContainer != nil {
// initContainer should have the same volumeMounts as mount container
initContainer.VolumeMounts = pod.Spec.Containers[0].VolumeMounts
pod.Spec.InitContainers = []corev1.Container{*initContainer}
}
return pod
}

Expand Down Expand Up @@ -146,6 +150,71 @@ func (r *BaseBuilder) genMountCommand() string {
return util.QuoteForShell(cmd)
}

// genInitContainer: generate init container
func (r *BaseBuilder) genInitContainer(cnGen func() corev1.Container) *corev1.Container {
if r.jfsSetting.SubPath == "" {
// do not need initContainer if no subpath
return nil
}
container := cnGen()
container.Name = "jfs-init"

initCmds := []string{
r.genInitCommand(),
}
// create subpath if readonly mount or in webhook mode
if util.ContainsString(r.jfsSetting.Options, "read-only") || util.ContainsString(r.jfsSetting.Options, "ro") || config.Webhook {
// generate mount command
initCmds = append(initCmds,
r.getJobCommand(),
fmt.Sprintf("if [ ! -d /mnt/jfs/%s ]; then mkdir -m 777 /mnt/jfs/%s; fi;", r.jfsSetting.SubPath, r.jfsSetting.SubPath),
"umount /mnt/jfs",
)
// set quota in webhook mode
if config.Webhook && r.capacity > 0 {
quotaPath := r.jfsSetting.SubPath
var subdir string
for _, o := range r.jfsSetting.Options {
pair := strings.Split(o, "=")
if len(pair) != 2 {
continue
}
if pair[0] == "subdir" {
subdir = path.Join("/", pair[1])
}
}
var setQuotaCmd string
targetPath := path.Join(subdir, quotaPath)
capacity := strconv.FormatInt(r.capacity, 10)
if r.jfsSetting.IsCe {
// juicefs quota; if [ $? -eq 0 ]; then juicefs quota set ${metaurl} --path ${path} --capacity ${capacity}; fi
cmdArgs := []string{
config.CeCliPath, "quota; if [ $? -eq 0 ]; then",
config.CeCliPath,
"quota", "set", "${metaurl}",
"--path", targetPath,
"--capacity", capacity,
"; fi",
}
setQuotaCmd = strings.Join(cmdArgs, " ")
} else {
cmdArgs := []string{
config.CliPath, "quota; if [ $? -eq 0 ]; then",
config.CliPath,
"quota", "set", r.jfsSetting.Name,
"--path", targetPath,
"--capacity", capacity,
"; fi",
}
setQuotaCmd = strings.Join(cmdArgs, " ")
}
initCmds = append(initCmds, setQuotaCmd)
}
}
container.Command = []string{"sh", "-c", strings.Join(initCmds, "\n")}
return &container
}

// genInitCommand generates init command
func (r *BaseBuilder) genInitCommand() string {
formatCmd := r.jfsSetting.FormatCmd
Expand All @@ -156,57 +225,6 @@ func (r *BaseBuilder) genInitCommand() string {
}
initCmds := []string{formatCmd}

// create subpath if readonly mount or in webhook mode
if r.jfsSetting.SubPath != "" {
if util.ContainsString(r.jfsSetting.Options, "read-only") || util.ContainsString(r.jfsSetting.Options, "ro") || config.Webhook {
// generate mount command
initCmds = append(initCmds,
r.getJobCommand(),
fmt.Sprintf("if [ ! -d /mnt/jfs/%s ]; then mkdir -m 777 /mnt/jfs/%s; fi;", r.jfsSetting.SubPath, r.jfsSetting.SubPath),
"umount /mnt/jfs",
)
// set quota in webhook mode
if config.Webhook && r.capacity > 0 {
quotaPath := r.jfsSetting.SubPath
var subdir string
for _, o := range r.jfsSetting.Options {
pair := strings.Split(o, "=")
if len(pair) != 2 {
continue
}
if pair[0] == "subdir" {
subdir = path.Join("/", pair[1])
}
}
var setQuotaCmd string
targetPath := path.Join(subdir, quotaPath)
capacity := strconv.FormatInt(r.capacity, 10)
if r.jfsSetting.IsCe {
// juicefs quota; if [ $? -eq 0 ]; then juicefs quota set ${metaurl} --path ${path} --capacity ${capacity}; fi
cmdArgs := []string{
config.CeCliPath, "quota; if [ $? -eq 0 ]; then",
config.CeCliPath,
"quota", "set", "${metaurl}",
"--path", targetPath,
"--capacity", capacity,
"; fi",
}
setQuotaCmd = strings.Join(cmdArgs, " ")
} else {
cmdArgs := []string{
config.CliPath, "quota; if [ $? -eq 0 ]; then",
config.CliPath,
"quota", "set", r.jfsSetting.Name,
"--path", targetPath,
"--capacity", capacity,
"; fi",
}
setQuotaCmd = strings.Join(cmdArgs, " ")
}
initCmds = append(initCmds, setQuotaCmd)
}
}
}
return strings.Join(initCmds, "\n")
}

Expand Down
10 changes: 8 additions & 2 deletions pkg/juicefs/mount/builder/job.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package builder
import (
"crypto/sha256"
"fmt"
"strings"

batchv1 "k8s.io/api/batch/v1"
corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -48,7 +49,10 @@ func (r *JobBuilder) NewJobForCreateVolume() *batchv1.Job {
jobName := GenJobNameByVolumeId(r.jfsSetting.VolumeId) + "-createvol"
job := r.newJob(jobName)
jobCmd := r.getCreateVolumeCmd()
job.Spec.Template.Spec.Containers[0].Command = []string{"sh", "-c", jobCmd}
initCmd := r.genInitCommand()
cmd := strings.Join([]string{initCmd, jobCmd}, "\n")
job.Spec.Template.Spec.Containers[0].Command = []string{"sh", "-c", cmd}

klog.Infof("create volume job cmd: %s", jobCmd)
return job
}
Expand All @@ -57,7 +61,9 @@ func (r *JobBuilder) NewJobForDeleteVolume() *batchv1.Job {
jobName := GenJobNameByVolumeId(r.jfsSetting.VolumeId) + "-delvol"
job := r.newJob(jobName)
jobCmd := r.getDeleteVolumeCmd()
job.Spec.Template.Spec.Containers[0].Command = []string{"sh", "-c", jobCmd}
initCmd := r.genInitCommand()
cmd := strings.Join([]string{initCmd, jobCmd}, "\n")
job.Spec.Template.Spec.Containers[0].Command = []string{"sh", "-c", cmd}
klog.Infof("delete volume job cmd: %s", jobCmd)
return job
}
Expand Down
75 changes: 10 additions & 65 deletions pkg/juicefs/mount/builder/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package builder
import (
"fmt"
"path/filepath"
"strings"

corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
Expand All @@ -43,8 +44,13 @@ func NewPodBuilder(setting *config.JfsSetting, capacity int64) *PodBuilder {
func (r *PodBuilder) NewMountPod(podName string) *corev1.Pod {
pod := r.genCommonJuicePod(r.genCommonContainer)

cmd := r.genMountCommand()
pod.Name = podName
mountCmd := r.genMountCommand()
cmd := mountCmd
initCmd := r.genInitCommand()
if initCmd != "" {
cmd = strings.Join([]string{initCmd, mountCmd}, "\n")
}
pod.Spec.Containers[0].Command = []string{"sh", "-c", cmd}
pod.Spec.Containers[0].Env = []corev1.EnvVar{{
Name: "JFS_FOREGROUND",
Expand All @@ -56,14 +62,6 @@ func (r *PodBuilder) NewMountPod(podName string) *corev1.Pod {
pod.Spec.Volumes = append(pod.Spec.Volumes, volumes...)
pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, volumeMounts...)

// generate initContainer
if r.jfsSetting.FormatCmd != "" {
initVolumeMounts := r.genInitVolumes()
initContainer := r.genInitContainer()
initContainer.VolumeMounts = append(initContainer.VolumeMounts, initVolumeMounts...)
pod.Spec.InitContainers = []corev1.Container{initContainer}
}

// add cache-dir hostpath & PVC volume
cacheVolumes, cacheVolumeMounts := r.genCacheDirVolumes()
pod.Spec.Volumes = append(pod.Spec.Volumes, cacheVolumes...)
Expand All @@ -74,6 +72,9 @@ func (r *PodBuilder) NewMountPod(podName string) *corev1.Pod {
pod.Spec.Volumes = append(pod.Spec.Volumes, mountVolumes...)
pod.Spec.Containers[0].VolumeMounts = append(pod.Spec.Containers[0].VolumeMounts, mountVolumeMounts...)

if len(pod.Spec.InitContainers) != 0 {
pod.Spec.InitContainers[0].VolumeMounts = append(pod.Spec.InitContainers[0].VolumeMounts, mountVolumeMounts...)
}
return pod
}

Expand Down Expand Up @@ -169,35 +170,9 @@ func (r *PodBuilder) genHostPathVolumes() (volumes []corev1.Volume, volumeMounts
return
}

// genInitContainer: generate init container
func (r *PodBuilder) genInitContainer() corev1.Container {
rootUser := int64(0)
isPrivileged := true
secretName := r.jfsSetting.SecretName
container := corev1.Container{
Name: "jfs-format",
Image: r.jfsSetting.Attr.Image,
SecurityContext: &corev1.SecurityContext{
Privileged: &isPrivileged,
RunAsUser: &rootUser,
},
}

initCmd := r.genInitCommand()
container.Command = []string{"sh", "-c", initCmd}

container.EnvFrom = append(container.EnvFrom, corev1.EnvFromSource{
SecretRef: &corev1.SecretEnvSource{LocalObjectReference: corev1.LocalObjectReference{
Name: secretName,
}},
})
return container
}

// genPodVolumes: generate volumes for mount pod
// 1. jfs dir: mount point used to propagate the mount point in the mount container to host
// 2. update db dir: mount updatedb.conf from host to mount pod
// 3. jfs config dir: mount jfs config dir as emptyDir to deliver config file to mount container
func (r *PodBuilder) genPodVolumes() ([]corev1.Volume, []corev1.VolumeMount) {
dir := corev1.HostPathDirectoryOrCreate
file := corev1.HostPathFileOrCreate
Expand Down Expand Up @@ -233,39 +208,9 @@ func (r *PodBuilder) genPodVolumes() ([]corev1.Volume, []corev1.VolumeMount) {
})
}

if r.jfsSetting.FormatCmd != "" {
// initContainer will generate xx.conf to share with mount container
volumes = append(volumes, corev1.Volume{
Name: JfsRootDirName,
VolumeSource: corev1.VolumeSource{
EmptyDir: &corev1.EmptyDirVolumeSource{},
},
})
volumeMounts = append(volumeMounts, corev1.VolumeMount{
Name: JfsRootDirName,
MountPath: "/root/.juicefs",
})
}

return volumes, volumeMounts
}

// genInitVolumes: generate volumes for initContainer in mount pod
// jfs config dir: mount jfs config dir as emptyDir to deliver config file to initContainer
func (r *PodBuilder) genInitVolumes() []corev1.VolumeMount {
volumeMounts := []corev1.VolumeMount{}

if r.jfsSetting.FormatCmd != "" {
// initContainer will generate xx.conf to share with mount container
volumeMounts = append(volumeMounts, corev1.VolumeMount{
Name: JfsRootDirName,
MountPath: "/root/.juicefs",
})
}

return volumeMounts
}

// genCleanCachePod: generate pod to clean cache in host
func (r *PodBuilder) genCleanCachePod() *corev1.Pod {
volumeMountPrefix := "/var/jfsCache"
Expand Down
19 changes: 5 additions & 14 deletions pkg/juicefs/mount/builder/pod_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,30 +174,21 @@ func Test_getCacheDirVolumes(t *testing.T) {
volumeMounts := []corev1.VolumeMount{{
Name: JfsDirName,
MountPath: config.PodMountBase,
}, {
Name: JfsRootDirName,
MountPath: "/root/.juicefs",
}}

volumes := []corev1.Volume{{
Name: JfsDirName,
VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{
Path: config.MountPointPath,
Type: &dir,
}}}, {
Name: JfsRootDirName,
VolumeSource: corev1.VolumeSource{HostPath: &corev1.HostPathVolumeSource{
Path: config.JFSConfigPath,
Type: &dir,
}},
}}
}}}}

s, _ := config.ParseSetting(map[string]string{"name": "test"}, nil, optionWithoutCacheDir, true)
r.jfsSetting = s
cacheVolumes, cacheVolumeMounts := r.genCacheDirVolumes()
volumes = append(volumes, cacheVolumes...)
volumeMounts = append(volumeMounts, cacheVolumeMounts...)
if len(volumes) != 3 || len(volumeMounts) != 3 {
if len(volumes) != 2 || len(volumeMounts) != 2 {
t.Error("getCacheDirVolumes can't work properly")
}

Expand All @@ -206,7 +197,7 @@ func Test_getCacheDirVolumes(t *testing.T) {
cacheVolumes, cacheVolumeMounts = r.genCacheDirVolumes()
volumes = append(volumes, cacheVolumes...)
volumeMounts = append(volumeMounts, cacheVolumeMounts...)
if len(volumes) != 4 || len(volumeMounts) != 4 {
if len(volumes) != 3 || len(volumeMounts) != 3 {
t.Error("getCacheDirVolumes can't work properly")
}

Expand All @@ -215,7 +206,7 @@ func Test_getCacheDirVolumes(t *testing.T) {
cacheVolumes, cacheVolumeMounts = r.genCacheDirVolumes()
volumes = append(volumes, cacheVolumes...)
volumeMounts = append(volumeMounts, cacheVolumeMounts...)
if len(volumes) != 6 || len(volumeMounts) != 6 {
if len(volumes) != 5 || len(volumeMounts) != 5 {
t.Error("getCacheDirVolumes can't work properly")
}

Expand All @@ -224,7 +215,7 @@ func Test_getCacheDirVolumes(t *testing.T) {
cacheVolumes, cacheVolumeMounts = r.genCacheDirVolumes()
volumes = append(volumes, cacheVolumes...)
volumeMounts = append(volumeMounts, cacheVolumeMounts...)
if len(volumes) != 7 || len(volumeMounts) != 7 {
if len(volumes) != 6 || len(volumeMounts) != 6 {
t.Error("getCacheDirVolumes can't work properly")
}
}
Expand Down
13 changes: 12 additions & 1 deletion pkg/juicefs/mount/builder/vci-serverless.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,6 @@ func NewVCIBuilder(setting *config.JfsSetting, capacity int64, app corev1.Pod, p
// 1. no hostpath
// 2. without privileged container
// 3. no propagationBidirectional
// 3. no initContainer
// 4. with env JFS_NO_UMOUNT=1
// 5. annotations for VCI
func (r *VCIBuilder) NewMountSidecar() *corev1.Pod {
Expand All @@ -70,6 +69,18 @@ func (r *VCIBuilder) NewMountSidecar() *corev1.Pod {
"mountPath" : "%s"
}]`, r.jfsSetting.MountPath),
}
if len(pod.Spec.InitContainers) != 0 {
pod.Annotations = map[string]string{
VCIPropagationConfig: VCIPropagationConfigValue,
VCIPropagation: fmt.Sprintf(`[{
"container": "jfs-mount",
"mountPath" : "%s"
}, {
"container": "jfs-init",
"mountPath" : "/mnt/jfs"
}]`, r.jfsSetting.MountPath),
}
}

pod.Spec.Containers[0].Lifecycle.PostStart = &corev1.Handler{
Exec: &corev1.ExecAction{Command: []string{"bash", "-c",
Expand Down
Loading

0 comments on commit c1ad4c3

Please sign in to comment.