refactor examples so to make FUSE containers be sidecars (restartable…

… init containers)

Makefile

@@ -130,31 +130,31 @@ push-examples: $(PUSH_EXAMPLES)
 
 define test-example-template
 ifneq ("$(EXAMPLE_TESTS)", "")
-EXAMPLE_TESTS += test-example-$(1)-$(2)-$(subst .,-,$(3))
+EXAMPLE_TESTS += test-example-$(1)-$(2)-$(6)
 else
-EXAMPLE_TESTS := test-example-$(1)-$(2)-$(subst .,-,$(3))
+EXAMPLE_TESTS := test-example-$(1)-$(2)-$(6)
 endif
 
-.PHONY: test-example-$1-$2
-test-example-$(1)-$(2)-$(subst .,-,$(3)):
-	./examples/check.sh ./$1/$2 $3 mfcp-example-$1-$2 $4 $5 $6 $7
+.PHONY: test-example-$1-$2-$(6)
+test-example-$(1)-$(2)-$(6):
+	./examples/check.sh ./$1/$2 mfcp-example-$1-$2 $3 $4 $5 $6
 endef
 
-$(eval $(call test-example-template,proxy,mountpoint-s3,deploy.yaml,starter,/test.txt,busybox,/data/subdir/test.txt))
-$(eval $(call test-example-template,proxy,goofys,deploy.yaml,starter,/test.txt,busybox,/data/subdir/test.txt))
-$(eval $(call test-example-template,proxy,s3fs,deploy.yaml,starter,/test.txt,busybox,/data/subdir/test.txt))
-$(eval $(call test-example-template,proxy,ros3fs,deploy.yaml,starter,/test.txt,busybox,/data/subdir/test.txt))
-$(eval $(call test-example-template,proxy,sshfs,deploy.yaml,starter,/root/sshfs-example/subdir/test.txt,busybox,/data/subdir/test.txt))
-$(eval $(call test-example-template,starter,ros3fs,deploy.yaml,starter,/test.txt,busybox,/data/subdir/test.txt))
-$(eval $(call test-example-template,starter,sshfs,deploy.yaml,starter,/root/sshfs-example/subdir/test.txt,busybox,/data/subdir/test.txt))
-ifdef TEST_SUBPATH
-$(eval $(call test-example-template,proxy,mountpoint-s3,deploy-subpath.yaml,starter,/test.txt,busybox,/data-subpath/test.txt))
-$(eval $(call test-example-template,proxy,goofys,deploy-subpath.yaml,starter,/test.txt,busybox,/data-subpath/test.txt))
-$(eval $(call test-example-template,proxy,s3fs,deploy-subpath.yaml,starter,/test.txt,busybox,/data-subpath/test.txt))
-$(eval $(call test-example-template,proxy,ros3fs,deploy-subpath.yaml,starter,/test.txt,busybox,/data-subpath/test.txt))
-$(eval $(call test-example-template,proxy,sshfs,deploy-subpath.yaml,starter,/root/sshfs-example/subdir/test.txt,busybox,/data-subpath/test.txt))
-$(eval $(call test-example-template,starter,ros3fs,deploy-subpath.yaml,starter,/test.txt,busybox,/data-subpath/test.txt))
-$(eval $(call test-example-template,starter,sshfs,deploy-subpath.yaml,starter,/root/sshfs-example/subdir/test.txt,busybox,/data-subpath/test.txt))
+$(eval $(call test-example-template,proxy,mountpoint-s3,starter,/test.txt,busybox,/data/subdir/test.txt))
+$(eval $(call test-example-template,proxy,goofys,starter,/test.txt,busybox,/data/subdir/test.txt))
+$(eval $(call test-example-template,proxy,s3fs,starter,/test.txt,busybox,/data/subdir/test.txt))
+$(eval $(call test-example-template,proxy,ros3fs,starter,/test.txt,busybox,/data/subdir/test.txt))
+$(eval $(call test-example-template,proxy,sshfs,starter,/root/sshfs-example/subdir/test.txt,busybox,/data/subdir/test.txt))
+$(eval $(call test-example-template,starter,ros3fs,starter,/test.txt,busybox,/data/subdir/test.txt))
+$(eval $(call test-example-template,starter,sshfs,starter,/root/sshfs-example/subdir/test.txt,busybox,/data/subdir/test.txt))
+ifndef SKIP_TEST_SUBPATH
+$(eval $(call test-example-template,proxy,mountpoint-s3,starter,/test.txt,busybox,/data-subpath/test.txt))
+$(eval $(call test-example-template,proxy,goofys,starter,/test.txt,busybox,/data-subpath/test.txt))
+$(eval $(call test-example-template,proxy,s3fs,starter,/test.txt,busybox,/data-subpath/test.txt))
+$(eval $(call test-example-template,proxy,ros3fs,starter,/test.txt,busybox,/data-subpath/test.txt))
+$(eval $(call test-example-template,proxy,sshfs,starter,/root/sshfs-example/subdir/test.txt,busybox,/data-subpath/test.txt))
+$(eval $(call test-example-template,starter,ros3fs,starter,/test.txt,busybox,/data-subpath/test.txt))
+$(eval $(call test-example-template,starter,sshfs,starter,/root/sshfs-example/subdir/test.txt,busybox,/data-subpath/test.txt))
 endif
 
 .PHONY: test-examples

README.md

@@ -95,37 +95,35 @@ $ kubectl delete -f ./examples/proxy/mountpoint-s3/deploy.yaml
 pod "mfcp-example-proxy-mountpoint-s3" deleted
 ```
 
-## NOTICE
-
-### fuse volume is mounted lazily
+## NOTICE: FUSE container should be a sidecar (a.k.a. restartable init container)
 
 meta-fuse-csi-plugin mounts FUSE implementations after the container started.
-Some applications may read the directory before mount.
-
-To avoid such race condition, there are two solutions.
-
-1. Wait for the FUSE impl is mounted. `examples/proxy/mountpoint-s3/deploy.yaml` and `examples/check.sh` do such delaying.
-  ```yaml
-    - image: busybox
-      name: busybox
-      command: ["/bin/ash"]
-      args: ["-c", "while [[ ! \"$(/bin/mount | grep fuse)\" ]]; do echo \"waiting for mount\" && sleep 1; done; sleep infinity"]
-  ```
-  or
-  ```bash
-  function wait_for_fuse_mounted() {
-      while [[ ! $(kubectl exec $1 -c $2 -- /bin/mount | grep fuse) ]]; do echo "waiting for mount" && sleep 1; done
-  }
-  ```
-1. Use [sidecar](https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/) container, a.k.a restartable init container (enabled by default since Kubernetes v1.30).
-   This can guarantees app containers can see the volume contents since the beggining. See `examples/proxy/mountpoint-s3/deploy-sidecar.yaml` for how to.
-   Please don't forget defining startup probe to make sure fuse volume is actually mounted before app containers are started.
-
-### `subPath` volume mount requires sidecar
-
-When fuse container is a normal container (i.e. not a sidecar), `subPath` volume mount creation by kubelet can race with actual fuse process startup.
-This race might cause that mounted `subPath` volume could be empty. Thus, when you use `subPath` volume mount, you have to make fuse container be a sidecar container.
-See `examples/proxy/mountpoint-s3/deploy-sidecar.yaml` for how to.
+Thus, the application container should start after the FUSE filesystem is surely mounted.
+
+To achieve this, FUSE container should be a [sidecar](https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/)
+container, a.k.a restartable init container (enabled by default since Kubernetes v1.30). Please don't forget defining startup
+probe to make sure fuse volume is actually mounted before app containers are started. See `examples/proxy/mountpoint-s3/deploy.yaml`
+for how to.
+
+If you can't use sidecar feature in your cluster, there can be a workaround. please wait for the FUSE impl is mounted in the application
+container like below:
+
+```yaml
+  - image: busybox
+    name: busybox
+    command: ["/bin/ash"]
+    args: ["-c", "while [[ ! \"$(/bin/mount | grep fuse)\" ]]; do echo \"waiting for mount\" && sleep 1; done; sleep infinity"]
+```
+or
+```bash
+function wait_for_fuse_mounted() {
+    while [[ ! $(kubectl exec $1 -c $2 -- /bin/mount | grep fuse) ]]; do echo "waiting for mount" && sleep 1; done
+}
+```
+
+Please remember that `subPath` doesn't work in this method because when FUSE container is a normal container (i.e. not a sidecar),
+`subPath` volume mount creation by kubelet can race with actual FUSE impl process startup. This race might cause that mounted
+`subPath` volume could be empty.
 
 ## Running E2E tests
 ### Tested Environment
@@ -139,10 +137,6 @@ You can run E2E tests with kind.
 
 ```console
 $ make test-e2e
-
-# if you test subpath volume mount, you can set TEST_SUBPATH=true
-# you will need kubernetes v1.30 or later because this tests needs sidecar containers
-$ TEST_SUBPATH=true make test-e2e
 ```
 
 ## How it works?

examples/check.sh

@@ -12,13 +12,13 @@ function wait_for_fuse_mounted() {
     while [[ ! $(kubectl exec $1 -c $2 -- /bin/mount | grep fuse) ]]; do echo "waiting for mount" && sleep 1; done
 }
 
+MAFNIFEST_FILENAME=deploy.yaml
 MANIFEST_DIR=$1 # path to example manifest
-MAFNIFEST_FILENAME=$2
-POD_NAME=$3
-PROVIDER_CONTAINER=$4
-PROVIDED_FILENAME=$5
-MOUNTED_CONTAINER=$6
-MOUNTED_FILENAME=$7
+POD_NAME=$2
+PROVIDER_CONTAINER=$3
+PROVIDED_FILENAME=$4
+MOUNTED_CONTAINER=$5
+MOUNTED_FILENAME=$6
 
 clean_up () {
     ARG=$?

examples/proxy/gcsfuse/deploy.yaml

@@ -5,7 +5,7 @@ metadata:
   namespace: csi-dev
 spec:
   terminationGracePeriodSeconds: 10
-  containers:
+  initContainers:
   - name: starter
     image: ghcr.io/pfnet-research/meta-fuse-csi-plugin/mfcp-example-proxy-gcsfuse:latest
     imagePullPolicy: IfNotPresent
@@ -19,6 +19,16 @@ spec:
       mountPath: /fusermount3-proxy
     - name: gcsfuse-temp
       mountPath: /gcsfuse-temp
+    - name: fuse-csi-ephemeral
+      mountPath: /data
+      readOnly: true
+      mountPropagation: HostToContainer
+    startupProbe:
+      exec:
+        command: ['sh', '-c', 'mount | grep /data | grep fuse']
+      failureThreshold: 300
+      periodSeconds: 1
+  containers:
   - image: busybox
     name: busybox
     command: ["sleep"]
@@ -27,7 +37,12 @@ spec:
     - name: fuse-csi-ephemeral
       mountPath: /data
       readOnly: true
-      mountPropagation: HostToContainer # TODO: need to validate in csi driver?
+      mountPropagation: HostToContainer
+    - name: fuse-csi-ephemeral
+      mountPath: /data-subpath
+      readOnly: true
+      subPath: subdir
+      mountPropagation: HostToContainer
   serviceAccountName: csi-dev-ksa
   volumes:
   - name: fuse-fd-passing

examples/proxy/goofys/deploy.yaml

@@ -5,12 +5,14 @@ metadata:
   namespace: default
 spec:
   terminationGracePeriodSeconds: 10
-  containers:
+  initContainers:
   - name: minio
+    restartPolicy: Always
     image: quay.io/minio/minio:latest
     command: ["/bin/bash"]
     args: ["-c", "minio server /data --console-address :9090"]
   - name: starter
+    restartPolicy: Always
     image: ghcr.io/pfnet-research/meta-fuse-csi-plugin/mfcp-example-proxy-goofys:latest
     imagePullPolicy: IfNotPresent
     command: ["/bin/bash"]
@@ -25,6 +27,16 @@ spec:
     volumeMounts:
     - name: fuse-fd-passing
       mountPath: /fusermount3-proxy
+    - name: fuse-csi-ephemeral
+      mountPath: /data
+      readOnly: true
+      mountPropagation: HostToContainer
+    startupProbe:
+      exec:
+        command: ['sh', '-c', 'mount | grep /data | grep fuse']
+      failureThreshold: 300
+      periodSeconds: 1
+  containers:
   - image: busybox
     name: busybox
     command: ["sleep"]
@@ -33,7 +45,12 @@ spec:
     - name: fuse-csi-ephemeral
       mountPath: /data
       readOnly: true
-      mountPropagation: HostToContainer # TODO: need to validate in csi driver?
+      mountPropagation: HostToContainer
+    - name: fuse-csi-ephemeral
+      mountPath: /data-subpath
+      readOnly: true
+      subPath: subdir
+      mountPropagation: HostToContainer
   volumes:
   - name: fuse-fd-passing
     emptyDir: {}