Merge pull request #577 from fidencio/topic/misc-snapshotter-ds-add-o…

…verlays-for-other-k8s-flavours

misc: snapshotter: Add `k3s` and `rke2` overlays to the snapshotter daemonset

.github/workflows/k8s-e2e.yml

@@ -33,7 +33,7 @@ jobs:
         run: |
           make
           cp bin/containerd-nydus-grpc ./
-          cp misc/snapshotter/* ./
+          cp -r misc/snapshotter/* ./
           ls -tl ./
           NYDUS_VER=v$(curl --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' -s "https://api.github.com/repos/dragonflyoss/nydus/releases/latest" | jq -r .tag_name | sed 's/^v//')
           docker build --build-arg NYDUS_VER=${NYDUS_VER} -t local-dev:e2e .

docs/setup_snapshotter_by_daemonset.md

@@ -27,10 +27,21 @@ Next, we can configure access control for nydus snapshotter.
 kubectl apply -f misc/snapshotter/nydus-snapshotter-rbac.yaml
 ```
 
-Afterward, we can deploy a DaemonSet for nydus snapshotter.
+Afterward, we can deploy a DaemonSet for nydus snapshotter, according to the kubernetes flavour you're using.
 
 ```bash
-kubectl apply -f misc/snapshotter/nydus-snapshotter.yaml
+# Vanilla kubernetes
+kubectl apply -f misc/snapshotter/base/nydus-snapshotter.yaml
+```
+
+```bash
+# k3s
+kubectl apply -k misc/snapshotter/overlays/k3s/
+```
+
+```bash
+# rke2
+kubectl apply -k misc/snapshotter/overlays/rke2/
 ```
 
 Then, we can confirm that nydus snapshotter is running through the DaemonSet.
@@ -75,7 +86,21 @@ Jan 17 16:14:23 worker containerd-nydus-grpc[1100169]: time="2024-01-17T16:14:23
 We use `preStop`` hook in the DaemonSet to uninstall nydus snapshotter and roll back the containerd configuration.
 
 ```bash
-$ kubectl delete -f misc/snapshotter/nydus-snapshotter.yaml 
+# Vanilla kubernetes
+$ kubectl delete -f misc/snapshotter/base/nydus-snapshotter.yaml 
+```
+
+```bash
+# k3s
+$ kubectl delete -k misc/snapshotter/overlays/k3s/ 
+```
+
+```bash
+# rke2
+$ kubectl delete -k misc/snapshotter/overlays/rke2/
+```
+
+```bash
 $ kubectl delete -f misc/snapshotter/nydus-snapshotter-rbac.yaml 
 $ systemd restart containerd.service
 ```

misc/snapshotter/Dockerfile

@@ -1,5 +1,6 @@
-FROM alpine:3.17.0 AS sourcer
+FROM alpine:3.17.0 AS base
 
+FROM base AS sourcer
 ARG NYDUS_VER=v2.1.5
 
 RUN apk add --no-cache curl && \
@@ -10,7 +11,15 @@ RUN apk add --no-cache curl && \
     rm nydus-static-$NYDUS_VER-linux-amd64.tgz && \
     mv nydus-static/* /
 
-FROM alpine:3.17.0
+FROM base AS kubectl-sourcer
+RUN apk add --no-cache curl && \
+    ARCH=$(uname -m) && \
+    if [ "${ARCH}" = "x86_64" ]; then ARCH=amd64; fi && \
+    if [ "${ARCH}" = "aarch64" ]; then ARCH=arm64; fi && \
+    curl -fL --progress-bar -o /usr/bin/kubectl https://storage.googleapis.com/kubernetes-release/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/${ARCH}/kubectl && \
+    chmod +x /usr/bin/kubectl
+
+FROM base
 ARG DESTINATION=/opt/nydus-artifacts
 ARG CONFIG_DESTINATION=${DESTINATION}/etc/nydus
 ARG BINARY_DESTINATION=${DESTINATION}/usr/local/bin
@@ -22,6 +31,7 @@ RUN apk add --no-cache libc6-compat bash
 VOLUME /var/lib/containerd-nydus /run/containerd-nydus
 
 COPY --from=sourcer /.nydus_version /.nydus_version
+COPY --from=kubectl-sourcer /usr/bin/kubectl /usr/bin/kubectl
 
 RUN mkdir -p ${CONFIG_DESTINATION} ${BINARY_DESTINATION} ${SCRIPT_DESTINATION} /var/lib/containerd-nydus/cache /tmp/blobs/
 COPY --from=sourcer /nydus* ${BINARY_DESTINATION}/

misc/snapshotter/base/kustomization.yaml

@@ -0,0 +1,5 @@
+resources:
+- nydus-snapshotter.yaml
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+

misc/snapshotter/nydus-snapshotter.yaml → misc/snapshotter/base/nydus-snapshotter.yaml

@@ -99,7 +99,7 @@ spec:
           mountPath: "/etc/nydus"
           mountPropagation: Bidirectional     
         - name: containerd-conf
-          mountPath: "/etc/containerd/config.toml"
+          mountPath: "/etc/containerd/"
         - name: local-bin
           mountPath: "/usr/local/bin/"
         - name: etc-systemd-system
@@ -130,7 +130,7 @@ spec:
           type: DirectoryOrCreate
       - name: containerd-conf
         hostPath:
-          path: /etc/containerd/config.toml
+          path: /etc/containerd/
       - name: local-bin
         hostPath:
           path: /usr/local/bin/

misc/snapshotter/overlays/k3s/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../base
+patches:
+- path: mount_k3s_conf.yaml

misc/snapshotter/overlays/k3s/mount_k3s_conf.yaml

@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nydus-snapshotter
+  namespace: nydus-system
+spec:
+  template:
+    spec:
+      volumes:
+        - name: containerd-conf
+          hostPath:
+            path: /var/lib/rancher/k3s/agent/etc/containerd/

misc/snapshotter/overlays/rke2/kustomization.yaml

@@ -0,0 +1,6 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ../../base
+patches:
+- path: mount_rke2_conf.yaml

misc/snapshotter/overlays/rke2/mount_rke2_conf.yaml

@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: nydus-snapshotter
+  namespace: nydus-system
+spec:
+  template:
+    spec:
+      volumes:
+        - name: containerd-conf
+          hostPath:
+            path: /var/lib/rancher/rke2/agent/etc/containerd/

misc/snapshotter/snapshotter.sh

@@ -12,7 +12,7 @@ SNAPSHOTTER_ARTIFACTS_DIR="/opt/nydus-artifacts"
 
 # Container runtime config, the default container runtime is containerd
 CONTAINER_RUNTIME="${CONTAINER_RUNTIME:-containerd}"
-CONTAINER_RUNTIME_CONFIG="${CONTAINER_RUNTIME_CONFIG:-/etc/containerd/config.toml}"
+CONTAINER_RUNTIME_CONFIG="/etc/containerd/config.toml"
 
 # Common nydus snapshotter config options
 FS_DRIVER="${FS_DRIVER:-fusedev}"
@@ -110,9 +110,6 @@ function fs_driver_handler() {
 function configure_snapshotter() {
 
     echo "configuring snapshotter"
-    if [ "${CONTAINER_RUNTIME}" != "containerd" ]; then
-        die "not supported container runtime: ${CONTAINER_RUNTIME}"
-    fi
 
     # Copy the container runtime config to a backup
     cp "$CONTAINER_RUNTIME_CONFIG" "$CONTAINER_RUNTIME_CONFIG".bak.nydus
@@ -176,11 +173,6 @@ function install_snapshotter() {
 
 function deploy_snapshotter() {
     echo "deploying snapshotter"
-    if [ ! -f "${CONTAINER_RUNTIME_CONFIG}" ] && [ "${CONTAINER_RUNTIME}" == "containerd" ]; then
-        mkdir -p /etc/containerd || true
-        containerd config default >/etc/containerd/config.toml
-    fi
-
     install_snapshotter
 
     COMMANDLINE="${SNAPSHOTTER_BINARY}"
@@ -196,7 +188,7 @@ function deploy_snapshotter() {
         echo "running snapshotter as standalone process"
         ${COMMANDLINE} &
     fi
-    wait_service_active 30 5 containerd
+    wait_service_active 30 5 ${CONTAINER_RUNTIME}
 
 }
 
@@ -218,7 +210,7 @@ function cleanup_snapshotter() {
     else
         kill -9 $pid || true
     fi
-    wait_service_active 30 5 containerd
+    wait_service_active 30 5 ${CONTAINER_RUNTIME}
     echo "Removing nydus-snapshotter artifacts from host"
     rm -f "${SNAPSHOTTER_BINARY}"
     rm -f "${NYDUS_BINARY_DIR}/nydus*"
@@ -227,13 +219,59 @@ function cleanup_snapshotter() {
     rm -rf "${NYDUS_LIB_DIR}/*"
 }
 
+function get_container_runtime() {
+    local runtime=$(kubectl get node ${NODE_NAME} -o jsonpath='{.status.nodeInfo.containerRuntimeVersion}')
+    if [ "$?" -ne 0 ]; then
+        die "\"$NODE_NAME\" is an invalid node name"
+    fi
+
+    if echo "$runtime" | grep -qE 'containerd.*-k3s'; then
+        if nsenter -t 1 -m systemctl is-active --quiet rke2-agent; then
+            echo "rke2-agent"
+        elif nsenter -t 1 -m systemctl is-active --quiet rke2-server; then
+            echo "rke2-server"
+        elif nsenter -t 1 -m systemctl is-active --quiet k3s-agent; then
+            echo "k3s-agent"
+        else
+            echo "k3s"
+        fi
+    elif nsenter -t 1 -m systemctl is-active --quiet k0scontroller; then
+        echo "k0s-controller"
+    elif nsenter -t 1 -m systemctl is-active --quiet k0sworker; then
+        echo "k0s-worker"
+    else
+        echo "$runtime" | awk -F '[:]' '{print $1}'
+    fi
+}
+
 function main() {
     # script requires that user is root
     euid=$(id -u)
     if [[ $euid -ne 0 ]]; then
         die "This script must be run as root"
     fi
 
+    CONTAINER_RUNTIME=$(get_container_runtime)
+    if [ "${CONTAINER_RUNTIME}" == "k3s" ] || [ "${CONTAINER_RUNTIME}" == "k3s-agent" ] || [ "${CONTAINER_RUNTIME}" == "rke2-agent" ] || [ "${CONTAINER_RUNTIME}" == "rke2-server" ]; then
+        CONTAINER_RUNTIME_CONFIG_TMPL="${CONTAINER_RUNTIME_CONFIG}.tmpl"
+        if [ ! -f "${CONTAINER_RUNTIME_CONFIG_TMPL}" ]; then
+            cp "${CONTAINER_RUNTIME_CONFIG}" "${CONTAINER_RUNTIME_CONFIG_TMPL}"
+        fi
+
+        CONTAINER_RUNTIME_CONFIG="${CONTAINER_RUNTIME_CONFIG_TMPL}"
+    elif [ "${CONTAINER_RUNTIME}" == "containerd" ]; then
+        if [ ! -f "${CONTAINER_RUNTIME_CONFIG}" ]; then
+            mkdir -p $(dirname ${CONTAINER_RUNTIME_CONFIG}) || true
+            if [ -x $(command -v ${CONTAINER_RUNTIME}) ]; then
+                ${CONTAINER_RUNTIME} config default > ${CONTAINER_RUNTIME_CONFIG}
+            else
+                die "Not able to find an executable ${CONTAINER_RUNTIME} binary to create the default config"
+            fi
+        fi
+    else
+        die "${CONTAINER_RUNTIME} is a unsupported containe runtime"
+    fi
+
     action=${1:-}
     if [ -z "$action" ]; then
         print_usage

tests/e2e/k8s/snapshotter-cri.yaml

@@ -9,8 +9,34 @@ kind: ServiceAccount
 metadata:
   name: nydus-snapshotter-sa
   namespace: nydus-system
+---
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: nydus-snapshotter-role
+rules:
+  - apiGroups: 
+    - ""
+    resources: 
+    - nodes
+    verbs:
+    - get
+    - patch
 
 ---
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: nydus-snapshotter-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: nydus-snapshotter-role
+subjects:
+  - kind: ServiceAccount
+    name: nydus-snapshotter-sa
+    namespace: nydus-system
+---
 apiVersion: v1
 kind: Pod
 metadata:
@@ -27,6 +53,10 @@ spec:
       image: "local-dev:e2e"
       imagePullPolicy: IfNotPresent
       env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         - name: FS_DRIVER
           valueFrom:
             configMapKeyRef:
@@ -80,7 +110,7 @@ spec:
           mountPath: "/etc/nydus"
           mountPropagation: Bidirectional
         - name: containerd-conf
-          mountPath: "/etc/containerd/config.toml"
+          mountPath: "/etc/containerd/"
         - name: local-bin
           mountPath: "/usr/local/bin/"
         - name: etc-systemd-system
@@ -113,7 +143,7 @@ spec:
         type: DirectoryOrCreate
     - name: containerd-conf
       hostPath:
-        path: /etc/containerd/config.toml
+        path: /etc/containerd/
     - name: local-bin
       hostPath:
         path: /usr/local/bin/

tests/e2e/k8s/snapshotter-kubeconf.yaml

@@ -23,6 +23,13 @@ rules:
       - get
       - list
       - watch
+  - apiGroups: 
+    - ""
+    resources: 
+    - nodes
+    verbs:
+    - get
+    - patch
 
 ---
 kind: ClusterRoleBinding
@@ -55,6 +62,10 @@ spec:
       image: "local-dev:e2e"
       imagePullPolicy: IfNotPresent
       env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              fieldPath: spec.nodeName
         - name: FS_DRIVER
           valueFrom:
             configMapKeyRef:
@@ -108,7 +119,7 @@ spec:
           mountPath: "/etc/nydus"
           mountPropagation: Bidirectional
         - name: containerd-conf
-          mountPath: "/etc/containerd/config.toml"
+          mountPath: "/etc/containerd/"
         - name: local-bin
           mountPath: "/usr/local/bin/"
         - name: etc-systemd-system
@@ -141,7 +152,7 @@ spec:
         type: DirectoryOrCreate
     - name: containerd-conf
       hostPath:
-        path: /etc/containerd/config.toml
+        path: /etc/containerd/
     - name: local-bin
       hostPath:
         path: /usr/local/bin/
@@ -254,4 +265,4 @@ data:
         "merging_size": 131072,
         "bandwidth_rate": 1048576
       }
-    }
+    }