juicedata · zwwhdls · Jul 6, 2023 · Jul 4, 2023 · Jul 4, 2023 · Jul 4, 2023
diff --git a/.github/scripts/deploy-csi-in-k8s.sh b/.github/scripts/deploy-csi-in-k8s.sh
@@ -3,6 +3,7 @@
 function main() {
   deployMode=$1
   withoutKubelet=$2
+  prepare_pkg
   echo "deployMode: " $deployMode
   echo "withoutKubelet: " $withoutKubelet
   if [ $withoutKubelet == "withoutkubelet" ]; then
@@ -16,6 +17,28 @@ function main() {
   fi
 }
 
+function prepare_pkg() {
+  # ceph
+  sudo apt install -y software-properties-common
+  sudo wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
+  sudo apt-add-repository 'deb https://download.ceph.com/debian-pacific/ buster main'
+  sudo apt-get update
+  sudo apt-get install -y librados-dev libcephfs-dev librbd-dev
+
+  # fdb
+  sudo mkdir -p /home/travis/.m2
+  sudo wget -O /home/travis/.m2/foundationdb-clients_6.3.23-1_amd64.deb https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-clients_6.3.23-1_amd64.deb
+  sudo dpkg -i /home/travis/.m2/foundationdb-clients_6.3.23-1_amd64.deb
+
+  # gluster
+  sudo wget -O - https://download.gluster.org/pub/gluster/glusterfs/7/rsa.pub | sudo apt-key add -
+  sudo mkdir mkdir /etc/apt/sources.list.d/gluster.list
+  sudo chmod 777 /etc/apt/sources.list.d/gluster.list
+  sudo echo deb [arch=amd64] https://download.gluster.org/pub/gluster/glusterfs/7/LATEST/Debian/buster/amd64/apt buster main > /etc/apt/sources.list.d/gluster.list
+  sudo apt-get update
+  sudo apt-get install -y uuid-dev libglusterfs-dev glusterfs-common
+}
+
 function deploy_csi() {
   sudo microk8s.kubectl delete -f ${GITHUB_WORKSPACE}/deploy/webhook.yaml
   sudo microk8s.kubectl label ns default juicefs.com/enable-injection=false

diff --git a/.github/scripts/test_case.py b/.github/scripts/test_case.py
@@ -1181,7 +1181,9 @@ def test_deployment_static_patch_pv():
 
 def test_dynamic_mount_image():
     LOG.info("[test case] Deployment set mount image in storageClass begin..")
-    mount_image = "juicedata/mount:v1.0.0-4.8.0"
+    mount_image = "juicedata/mount:ee-nightly"
+    if IS_CE:
+        mount_image = "juicedata/mount:ce-nightly"
     # deploy sc
     sc_name = "mount-image-dynamic"
     sc = StorageClass(name=sc_name, secret_name=SECRET_NAME,
@@ -1253,7 +1255,9 @@ def test_dynamic_mount_image():
 
 def test_static_mount_image():
     LOG.info("[test case] Deployment set mount image in PV begin..")
-    mount_image = "juicedata/mount:v1.0.0-4.8.0"
+    mount_image = "juicedata/mount:ee-nightly"
+    if IS_CE:
+        mount_image = "juicedata/mount:ce-nightly"
     # deploy pv
     pv_name = "mount-image-pv"
     pv = PV(name=pv_name, access_mode="ReadWriteMany", volume_handle=pv_name,

diff --git a/.github/workflows/nightly.yaml b/.github/workflows/nightly.yaml
@@ -19,8 +19,102 @@ jobs:
           value=`printf '%s\n' "${testmode[@]}" | jq -R . | jq -cs .`
           echo "value: $value"
           echo "matrix=$value" >> $GITHUB_OUTPUT
+
+          wtestmode=("pod" "pod-mount-share" "pod-provisioner")
+          value=`printf '%s\n' "${wtestmode[@]}" | jq -R . | jq -cs .`
+          echo "value without kubelet: $value"
+          echo "wmatrix=$value" >> $GITHUB_OUTPUT
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
+      wmatrix: ${{ steps.set-matrix.outputs.wmatrix }}
+
+  e2e-ce-without-kubelet-test:
+    runs-on: ubuntu-latest
+    needs: build-matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        testmode: ${{ fromJson(needs.build-matrix.outputs.wmatrix) }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build docker image nightly
+        #        env:
+        #          JFSCHAN: beta
+        run: |
+          make image-nightly
+      - name: Prepare microk8s environment
+        run: |
+          ${GITHUB_WORKSPACE}/.github/scripts/k8s-deps.sh
+          docker image save -o juicefs-csi-driver-nightly.tar juicedata/juicefs-csi-driver:nightly
+          sudo microk8s.ctr image import juicefs-csi-driver-nightly.tar
+          rm -f juicefs-csi-driver-nightly.tar
+      - name: Deploy JuiceFS CSI
+        run: |
+          testmode=${{matrix.testmode}}
+          export dev_tag=nightly
+          .github/scripts/deploy-csi-in-k8s.sh ${testmode} withoutkubelet
+      - name: Run e2e test
+        env:
+          JUICEFS_STORAGE: s3
+          JUICEFS_BUCKET: "http://juicefs-bucket.minio.default.svc.cluster.local:9000"
+          JUICEFS_ACCESS_KEY: "minioadmin"
+          JUICEFS_SECRET_KEY: "minioadmin"
+          JUICEFS_NAME: "ce-secret"
+          JUICEFS_META_URL: "redis://redis.default.svc.cluster.local:6379/1"
+          JUICEFS_MODE: ce
+          TEST_MODE: ${{matrix.testmode}}
+        run: |
+          cd ${GITHUB_WORKSPACE}/.github/scripts/
+          python3 e2e-test.py
+      - name: Setup upterm session
+        if: ${{ failure() }}
+        timeout-minutes: 60
+        uses: lhotari/action-upterm@v1
+
+  e2e-ee-without-kubelet-test:
+    runs-on: ubuntu-latest
+    needs: build-matrix
+    strategy:
+      fail-fast: false
+      matrix:
+        testmode: ${{ fromJson(needs.build-matrix.outputs.wmatrix) }}
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build docker image nightly
+        #        env:
+        #          JFSCHAN: beta
+        run: |
+          make image-nightly
+      - name: Prepare microk8s environment
+        run: |
+          ${GITHUB_WORKSPACE}/.github/scripts/k8s-deps.sh
+          docker image save -o juicefs-csi-driver-nightly.tar juicedata/juicefs-csi-driver:nightly
+          sudo microk8s.ctr image import juicefs-csi-driver-nightly.tar
+          rm -f juicefs-csi-driver-nightly.tar
+      - name: Deploy JuiceFS CSI
+        run: |
+          testmode=${{matrix.testmode}}
+          cd ${GITHUB_WORKSPACE}
+          export dev_tag=nightly
+          .github/scripts/deploy-csi-in-k8s.sh ${testmode} withoutkubelet
+      - name: Run e2e test
+        env:
+          JUICEFS_TOKEN: ${{ secrets.JUICEFS_CI_VOLUME_TOKEN }}
+          JUICEFS_STORAGE: s3
+          JUICEFS_BUCKET: "http://juicefs-bucket.minio.default.svc.cluster.local:9000"
+          JUICEFS_ACCESS_KEY: "minioadmin"
+          JUICEFS_SECRET_KEY: "minioadmin"
+          JUICEFS_NAME: "csi-ci"
+          JUICEFS_META_URL: ""
+          JUICEFS_MODE: "ee"
+          TEST_MODE: ${{matrix.testmode}}
+        run: |
+          cd ${GITHUB_WORKSPACE}/.github/scripts/
+          python3 e2e-test.py
+      - name: Setup upterm session
+        if: ${{ failure() }}
+        timeout-minutes: 60
+        uses: lhotari/action-upterm@v1
 
   e2e-ce-test:
     runs-on: ubuntu-latest

diff --git a/Makefile b/Makefile
@@ -184,7 +184,7 @@ csi-slim-image-version:
 .PHONY: ce-image
 ce-image:
 	docker build -f docker/ce.juicefs.Dockerfile -t $(REGISTRY)/$(JUICEFS_IMAGE):$(CE_VERSION) \
-        --build-arg JUICEFS_REPO_REF=$(CE_JUICEFS_VERSION) .
+        --build-arg JUICEFS_REPO_REF=$(CE_JUICEFS_VERSION) --build-arg TARGETARCH=$(TARGETARCH) .
 	docker push $(REGISTRY)/$(JUICEFS_IMAGE):$(CE_VERSION)
 
 .PHONY: ce-image-buildx
@@ -196,7 +196,7 @@ ce-image-buildx:
 .PHONY: ee-image
 ee-image:
 	docker build -f docker/ee.juicefs.Dockerfile -t $(REGISTRY)/$(JUICEFS_IMAGE):$(EE_VERSION) \
-        --build-arg=JFS_AUTO_UPGRADE=disabled --build-arg JFSCHAN=$(JFS_CHAN) .
+        --build-arg=JFS_AUTO_UPGRADE=disabled --build-arg JFSCHAN=$(JFS_CHAN) --build-arg TARGETARCH=$(TARGETARCH) .
 	docker push $(REGISTRY)/$(JUICEFS_IMAGE):$(EE_VERSION)
 
 .PHONY: ee-image-buildx

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -12,26 +12,31 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.18-buster as builder
+FROM golang:1.19-buster as builder
 
 ARG GOPROXY
+ARG TARGETARCH
 ARG JUICEFS_REPO_BRANCH=main
 ARG JUICEFS_REPO_REF=${JUICEFS_REPO_BRANCH}
 ARG JUICEFS_CSI_REPO_REF=master
 
 RUN apt update && apt install -y software-properties-common && apt update && \
     wget -q -O- 'https://download.ceph.com/keys/release.asc' | apt-key add - && \
     apt-add-repository 'deb https://download.ceph.com/debian-pacific/ buster main' && \
+    wget -O - https://download.gluster.org/pub/gluster/glusterfs/7/rsa.pub | apt-key add - && \
+    echo deb [arch=${TARGETARCH}] https://download.gluster.org/pub/gluster/glusterfs/7/LATEST/Debian/buster/${TARGETARCH}/apt buster main > /etc/apt/sources.list.d/gluster.list && \
     apt update
 
 WORKDIR /workspace
 ENV GOPROXY=${GOPROXY:-https://proxy.golang.org}
-RUN apt-get update && apt-get install -y musl-tools upx-ucl librados-dev libcephfs-dev librbd-dev && \
+RUN apt-get update && apt-get install -y musl-tools upx-ucl librados-dev libcephfs-dev librbd-dev uuid-dev libglusterfs-dev glusterfs-common && mkdir -p /home/travis/.m2 && \
+    wget -O /home/travis/.m2/foundationdb-clients_6.3.23-1_${TARGETARCH}.deb https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-clients_6.3.23-1_${TARGETARCH}.deb && \
+    dpkg -i /home/travis/.m2/foundationdb-clients_6.3.23-1_${TARGETARCH}.deb && \
     git clone https://github.com/juicedata/juicefs-csi-driver && \
     cd juicefs-csi-driver && git checkout $JUICEFS_CSI_REPO_REF && make && \
     cd /workspace && git clone --branch=$JUICEFS_REPO_BRANCH https://github.com/juicedata/juicefs && \
     cd juicefs && git checkout $JUICEFS_REPO_REF && go get github.com/ceph/[email protected] && go mod tidy && \
-    make juicefs.ceph && mv juicefs.ceph juicefs
+    make juicefs.all && mv juicefs.all juicefs
 
 FROM python:3.8-slim-buster
 
@@ -54,9 +59,13 @@ RUN chmod +x /tini
 RUN apt update && apt install -y software-properties-common wget gnupg gnupg2 && apt update && \
     wget -q -O- 'https://download.ceph.com/keys/release.asc' | apt-key add - && \
     apt-add-repository 'deb https://download.ceph.com/debian-pacific/ buster main' && \
+    wget -O - https://download.gluster.org/pub/gluster/glusterfs/7/rsa.pub | apt-key add - && \
+    echo deb [arch=${TARGETARCH}] https://download.gluster.org/pub/gluster/glusterfs/7/LATEST/Debian/buster/${TARGETARCH}/apt buster main > /etc/apt/sources.list.d/gluster.list && \
     apt update
 
-RUN apt-get update && apt-get install -y librados2 librados-dev libcephfs-dev librbd-dev curl fuse procps iputils-ping strace iproute2 net-tools tcpdump lsof && \
+RUN apt-get update && apt-get install -y librados2 librados-dev libcephfs-dev librbd-dev curl fuse procps iputils-ping strace iproute2 net-tools tcpdump lsof uuid-dev libglusterfs-dev glusterfs-common && \
+    mkdir -p /home/travis/.m2 && wget -O /home/travis/.m2/foundationdb-clients_6.3.23-1_${TARGETARCH}.deb https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-clients_6.3.23-1_${TARGETARCH}.deb && \
+    dpkg -i /home/travis/.m2/foundationdb-clients_6.3.23-1_${TARGETARCH}.deb && \
     rm -rf /var/cache/apt/* && \
     curl -sSL https://juicefs.com/static/juicefs -o ${JUICEFS_CLI} && chmod +x ${JUICEFS_CLI} && \
     mkdir -p /root/.juicefs && \

diff --git a/docker/ce.juicefs.Dockerfile b/docker/ce.juicefs.Dockerfile
@@ -12,18 +12,24 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM golang:1.18-buster
+FROM golang:1.19-buster
 
 ARG GOPROXY
+ARG TARGETARCH
 ARG JUICEFS_REPO_URL=https://github.com/juicedata/juicefs
 ARG JUICEFS_REPO_BRANCH=main
 ARG JUICEFS_REPO_REF=${JUICEFS_REPO_BRANCH}
 
 WORKDIR /workspace
 ENV GOPROXY=${GOPROXY:-https://proxy.golang.org}
-RUN apt-get update && apt-get install -y musl-tools upx-ucl librados-dev libcephfs-dev librbd-dev && \
+RUN mkdir -p /home/travis/.m2 && \
+    wget -O /home/travis/.m2/foundationdb-clients_6.3.23-1_${TARGETARCH}.deb https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-clients_6.3.23-1_${TARGETARCH}.deb && \
+    dpkg -i /home/travis/.m2/foundationdb-clients_6.3.23-1_${TARGETARCH}.deb && \
+    wget -O - https://download.gluster.org/pub/gluster/glusterfs/7/rsa.pub | apt-key add - && \
+    echo deb [arch=${TARGETARCH}] https://download.gluster.org/pub/gluster/glusterfs/7/LATEST/Debian/buster/${TARGETARCH}/apt buster main > /etc/apt/sources.list.d/gluster.list && \
+    apt-get update && apt-get install -y musl-tools upx-ucl librados-dev libcephfs-dev librbd-dev uuid-dev libglusterfs-dev glusterfs-common && \
     cd /workspace && git clone --branch=$JUICEFS_REPO_BRANCH $JUICEFS_REPO_URL && \
     cd juicefs && git checkout $JUICEFS_REPO_REF && go get github.com/ceph/[email protected] && go mod tidy && \
-    make juicefs.ceph && mv juicefs.ceph juicefs && mv juicefs /usr/local/bin/juicefs
+    make juicefs.all && mv juicefs.all juicefs && mv juicefs /usr/local/bin/juicefs
 
 RUN ln -s /usr/local/bin/juicefs /bin/mount.juicefs && /usr/local/bin/juicefs --version
diff --git a/pkg/juicefs/juicefs.go b/pkg/juicefs/juicefs.go
@@ -740,19 +740,6 @@ func (j *juicefs) SetQuota(ctx context.Context, secrets map[string]string, jfsSe
 		return "", fmt.Errorf("capacity %d is too small, at least 1GiB for quota", capacity)
 	}
 
-	version, err := j.version(ctx, jfsSetting)
-	if err != nil {
-		return "", err
-	}
-	if jfsSetting.IsCe && version.LessThan(&clientVersion{1, 1, 0, ""}) {
-		klog.Infof("juicefs-ce version %s does not support quota, skipped", version)
-		return "", nil
-	}
-	if !jfsSetting.IsCe && version.LessThan(&clientVersion{4, 9, 2, ""}) {
-		klog.Infof("juicefs-ee version %s does not support quota, skipped", version)
-		return "", nil
-	}
-
 	var args, cmdArgs []string
 	if jfsSetting.IsCe {
 		args = []string{"quota", "set", secrets["metaurl"], "--path", quotaPath, "--capacity", strconv.FormatInt(cap, 10)}
@@ -766,6 +753,7 @@ func (j *juicefs) SetQuota(ctx context.Context, secrets map[string]string, jfsSe
 	defer cmdCancel()
 
 	var res []byte
+	var err error
 	if jfsSetting.IsCe {
 		res, err = j.Exec.CommandContext(cmdCtx, config.CeCliPath, args...).CombinedOutput()
 	} else {