Backports 1.1.3

.github/scripts/command/sync_minio.sh

@@ -79,11 +79,12 @@ test_sync_external_link(){
     [ -z $(./mc cat myminio/myjfs/hello) ]
 }
 
+# list object should be skipped when encountering a loop symlink
 test_sync_loop_symlink(){
     prepare_test
     touch hello
     ln -s hello /jfs/hello
-    ./juicefs sync minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/ && exit 1 || true
+    ./juicefs sync minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/
     rm -rf /jfs/hello
     ./juicefs sync minio://minioadmin:minioadmin@localhost:9005/myjfs/ minio://minioadmin:minioadmin@localhost:9000/myjfs/
 }
@@ -107,6 +108,18 @@ test_sync_deep_symlink(){
     done
 }
 
+test_sync_list_object_symlink(){
+    prepare_test
+    cd /jfs
+    mkdir dir1
+    mkdir -p dir2/src_dir
+    echo abc > dir2/src_dir/afile
+    ln -s ./../dir2/src_dir dir1/symlink_dir
+    cd -
+    ./juicefs sync minio://minioadmin:minioadmin@localhost:9005/myjfs/dir1/ minio://minioadmin:minioadmin@localhost:9000/myjfs/dir3/
+    ./mc cat myminio/myjfs/dir3/symlink_dir/afile | grep abc || (echo "content should be abc" && exit 1)
+}
+
 prepare_test(){
     umount_jfs /jfs $META_URL
     python3 .github/scripts/flush_meta.py $META_URL

cmd/object.go

@@ -96,7 +96,7 @@ func (f *jFile) Close() error {
 }
 
 func (j *juiceFS) Get(key string, off, limit int64) (io.ReadCloser, error) {
-	f, err := j.jfs.Open(ctx, j.path(key), 0)
+	f, err := j.jfs.Open(ctx, j.path(key), vfs.MODE_MASK_R)
 	if err != 0 {
 		return nil, err
 	}

go.mod

@@ -67,12 +67,12 @@ require (
 	go.etcd.io/etcd/client/v3 v3.5.9
 	go.uber.org/automaxprocs v1.5.2
 	go.uber.org/zap v1.20.0
-	golang.org/x/crypto v0.17.0
-	golang.org/x/net v0.17.0
+	golang.org/x/crypto v0.21.0
+	golang.org/x/net v0.23.0
 	golang.org/x/oauth2 v0.7.0
 	golang.org/x/sync v0.2.0
-	golang.org/x/sys v0.15.0
-	golang.org/x/term v0.15.0
+	golang.org/x/sys v0.18.0
+	golang.org/x/term v0.18.0
 	golang.org/x/text v0.14.0
 	google.golang.org/api v0.94.0
 	google.golang.org/protobuf v1.30.0

go.sum

@@ -1123,8 +1123,8 @@ golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20220112180741-5e0467b6c7ce/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
 golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
-golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
-golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1225,8 +1225,8 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
+golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
+golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1369,15 +1369,15 @@ golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4=
-golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
+golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8=
+golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

pkg/fs/fs.go

@@ -575,7 +575,7 @@ func (fs *FileSystem) CopyFileRange(ctx meta.Context, src string, soff uint64, d
 	if err != 0 {
 		return
 	}
-	err = fs.m.CopyFileRange(ctx, sfi.inode, soff, dfi.inode, doff, size, 0, &written)
+	err = fs.m.CopyFileRange(ctx, sfi.inode, soff, dfi.inode, doff, size, 0, &written, nil)
 	return
 }
 

pkg/fuse/fuse.go

@@ -354,11 +354,12 @@ func (fs *fileSystem) ReadDir(cancel <-chan struct{}, in *fuse.ReadIn, out *fuse
 	defer releaseContext(ctx)
 	entries, _, err := fs.v.Readdir(ctx, Ino(in.NodeId), in.Size, int(in.Offset), in.Fh, false)
 	var de fuse.DirEntry
-	for _, e := range entries {
+	for i, e := range entries {
 		de.Ino = uint64(e.Inode)
 		de.Name = string(e.Name)
 		de.Mode = e.Attr.SMode()
 		if !out.AddDirEntry(de) {
+			fs.v.UpdateReaddirOffset(ctx, Ino(in.NodeId), in.Fh, int(in.Offset)+i)
 			break
 		}
 	}
@@ -371,12 +372,13 @@ func (fs *fileSystem) ReadDirPlus(cancel <-chan struct{}, in *fuse.ReadIn, out *
 	entries, readAt, err := fs.v.Readdir(ctx, Ino(in.NodeId), in.Size, int(in.Offset), in.Fh, true)
 	ctx.start = readAt
 	var de fuse.DirEntry
-	for _, e := range entries {
+	for i, e := range entries {
 		de.Ino = uint64(e.Inode)
 		de.Name = string(e.Name)
 		de.Mode = e.Attr.SMode()
 		eo := out.AddDirLookupEntry(de)
 		if eo == nil {
+			fs.v.UpdateReaddirOffset(ctx, Ino(in.NodeId), in.Fh, int(in.Offset)+i)
 			break
 		}
 		if e.Attr.Full {

pkg/gateway/gateway.go

@@ -157,7 +157,7 @@ func (n *jfsObjects) isValidBucketName(bucket string) error {
 }
 
 func (n *jfsObjects) path(p ...string) string {
-	if len(p) > 0 && p[0] == n.conf.Format.Name {
+	if !n.gConf.MultiBucket && len(p) > 0 && p[0] == n.conf.Format.Name {
 		p = p[1:]
 	}
 	return sep + minio.PathJoin(p...)
@@ -303,6 +303,14 @@ func (n *jfsObjects) listDirFactory() minio.ListDirFunc {
 			if root && len(fi.Name()) == len(metaBucket) && fi.Name() == metaBucket {
 				continue
 			}
+			if stat, ok := fi.(*fs.FileStat); ok && stat.IsSymlink() {
+				var err syscall.Errno
+				p := n.path(bucket, prefixDir, fi.Name())
+				if fi, err = n.fs.Stat(mctx, p); err != 0 {
+					logger.Errorf("stat %s: %s", p, err)
+					continue
+				}
+			}
 			if fi.IsDir() {
 				entries = append(entries, fi.Name()+sep)
 			} else {
@@ -468,7 +476,7 @@ func (n *jfsObjects) GetObjectNInfo(ctx context.Context, bucket, object string,
 	if err != nil {
 		return
 	}
-	f, eno := n.fs.Open(mctx, n.path(bucket, object), 0)
+	f, eno := n.fs.Open(mctx, n.path(bucket, object), vfs.MODE_MASK_R)
 	if eno != 0 {
 		return nil, jfsToObjectErr(ctx, eno, bucket, object)
 	}
@@ -601,8 +609,15 @@ func (n *jfsObjects) GetObjectInfo(ctx context.Context, bucket, object string, o
 		err = jfsToObjectErr(ctx, eno, bucket, object)
 		return
 	}
-	// put /dir1/key1; head /dir1 return 404; head /dir1/ return 200
-	if strings.HasSuffix(object, sep) && !fi.IsDir() || !strings.HasSuffix(object, sep) && fi.IsDir() {
+	// put /dir1/key1; head /dir1 return 404; head /dir1/ return 404; head /dir1/key1 return 200
+	// put /dir1/key1/; head /dir1/key1 return 404; head /dir1/key1/ return 200
+	var isObject bool
+	if strings.HasSuffix(object, sep) && fi.IsDir() && fi.Atime() == 0 {
+		isObject = true
+	} else if !strings.HasSuffix(object, sep) && !fi.IsDir() {
+		isObject = true
+	}
+	if !isObject {
 		err = jfsToObjectErr(ctx, syscall.ENOENT, bucket, object)
 		return
 	}
@@ -925,7 +940,7 @@ func (n *jfsObjects) CompleteMultipartUpload(ctx context.Context, bucket, object
 	var total uint64
 	for _, part := range parts {
 		p := n.ppath(bucket, uploadID, strconv.Itoa(part.PartNumber))
-		copied, eno := n.fs.CopyFileRange(mctx, p, 0, tmp, total, 1<<30)
+		copied, eno := n.fs.CopyFileRange(mctx, p, 0, tmp, total, 5<<30)
 		if eno != 0 {
 			err = jfsToObjectErr(ctx, eno, bucket, object, uploadID)
 			logger.Errorf("merge parts: %s", err)

pkg/meta/base.go

@@ -708,8 +708,9 @@ func (m *baseMeta) getQuotaParent(ctx Context, inode Ino) Ino {
 		if inode <= RootInode {
 			break
 		}
+		lastInode := inode
 		if inode, st = m.getDirParent(ctx, inode); st != 0 {
-			logger.Warnf("Get directory parent of inode %d: %s", inode, st)
+			logger.Warnf("Get directory parent of inode %d: %s", lastInode, st)
 			break
 		}
 	}
@@ -732,8 +733,9 @@ func (m *baseMeta) checkDirQuota(ctx Context, inode Ino, space, inodes int64) bo
 		if inode <= RootInode {
 			break
 		}
+		lastInode := inode
 		if inode, st = m.getDirParent(ctx, inode); st != 0 {
-			logger.Warnf("Get directory parent of inode %d: %s", inode, st)
+			logger.Warnf("Get directory parent of inode %d: %s", lastInode, st)
 			break
 		}
 	}
@@ -756,8 +758,9 @@ func (m *baseMeta) updateDirQuota(ctx Context, inode Ino, space, inodes int64) {
 		if inode <= RootInode {
 			break
 		}
+		lastInode := inode
 		if inode, st = m.getDirParent(ctx, inode); st != 0 {
-			logger.Warnf("Get directory parent of inode %d: %s", inode, st)
+			logger.Warnf("Get directory parent of inode %d: %s", lastInode, st)
 			break
 		}
 	}
@@ -986,25 +989,20 @@ func (m *baseMeta) StatFS(ctx Context, ino Ino, totalspace, availspace, iused, i
 		if q == nil {
 			continue
 		}
+		q.sanitize()
 		if usage == nil {
 			usage = q
 		}
 		if q.MaxSpace > 0 {
-			ls := q.MaxSpace - q.UsedSpace
-			if ls < 0 {
-				ls = 0
-			}
-			if uint64(ls) < *availspace {
-				*availspace = uint64(ls)
+			ls := uint64(q.MaxSpace - q.UsedSpace)
+			if ls < *availspace {
+				*availspace = ls
 			}
 		}
 		if q.MaxInodes > 0 {
-			li := q.MaxInodes - q.UsedInodes
-			if li < 0 {
-				li = 0
-			}
-			if uint64(li) < *iavail {
-				*iavail = uint64(li)
+			li := uint64(q.MaxInodes - q.UsedInodes)
+			if li < *iavail {
+				*iavail = li
 			}
 		}
 	}
@@ -1402,6 +1400,9 @@ func (m *baseMeta) Link(ctx Context, inode, parent Ino, name string, attr *Attr)
 	if name == "" {
 		return syscall.ENOENT
 	}
+	if name == "." || name == ".." {
+		return syscall.EEXIST
+	}
 
 	defer m.timeit("Link", time.Now())
 	if attr == nil {
@@ -1647,22 +1648,23 @@ func (m *baseMeta) touchAtime(ctx Context, inode Ino, attr *Attr) {
 	}
 }
 
-func (m *baseMeta) Open(ctx Context, inode Ino, flags uint32, attr *Attr) (rerr syscall.Errno) {
+func (m *baseMeta) Open(ctx Context, inode Ino, flags uint32, attr *Attr) (st syscall.Errno) {
 	if m.conf.ReadOnly && flags&(syscall.O_WRONLY|syscall.O_RDWR|syscall.O_TRUNC|syscall.O_APPEND) != 0 {
 		return syscall.EROFS
 	}
 	defer func() {
-		if rerr == 0 {
+		if st == 0 {
 			m.touchAtime(ctx, inode, attr)
 		}
 	}()
 	if m.conf.OpenCache > 0 && m.of.OpenCheck(inode, attr) {
 		return 0
 	}
-	var err syscall.Errno
 	// attr may be valid, see fs.Open()
 	if attr != nil && !attr.Full {
-		err = m.GetAttr(ctx, inode, attr)
+		if st = m.GetAttr(ctx, inode, attr); st != 0 {
+			return
+		}
 	}
 	var mmask uint8 = 0
 	switch flags & (syscall.O_RDONLY | syscall.O_WRONLY | syscall.O_RDWR) {
@@ -1673,7 +1675,7 @@ func (m *baseMeta) Open(ctx Context, inode Ino, flags uint32, attr *Attr) (rerr
 	case syscall.O_RDWR:
 		mmask = MODE_MASK_R | MODE_MASK_W
 	}
-	if rerr = m.Access(ctx, inode, mmask, attr); rerr != 0 {
+	if st = m.Access(ctx, inode, mmask, attr); st != 0 {
 		return
 	}
 
@@ -1690,10 +1692,8 @@ func (m *baseMeta) Open(ctx Context, inode Ino, flags uint32, attr *Attr) (rerr
 			return syscall.EPERM
 		}
 	}
-	if err == 0 {
-		m.of.Open(inode, attr)
-	}
-	return err
+	m.of.Open(inode, attr)
+	return 0
 }
 
 func (m *baseMeta) InvalidateChunkCache(ctx Context, inode Ino, indx uint32) syscall.Errno {