refactor

juicedata · Nov 14, 2024 · 5b5c9fe · 5b5c9fe
1 parent 04f7998
commit 5b5c9fe
Show file tree

Hide file tree

Showing 39 changed files with 163 additions and 266 deletions.
diff --git a/cmd/objbench.go b/cmd/objbench.go
@@ -948,7 +948,7 @@ func functionalTesting(blob object.ObjectStorage, result *[][]string, colorful b
 		if err := blob.Put(key, bytes.NewReader([]byte("1"))); err != nil {
 			return fmt.Errorf("put encode file failed: %s", err)
 		} else {
-			if resp, err := blob.List("", "测试编码文件", "", 1, true); err != nil && err != utils.ENOTSUP {
+			if resp, _, _, err := blob.List("", "测试编码文件", "", "", 1, true); err != nil && err != utils.ENOTSUP {
 				return fmt.Errorf("list encode file failed %s", err)
 			} else if len(resp) == 1 && resp[0].Key() != key {
 				return fmt.Errorf("list encode file failed: expect key %s, but got %s", key, resp[0].Key())

diff --git a/cmd/object.go b/cmd/object.go
@@ -214,9 +214,9 @@ func (j *juiceFS) Head(key string) (object.Object, error) {
 	return &jObj{key, fi}, nil
 }
 
-func (j *juiceFS) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]object.Object, error) {
+func (j *juiceFS) List(prefix, marker, token, delimiter string, limit int64, followLink bool) ([]object.Object, bool, string, error) {
 	if delimiter != "/" {
-		return nil, utils.ENOTSUP
+		return nil, false, "", utils.ENOTSUP
 	}
 	dir := j.path(prefix)
 	var objs []object.Object
@@ -229,18 +229,18 @@ func (j *juiceFS) List(prefix, marker, delimiter string, limit int64, followLink
 		obj, err := j.Head(prefix)
 		if err != nil {
 			if os.IsNotExist(err) {
-				return nil, nil
+				return nil, false, "", nil
 			}
-			return nil, err
+			return nil, false, "", err
 		}
 		objs = append(objs, obj)
 	}
 	entries, err := j.readDirSorted(dir, followLink)
 	if err != 0 {
 		if err == syscall.ENOENT {
-			return nil, nil
+			return nil, false, "", nil
 		}
-		return nil, err
+		return nil, false, "", err
 	}
 	for _, e := range entries {
 		key := dir[1:] + e.name
@@ -253,7 +253,11 @@ func (j *juiceFS) List(prefix, marker, delimiter string, limit int64, followLink
 			break
 		}
 	}
-	return objs, nil
+	var nextMarker string
+	if len(objs) > 0 {
+		nextMarker = objs[len(objs)-1].Key()
+	}
+	return objs, nextMarker != "", nextMarker, nil
 }
 
 type mEntry struct {

diff --git a/pkg/object/azure.go b/pkg/object/azure.go
@@ -39,7 +39,6 @@ import (
 
 type wasb struct {
 	DefaultObjectStorage
-	hasV2
 	container *container.Client
 	azblobCli *azblob.Client
 	sc        string
@@ -136,11 +135,7 @@ func (b *wasb) Delete(key string, getters ...AttrGetter) error {
 	return err
 }
 
-func (b *wasb) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
-	return b.hasV2.List(b, prefix, marker, delimiter, limit, followLink)
-}
-
-func (b *wasb) ListV2(prefix, startAfter, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {
+func (b *wasb) List(prefix, startAfter, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {
 	if delimiter != "" {
 		return nil, false, "", notSupported
 	}

diff --git a/pkg/object/b2.go b/pkg/object/b2.go
@@ -33,7 +33,6 @@ import (
 
 type b2client struct {
 	DefaultObjectStorage
-	hasV2
 	bucket *backblaze.Bucket
 }
 
@@ -122,11 +121,7 @@ func (c *b2client) Delete(key string, getters ...AttrGetter) error {
 	return err
 }
 
-func (c *b2client) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
-	return c.hasV2.List(c, prefix, marker, delimiter, limit, followLink)
-}
-
-func (c *b2client) ListV2(prefix, startAfter, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {
+func (c *b2client) List(prefix, startAfter, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {
 	if limit > 1000 {
 		limit = 1000
 	}

diff --git a/pkg/object/bos.go b/pkg/object/bos.go
@@ -147,11 +147,7 @@ func (q *bosclient) Delete(key string, getters ...AttrGetter) error {
 	return err
 }
 
-func (q *bosclient) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
-	return retryListV2(q, prefix, marker, delimiter, limit, followLink)
-}
-
-func (q *bosclient) ListV2(prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {
+func (q *bosclient) List(prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {
 	if limit > 1000 {
 		limit = 1000
 	}

diff --git a/pkg/object/bunny.go b/pkg/object/bunny.go
@@ -77,7 +77,7 @@ func (b *bunnyClient) Delete(key string, getters ...AttrGetter) error {
 	return err
 }
 
-func (b *bunnyClient) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
+func (b *bunnyClient) List(prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {
 	if delimiter != "/" {
 		return nil, notSupported
 	}

diff --git a/pkg/object/cos.go b/pkg/object/cos.go
@@ -166,11 +166,7 @@ func (c *COS) Delete(key string, getters ...AttrGetter) error {
 	return err
 }
 
-func (c *COS) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
-	return retryListV2(c, prefix, marker, delimiter, limit, followLink)
-}
-
-func (c *COS) ListV2(prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {
+func (c *COS) List(prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {
 	param := cos.BucketGetOptions{
 		Prefix:       prefix,
 		Marker:       start,

diff --git a/pkg/object/dragonfly.go b/pkg/object/dragonfly.go
@@ -177,7 +177,7 @@ func (d *dragonfly) String() string {
 
 // Create creates the object if it does not exist.
 func (d *dragonfly) Create() error {
-	if _, err := d.List("", "", "", 1, false); err == nil {
+	if _, _, _, err := d.List("", "", "", "", 1, false); err == nil {
 		return nil
 	}
 
@@ -438,14 +438,14 @@ func (d *dragonfly) Delete(key string, getters ...AttrGetter) error {
 }
 
 // List lists the objects with the given prefix.
-func (d *dragonfly) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
+func (d *dragonfly) List(prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {
 	if limit > MaxGetObjectMetadatasLimit {
 		limit = MaxGetObjectMetadatasLimit
 	}
 
 	u, err := url.Parse(d.endpoint)
 	if err != nil {
-		return nil, err
+		return nil, false, "", err
 	}
 
 	u.Path = path.Join("buckets", d.bucket, "metadatas")
@@ -469,23 +469,23 @@ func (d *dragonfly) List(prefix, marker, delimiter string, limit int64, followLi
 	u.RawQuery = query.Encode()
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
 	if err != nil {
-		return nil, err
+		return nil, false, "", err
 	}
 
 	// List object.
 	resp, err := d.client.Do(req)
 	if err != nil {
-		return nil, err
+		return nil, false, "", err
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode/100 != 2 {
-		return nil, fmt.Errorf("bad response status %s", resp.Status)
+		return nil, false, "", fmt.Errorf("bad response status %s", resp.Status)
 	}
 
 	var objectMetadatas ObjectMetadatas
 	if err := json.NewDecoder(resp.Body).Decode(&objectMetadatas); err != nil {
-		return nil, err
+		return nil, false, "", err
 	}
 
 	objs := make([]Object, 0, len(objectMetadatas.Metadatas))
@@ -506,7 +506,8 @@ func (d *dragonfly) List(prefix, marker, delimiter string, limit int64, followLi
 		sort.Slice(objs, func(i, j int) bool { return objs[i].Key() < objs[j].Key() })
 	}
 
-	return objs, err
+	hasMore, nextMarker := generateListResult(objs)
+	return objs, hasMore, nextMarker, err
 }
 
 // newDragonfly creates a new dragonfly object storage.

diff --git a/pkg/object/etcd.go b/pkg/object/etcd.go
@@ -111,11 +111,7 @@ func genNextKey(key string) string {
 	return string(next)
 }
 
-func (c *etcdClient) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
-	return retryListV2(c, prefix, marker, delimiter, limit, followLink)
-}
-
-func (c *etcdClient) ListV2(prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {
+func (c *etcdClient) List(prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {
 	if delimiter != "" {
 		return nil, false, "", notSupported
 	}

diff --git a/pkg/object/file.go b/pkg/object/file.go
@@ -268,9 +268,9 @@ func readDirSorted(dir string, followLink bool) ([]*mEntry, error) {
 	return mEntries, err
 }
 
-func (d *filestore) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
+func (d *filestore) List(prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {
 	if delimiter != "/" {
-		return nil, notSupported
+		return nil, false, "", notSupported
 	}
 	var dir string = d.root + prefix
 	var objs []Object
@@ -283,22 +283,22 @@ func (d *filestore) List(prefix, marker, delimiter string, limit int64, followLi
 		obj, err := d.Head(prefix)
 		if err != nil {
 			if os.IsNotExist(err) {
-				return nil, nil
+				return nil, false, "", nil
 			}
-			return nil, err
+			return nil, false, "", err
 		}
 		objs = append(objs, obj)
 	}
 	entries, err := readDirSorted(dir, followLink)
 	if err != nil {
 		if os.IsPermission(err) {
 			logger.Warnf("skip %s: %s", dir, err)
-			return nil, nil
+			return nil, false, "", nil
 		}
 		if os.IsNotExist(err) {
-			return nil, nil
+			return nil, false, "", nil
 		}
-		return nil, err
+		return nil, false, "", err
 	}
 	for _, e := range entries {
 		p := path.Join(dir, e.Name())
@@ -319,7 +319,8 @@ func (d *filestore) List(prefix, marker, delimiter string, limit int64, followLi
 			break
 		}
 	}
-	return objs, nil
+	hasMore, nextMarker := generateListResult(objs)
+	return objs, hasMore, nextMarker, err
 }
 
 func (d *filestore) Chmod(key string, mode os.FileMode) error {

diff --git a/pkg/object/gluster.go b/pkg/object/gluster.go
@@ -198,9 +198,9 @@ func (g *gluster) readDirSorted(dirname string, followLink bool) ([]*mEntry, err
 	return mEntries, err
 }
 
-func (g *gluster) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
+func (g *gluster) List(prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {
 	if delimiter != "/" {
-		return nil, notSupported
+		return nil, false, "", notSupported
 	}
 	var dir string = prefix
 	var objs []Object
@@ -213,22 +213,22 @@ func (g *gluster) List(prefix, marker, delimiter string, limit int64, followLink
 		obj, err := g.Head(prefix)
 		if err != nil {
 			if os.IsNotExist(err) {
-				return nil, nil
+				return nil, false, "", nil
 			}
-			return nil, err
+			return nil, false, "", err
 		}
 		objs = append(objs, obj)
 	}
 	entries, err := g.readDirSorted(dir, followLink)
 	if err != nil {
 		if os.IsPermission(err) {
 			logger.Warnf("skip %s: %s", dir, err)
-			return nil, nil
+			return nil, false, "", nil
 		}
 		if os.IsNotExist(err) {
-			return nil, nil
+			return nil, false, "", nil
 		}
-		return nil, err
+		return nil, false, "", err
 	}
 	for _, e := range entries {
 		p := filepath.Join(dir, e.Name())
@@ -246,7 +246,8 @@ func (g *gluster) List(prefix, marker, delimiter string, limit int64, followLink
 			break
 		}
 	}
-	return objs, nil
+	hasMore, nextMarker := generateListResult(objs)
+	return objs, hasMore, nextMarker, nil
 }
 
 func (g *gluster) Chtimes(path string, mtime time.Time) error {

diff --git a/pkg/object/gs.go b/pkg/object/gs.go
@@ -40,7 +40,6 @@ import (
 
 type gs struct {
 	DefaultObjectStorage
-	hasV2
 	clients []*storage.Client
 	index   uint64
 	bucket  string
@@ -62,10 +61,9 @@ func (g *gs) getClient() *storage.Client {
 
 func (g *gs) Create() error {
 	// check if the bucket is already exists
-	if objs, err := g.List("", "", "", 1, true); err == nil && len(objs) > 0 {
+	if objs, _, _, err := g.List("", "", "", "", 1, true); err == nil && len(objs) > 0 {
 		return nil
 	}
-
 	projectID := os.Getenv("GOOGLE_CLOUD_PROJECT")
 	if projectID == "" {
 		projectID, _ = metadata.ProjectID()
@@ -166,11 +164,7 @@ func (g *gs) Delete(key string, getters ...AttrGetter) error {
 	return nil
 }
 
-func (g *gs) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
-	return g.hasV2.List(g, prefix, marker, delimiter, limit, followLink)
-}
-
-func (g *gs) ListV2(prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {
+func (g *gs) List(prefix, start, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {
 	objectIterator := g.getClient().Bucket(g.bucket).Objects(ctx, &storage.Query{Prefix: prefix, Delimiter: delimiter, StartOffset: start})
 	pager := iterator.NewPager(objectIterator, int(limit), token)
 	var entries []*storage.ObjectAttrs

diff --git a/pkg/object/hdfs.go b/pkg/object/hdfs.go
@@ -186,9 +186,9 @@ func (h *hdfsclient) Delete(key string, getters ...AttrGetter) error {
 	return err
 }
 
-func (h *hdfsclient) List(prefix, marker, delimiter string, limit int64, followLink bool) ([]Object, error) {
+func (h *hdfsclient) List(prefix, marker, token, delimiter string, limit int64, followLink bool) ([]Object, bool, string, error) {
 	if delimiter != "/" {
-		return nil, notSupported
+		return nil, false, "", notSupported
 	}
 	dir := h.path(prefix)
 	var objs []Object
@@ -201,9 +201,9 @@ func (h *hdfsclient) List(prefix, marker, delimiter string, limit int64, followL
 		obj, err := h.Head(prefix)
 		if err != nil {
 			if os.IsNotExist(err) {
-				return nil, nil
+				return nil, false, "", nil
 			}
-			return nil, err
+			return nil, false, "", err
 		}
 		objs = append(objs, obj)
 	}
@@ -216,12 +216,12 @@ func (h *hdfsclient) List(prefix, marker, delimiter string, limit int64, followL
 	if err != nil {
 		if os.IsPermission(err) {
 			logger.Warnf("skip %s: %s", dir, err)
-			return nil, nil
+			return nil, false, "", nil
 		}
 		if os.IsNotExist(err) {
-			return nil, nil
+			return nil, false, "", nil
 		}
-		return nil, err
+		return nil, false, "", err
 	}
 
 	// make sure they are ordered in full path
@@ -252,7 +252,8 @@ func (h *hdfsclient) List(prefix, marker, delimiter string, limit int64, followL
 			break
 		}
 	}
-	return objs, nil
+	hasMore, nextMarker := generateListResult(objs)
+	return objs, hasMore, nextMarker, nil
 }
 
 func (h *hdfsclient) Chtimes(key string, mtime time.Time) error {