cmd/dump&load: support zstd compression, and adjust the compression l…

…evel (#4921)

cmd/dump.go

@@ -18,10 +18,12 @@ package cmd
 
 import (
 	"compress/gzip"
+	"errors"
 	"io"
 	"os"
 	"strings"
 
+	"github.com/DataDog/zstd"
 	"github.com/juicedata/juicefs/pkg/meta"
 	"github.com/urfave/cli/v2"
 )
@@ -78,15 +80,12 @@ func dumpMeta(m meta.Meta, dst string, threads int, keepSecret, fast, skipTrash
 		w = os.Stdout
 	} else {
 		tmp := dst + ".tmp"
-		fp, e := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+		fp, e := os.Create(tmp)
 		if e != nil {
 			return e
 		}
 		defer func() {
-			e := fp.Close()
-			if err == nil {
-				err = e
-			}
+			err = errors.Join(err, fp.Close())
 			if err == nil {
 				err = os.Rename(tmp, dst)
 			} else {
@@ -95,14 +94,15 @@ func dumpMeta(m meta.Meta, dst string, threads int, keepSecret, fast, skipTrash
 		}()
 
 		if strings.HasSuffix(dst, ".gz") {
-			zw := gzip.NewWriter(fp)
+			w, _ = gzip.NewWriterLevel(fp, gzip.BestSpeed)
+			defer func() {
+				err = errors.Join(err, w.Close())
+			}()
+		} else if strings.HasSuffix(dst, ".zstd") {
+			w = zstd.NewWriterLevel(fp, zstd.BestSpeed)
 			defer func() {
-				e := zw.Close()
-				if err == nil {
-					err = e
-				}
+				err = errors.Join(err, w.Close())
 			}()
-			w = zw
 		} else {
 			w = fp
 		}

cmd/load.go

@@ -26,6 +26,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/DataDog/zstd"
 	"github.com/juicedata/juicefs/pkg/object"
 
 	"github.com/juicedata/juicefs/pkg/meta"
@@ -122,6 +123,9 @@ func load(ctx *cli.Context) error {
 				return err
 			}
 			defer r.Close()
+		} else if strings.HasSuffix(src, ".zstd") {
+			r = zstd.NewReader(fp)
+			defer r.Close()
 		} else {
 			r = fp
 		}

pkg/vfs/backup.go

@@ -98,7 +98,7 @@ func backup(m meta.Meta, blob object.ObjectStorage, now time.Time, fast, skipTra
 	}
 	defer os.Remove(fp.Name())
 	defer fp.Close()
-	zw := gzip.NewWriter(fp)
+	zw, _ := gzip.NewWriterLevel(fp, gzip.BestSpeed)
 	var threads = 2
 	if m.Name() == "tikv" {
 		threads = 10