From 4c1d76091f9c3ea9c9813d3d161479cb61d0471f Mon Sep 17 00:00:00 2001 From: jiefenghuang Date: Thu, 31 Oct 2024 10:28:25 +0800 Subject: [PATCH] sql backup Signed-off-by: jiefenghuang --- pkg/meta/dump_v2.go | 313 ++++++ pkg/meta/interface.go | 3 + pkg/meta/load_dump_test.go | 93 +- pkg/meta/pb/backup.pb.go | 2098 ++++++++++++++++++++++++++++++++++++ pkg/meta/pb/backup.proto | 190 ++++ pkg/meta/redis.go | 74 ++ pkg/meta/sql.go | 66 +- pkg/meta/sql_bak.go | 1313 ++++++++++++++++++++++ pkg/meta/tkv.go | 8 + 9 files changed, 4105 insertions(+), 53 deletions(-) create mode 100644 pkg/meta/dump_v2.go create mode 100644 pkg/meta/pb/backup.pb.go create mode 100644 pkg/meta/pb/backup.proto create mode 100644 pkg/meta/sql_bak.go diff --git a/pkg/meta/dump_v2.go b/pkg/meta/dump_v2.go new file mode 100644 index 000000000000..3193be9ad4ea --- /dev/null +++ b/pkg/meta/dump_v2.go @@ -0,0 +1,313 @@ +package meta + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "sync" + + "github.com/juicedata/juicefs/pkg/meta/pb" + "google.golang.org/protobuf/proto" +) + +const ( + BakMagic = 0x747083 + BakVersion = 1 + BakHeaderSize = 4096 +) + +type BakFormat struct { + Header BakHeader + // Format pb.Format + // Counters pb.Counters + // Sustaineds pb.Sustaineds + // DelFiles pb.DelFiles + // Acls pb.Acls + // repeated Header.NodeBatchNum times: size(pb.NodeBatch), pb.NodeBatch, ... + // repeated Header.ChunkBatchNum times: size(pb.ChunkBatch), pb.ChunkBatch, ... + // repeated Header.EdgeBatchNum times: size(pb.EdgeBatch), pb.EdgeBatch, ... + // repeated Header.SliceRefBatchNum times: size(pb.SliceRefBatch), pb.SliceRefBatch, ... + // repeated Header.SymlinkBatchNum times: size(pb.SymlinkBatch), pb.SymlinkBatch, ... + // repeated Header.XattrBatchNum times: size(pb.XattrBatch), pb.XattrBatch, ... + // repeated Header.QuotaBatchNum times: size(pb.QuotaBatch), pb.QuotaBatch, ... + // repeated Header.StatBatchNum times: size(pb.StatBatch), pb.StatBatch, ... +} + +func newBakFormat() *BakFormat { + return &BakFormat{ + Header: BakHeader{ + Magic: BakMagic, + Version: BakVersion, + }, + } +} + +func (f *BakFormat) seekForWrite(w io.Seeker) { + w.Seek(BakHeaderSize, io.SeekStart) +} + +func (f *BakFormat) writeData(w io.Writer, name string, data []byte) (int, error) { + n, err := w.Write(data) + if err != nil && n != len(data) { + return n, fmt.Errorf("write %s failed: err %v, write len %d, expect len %d", name, err, n, len(data)) + } + return n, nil +} + +func (f *BakFormat) readData(r io.Reader, name string, size int) ([]byte, error) { + data := make([]byte, size) + n, err := r.Read(data) + if err != nil && n != size { + return nil, fmt.Errorf("read %s failed: err %v, read len %d, expect len %d", name, err, n, size) + } + return data, nil +} + +func (f *BakFormat) writeSeg(w io.Writer, m proto.Message) error { + if m == nil { + return nil + } + data, err := proto.Marshal(m) + if err != nil { + return err + } + n, err := f.writeData(w, string(proto.MessageName(m)), data) + if err != nil { + return err + } + f.Header.setSize(string(proto.MessageName(m).Name()), uint32(n)) + return nil +} + +func (f *BakFormat) readSeg(r io.Reader, name string, size int, msg proto.Message) error { + if size == 0 { + return nil + } + data, err := f.readData(r, name, size) + if err != nil { + return err + } + // skip nil message + if msg == nil { + return nil + } + return proto.Unmarshal(data, msg) +} + +func (f *BakFormat) writeBatchSeg(w io.Writer, m proto.Message) error { + if m == nil { + return nil + } + + data, err := proto.Marshal(m) + if err != nil { + return err + } + + name := string(proto.MessageName(m).Name()) + // batch size + n1, err := f.writeData(w, name+" size", binary.LittleEndian.AppendUint32(nil, uint32(len(data)))) + if err != nil { + return nil + } + // batch msg + n2, err := f.writeData(w, name, data) + if err != nil { + return err + } + f.Header.addBatch(name, uint32(n1+n2)) + return nil +} + +func (f *BakFormat) readBatchSeg(r io.Reader, name string, msg proto.Message) error { + sd, err := f.readData(r, name+" size", 4) + if err != nil { + return err + } + + size := int(binary.LittleEndian.Uint32(sd)) + data, err := f.readData(r, name, size) + if err != nil { + return err + } + return proto.Unmarshal(data, msg) +} + +func (f *BakFormat) writeHeader(w io.WriteSeeker) error { + w.Seek(0, io.SeekStart) + data, err := f.Header.marshal() + if err != nil { + return err + } + _, err = f.writeData(w, "Header", data) + return err +} + +func (f *BakFormat) readHeader(r io.Reader) error { + data, err := f.readData(r, "Header", BakHeaderSize) + if err != nil { + return err + } + if err = f.Header.unmarshal(data); err != nil { + return err + } + + // check TODO + if f.Header.Magic != BakMagic { + return fmt.Errorf("this binary file may not be a juicefs backup file") + } + return nil +} + +type BakHeader struct { + Magic uint32 + Version uint32 + Checksum uint32 + FormatSize uint32 + CounterSize uint32 + SustainedSize uint32 + DelFileSize uint32 + AclSize uint32 + NodeSize uint32 + NodeBatchNum uint32 + ChunkSize uint32 + ChunkBatchNum uint32 + EdgeSize uint32 + EdgeBatchNum uint32 + SliceRefSize uint32 + SliceRefBatchNum uint32 + SymlinkSize uint32 + SymlinkBatchNum uint32 + XattrSize uint32 + XattrBatchNum uint32 + QuotaSize uint32 + QuotaBatchNum uint32 + StatSize uint32 + StatBatchNum uint32 + _ [BakHeaderSize - 96]byte +} + +func (h *BakHeader) addBatch(name string, size uint32) { + switch name { + case "NodeBatch": + h.NodeSize += size + h.NodeBatchNum++ + case "ChunkBatch": + h.ChunkSize += size + h.ChunkBatchNum++ + case "EdgeBatch": + h.EdgeSize += size + h.EdgeBatchNum++ + case "SliceRefBatch": + h.SliceRefSize += size + h.SliceRefBatchNum++ + case "SymlinkBatch": + h.SymlinkSize += size + h.SymlinkBatchNum++ + case "XattrBatch": + h.XattrSize += size + h.XattrBatchNum++ + case "QuotaBatch": + h.QuotaSize += size + h.QuotaBatchNum++ + case "StatBatch": + h.StatSize += size + h.StatBatchNum++ + } +} + +func (h *BakHeader) setSize(name string, n uint32) { + switch name { + case "Format": + h.FormatSize = n + case "Counters": + h.CounterSize = n + case "Sustaineds": + h.SustainedSize = n + case "DelFiles": + h.DelFileSize = n + case "Acls": + h.AclSize = n + } +} + +func (h *BakHeader) marshal() ([]byte, error) { + buff := bytes.NewBuffer(make([]byte, 0, BakHeaderSize)) + if err := binary.Write(buff, binary.LittleEndian, h); err != nil { + return nil, err + } + data := buff.Bytes() + if len(data) != BakHeaderSize { + return nil, fmt.Errorf("header size is %d, expect %d", len(data), BakHeaderSize) + } + return data, nil +} + +func (h *BakHeader) unmarshal(data []byte) error { + buff := bytes.NewBuffer(data) + return binary.Read(buff, binary.LittleEndian, h) +} + +func newPBFormat(f *Format, keepSecret bool) *pb.Format { + pf := &pb.Format{} + pf.Name = f.Name + pf.Uuid = f.UUID + pf.Storage = f.Storage + pf.StorageClass = f.StorageClass + pf.Bucket = f.Bucket + pf.AccessKey = f.AccessKey + pf.SecretKey = f.SecretKey + pf.SessionToken = f.SessionToken + pf.BlockSize = int32(f.BlockSize) + pf.Compression = f.Compression + pf.Shards = int32(f.Shards) + pf.HashPrefix = f.HashPrefix + pf.Capacity = f.Capacity + pf.Inodes = f.Inodes + pf.EncryptKey = f.EncryptKey + pf.EncryptAlgo = f.EncryptAlgo + pf.KeyEncrypted = f.KeyEncrypted + pf.UploadLimit = f.UploadLimit + pf.DownloadLimit = f.DownloadLimit + pf.TrashDays = int32(f.TrashDays) + pf.MetaVersion = int32(f.MetaVersion) + pf.MinClientVersion = f.MinClientVersion + pf.MaxClientVersion = f.MaxClientVersion + pf.DirStats = f.DirStats + pf.EnableAcl = f.EnableACL + + if !keepSecret { + removeKey := func(key *string, name string) { + if *key == "" { + *key = "remove" + logger.Warnf("%s is removed for the sake of safety", name) + } + } + removeKey(&pf.SecretKey, "Secret Key") + removeKey(&pf.SessionToken, "Session Token") + removeKey(&pf.EncryptKey, "Encrypt Key") + } + return pf +} + +func newPBPool(msg proto.Message) *sync.Pool { + pr := msg.ProtoReflect() + return &sync.Pool{ + New: func() interface{} { + return pr.New().Interface() + }, + } +} + +// TODO +func getCounterFields(c *pb.Counters) map[string]*int64 { + return map[string]*int64{ + "usedSpace": &c.UsedSpace, + "totalInodes": &c.UsedInodes, + "nextInode": &c.NextInode, + "nextChunk": &c.NextChunk, + "nextSession": &c.NextSession, + "nextTrash": &c.NextTrash, + } +} diff --git a/pkg/meta/interface.go b/pkg/meta/interface.go index 2bf960c5ab81..a72a9ffe73b8 100644 --- a/pkg/meta/interface.go +++ b/pkg/meta/interface.go @@ -452,6 +452,9 @@ type Meta interface { DumpMeta(w io.Writer, root Ino, threads int, keepSecret, fast, skipTrash bool) error LoadMeta(r io.Reader) error + DumpMetaV2(ctx Context, w io.WriteSeeker, opt *DumpOption) (err error) + LoadMetaV2(ctx Context, r io.Reader, opt *LoadOption) error + // getBase return the base engine. getBase() *baseMeta InitMetrics(registerer prometheus.Registerer) diff --git a/pkg/meta/load_dump_test.go b/pkg/meta/load_dump_test.go index 2c3f41b961b4..7f826181c86a 100644 --- a/pkg/meta/load_dump_test.go +++ b/pkg/meta/load_dump_test.go @@ -25,12 +25,13 @@ import ( "path" "strings" "testing" + "time" "golang.org/x/text/encoding/simplifiedchinese" "golang.org/x/text/transform" ) -const sampleFile = "metadata.sample" +const sampleFile = "../../1M_files_in_one_dir.dump" const subSampleFile = "metadata-sub.sample" func TestEscape(t *testing.T) { @@ -88,20 +89,7 @@ func GbkToUtf8(s []byte) ([]byte, error) { return d, nil } -func testLoad(t *testing.T, uri, fname string) Meta { - m := NewClient(uri, nil) - if err := m.Reset(); err != nil { - t.Fatalf("reset meta: %s", err) - } - fp, err := os.Open(fname) - if err != nil { - t.Fatalf("open file: %s", fname) - } - defer fp.Close() - if err = m.LoadMeta(fp); err != nil { - t.Fatalf("load meta: %s", err) - } - +func checkMeta(t *testing.T, m Meta) { ctx := Background var entries []*Entry if st := m.Readdir(ctx, 1, 1, &entries); st != 0 { @@ -207,7 +195,22 @@ func testLoad(t *testing.T, uri, fname string) Meta { if st := m.GetXattr(ctx, 3, "dk", &value); st != 0 || string(value) != "果汁%25" { t.Fatalf("getxattr: %s %v", st, value) } +} +func testLoad(t *testing.T, uri, fname string) Meta { + m := NewClient(uri, nil) + if err := m.Reset(); err != nil { + t.Fatalf("reset meta: %s", err) + } + fp, err := os.Open(fname) + if err != nil { + t.Fatalf("open file: %s", fname) + } + defer fp.Close() + if err = m.LoadMeta(fp); err != nil { + t.Fatalf("load meta: %s", err) + } + // checkMeta(t, m) return m } @@ -282,10 +285,62 @@ func testLoadDump(t *testing.T, name, addr string) { } func TestLoadDump(t *testing.T) { //skip mutate - testLoadDump(t, "redis", "redis://127.0.0.1/10") - testLoadDump(t, "mysql", "mysql://root:@/dev") - testLoadDump(t, "badger", "badger://jfs-load-dump") - testLoadDump(t, "tikv", "tikv://127.0.0.1:2379/jfs-load-dump") + // testLoadDump(t, "redis", "redis://127.0.0.1/10") + testLoadDump(t, "mysql", "mysql://root:123456@/dev") + // testLoadDump(t, "badger", "badger://jfs-load-dump") + // testLoadDump(t, "tikv", "tikv://127.0.0.1:2379/jfs-load-dump") +} + +func testDumpV2(t *testing.T, m Meta, result string) { + fp, err := os.OpenFile(result, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + t.Fatalf("open file %s: %s", result, err) + } + defer fp.Close() + if _, err = m.Load(true); err != nil { + t.Fatalf("load setting: %s", err) + } + if err = m.DumpMetaV2(Background, fp, &DumpOption{CoNum: 10}); err != nil { + t.Fatalf("dump meta: %s", err) + } + fp.Sync() +} + +func testLoadV2(t *testing.T, uri, fname string) Meta { + m := NewClient(uri, nil) + if err := m.Reset(); err != nil { + t.Fatalf("reset meta: %s", err) + } + fp, err := os.Open(fname) + if err != nil { + t.Fatalf("open file: %s", fname) + } + defer fp.Close() + if err = m.LoadMetaV2(Background, fp, &LoadOption{CoNum: 10}); err != nil { + t.Fatalf("load meta: %s", err) + } + // checkMeta(t, m) + return m +} + +func testLoadDumpV2(t *testing.T, name, addr string) { + t.Run("Metadata Engine: "+name, func(t *testing.T) { + start := time.Now() + m := testLoad(t, fmt.Sprintf("%s%s", addr, "dev"), sampleFile) + t.Logf("load meta: %v", time.Since(start)) + start = time.Now() + testDumpV2(t, m, "test.dump") + m.Shutdown() + t.Logf("dump meta: %v", time.Since(start)) + start = time.Now() + m = testLoadV2(t, fmt.Sprintf("%s%s", addr, "dev2"), "test.dump") + m.Shutdown() + t.Logf("load meta v2: %v", time.Since(start)) + }) +} + +func TestLoadDumpV2(t *testing.T) { + testLoadDumpV2(t, "mysql", "mysql://root:123456@/") } func TestLoadDumpSlow(t *testing.T) { //skip mutate diff --git a/pkg/meta/pb/backup.pb.go b/pkg/meta/pb/backup.pb.go new file mode 100644 index 000000000000..c05649a56512 --- /dev/null +++ b/pkg/meta/pb/backup.pb.go @@ -0,0 +1,2098 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.35.1 +// protoc v3.12.4 +// source: pkg/meta/pb/backup.proto + +package pb + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Format struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Uuid string `protobuf:"bytes,2,opt,name=uuid,proto3" json:"uuid,omitempty"` + Storage string `protobuf:"bytes,3,opt,name=storage,proto3" json:"storage,omitempty"` + StorageClass string `protobuf:"bytes,4,opt,name=storage_class,json=storageClass,proto3" json:"storage_class,omitempty"` + Bucket string `protobuf:"bytes,5,opt,name=bucket,proto3" json:"bucket,omitempty"` + AccessKey string `protobuf:"bytes,6,opt,name=access_key,json=accessKey,proto3" json:"access_key,omitempty"` + SecretKey string `protobuf:"bytes,7,opt,name=secret_key,json=secretKey,proto3" json:"secret_key,omitempty"` + SessionToken string `protobuf:"bytes,8,opt,name=session_token,json=sessionToken,proto3" json:"session_token,omitempty"` + BlockSize int32 `protobuf:"varint,9,opt,name=block_size,json=blockSize,proto3" json:"block_size,omitempty"` + Compression string `protobuf:"bytes,10,opt,name=compression,proto3" json:"compression,omitempty"` + Shards int32 `protobuf:"varint,11,opt,name=shards,proto3" json:"shards,omitempty"` + HashPrefix bool `protobuf:"varint,12,opt,name=hash_prefix,json=hashPrefix,proto3" json:"hash_prefix,omitempty"` + Capacity uint64 `protobuf:"varint,13,opt,name=capacity,proto3" json:"capacity,omitempty"` + Inodes uint64 `protobuf:"varint,14,opt,name=inodes,proto3" json:"inodes,omitempty"` + EncryptKey string `protobuf:"bytes,15,opt,name=encrypt_key,json=encryptKey,proto3" json:"encrypt_key,omitempty"` + EncryptAlgo string `protobuf:"bytes,16,opt,name=encrypt_algo,json=encryptAlgo,proto3" json:"encrypt_algo,omitempty"` + KeyEncrypted bool `protobuf:"varint,17,opt,name=key_encrypted,json=keyEncrypted,proto3" json:"key_encrypted,omitempty"` + UploadLimit int64 `protobuf:"varint,18,opt,name=upload_limit,json=uploadLimit,proto3" json:"upload_limit,omitempty"` // Mbps + DownloadLimit int64 `protobuf:"varint,19,opt,name=download_limit,json=downloadLimit,proto3" json:"download_limit,omitempty"` // Mbps + TrashDays int32 `protobuf:"varint,20,opt,name=trash_days,json=trashDays,proto3" json:"trash_days,omitempty"` + MetaVersion int32 `protobuf:"varint,21,opt,name=meta_version,json=metaVersion,proto3" json:"meta_version,omitempty"` + MinClientVersion string `protobuf:"bytes,22,opt,name=min_client_version,json=minClientVersion,proto3" json:"min_client_version,omitempty"` + MaxClientVersion string `protobuf:"bytes,23,opt,name=max_client_version,json=maxClientVersion,proto3" json:"max_client_version,omitempty"` + DirStats bool `protobuf:"varint,24,opt,name=dir_stats,json=dirStats,proto3" json:"dir_stats,omitempty"` + EnableAcl bool `protobuf:"varint,25,opt,name=enable_acl,json=enableAcl,proto3" json:"enable_acl,omitempty"` +} + +func (x *Format) Reset() { + *x = Format{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Format) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Format) ProtoMessage() {} + +func (x *Format) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Format.ProtoReflect.Descriptor instead. +func (*Format) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{0} +} + +func (x *Format) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Format) GetUuid() string { + if x != nil { + return x.Uuid + } + return "" +} + +func (x *Format) GetStorage() string { + if x != nil { + return x.Storage + } + return "" +} + +func (x *Format) GetStorageClass() string { + if x != nil { + return x.StorageClass + } + return "" +} + +func (x *Format) GetBucket() string { + if x != nil { + return x.Bucket + } + return "" +} + +func (x *Format) GetAccessKey() string { + if x != nil { + return x.AccessKey + } + return "" +} + +func (x *Format) GetSecretKey() string { + if x != nil { + return x.SecretKey + } + return "" +} + +func (x *Format) GetSessionToken() string { + if x != nil { + return x.SessionToken + } + return "" +} + +func (x *Format) GetBlockSize() int32 { + if x != nil { + return x.BlockSize + } + return 0 +} + +func (x *Format) GetCompression() string { + if x != nil { + return x.Compression + } + return "" +} + +func (x *Format) GetShards() int32 { + if x != nil { + return x.Shards + } + return 0 +} + +func (x *Format) GetHashPrefix() bool { + if x != nil { + return x.HashPrefix + } + return false +} + +func (x *Format) GetCapacity() uint64 { + if x != nil { + return x.Capacity + } + return 0 +} + +func (x *Format) GetInodes() uint64 { + if x != nil { + return x.Inodes + } + return 0 +} + +func (x *Format) GetEncryptKey() string { + if x != nil { + return x.EncryptKey + } + return "" +} + +func (x *Format) GetEncryptAlgo() string { + if x != nil { + return x.EncryptAlgo + } + return "" +} + +func (x *Format) GetKeyEncrypted() bool { + if x != nil { + return x.KeyEncrypted + } + return false +} + +func (x *Format) GetUploadLimit() int64 { + if x != nil { + return x.UploadLimit + } + return 0 +} + +func (x *Format) GetDownloadLimit() int64 { + if x != nil { + return x.DownloadLimit + } + return 0 +} + +func (x *Format) GetTrashDays() int32 { + if x != nil { + return x.TrashDays + } + return 0 +} + +func (x *Format) GetMetaVersion() int32 { + if x != nil { + return x.MetaVersion + } + return 0 +} + +func (x *Format) GetMinClientVersion() string { + if x != nil { + return x.MinClientVersion + } + return "" +} + +func (x *Format) GetMaxClientVersion() string { + if x != nil { + return x.MaxClientVersion + } + return "" +} + +func (x *Format) GetDirStats() bool { + if x != nil { + return x.DirStats + } + return false +} + +func (x *Format) GetEnableAcl() bool { + if x != nil { + return x.EnableAcl + } + return false +} + +type Counters struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + UsedSpace int64 `protobuf:"varint,1,opt,name=usedSpace,proto3" json:"usedSpace,omitempty"` + UsedInodes int64 `protobuf:"varint,2,opt,name=usedInodes,proto3" json:"usedInodes,omitempty"` + NextInode int64 `protobuf:"varint,3,opt,name=nextInode,proto3" json:"nextInode,omitempty"` + NextChunk int64 `protobuf:"varint,4,opt,name=nextChunk,proto3" json:"nextChunk,omitempty"` + NextSession int64 `protobuf:"varint,5,opt,name=nextSession,proto3" json:"nextSession,omitempty"` + NextTrash int64 `protobuf:"varint,6,opt,name=nextTrash,proto3" json:"nextTrash,omitempty"` +} + +func (x *Counters) Reset() { + *x = Counters{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Counters) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Counters) ProtoMessage() {} + +func (x *Counters) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Counters.ProtoReflect.Descriptor instead. +func (*Counters) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{1} +} + +func (x *Counters) GetUsedSpace() int64 { + if x != nil { + return x.UsedSpace + } + return 0 +} + +func (x *Counters) GetUsedInodes() int64 { + if x != nil { + return x.UsedInodes + } + return 0 +} + +func (x *Counters) GetNextInode() int64 { + if x != nil { + return x.NextInode + } + return 0 +} + +func (x *Counters) GetNextChunk() int64 { + if x != nil { + return x.NextChunk + } + return 0 +} + +func (x *Counters) GetNextSession() int64 { + if x != nil { + return x.NextSession + } + return 0 +} + +func (x *Counters) GetNextTrash() int64 { + if x != nil { + return x.NextTrash + } + return 0 +} + +type Sustained struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Sid uint64 `protobuf:"varint,1,opt,name=sid,proto3" json:"sid,omitempty"` + Inodes []uint64 `protobuf:"varint,2,rep,packed,name=inodes,proto3" json:"inodes,omitempty"` +} + +func (x *Sustained) Reset() { + *x = Sustained{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Sustained) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Sustained) ProtoMessage() {} + +func (x *Sustained) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Sustained.ProtoReflect.Descriptor instead. +func (*Sustained) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{2} +} + +func (x *Sustained) GetSid() uint64 { + if x != nil { + return x.Sid + } + return 0 +} + +func (x *Sustained) GetInodes() []uint64 { + if x != nil { + return x.Inodes + } + return nil +} + +type Sustaineds struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Sustaineds []*Sustained `protobuf:"bytes,1,rep,name=sustaineds,proto3" json:"sustaineds,omitempty"` +} + +func (x *Sustaineds) Reset() { + *x = Sustaineds{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Sustaineds) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Sustaineds) ProtoMessage() {} + +func (x *Sustaineds) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Sustaineds.ProtoReflect.Descriptor instead. +func (*Sustaineds) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{3} +} + +func (x *Sustaineds) GetSustaineds() []*Sustained { + if x != nil { + return x.Sustaineds + } + return nil +} + +type DelFile struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Inode uint64 `protobuf:"varint,1,opt,name=inode,proto3" json:"inode,omitempty"` + Length uint64 `protobuf:"varint,2,opt,name=length,proto3" json:"length,omitempty"` + Expire int64 `protobuf:"varint,3,opt,name=expire,proto3" json:"expire,omitempty"` +} + +func (x *DelFile) Reset() { + *x = DelFile{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DelFile) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DelFile) ProtoMessage() {} + +func (x *DelFile) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DelFile.ProtoReflect.Descriptor instead. +func (*DelFile) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{4} +} + +func (x *DelFile) GetInode() uint64 { + if x != nil { + return x.Inode + } + return 0 +} + +func (x *DelFile) GetLength() uint64 { + if x != nil { + return x.Length + } + return 0 +} + +func (x *DelFile) GetExpire() int64 { + if x != nil { + return x.Expire + } + return 0 +} + +type DelFiles struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Files []*DelFile `protobuf:"bytes,1,rep,name=files,proto3" json:"files,omitempty"` +} + +func (x *DelFiles) Reset() { + *x = DelFiles{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DelFiles) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DelFiles) ProtoMessage() {} + +func (x *DelFiles) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DelFiles.ProtoReflect.Descriptor instead. +func (*DelFiles) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{5} +} + +func (x *DelFiles) GetFiles() []*DelFile { + if x != nil { + return x.Files + } + return nil +} + +type Quota struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Inode uint64 `protobuf:"varint,1,opt,name=inode,proto3" json:"inode,omitempty"` + MaxSpace int64 `protobuf:"varint,2,opt,name=maxSpace,proto3" json:"maxSpace,omitempty"` + MaxInodes int64 `protobuf:"varint,3,opt,name=maxInodes,proto3" json:"maxInodes,omitempty"` + UsedSpace int64 `protobuf:"varint,4,opt,name=usedSpace,proto3" json:"usedSpace,omitempty"` + UsedInodes int64 `protobuf:"varint,5,opt,name=usedInodes,proto3" json:"usedInodes,omitempty"` +} + +func (x *Quota) Reset() { + *x = Quota{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Quota) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Quota) ProtoMessage() {} + +func (x *Quota) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Quota.ProtoReflect.Descriptor instead. +func (*Quota) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{6} +} + +func (x *Quota) GetInode() uint64 { + if x != nil { + return x.Inode + } + return 0 +} + +func (x *Quota) GetMaxSpace() int64 { + if x != nil { + return x.MaxSpace + } + return 0 +} + +func (x *Quota) GetMaxInodes() int64 { + if x != nil { + return x.MaxInodes + } + return 0 +} + +func (x *Quota) GetUsedSpace() int64 { + if x != nil { + return x.UsedSpace + } + return 0 +} + +func (x *Quota) GetUsedInodes() int64 { + if x != nil { + return x.UsedInodes + } + return 0 +} + +type QuotaBatch struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Batch []*Quota `protobuf:"bytes,1,rep,name=batch,proto3" json:"batch,omitempty"` +} + +func (x *QuotaBatch) Reset() { + *x = QuotaBatch{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *QuotaBatch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*QuotaBatch) ProtoMessage() {} + +func (x *QuotaBatch) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use QuotaBatch.ProtoReflect.Descriptor instead. +func (*QuotaBatch) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{7} +} + +func (x *QuotaBatch) GetBatch() []*Quota { + if x != nil { + return x.Batch + } + return nil +} + +type Stat struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Inode uint64 `protobuf:"varint,1,opt,name=inode,proto3" json:"inode,omitempty"` + DataLength int64 `protobuf:"varint,2,opt,name=dataLength,proto3" json:"dataLength,omitempty"` + UsedSpace int64 `protobuf:"varint,3,opt,name=usedSpace,proto3" json:"usedSpace,omitempty"` + UsedInodes int64 `protobuf:"varint,4,opt,name=usedInodes,proto3" json:"usedInodes,omitempty"` +} + +func (x *Stat) Reset() { + *x = Stat{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Stat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Stat) ProtoMessage() {} + +func (x *Stat) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Stat.ProtoReflect.Descriptor instead. +func (*Stat) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{8} +} + +func (x *Stat) GetInode() uint64 { + if x != nil { + return x.Inode + } + return 0 +} + +func (x *Stat) GetDataLength() int64 { + if x != nil { + return x.DataLength + } + return 0 +} + +func (x *Stat) GetUsedSpace() int64 { + if x != nil { + return x.UsedSpace + } + return 0 +} + +func (x *Stat) GetUsedInodes() int64 { + if x != nil { + return x.UsedInodes + } + return 0 +} + +type StatBatch struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Batch []*Stat `protobuf:"bytes,1,rep,name=batch,proto3" json:"batch,omitempty"` +} + +func (x *StatBatch) Reset() { + *x = StatBatch{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *StatBatch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*StatBatch) ProtoMessage() {} + +func (x *StatBatch) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use StatBatch.ProtoReflect.Descriptor instead. +func (*StatBatch) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{9} +} + +func (x *StatBatch) GetBatch() []*Stat { + if x != nil { + return x.Batch + } + return nil +} + +type Edge struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Parent uint64 `protobuf:"varint,1,opt,name=parent,proto3" json:"parent,omitempty"` + Inode uint64 `protobuf:"varint,2,opt,name=inode,proto3" json:"inode,omitempty"` + Name []byte `protobuf:"bytes,3,opt,name=name,proto3" json:"name,omitempty"` + Type uint32 `protobuf:"varint,4,opt,name=type,proto3" json:"type,omitempty"` +} + +func (x *Edge) Reset() { + *x = Edge{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Edge) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Edge) ProtoMessage() {} + +func (x *Edge) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Edge.ProtoReflect.Descriptor instead. +func (*Edge) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{10} +} + +func (x *Edge) GetParent() uint64 { + if x != nil { + return x.Parent + } + return 0 +} + +func (x *Edge) GetInode() uint64 { + if x != nil { + return x.Inode + } + return 0 +} + +func (x *Edge) GetName() []byte { + if x != nil { + return x.Name + } + return nil +} + +func (x *Edge) GetType() uint32 { + if x != nil { + return x.Type + } + return 0 +} + +type EdgeBatch struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Batch []*Edge `protobuf:"bytes,1,rep,name=batch,proto3" json:"batch,omitempty"` +} + +func (x *EdgeBatch) Reset() { + *x = EdgeBatch{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EdgeBatch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EdgeBatch) ProtoMessage() {} + +func (x *EdgeBatch) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EdgeBatch.ProtoReflect.Descriptor instead. +func (*EdgeBatch) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{11} +} + +func (x *EdgeBatch) GetBatch() []*Edge { + if x != nil { + return x.Batch + } + return nil +} + +type Node struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Inode uint64 `protobuf:"varint,1,opt,name=inode,proto3" json:"inode,omitempty"` + Type uint32 `protobuf:"varint,2,opt,name=type,proto3" json:"type,omitempty"` + Flags uint32 `protobuf:"varint,3,opt,name=flags,proto3" json:"flags,omitempty"` + Mode uint32 `protobuf:"varint,4,opt,name=mode,proto3" json:"mode,omitempty"` + Uid uint32 `protobuf:"varint,5,opt,name=uid,proto3" json:"uid,omitempty"` + Gid uint32 `protobuf:"varint,6,opt,name=gid,proto3" json:"gid,omitempty"` + Atime int64 `protobuf:"varint,7,opt,name=atime,proto3" json:"atime,omitempty"` + Mtime int64 `protobuf:"varint,8,opt,name=mtime,proto3" json:"mtime,omitempty"` + Ctime int64 `protobuf:"varint,9,opt,name=ctime,proto3" json:"ctime,omitempty"` + AtimeNsec int32 `protobuf:"varint,10,opt,name=atime_nsec,json=atimeNsec,proto3" json:"atime_nsec,omitempty"` + MtimeNsec int32 `protobuf:"varint,11,opt,name=mtime_nsec,json=mtimeNsec,proto3" json:"mtime_nsec,omitempty"` + CtimeNsec int32 `protobuf:"varint,12,opt,name=ctime_nsec,json=ctimeNsec,proto3" json:"ctime_nsec,omitempty"` + Nlink uint32 `protobuf:"varint,13,opt,name=nlink,proto3" json:"nlink,omitempty"` + Length uint64 `protobuf:"varint,14,opt,name=length,proto3" json:"length,omitempty"` + Rdev uint32 `protobuf:"varint,15,opt,name=rdev,proto3" json:"rdev,omitempty"` + Parent uint64 `protobuf:"varint,16,opt,name=parent,proto3" json:"parent,omitempty"` + AccessAclId uint32 `protobuf:"varint,17,opt,name=access_acl_id,json=accessAclId,proto3" json:"access_acl_id,omitempty"` + DefaultAclId uint32 `protobuf:"varint,18,opt,name=default_acl_id,json=defaultAclId,proto3" json:"default_acl_id,omitempty"` +} + +func (x *Node) Reset() { + *x = Node{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Node) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Node) ProtoMessage() {} + +func (x *Node) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Node.ProtoReflect.Descriptor instead. +func (*Node) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{12} +} + +func (x *Node) GetInode() uint64 { + if x != nil { + return x.Inode + } + return 0 +} + +func (x *Node) GetType() uint32 { + if x != nil { + return x.Type + } + return 0 +} + +func (x *Node) GetFlags() uint32 { + if x != nil { + return x.Flags + } + return 0 +} + +func (x *Node) GetMode() uint32 { + if x != nil { + return x.Mode + } + return 0 +} + +func (x *Node) GetUid() uint32 { + if x != nil { + return x.Uid + } + return 0 +} + +func (x *Node) GetGid() uint32 { + if x != nil { + return x.Gid + } + return 0 +} + +func (x *Node) GetAtime() int64 { + if x != nil { + return x.Atime + } + return 0 +} + +func (x *Node) GetMtime() int64 { + if x != nil { + return x.Mtime + } + return 0 +} + +func (x *Node) GetCtime() int64 { + if x != nil { + return x.Ctime + } + return 0 +} + +func (x *Node) GetAtimeNsec() int32 { + if x != nil { + return x.AtimeNsec + } + return 0 +} + +func (x *Node) GetMtimeNsec() int32 { + if x != nil { + return x.MtimeNsec + } + return 0 +} + +func (x *Node) GetCtimeNsec() int32 { + if x != nil { + return x.CtimeNsec + } + return 0 +} + +func (x *Node) GetNlink() uint32 { + if x != nil { + return x.Nlink + } + return 0 +} + +func (x *Node) GetLength() uint64 { + if x != nil { + return x.Length + } + return 0 +} + +func (x *Node) GetRdev() uint32 { + if x != nil { + return x.Rdev + } + return 0 +} + +func (x *Node) GetParent() uint64 { + if x != nil { + return x.Parent + } + return 0 +} + +func (x *Node) GetAccessAclId() uint32 { + if x != nil { + return x.AccessAclId + } + return 0 +} + +func (x *Node) GetDefaultAclId() uint32 { + if x != nil { + return x.DefaultAclId + } + return 0 +} + +type NodeBatch struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Batch []*Node `protobuf:"bytes,1,rep,name=batch,proto3" json:"batch,omitempty"` +} + +func (x *NodeBatch) Reset() { + *x = NodeBatch{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *NodeBatch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NodeBatch) ProtoMessage() {} + +func (x *NodeBatch) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NodeBatch.ProtoReflect.Descriptor instead. +func (*NodeBatch) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{13} +} + +func (x *NodeBatch) GetBatch() []*Node { + if x != nil { + return x.Batch + } + return nil +} + +type Slice struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id uint64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Pos uint32 `protobuf:"varint,2,opt,name=pos,proto3" json:"pos,omitempty"` + Size uint32 `protobuf:"varint,3,opt,name=size,proto3" json:"size,omitempty"` + Off uint32 `protobuf:"varint,4,opt,name=off,proto3" json:"off,omitempty"` + Len uint32 `protobuf:"varint,5,opt,name=len,proto3" json:"len,omitempty"` +} + +func (x *Slice) Reset() { + *x = Slice{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[14] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Slice) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Slice) ProtoMessage() {} + +func (x *Slice) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[14] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Slice.ProtoReflect.Descriptor instead. +func (*Slice) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{14} +} + +func (x *Slice) GetId() uint64 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *Slice) GetPos() uint32 { + if x != nil { + return x.Pos + } + return 0 +} + +func (x *Slice) GetSize() uint32 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *Slice) GetOff() uint32 { + if x != nil { + return x.Off + } + return 0 +} + +func (x *Slice) GetLen() uint32 { + if x != nil { + return x.Len + } + return 0 +} + +type Chunk struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Inode uint64 `protobuf:"varint,1,opt,name=inode,proto3" json:"inode,omitempty"` + Index uint32 `protobuf:"varint,2,opt,name=index,proto3" json:"index,omitempty"` + Slices []*Slice `protobuf:"bytes,3,rep,name=slices,proto3" json:"slices,omitempty"` +} + +func (x *Chunk) Reset() { + *x = Chunk{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[15] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Chunk) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Chunk) ProtoMessage() {} + +func (x *Chunk) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[15] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Chunk.ProtoReflect.Descriptor instead. +func (*Chunk) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{15} +} + +func (x *Chunk) GetInode() uint64 { + if x != nil { + return x.Inode + } + return 0 +} + +func (x *Chunk) GetIndex() uint32 { + if x != nil { + return x.Index + } + return 0 +} + +func (x *Chunk) GetSlices() []*Slice { + if x != nil { + return x.Slices + } + return nil +} + +type ChunkBatch struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Batch []*Chunk `protobuf:"bytes,1,rep,name=batch,proto3" json:"batch,omitempty"` +} + +func (x *ChunkBatch) Reset() { + *x = ChunkBatch{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[16] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ChunkBatch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChunkBatch) ProtoMessage() {} + +func (x *ChunkBatch) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[16] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChunkBatch.ProtoReflect.Descriptor instead. +func (*ChunkBatch) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{16} +} + +func (x *ChunkBatch) GetBatch() []*Chunk { + if x != nil { + return x.Batch + } + return nil +} + +type SliceRef struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id uint64 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Size uint32 `protobuf:"varint,2,opt,name=size,proto3" json:"size,omitempty"` + Refs int64 `protobuf:"varint,3,opt,name=refs,proto3" json:"refs,omitempty"` +} + +func (x *SliceRef) Reset() { + *x = SliceRef{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[17] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SliceRef) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SliceRef) ProtoMessage() {} + +func (x *SliceRef) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[17] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SliceRef.ProtoReflect.Descriptor instead. +func (*SliceRef) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{17} +} + +func (x *SliceRef) GetId() uint64 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *SliceRef) GetSize() uint32 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *SliceRef) GetRefs() int64 { + if x != nil { + return x.Refs + } + return 0 +} + +type SliceRefBatch struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Batch []*SliceRef `protobuf:"bytes,1,rep,name=batch,proto3" json:"batch,omitempty"` +} + +func (x *SliceRefBatch) Reset() { + *x = SliceRefBatch{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SliceRefBatch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SliceRefBatch) ProtoMessage() {} + +func (x *SliceRefBatch) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[18] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SliceRefBatch.ProtoReflect.Descriptor instead. +func (*SliceRefBatch) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{18} +} + +func (x *SliceRefBatch) GetBatch() []*SliceRef { + if x != nil { + return x.Batch + } + return nil +} + +type Symlink struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Inode uint64 `protobuf:"varint,1,opt,name=inode,proto3" json:"inode,omitempty"` + Target []byte `protobuf:"bytes,2,opt,name=target,proto3" json:"target,omitempty"` +} + +func (x *Symlink) Reset() { + *x = Symlink{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[19] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Symlink) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Symlink) ProtoMessage() {} + +func (x *Symlink) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[19] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Symlink.ProtoReflect.Descriptor instead. +func (*Symlink) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{19} +} + +func (x *Symlink) GetInode() uint64 { + if x != nil { + return x.Inode + } + return 0 +} + +func (x *Symlink) GetTarget() []byte { + if x != nil { + return x.Target + } + return nil +} + +type SymlinkBatch struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Batch []*Symlink `protobuf:"bytes,1,rep,name=batch,proto3" json:"batch,omitempty"` +} + +func (x *SymlinkBatch) Reset() { + *x = SymlinkBatch{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[20] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SymlinkBatch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SymlinkBatch) ProtoMessage() {} + +func (x *SymlinkBatch) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[20] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SymlinkBatch.ProtoReflect.Descriptor instead. +func (*SymlinkBatch) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{20} +} + +func (x *SymlinkBatch) GetBatch() []*Symlink { + if x != nil { + return x.Batch + } + return nil +} + +type Xattr struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Inode uint64 `protobuf:"varint,1,opt,name=inode,proto3" json:"inode,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Value []byte `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` +} + +func (x *Xattr) Reset() { + *x = Xattr{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[21] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Xattr) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Xattr) ProtoMessage() {} + +func (x *Xattr) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[21] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Xattr.ProtoReflect.Descriptor instead. +func (*Xattr) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{21} +} + +func (x *Xattr) GetInode() uint64 { + if x != nil { + return x.Inode + } + return 0 +} + +func (x *Xattr) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Xattr) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +type XattrBatch struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Batch []*Xattr `protobuf:"bytes,1,rep,name=batch,proto3" json:"batch,omitempty"` +} + +func (x *XattrBatch) Reset() { + *x = XattrBatch{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[22] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *XattrBatch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*XattrBatch) ProtoMessage() {} + +func (x *XattrBatch) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[22] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use XattrBatch.ProtoReflect.Descriptor instead. +func (*XattrBatch) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{22} +} + +func (x *XattrBatch) GetBatch() []*Xattr { + if x != nil { + return x.Batch + } + return nil +} + +type AclEntry struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id uint32 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Perm uint32 `protobuf:"varint,2,opt,name=perm,proto3" json:"perm,omitempty"` +} + +func (x *AclEntry) Reset() { + *x = AclEntry{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[23] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AclEntry) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AclEntry) ProtoMessage() {} + +func (x *AclEntry) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[23] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AclEntry.ProtoReflect.Descriptor instead. +func (*AclEntry) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{23} +} + +func (x *AclEntry) GetId() uint32 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *AclEntry) GetPerm() uint32 { + if x != nil { + return x.Perm + } + return 0 +} + +type Acl struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Id uint32 `protobuf:"varint,1,opt,name=id,proto3" json:"id,omitempty"` + Owner uint32 `protobuf:"varint,2,opt,name=owner,proto3" json:"owner,omitempty"` + Group uint32 `protobuf:"varint,3,opt,name=group,proto3" json:"group,omitempty"` + Other uint32 `protobuf:"varint,4,opt,name=other,proto3" json:"other,omitempty"` + Mask uint32 `protobuf:"varint,5,opt,name=mask,proto3" json:"mask,omitempty"` + Users []*AclEntry `protobuf:"bytes,6,rep,name=users,proto3" json:"users,omitempty"` + Groups []*AclEntry `protobuf:"bytes,7,rep,name=groups,proto3" json:"groups,omitempty"` +} + +func (x *Acl) Reset() { + *x = Acl{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[24] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Acl) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Acl) ProtoMessage() {} + +func (x *Acl) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[24] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Acl.ProtoReflect.Descriptor instead. +func (*Acl) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{24} +} + +func (x *Acl) GetId() uint32 { + if x != nil { + return x.Id + } + return 0 +} + +func (x *Acl) GetOwner() uint32 { + if x != nil { + return x.Owner + } + return 0 +} + +func (x *Acl) GetGroup() uint32 { + if x != nil { + return x.Group + } + return 0 +} + +func (x *Acl) GetOther() uint32 { + if x != nil { + return x.Other + } + return 0 +} + +func (x *Acl) GetMask() uint32 { + if x != nil { + return x.Mask + } + return 0 +} + +func (x *Acl) GetUsers() []*AclEntry { + if x != nil { + return x.Users + } + return nil +} + +func (x *Acl) GetGroups() []*AclEntry { + if x != nil { + return x.Groups + } + return nil +} + +type Acls struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Acls []*Acl `protobuf:"bytes,1,rep,name=acls,proto3" json:"acls,omitempty"` +} + +func (x *Acls) Reset() { + *x = Acls{} + mi := &file_pkg_meta_pb_backup_proto_msgTypes[25] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Acls) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Acls) ProtoMessage() {} + +func (x *Acls) ProtoReflect() protoreflect.Message { + mi := &file_pkg_meta_pb_backup_proto_msgTypes[25] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Acls.ProtoReflect.Descriptor instead. +func (*Acls) Descriptor() ([]byte, []int) { + return file_pkg_meta_pb_backup_proto_rawDescGZIP(), []int{25} +} + +func (x *Acls) GetAcls() []*Acl { + if x != nil { + return x.Acls + } + return nil +} + +var File_pkg_meta_pb_backup_proto protoreflect.FileDescriptor + +var file_pkg_meta_pb_backup_proto_rawDesc = []byte{ + 0x0a, 0x18, 0x70, 0x6b, 0x67, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x2f, 0x70, 0x62, 0x2f, 0x62, 0x61, + 0x63, 0x6b, 0x75, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x02, 0x70, 0x62, 0x22, 0xa5, + 0x06, 0x0a, 0x06, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, + 0x04, 0x75, 0x75, 0x69, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x75, 0x75, 0x69, + 0x64, 0x12, 0x18, 0x0a, 0x07, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x07, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x73, + 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x0c, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73, + 0x12, 0x16, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x61, 0x63, 0x63, 0x65, + 0x73, 0x73, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x61, 0x63, + 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x65, 0x63, 0x72, 0x65, + 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x73, 0x65, 0x63, + 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x23, 0x0a, 0x0d, 0x73, 0x65, 0x73, 0x73, 0x69, 0x6f, + 0x6e, 0x5f, 0x74, 0x6f, 0x6b, 0x65, 0x6e, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x73, + 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e, 0x12, 0x1d, 0x0a, 0x0a, 0x62, + 0x6c, 0x6f, 0x63, 0x6b, 0x5f, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x09, 0x62, 0x6c, 0x6f, 0x63, 0x6b, 0x53, 0x69, 0x7a, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x63, 0x6f, + 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x63, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x06, + 0x73, 0x68, 0x61, 0x72, 0x64, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x73, 0x68, + 0x61, 0x72, 0x64, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x68, 0x61, 0x73, 0x68, 0x5f, 0x70, 0x72, 0x65, + 0x66, 0x69, 0x78, 0x18, 0x0c, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x68, 0x61, 0x73, 0x68, 0x50, + 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, + 0x79, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x04, 0x52, 0x08, 0x63, 0x61, 0x70, 0x61, 0x63, 0x69, 0x74, + 0x79, 0x12, 0x16, 0x0a, 0x06, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x0e, 0x20, 0x01, 0x28, + 0x04, 0x52, 0x06, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x65, 0x6e, 0x63, + 0x72, 0x79, 0x70, 0x74, 0x5f, 0x6b, 0x65, 0x79, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x65, 0x6e, + 0x63, 0x72, 0x79, 0x70, 0x74, 0x5f, 0x61, 0x6c, 0x67, 0x6f, 0x18, 0x10, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0b, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x41, 0x6c, 0x67, 0x6f, 0x12, 0x23, 0x0a, + 0x0d, 0x6b, 0x65, 0x79, 0x5f, 0x65, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x65, 0x64, 0x18, 0x11, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x0c, 0x6b, 0x65, 0x79, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, + 0x65, 0x64, 0x12, 0x21, 0x0a, 0x0c, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x5f, 0x6c, 0x69, 0x6d, + 0x69, 0x74, 0x18, 0x12, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x75, 0x70, 0x6c, 0x6f, 0x61, 0x64, + 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x64, 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, + 0x64, 0x5f, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x18, 0x13, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0d, 0x64, + 0x6f, 0x77, 0x6e, 0x6c, 0x6f, 0x61, 0x64, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x12, 0x1d, 0x0a, 0x0a, + 0x74, 0x72, 0x61, 0x73, 0x68, 0x5f, 0x64, 0x61, 0x79, 0x73, 0x18, 0x14, 0x20, 0x01, 0x28, 0x05, + 0x52, 0x09, 0x74, 0x72, 0x61, 0x73, 0x68, 0x44, 0x61, 0x79, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x6d, + 0x65, 0x74, 0x61, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x15, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x0b, 0x6d, 0x65, 0x74, 0x61, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2c, + 0x0a, 0x12, 0x6d, 0x69, 0x6e, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x76, 0x65, 0x72, + 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x16, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6d, 0x69, 0x6e, 0x43, + 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x2c, 0x0a, 0x12, + 0x6d, 0x61, 0x78, 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x76, 0x65, 0x72, 0x73, 0x69, + 0x6f, 0x6e, 0x18, 0x17, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x6d, 0x61, 0x78, 0x43, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x1b, 0x0a, 0x09, 0x64, 0x69, + 0x72, 0x5f, 0x73, 0x74, 0x61, 0x74, 0x73, 0x18, 0x18, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x64, + 0x69, 0x72, 0x53, 0x74, 0x61, 0x74, 0x73, 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x6e, 0x61, 0x62, 0x6c, + 0x65, 0x5f, 0x61, 0x63, 0x6c, 0x18, 0x19, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x65, 0x6e, 0x61, + 0x62, 0x6c, 0x65, 0x41, 0x63, 0x6c, 0x22, 0xc4, 0x01, 0x0a, 0x08, 0x43, 0x6f, 0x75, 0x6e, 0x74, + 0x65, 0x72, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x75, 0x73, 0x65, 0x64, 0x53, 0x70, 0x61, 0x63, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x75, 0x73, 0x65, 0x64, 0x53, 0x70, 0x61, 0x63, + 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x75, 0x73, 0x65, 0x64, 0x49, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x75, 0x73, 0x65, 0x64, 0x49, 0x6e, 0x6f, 0x64, 0x65, + 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x6e, 0x65, 0x78, 0x74, 0x49, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x6e, 0x65, 0x78, 0x74, 0x49, 0x6e, 0x6f, 0x64, 0x65, 0x12, + 0x1c, 0x0a, 0x09, 0x6e, 0x65, 0x78, 0x74, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x18, 0x04, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x09, 0x6e, 0x65, 0x78, 0x74, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x20, 0x0a, + 0x0b, 0x6e, 0x65, 0x78, 0x74, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x05, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x0b, 0x6e, 0x65, 0x78, 0x74, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x12, + 0x1c, 0x0a, 0x09, 0x6e, 0x65, 0x78, 0x74, 0x54, 0x72, 0x61, 0x73, 0x68, 0x18, 0x06, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x09, 0x6e, 0x65, 0x78, 0x74, 0x54, 0x72, 0x61, 0x73, 0x68, 0x22, 0x35, 0x0a, + 0x09, 0x53, 0x75, 0x73, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x73, 0x69, + 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x03, 0x73, 0x69, 0x64, 0x12, 0x16, 0x0a, 0x06, + 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x04, 0x52, 0x06, 0x69, 0x6e, + 0x6f, 0x64, 0x65, 0x73, 0x22, 0x3b, 0x0a, 0x0a, 0x53, 0x75, 0x73, 0x74, 0x61, 0x69, 0x6e, 0x65, + 0x64, 0x73, 0x12, 0x2d, 0x0a, 0x0a, 0x73, 0x75, 0x73, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x64, 0x73, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x70, 0x62, 0x2e, 0x53, 0x75, 0x73, 0x74, + 0x61, 0x69, 0x6e, 0x65, 0x64, 0x52, 0x0a, 0x73, 0x75, 0x73, 0x74, 0x61, 0x69, 0x6e, 0x65, 0x64, + 0x73, 0x22, 0x4f, 0x0a, 0x07, 0x44, 0x65, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x69, 0x6e, 0x6f, + 0x64, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, 0x16, 0x0a, 0x06, 0x65, 0x78, + 0x70, 0x69, 0x72, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x06, 0x65, 0x78, 0x70, 0x69, + 0x72, 0x65, 0x22, 0x2d, 0x0a, 0x08, 0x44, 0x65, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x21, + 0x0a, 0x05, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, + 0x70, 0x62, 0x2e, 0x44, 0x65, 0x6c, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x05, 0x66, 0x69, 0x6c, 0x65, + 0x73, 0x22, 0x95, 0x01, 0x0a, 0x05, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x12, 0x14, 0x0a, 0x05, 0x69, + 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x69, 0x6e, 0x6f, 0x64, + 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x61, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x03, 0x52, 0x08, 0x6d, 0x61, 0x78, 0x53, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1c, 0x0a, + 0x09, 0x6d, 0x61, 0x78, 0x49, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, + 0x52, 0x09, 0x6d, 0x61, 0x78, 0x49, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x75, + 0x73, 0x65, 0x64, 0x53, 0x70, 0x61, 0x63, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, + 0x75, 0x73, 0x65, 0x64, 0x53, 0x70, 0x61, 0x63, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x75, 0x73, 0x65, + 0x64, 0x49, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x75, + 0x73, 0x65, 0x64, 0x49, 0x6e, 0x6f, 0x64, 0x65, 0x73, 0x22, 0x2d, 0x0a, 0x0a, 0x51, 0x75, 0x6f, + 0x74, 0x61, 0x42, 0x61, 0x74, 0x63, 0x68, 0x12, 0x1f, 0x0a, 0x05, 0x62, 0x61, 0x74, 0x63, 0x68, + 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x70, 0x62, 0x2e, 0x51, 0x75, 0x6f, 0x74, + 0x61, 0x52, 0x05, 0x62, 0x61, 0x74, 0x63, 0x68, 0x22, 0x7a, 0x0a, 0x04, 0x53, 0x74, 0x61, 0x74, + 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x64, 0x61, 0x74, 0x61, 0x4c, 0x65, + 0x6e, 0x67, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x64, 0x61, 0x74, 0x61, + 0x4c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, 0x1c, 0x0a, 0x09, 0x75, 0x73, 0x65, 0x64, 0x53, 0x70, + 0x61, 0x63, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x09, 0x75, 0x73, 0x65, 0x64, 0x53, + 0x70, 0x61, 0x63, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x75, 0x73, 0x65, 0x64, 0x49, 0x6e, 0x6f, 0x64, + 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0a, 0x75, 0x73, 0x65, 0x64, 0x49, 0x6e, + 0x6f, 0x64, 0x65, 0x73, 0x22, 0x2b, 0x0a, 0x09, 0x53, 0x74, 0x61, 0x74, 0x42, 0x61, 0x74, 0x63, + 0x68, 0x12, 0x1e, 0x0a, 0x05, 0x62, 0x61, 0x74, 0x63, 0x68, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x08, 0x2e, 0x70, 0x62, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x52, 0x05, 0x62, 0x61, 0x74, 0x63, + 0x68, 0x22, 0x5c, 0x0a, 0x04, 0x45, 0x64, 0x67, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x61, 0x72, + 0x65, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, + 0x74, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x04, + 0x52, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, + 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x22, + 0x2b, 0x0a, 0x09, 0x45, 0x64, 0x67, 0x65, 0x42, 0x61, 0x74, 0x63, 0x68, 0x12, 0x1e, 0x0a, 0x05, + 0x62, 0x61, 0x74, 0x63, 0x68, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x70, 0x62, + 0x2e, 0x45, 0x64, 0x67, 0x65, 0x52, 0x05, 0x62, 0x61, 0x74, 0x63, 0x68, 0x22, 0xc1, 0x03, 0x0a, + 0x04, 0x4e, 0x6f, 0x64, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x74, + 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, + 0x14, 0x0a, 0x05, 0x66, 0x6c, 0x61, 0x67, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, + 0x66, 0x6c, 0x61, 0x67, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6d, 0x6f, 0x64, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x75, 0x69, 0x64, + 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x75, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x67, + 0x69, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x67, 0x69, 0x64, 0x12, 0x14, 0x0a, + 0x05, 0x61, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x61, 0x74, + 0x69, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x03, 0x52, 0x05, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x63, 0x74, 0x69, + 0x6d, 0x65, 0x18, 0x09, 0x20, 0x01, 0x28, 0x03, 0x52, 0x05, 0x63, 0x74, 0x69, 0x6d, 0x65, 0x12, + 0x1d, 0x0a, 0x0a, 0x61, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x65, 0x63, 0x18, 0x0a, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x09, 0x61, 0x74, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x65, 0x63, 0x12, 0x1d, + 0x0a, 0x0a, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x65, 0x63, 0x18, 0x0b, 0x20, 0x01, + 0x28, 0x05, 0x52, 0x09, 0x6d, 0x74, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x65, 0x63, 0x12, 0x1d, 0x0a, + 0x0a, 0x63, 0x74, 0x69, 0x6d, 0x65, 0x5f, 0x6e, 0x73, 0x65, 0x63, 0x18, 0x0c, 0x20, 0x01, 0x28, + 0x05, 0x52, 0x09, 0x63, 0x74, 0x69, 0x6d, 0x65, 0x4e, 0x73, 0x65, 0x63, 0x12, 0x14, 0x0a, 0x05, + 0x6e, 0x6c, 0x69, 0x6e, 0x6b, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6e, 0x6c, 0x69, + 0x6e, 0x6b, 0x12, 0x16, 0x0a, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x0e, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x06, 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x64, + 0x65, 0x76, 0x18, 0x0f, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x72, 0x64, 0x65, 0x76, 0x12, 0x16, + 0x0a, 0x06, 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x18, 0x10, 0x20, 0x01, 0x28, 0x04, 0x52, 0x06, + 0x70, 0x61, 0x72, 0x65, 0x6e, 0x74, 0x12, 0x22, 0x0a, 0x0d, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, + 0x5f, 0x61, 0x63, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0b, 0x61, + 0x63, 0x63, 0x65, 0x73, 0x73, 0x41, 0x63, 0x6c, 0x49, 0x64, 0x12, 0x24, 0x0a, 0x0e, 0x64, 0x65, + 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x61, 0x63, 0x6c, 0x5f, 0x69, 0x64, 0x18, 0x12, 0x20, 0x01, + 0x28, 0x0d, 0x52, 0x0c, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x41, 0x63, 0x6c, 0x49, 0x64, + 0x22, 0x2b, 0x0a, 0x09, 0x4e, 0x6f, 0x64, 0x65, 0x42, 0x61, 0x74, 0x63, 0x68, 0x12, 0x1e, 0x0a, + 0x05, 0x62, 0x61, 0x74, 0x63, 0x68, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x08, 0x2e, 0x70, + 0x62, 0x2e, 0x4e, 0x6f, 0x64, 0x65, 0x52, 0x05, 0x62, 0x61, 0x74, 0x63, 0x68, 0x22, 0x61, 0x0a, + 0x05, 0x53, 0x6c, 0x69, 0x63, 0x65, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x02, 0x69, 0x64, 0x12, 0x10, 0x0a, 0x03, 0x70, 0x6f, 0x73, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0d, 0x52, 0x03, 0x70, 0x6f, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x10, 0x0a, 0x03, + 0x6f, 0x66, 0x66, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6f, 0x66, 0x66, 0x12, 0x10, + 0x0a, 0x03, 0x6c, 0x65, 0x6e, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x03, 0x6c, 0x65, 0x6e, + 0x22, 0x56, 0x0a, 0x05, 0x43, 0x68, 0x75, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x6f, + 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x12, + 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, + 0x69, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x21, 0x0a, 0x06, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x73, 0x18, + 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x70, 0x62, 0x2e, 0x53, 0x6c, 0x69, 0x63, 0x65, + 0x52, 0x06, 0x73, 0x6c, 0x69, 0x63, 0x65, 0x73, 0x22, 0x2d, 0x0a, 0x0a, 0x43, 0x68, 0x75, 0x6e, + 0x6b, 0x42, 0x61, 0x74, 0x63, 0x68, 0x12, 0x1f, 0x0a, 0x05, 0x62, 0x61, 0x74, 0x63, 0x68, 0x18, + 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x70, 0x62, 0x2e, 0x43, 0x68, 0x75, 0x6e, 0x6b, + 0x52, 0x05, 0x62, 0x61, 0x74, 0x63, 0x68, 0x22, 0x42, 0x0a, 0x08, 0x53, 0x6c, 0x69, 0x63, 0x65, + 0x52, 0x65, 0x66, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, + 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x04, 0x73, 0x69, 0x7a, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x72, 0x65, 0x66, 0x73, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x03, 0x52, 0x04, 0x72, 0x65, 0x66, 0x73, 0x22, 0x33, 0x0a, 0x0d, 0x53, + 0x6c, 0x69, 0x63, 0x65, 0x52, 0x65, 0x66, 0x42, 0x61, 0x74, 0x63, 0x68, 0x12, 0x22, 0x0a, 0x05, + 0x62, 0x61, 0x74, 0x63, 0x68, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x70, 0x62, + 0x2e, 0x53, 0x6c, 0x69, 0x63, 0x65, 0x52, 0x65, 0x66, 0x52, 0x05, 0x62, 0x61, 0x74, 0x63, 0x68, + 0x22, 0x37, 0x0a, 0x07, 0x53, 0x79, 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x12, 0x14, 0x0a, 0x05, 0x69, + 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x69, 0x6e, 0x6f, 0x64, + 0x65, 0x12, 0x16, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0c, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x22, 0x31, 0x0a, 0x0c, 0x53, 0x79, 0x6d, + 0x6c, 0x69, 0x6e, 0x6b, 0x42, 0x61, 0x74, 0x63, 0x68, 0x12, 0x21, 0x0a, 0x05, 0x62, 0x61, 0x74, + 0x63, 0x68, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x70, 0x62, 0x2e, 0x53, 0x79, + 0x6d, 0x6c, 0x69, 0x6e, 0x6b, 0x52, 0x05, 0x62, 0x61, 0x74, 0x63, 0x68, 0x22, 0x47, 0x0a, 0x05, + 0x58, 0x61, 0x74, 0x74, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x04, 0x52, 0x05, 0x69, 0x6e, 0x6f, 0x64, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, + 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x2d, 0x0a, 0x0a, 0x58, 0x61, 0x74, 0x74, 0x72, 0x42, 0x61, + 0x74, 0x63, 0x68, 0x12, 0x1f, 0x0a, 0x05, 0x62, 0x61, 0x74, 0x63, 0x68, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x09, 0x2e, 0x70, 0x62, 0x2e, 0x58, 0x61, 0x74, 0x74, 0x72, 0x52, 0x05, 0x62, + 0x61, 0x74, 0x63, 0x68, 0x22, 0x2e, 0x0a, 0x08, 0x41, 0x63, 0x6c, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, 0x64, + 0x12, 0x12, 0x0a, 0x04, 0x70, 0x65, 0x72, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, + 0x70, 0x65, 0x72, 0x6d, 0x22, 0xb5, 0x01, 0x0a, 0x03, 0x41, 0x63, 0x6c, 0x12, 0x0e, 0x0a, 0x02, + 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x02, 0x69, 0x64, 0x12, 0x14, 0x0a, 0x05, + 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6f, 0x77, 0x6e, + 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0d, 0x52, 0x05, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x12, 0x14, 0x0a, 0x05, 0x6f, 0x74, 0x68, 0x65, + 0x72, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x05, 0x6f, 0x74, 0x68, 0x65, 0x72, 0x12, 0x12, + 0x0a, 0x04, 0x6d, 0x61, 0x73, 0x6b, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x04, 0x6d, 0x61, + 0x73, 0x6b, 0x12, 0x22, 0x0a, 0x05, 0x75, 0x73, 0x65, 0x72, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x0c, 0x2e, 0x70, 0x62, 0x2e, 0x41, 0x63, 0x6c, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, + 0x05, 0x75, 0x73, 0x65, 0x72, 0x73, 0x12, 0x24, 0x0a, 0x06, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, + 0x18, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x70, 0x62, 0x2e, 0x41, 0x63, 0x6c, 0x45, + 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x67, 0x72, 0x6f, 0x75, 0x70, 0x73, 0x22, 0x23, 0x0a, 0x04, + 0x41, 0x63, 0x6c, 0x73, 0x12, 0x1b, 0x0a, 0x04, 0x61, 0x63, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x07, 0x2e, 0x70, 0x62, 0x2e, 0x41, 0x63, 0x6c, 0x52, 0x04, 0x61, 0x63, 0x6c, + 0x73, 0x42, 0x06, 0x5a, 0x04, 0x2e, 0x2f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x33, +} + +var ( + file_pkg_meta_pb_backup_proto_rawDescOnce sync.Once + file_pkg_meta_pb_backup_proto_rawDescData = file_pkg_meta_pb_backup_proto_rawDesc +) + +func file_pkg_meta_pb_backup_proto_rawDescGZIP() []byte { + file_pkg_meta_pb_backup_proto_rawDescOnce.Do(func() { + file_pkg_meta_pb_backup_proto_rawDescData = protoimpl.X.CompressGZIP(file_pkg_meta_pb_backup_proto_rawDescData) + }) + return file_pkg_meta_pb_backup_proto_rawDescData +} + +var file_pkg_meta_pb_backup_proto_msgTypes = make([]protoimpl.MessageInfo, 26) +var file_pkg_meta_pb_backup_proto_goTypes = []any{ + (*Format)(nil), // 0: pb.Format + (*Counters)(nil), // 1: pb.Counters + (*Sustained)(nil), // 2: pb.Sustained + (*Sustaineds)(nil), // 3: pb.Sustaineds + (*DelFile)(nil), // 4: pb.DelFile + (*DelFiles)(nil), // 5: pb.DelFiles + (*Quota)(nil), // 6: pb.Quota + (*QuotaBatch)(nil), // 7: pb.QuotaBatch + (*Stat)(nil), // 8: pb.Stat + (*StatBatch)(nil), // 9: pb.StatBatch + (*Edge)(nil), // 10: pb.Edge + (*EdgeBatch)(nil), // 11: pb.EdgeBatch + (*Node)(nil), // 12: pb.Node + (*NodeBatch)(nil), // 13: pb.NodeBatch + (*Slice)(nil), // 14: pb.Slice + (*Chunk)(nil), // 15: pb.Chunk + (*ChunkBatch)(nil), // 16: pb.ChunkBatch + (*SliceRef)(nil), // 17: pb.SliceRef + (*SliceRefBatch)(nil), // 18: pb.SliceRefBatch + (*Symlink)(nil), // 19: pb.Symlink + (*SymlinkBatch)(nil), // 20: pb.SymlinkBatch + (*Xattr)(nil), // 21: pb.Xattr + (*XattrBatch)(nil), // 22: pb.XattrBatch + (*AclEntry)(nil), // 23: pb.AclEntry + (*Acl)(nil), // 24: pb.Acl + (*Acls)(nil), // 25: pb.Acls +} +var file_pkg_meta_pb_backup_proto_depIdxs = []int32{ + 2, // 0: pb.Sustaineds.sustaineds:type_name -> pb.Sustained + 4, // 1: pb.DelFiles.files:type_name -> pb.DelFile + 6, // 2: pb.QuotaBatch.batch:type_name -> pb.Quota + 8, // 3: pb.StatBatch.batch:type_name -> pb.Stat + 10, // 4: pb.EdgeBatch.batch:type_name -> pb.Edge + 12, // 5: pb.NodeBatch.batch:type_name -> pb.Node + 14, // 6: pb.Chunk.slices:type_name -> pb.Slice + 15, // 7: pb.ChunkBatch.batch:type_name -> pb.Chunk + 17, // 8: pb.SliceRefBatch.batch:type_name -> pb.SliceRef + 19, // 9: pb.SymlinkBatch.batch:type_name -> pb.Symlink + 21, // 10: pb.XattrBatch.batch:type_name -> pb.Xattr + 23, // 11: pb.Acl.users:type_name -> pb.AclEntry + 23, // 12: pb.Acl.groups:type_name -> pb.AclEntry + 24, // 13: pb.Acls.acls:type_name -> pb.Acl + 14, // [14:14] is the sub-list for method output_type + 14, // [14:14] is the sub-list for method input_type + 14, // [14:14] is the sub-list for extension type_name + 14, // [14:14] is the sub-list for extension extendee + 0, // [0:14] is the sub-list for field type_name +} + +func init() { file_pkg_meta_pb_backup_proto_init() } +func file_pkg_meta_pb_backup_proto_init() { + if File_pkg_meta_pb_backup_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_pkg_meta_pb_backup_proto_rawDesc, + NumEnums: 0, + NumMessages: 26, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_pkg_meta_pb_backup_proto_goTypes, + DependencyIndexes: file_pkg_meta_pb_backup_proto_depIdxs, + MessageInfos: file_pkg_meta_pb_backup_proto_msgTypes, + }.Build() + File_pkg_meta_pb_backup_proto = out.File + file_pkg_meta_pb_backup_proto_rawDesc = nil + file_pkg_meta_pb_backup_proto_goTypes = nil + file_pkg_meta_pb_backup_proto_depIdxs = nil +} diff --git a/pkg/meta/pb/backup.proto b/pkg/meta/pb/backup.proto new file mode 100644 index 000000000000..eec3466f942b --- /dev/null +++ b/pkg/meta/pb/backup.proto @@ -0,0 +1,190 @@ +syntax = "proto3"; +package pb; +option go_package = "./pb"; + +/* +1. install protocol buffer compiler +2. install Go protoc plugin (protoc-gen-go) +3. exec: protoc --go_out=pkg/meta pkg/meta/pb/backup.proto in main directory +*/ + +message Format { + string name = 1; + string uuid = 2; + string storage = 3; + string storage_class = 4; + string bucket = 5; + string access_key = 6; + string secret_key = 7; + string session_token = 8; + int32 block_size = 9; + string compression = 10; + int32 shards = 11; + bool hash_prefix = 12; + uint64 capacity = 13; + uint64 inodes = 14; + string encrypt_key = 15; + string encrypt_algo = 16; + bool key_encrypted = 17; + int64 upload_limit = 18; // Mbps + int64 download_limit = 19; // Mbps + int32 trash_days = 20; + int32 meta_version = 21; + string min_client_version = 22; + string max_client_version = 23; + bool dir_stats = 24; + bool enable_acl = 25; +} + +message Counters { + int64 usedSpace = 1; + int64 usedInodes = 2; + int64 nextInode = 3; + int64 nextChunk = 4; + int64 nextSession = 5; + int64 nextTrash = 6; +} + +message Sustained { + uint64 sid = 1; + repeated uint64 inodes = 2; +} + +message Sustaineds { + repeated Sustained sustaineds = 1; +} + +message DelFile { + uint64 inode = 1; + uint64 length = 2; + int64 expire = 3; +} + +message DelFiles { + repeated DelFile files = 1; +} + +message Quota { + uint64 inode = 1; + int64 maxSpace = 2; + int64 maxInodes = 3; + int64 usedSpace = 4; + int64 usedInodes = 5; +} + +message QuotaBatch { + repeated Quota batch = 1; +} + +message Stat { + uint64 inode = 1; + int64 dataLength = 2; + int64 usedSpace = 3; + int64 usedInodes = 4; +} + +message StatBatch { + repeated Stat batch = 1; +} + +message Edge { + uint64 parent = 1; + uint64 inode = 2; + bytes name = 3; + uint32 type = 4; +} + +message EdgeBatch { + repeated Edge batch = 1; +} + +message Node { + uint64 inode = 1; + uint32 type = 2; + uint32 flags = 3; + uint32 mode = 4; + uint32 uid = 5; + uint32 gid = 6; + int64 atime = 7; + int64 mtime = 8; + int64 ctime = 9; + int32 atime_nsec = 10; + int32 mtime_nsec = 11; + int32 ctime_nsec = 12; + uint32 nlink = 13; + uint64 length = 14; + uint32 rdev = 15; + uint64 parent = 16; + uint32 access_acl_id = 17; + uint32 default_acl_id = 18; +} + +message NodeBatch { + repeated Node batch = 1; +} + +message Slice { + uint64 id = 1; + uint32 pos = 2; + uint32 size = 3; + uint32 off = 4; + uint32 len = 5; +} + +message Chunk { + uint64 inode = 1; + uint32 index = 2; + repeated Slice slices = 3; +} + +message ChunkBatch { + repeated Chunk batch = 1; +} + +message SliceRef { + uint64 id = 1; + uint32 size = 2; + int64 refs = 3; +} + +message SliceRefBatch { + repeated SliceRef batch = 1; +} + +message Symlink { + uint64 inode = 1; + bytes target = 2; +} + +message SymlinkBatch { + repeated Symlink batch = 1; +} + +message Xattr { + uint64 inode = 1; + string name = 2; + bytes value = 3; +} + +message XattrBatch { + repeated Xattr batch = 1; +} + +message AclEntry { + uint32 id = 1; + uint32 perm = 2; +} + +message Acl { + uint32 id = 1; + uint32 owner = 2; + uint32 group = 3; + uint32 other = 4; + uint32 mask = 5; + repeated AclEntry users = 6; + repeated AclEntry groups = 7; +} + +message Acls { + repeated Acl acls = 1; +} \ No newline at end of file diff --git a/pkg/meta/redis.go b/pkg/meta/redis.go index 4c70d6f527e5..913d1bf5c689 100644 --- a/pkg/meta/redis.go +++ b/pkg/meta/redis.go @@ -48,6 +48,7 @@ import ( "github.com/juicedata/juicefs/pkg/utils" "github.com/pkg/errors" "github.com/redis/go-redis/v9" + "google.golang.org/protobuf/proto" ) /* @@ -3544,6 +3545,7 @@ func (m *redisMeta) checkServerConfig() { } func (m *redisMeta) dumpEntries(es ...*DumpedEntry) error { + logger.Infof("Dump entries: %+v", es) ctx := Background var keys []string for _, e := range es { @@ -3656,6 +3658,7 @@ func (m *redisMeta) dumpEntries(es ...*DumpedEntry) error { } case TypeDirectory: keys, cursor, err := dr[i].Result() + logger.Infof("inode %d, cursor %d, keys %d err %v", inode, cursor, len(keys), err) if err != nil { return err } @@ -4715,3 +4718,74 @@ func (s *redisDirHandler) List(ctx Context, offset int) ([]*Entry, syscall.Errno func (s *redisDirHandler) Read(offset int) { s.readOff = offset - len(s.initEntries) } + +func (m *redisMeta) DumpMetaV2(ctx Context, w io.WriteSeeker, opt *DumpOption) (err error) { + opt = opt.check() + + bak := newBakFormat() + bak.seekForWrite(w) + + type result struct { + seg segWriter + msg proto.Message + } + + ch := make(chan *result, 100) + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + defer close(ch) + for _, seg := range []iDumpSeg{ + newFormatDS(bak, m.getFormat(), opt.KeepSecret), + newCounterDS(bak), + newSustainedDS(bak), + newDelFileDS(bak), + newAclDS(bak), + } { + rows := seg.newData() + if err := queryAll(rows); err != nil { + logger.Errorf("query %s err: %v", seg, err) + ctx.Cancel() + return + } + msg := seg.encode(rows) + if msg == nil { + continue + } + select { + case <-ctx.Done(): + return + case ch <- &result{seg, msg}: + } + } + }() + + finished := false + for !finished { + select { + case <-ctx.Done(): + wg.Wait() + return ctx.Err() + case res, ok := <-ch: + if !ok { + finished = true + break + } + if err := res.seg.write(w, res.msg); err != nil { + logger.Errorf("write %s err: %v", res.seg, err) + ctx.Cancel() + wg.Wait() + return err + } + res.seg.release(res.msg) + } + } + + wg.Wait() + return bak.writeHeader(w) +} + +func (m *redisMeta) LoadMetaV2(ctx Context, r io.Reader, opt *LoadOption) error { + return nil +} diff --git a/pkg/meta/sql.go b/pkg/meta/sql.go index b9bed46f1fd6..5dec77114b79 100644 --- a/pkg/meta/sql.go +++ b/pkg/meta/sql.go @@ -355,7 +355,7 @@ func (m *dbMeta) syncTable(beans ...interface{}) error { return err } -func (m *dbMeta) doInit(format *Format, force bool) error { +func (m *dbMeta) syncAllTables() error { if err := m.syncTable(new(setting), new(counter)); err != nil { return fmt.Errorf("create table setting, counter: %s", err) } @@ -383,7 +383,13 @@ func (m *dbMeta) doInit(format *Format, force bool) error { if err := m.syncTable(new(acl)); err != nil { return fmt.Errorf("create table acl: %s", err) } + return nil +} +func (m *dbMeta) doInit(format *Format, force bool) error { + if err := m.syncAllTables(); err != nil { + return err + } var s = setting{Name: "format"} var ok bool err := m.roTxn(func(ses *xorm.Session) (err error) { @@ -3870,7 +3876,20 @@ func (m *dbMeta) loadEntry(e *DumpedEntry, chs []chan interface{}, aclMaxId *uin chs[0] <- n } -func (m *dbMeta) LoadMeta(r io.Reader) error { +func (m *dbMeta) getTxnBatchNum() int { + switch m.Name() { + case "sqlite3": + return 999 / MaxFieldsCountOfTable + case "mysql": + return 65535 / MaxFieldsCountOfTable + case "postgres": + return 1000 + default: + return 1000 + } +} + +func (m *dbMeta) checkAddr() error { tables, err := m.db.DBMetas() if err != nil { return err @@ -3880,41 +3899,20 @@ func (m *dbMeta) LoadMeta(r io.Reader) error { if !strings.Contains(addr, "://") { addr = fmt.Sprintf("%s://%s", m.Name(), addr) } - return fmt.Errorf("Database %s is not empty", addr) - } - if err = m.syncTable(new(setting), new(counter)); err != nil { - return fmt.Errorf("create table setting, counter: %s", err) - } - if err = m.syncTable(new(node), new(edge), new(symlink), new(xattr)); err != nil { - return fmt.Errorf("create table node, edge, symlink, xattr: %s", err) - } - if err = m.syncTable(new(chunk), new(sliceRef), new(delslices)); err != nil { - return fmt.Errorf("create table chunk, chunk_ref, delslices: %s", err) - } - if err = m.syncTable(new(session2), new(sustained), new(delfile)); err != nil { - return fmt.Errorf("create table session2, sustaind, delfile: %s", err) - } - if err = m.syncTable(new(flock), new(plock), new(dirQuota)); err != nil { - return fmt.Errorf("create table flock, plock, dirQuota: %s", err) - } - if err := m.syncTable(new(dirStats)); err != nil { - return fmt.Errorf("create table dirStats: %s", err) + return fmt.Errorf("database %s is not empty", addr) } - if err := m.syncTable(new(detachedNode)); err != nil { - return fmt.Errorf("create table detachedNode: %s", err) - } - if err = m.syncTable(new(acl)); err != nil { - return fmt.Errorf("create table acl: %s", err) + return nil +} + +func (m *dbMeta) LoadMeta(r io.Reader) error { + if err := m.checkAddr(); err != nil { + return err } - var batch int - switch m.Name() { - case "sqlite3": - batch = 999 / MaxFieldsCountOfTable - case "mysql": - batch = 65535 / MaxFieldsCountOfTable - case "postgres": - batch = 1000 + if err := m.syncAllTables(); err != nil { + return err } + + batch := m.getTxnBatchNum() chs := make([]chan interface{}, 6) // node, edge, chunk, chunkRef, xattr, others insert := func(index int, beans []interface{}) error { return m.txn(func(s *xorm.Session) error { diff --git a/pkg/meta/sql_bak.go b/pkg/meta/sql_bak.go new file mode 100644 index 000000000000..92183d824794 --- /dev/null +++ b/pkg/meta/sql_bak.go @@ -0,0 +1,1313 @@ +package meta + +import ( + "encoding/binary" + "fmt" + "io" + "sync" + "sync/atomic" + + "github.com/juicedata/juicefs/pkg/meta/pb" + "github.com/juicedata/juicefs/pkg/utils" + "golang.org/x/sync/errgroup" + "google.golang.org/protobuf/proto" + "xorm.io/xorm" +) + +type DumpOption struct { + KeepSecret bool + CoNum int + // SkipTrash bool +} + +func (opt *DumpOption) check() *DumpOption { + if opt == nil { + opt = &DumpOption{} + } + if opt.CoNum < 1 { + opt.CoNum = 1 + } + return opt +} + +type segNewData interface { + newData() interface{} +} + +type segWriter interface { + write(w io.Writer, msg proto.Message) error + release(msg proto.Message) +} + +type segEncoder interface { + encode(interface{}) proto.Message + count(proto.Message) int +} + +type iDumpSeg interface { + String() string + query() proto.Message + segNewData + segEncoder + segWriter +} + +type dumpSeg struct { + iDumpSeg + name string + bak *BakFormat +} + +func (s *dumpSeg) String() string { return s.name } +func (s *dumpSeg) write(w io.Writer, msg proto.Message) error { return s.bak.writeSeg(w, msg) } +func (s *dumpSeg) release(msg proto.Message) {} + +type formatDS struct { + dumpSeg + f *Format + keepSecret bool +} + +func newFormatDS(bak *BakFormat, f *Format, keepSecret bool) iDumpSeg { + return &formatDS{dumpSeg{name: "Format", bak: bak}, f, keepSecret} +} + +func (s *formatDS) newData() interface{} { return nil } + +func (s *formatDS) encode(rows interface{}) proto.Message { + return newPBFormat(s.f, s.keepSecret) +} + +func (s *formatDS) count(msg proto.Message) int { return 1 } + +type counterDS struct { + dumpSeg +} + +func newCounterDS(bak *BakFormat) iDumpSeg { + return &counterDS{dumpSeg: dumpSeg{name: "Counter", bak: bak}} +} + +func (s *counterDS) newData() interface{} { + return &[]counter{} +} + +func (s *counterDS) encode(rows interface{}) proto.Message { + crows := *(rows.(*[]counter)) + if len(crows) == 0 { + return nil + } + counters := &pb.Counters{} + fieldMap := getCounterFields(counters) + for _, row := range crows { + if fieldPtr, ok := fieldMap[row.Name]; ok { + *fieldPtr = row.Value + } + } + return counters +} + +func (s *counterDS) count(msg proto.Message) int { return 6 } + +type delFileDS struct { + dumpSeg +} + +func newDelFileDS(bak *BakFormat) iDumpSeg { + return &delFileDS{dumpSeg: dumpSeg{name: "DelFiles", bak: bak}} +} + +func (s *delFileDS) newData() interface{} { return &[]delfile{} } + +func (s *delFileDS) encode(rows interface{}) proto.Message { + drows := *(rows.(*[]delfile)) + if len(drows) == 0 { + return nil + } + delFiles := &pb.DelFiles{ + Files: make([]*pb.DelFile, 0, len(drows)), + } + for _, row := range drows { + delFiles.Files = append(delFiles.Files, &pb.DelFile{Inode: uint64(row.Inode), Length: row.Length, Expire: row.Expire}) + } + return delFiles +} + +func (s *delFileDS) count(msg proto.Message) int { return len(msg.(*pb.DelFiles).Files) } + +type sustainedDS struct { + dumpSeg +} + +func newSustainedDS(bak *BakFormat) iDumpSeg { + return &sustainedDS{dumpSeg: dumpSeg{name: "Sustaineds", bak: bak}} +} + +func (s *sustainedDS) newData() interface{} { return &[]sustained{} } + +func (s *sustainedDS) encode(rows interface{}) proto.Message { + srows := *(rows.(*[]sustained)) + if len(srows) == 0 { + return nil + } + ss := make(map[uint64][]uint64) + for _, row := range srows { + ss[row.Sid] = append(ss[row.Sid], uint64(row.Inode)) + } + + pss := &pb.Sustaineds{ + Sustaineds: make([]*pb.Sustained, 0, len(ss)), + } + for k, v := range ss { + pss.Sustaineds = append(pss.Sustaineds, &pb.Sustained{Sid: k, Inodes: v}) + } + return pss +} + +func (s *sustainedDS) count(msg proto.Message) int { return len(msg.(*pb.Sustaineds).Sustaineds) } + +type aclDS struct { + dumpSeg +} + +func newAclDS(bak *BakFormat) iDumpSeg { + return &aclDS{dumpSeg: dumpSeg{name: "Acls", bak: bak}} +} + +func (s *aclDS) newData() interface{} { return &[]acl{} } + +func (s *aclDS) encode(rows interface{}) proto.Message { + acls := *(rows.(*[]acl)) + if len(acls) == 0 { + return nil + } + pas := &pb.Acls{ + Acls: make([]*pb.Acl, 0, len(acls)), + } + for _, a := range acls { + pa := &pb.Acl{ + Id: a.Id, + Owner: uint32(a.Owner), + Group: uint32(a.Group), + Other: uint32(a.Other), + Mask: uint32(a.Mask), + } + r := utils.ReadBuffer(a.NamedUsers) + for r.HasMore() { + pa.Users = append(pa.Users, &pb.AclEntry{ + Id: r.Get32(), + Perm: uint32(r.Get16()), + }) + } + r = utils.ReadBuffer(a.NamedGroups) + for r.HasMore() { + pa.Groups = append(pa.Groups, &pb.AclEntry{ + Id: r.Get32(), + Perm: uint32(r.Get16()), + }) + } + pas.Acls = append(pas.Acls, pa) + } + return pas +} + +func (s *aclDS) count(msg proto.Message) int { return len(msg.(*pb.Acls).Acls) } + +type iDumpBatchSeg interface { + String() string + segNewData + segEncoder + segWriter +} + +type dumpBatchSeg struct { + iDumpBatchSeg + bak *BakFormat + name string + pools []*sync.Pool +} + +func (s *dumpBatchSeg) String() string { return s.name } + +func (s *dumpBatchSeg) write(w io.Writer, msg proto.Message) error { + return s.bak.writeBatchSeg(w, msg) +} + +type edgeDBS struct { + dumpBatchSeg +} + +func newEdgeDBS(bak *BakFormat) iDumpBatchSeg { + return &edgeDBS{dumpBatchSeg{name: "Edges", bak: bak, pools: []*sync.Pool{{New: func() interface{} { return &pb.Edge{} }}}}} +} + +func (s *edgeDBS) newData() interface{} { return &[]edge{} } + +func (s *edgeDBS) encode(rows interface{}) proto.Message { + edges := *(rows.(*[]edge)) + if len(edges) == 0 { + return nil + } + pes := &pb.EdgeBatch{ + Batch: make([]*pb.Edge, 0, len(edges)), + } + var pe *pb.Edge + for _, e := range edges { + pe = s.pools[0].Get().(*pb.Edge) + pe.Parent = uint64(e.Parent) + pe.Inode = uint64(e.Inode) + pe.Name = e.Name + pe.Type = uint32(e.Type) + pes.Batch = append(pes.Batch, pe) + } + return pes +} + +func (s *edgeDBS) count(msg proto.Message) int { return len(msg.(*pb.EdgeBatch).Batch) } + +func (s *edgeDBS) release(msg proto.Message) { + pes := msg.(*pb.EdgeBatch) + for _, pe := range pes.Batch { + s.pools[0].Put(pe) + } + pes.Batch = nil +} + +type nodeDBS struct { + dumpBatchSeg +} + +func newNodeDBS(bak *BakFormat) iDumpBatchSeg { + return &nodeDBS{dumpBatchSeg{name: "Nodes", bak: bak, pools: []*sync.Pool{{New: func() interface{} { return &pb.Node{} }}}}} +} + +func (s *nodeDBS) newData() interface{} { return &[]node{} } + +func (s *nodeDBS) encode(rows interface{}) proto.Message { + nodes := *(rows.(*[]node)) + if len(nodes) == 0 { + return nil + } + pns := &pb.NodeBatch{ + Batch: make([]*pb.Node, 0, len(nodes)), + } + var pn *pb.Node + for _, n := range nodes { + pn = s.pools[0].Get().(*pb.Node) + pn.Inode = uint64(n.Inode) + pn.Type = uint32(n.Type) + pn.Flags = uint32(n.Flags) + pn.Mode = uint32(n.Mode) + pn.Uid = n.Uid + pn.Gid = n.Gid + pn.Atime = n.Atime + pn.Mtime = n.Mtime + pn.Ctime = n.Ctime + pn.AtimeNsec = int32(n.Atimensec) + pn.MtimeNsec = int32(n.Mtimensec) + pn.CtimeNsec = int32(n.Ctimensec) + pn.Nlink = n.Nlink + pn.Length = n.Length + pn.Rdev = n.Rdev + pn.Parent = uint64(n.Parent) + pn.AccessAclId = n.AccessACLId + pn.DefaultAclId = n.DefaultACLId + pns.Batch = append(pns.Batch, pn) + } + return pns +} + +func (s *nodeDBS) count(msg proto.Message) int { return len(msg.(*pb.NodeBatch).Batch) } + +func (s *nodeDBS) release(msg proto.Message) { + pns := msg.(*pb.NodeBatch) + for _, pn := range pns.Batch { + s.pools[0].Put(pn) + } + pns.Batch = nil +} + +type chunkDBS struct { + dumpBatchSeg +} + +func newChunkDBS(bak *BakFormat) iDumpBatchSeg { + return &chunkDBS{dumpBatchSeg{name: "Chunks", bak: bak, pools: []*sync.Pool{{New: func() interface{} { return &pb.Chunk{} }}, {New: func() interface{} { return &pb.Slice{} }}}}} +} + +func (s *chunkDBS) newData() interface{} { return &[]chunk{} } + +func (s *chunkDBS) encode(rows interface{}) proto.Message { + chunks := *(rows.(*[]chunk)) + if len(chunks) == 0 { + return nil + } + pcs := &pb.ChunkBatch{ + Batch: make([]*pb.Chunk, 0, len(chunks)), + } + var pc *pb.Chunk + for _, c := range chunks { + pc = s.pools[0].Get().(*pb.Chunk) + pc.Inode = uint64(c.Inode) + pc.Index = c.Indx + + n := len(c.Slices) / sliceBytes + pc.Slices = make([]*pb.Slice, 0, n) + var ps *pb.Slice + for i := 0; i < n; i++ { + ps = s.pools[1].Get().(*pb.Slice) + rb := utils.ReadBuffer(c.Slices[i*sliceBytes:]) + ps.Pos = rb.Get32() + ps.Id = rb.Get64() + ps.Size = rb.Get32() + ps.Off = rb.Get32() + ps.Len = rb.Get32() + pc.Slices = append(pc.Slices, ps) + } + pcs.Batch = append(pcs.Batch, pc) + } + return pcs +} + +func (s *chunkDBS) count(msg proto.Message) int { return len(msg.(*pb.ChunkBatch).Batch) } + +func (s *chunkDBS) release(msg proto.Message) { + pcs := msg.(*pb.ChunkBatch) + for _, pc := range pcs.Batch { + for _, ps := range pc.Slices { + s.pools[1].Put(ps) + } + pc.Slices = nil + s.pools[0].Put(pc) + } + pcs.Batch = nil +} + +type sliceRefDBS struct { + dumpBatchSeg +} + +func newSliceRefDBS(bak *BakFormat) iDumpBatchSeg { + return &sliceRefDBS{dumpBatchSeg{name: "SliceRefs", bak: bak, pools: []*sync.Pool{{New: func() interface{} { return &pb.SliceRef{} }}}}} +} + +func (s *sliceRefDBS) newData() interface{} { return &[]sliceRef{} } + +func (s *sliceRefDBS) encode(rows interface{}) proto.Message { + sliceRefs := *(rows.(*[]sliceRef)) + if len(sliceRefs) == 0 { + return nil + } + psrs := &pb.SliceRefBatch{ + Batch: make([]*pb.SliceRef, 0, len(sliceRefs)), + } + var psr *pb.SliceRef + for _, sr := range sliceRefs { + psr = s.pools[0].Get().(*pb.SliceRef) + psr.Id = sr.Id + psr.Size = sr.Size + psr.Refs = int64(sr.Refs) + psrs.Batch = append(psrs.Batch, psr) + } + return psrs +} + +func (s *sliceRefDBS) count(msg proto.Message) int { return len(msg.(*pb.SliceRefBatch).Batch) } + +func (s *sliceRefDBS) release(msg proto.Message) { + psrs := msg.(*pb.SliceRefBatch) + for _, psr := range psrs.Batch { + s.pools[0].Put(psr) + } + psrs.Batch = nil +} + +type symlinkDBS struct { + dumpBatchSeg +} + +func newSymlinkDBS(bak *BakFormat) iDumpBatchSeg { + return &symlinkDBS{dumpBatchSeg{name: "Symlinks", bak: bak, pools: []*sync.Pool{{New: func() interface{} { return &pb.Symlink{} }}}}} +} + +func (s *symlinkDBS) newData() interface{} { return &[]symlink{} } + +func (s *symlinkDBS) encode(rows interface{}) proto.Message { + symlinks := *(rows.(*[]symlink)) + if len(symlinks) == 0 { + return nil + } + pss := &pb.SymlinkBatch{ + Batch: make([]*pb.Symlink, 0, len(symlinks)), + } + var ps *pb.Symlink + for _, sl := range symlinks { + ps = s.pools[0].Get().(*pb.Symlink) + ps.Inode = uint64(sl.Inode) + ps.Target = sl.Target + pss.Batch = append(pss.Batch, ps) + } + return pss +} + +func (s *symlinkDBS) count(msg proto.Message) int { return len(msg.(*pb.SymlinkBatch).Batch) } + +func (s *symlinkDBS) release(msg proto.Message) { + pss := msg.(*pb.SymlinkBatch) + for _, ps := range pss.Batch { + s.pools[0].Put(ps) + } + pss.Batch = nil +} + +type xattrDBS struct { + dumpBatchSeg +} + +func newXattrDBS(bak *BakFormat) iDumpBatchSeg { + return &xattrDBS{dumpBatchSeg{name: "Xattrs", bak: bak, pools: []*sync.Pool{{New: func() interface{} { return &pb.Xattr{} }}}}} +} + +func (s *xattrDBS) newData() interface{} { return &[]xattr{} } + +func (s *xattrDBS) encode(rows interface{}) proto.Message { + xattrs := *(rows.(*[]xattr)) + if len(xattrs) == 0 { + return nil + } + pxs := &pb.XattrBatch{ + Batch: make([]*pb.Xattr, 0, len(xattrs)), + } + var px *pb.Xattr + for _, x := range xattrs { + px = s.pools[0].Get().(*pb.Xattr) + px.Inode = uint64(x.Inode) + px.Name = x.Name + px.Value = x.Value + pxs.Batch = append(pxs.Batch, px) + } + return pxs +} + +func (s *xattrDBS) count(msg proto.Message) int { return len(msg.(*pb.XattrBatch).Batch) } + +func (s *xattrDBS) release(msg proto.Message) { + pxs := msg.(*pb.XattrBatch) + for _, px := range pxs.Batch { + s.pools[0].Put(px) + } + pxs.Batch = nil +} + +type quotaDBS struct { + dumpBatchSeg +} + +func newQuotaDBS(bak *BakFormat) iDumpBatchSeg { + return "aDBS{dumpBatchSeg{name: "Quotas", bak: bak, pools: []*sync.Pool{{New: func() interface{} { return &pb.Quota{} }}}}} +} + +func (s *quotaDBS) newData() interface{} { return &[]dirQuota{} } + +func (s *quotaDBS) encode(rows interface{}) proto.Message { + quotas := *(rows.(*[]dirQuota)) + if len(quotas) == 0 { + return nil + } + pqs := &pb.QuotaBatch{ + Batch: make([]*pb.Quota, 0, len(quotas)), + } + var pq *pb.Quota + for _, q := range quotas { + pq = s.pools[0].Get().(*pb.Quota) + pq.Inode = uint64(q.Inode) + pq.MaxSpace = q.MaxSpace + pq.MaxInodes = q.MaxInodes + pq.UsedSpace = q.UsedSpace + pq.UsedInodes = q.UsedInodes + pqs.Batch = append(pqs.Batch, pq) + } + return pqs +} + +func (s *quotaDBS) count(msg proto.Message) int { return len(msg.(*pb.QuotaBatch).Batch) } + +func (s *quotaDBS) release(msg proto.Message) { + pqs := msg.(*pb.QuotaBatch) + for _, pq := range pqs.Batch { + s.pools[0].Put(pq) + } + pqs.Batch = nil +} + +type statDBS struct { + dumpBatchSeg +} + +func newStatDBS(bak *BakFormat) iDumpBatchSeg { + return &statDBS{dumpBatchSeg{name: "Stats", bak: bak, pools: []*sync.Pool{{New: func() interface{} { return &pb.Stat{} }}}}} +} + +func (s *statDBS) newData() interface{} { return &[]dirStats{} } + +func (s *statDBS) encode(rows interface{}) proto.Message { + stats := *(rows.(*[]dirStats)) + if len(stats) == 0 { + return nil + } + pss := &pb.StatBatch{ + Batch: make([]*pb.Stat, 0, len(stats)), + } + var ps *pb.Stat + for _, st := range stats { + ps = s.pools[0].Get().(*pb.Stat) + ps.Inode = uint64(st.Inode) + ps.DataLength = st.DataLength + ps.UsedInodes = st.UsedInodes + ps.UsedSpace = st.UsedSpace + pss.Batch = append(pss.Batch, ps) + } + return pss +} + +func (s *statDBS) count(msg proto.Message) int { return len(msg.(*pb.StatBatch).Batch) } + +func (s *statDBS) release(msg proto.Message) { + pss := msg.(*pb.StatBatch) + for _, ps := range pss.Batch { + s.pools[0].Put(ps) + } + pss.Batch = nil +} + +func (m *dbMeta) DumpMetaV2(ctx Context, w io.WriteSeeker, opt *DumpOption) (err error) { + opt = opt.check() + + bak := newBakFormat() + bak.seekForWrite(w) + + queryAll := func(rows interface{}) error { + if rows == nil { + return nil + } + return m.roTxn(func(s *xorm.Session) error { + return s.Find(rows) + }) + } + + queryPage := func(rows interface{}, limit, start int) error { + return m.roTxn(func(s *xorm.Session) error { + return s.Limit(limit, start).Find(rows) + }) + } + + type result struct { + seg segWriter + msg proto.Message + } + + ch := make(chan *result, 100) + wg := &sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + defer close(ch) + for _, seg := range []iDumpSeg{ + newFormatDS(bak, m.getFormat(), opt.KeepSecret), + newCounterDS(bak), + newSustainedDS(bak), + newDelFileDS(bak), + newAclDS(bak), + } { + // 1. 不同engine查询实现 -> proto.Message + // 1.1 sql: newData + encode, 查询语句相同, 只是面向对象不同 + // 1.2 redis: 不同查询语句, 转换成proto.Message + // 1.3 tkv: 不同遍历语句, 转换成proto.Message + // 2. proto.Message 写入到 w + + // TODO: 不同存储引擎创建不同seg实现, 其中的查询实现不同 + + rows := seg.newData() + if err := queryAll(rows); err != nil { + logger.Errorf("query %s err: %v", seg, err) + ctx.Cancel() + return + } + msg := seg.encode(rows) + if msg == nil { + continue + } + select { + case <-ctx.Done(): + return + case ch <- &result{seg, msg}: + } + } + + for _, seg := range []iDumpBatchSeg{ + newNodeDBS(bak), + newChunkDBS(bak), + newEdgeDBS(bak), + newSliceRefDBS(bak), + newSymlinkDBS(bak), + newXattrDBS(bak), + newQuotaDBS(bak), + newStatDBS(bak), + } { + // 不同存储引擎, batch的处理实现不同 + // 内部的查询实现也要区分 + eg, nCtx := errgroup.WithContext(ctx) + eg.SetLimit(opt.CoNum) + + taskFinished := false + limit := 40960 + sum := int64(0) + for start := 0; !taskFinished; start += limit { + nSeg, nStart := seg, start + eg.Go(func() error { + rows := nSeg.newData() + if err := queryPage(rows, limit, int(nStart)); err != nil { + taskFinished = true + return err + } + msg := nSeg.encode(rows) + if msg == nil { + taskFinished = true // real end + return nil + } + atomic.AddInt64(&sum, int64(nSeg.count(msg))) + select { + case <-nCtx.Done(): + taskFinished = true + return nCtx.Err() + case ch <- &result{nSeg, msg}: + } + return nil + }) + } + if err := eg.Wait(); err != nil { + logger.Errorf("query %s err: %v", seg, err) + ctx.Cancel() + return + } + logger.Infof("dump %s total num %d", seg, sum) // TODO hjf debug + } + }() + + finished := false + for !finished { + select { + case <-ctx.Done(): + wg.Wait() + return ctx.Err() + case res, ok := <-ch: + if !ok { + finished = true + break + } + if err := res.seg.write(w, res.msg); err != nil { + logger.Errorf("write %s err: %v", res.seg, err) + ctx.Cancel() + wg.Wait() + return err + } + res.seg.release(res.msg) + } + } + + wg.Wait() + return bak.writeHeader(w) +} + +type LoadOption struct { + CoNum int +} + +func (opt *LoadOption) check() *LoadOption { + if opt == nil { + opt = &LoadOption{} + } + if opt.CoNum < 1 { + opt.CoNum = 1 + } + return opt +} + +type segReader interface { + read(r io.Reader, msg proto.Message) error +} + +type segDecoder interface { + decode(proto.Message) []interface{} + count(proto.Message) int + release(rows []interface{}) +} + +type iLoadSeg interface { + String() string + newMsg() proto.Message + segReader + segDecoder +} + +type loadSeg struct { + iLoadSeg + bak *BakFormat + name string + size uint32 +} + +func (s *loadSeg) String() string { return s.name } + +func (s *loadSeg) read(r io.Reader, msg proto.Message) error { + return s.bak.readSeg(r, s.name, int(s.size), msg) +} + +func (s *loadSeg) release(rows []interface{}) {} + +type formatLS struct { + loadSeg +} + +func newFormatLS(bak *BakFormat) iLoadSeg { + return &formatLS{loadSeg{bak: bak, name: "Format", size: bak.Header.FormatSize}} +} + +func (s *formatLS) newMsg() proto.Message { return nil } +func (s *formatLS) decode(msg proto.Message) []interface{} { return nil } +func (s *formatLS) count(msg proto.Message) int { return 1 } + +type counterLS struct { + loadSeg +} + +func newCounterLS(bak *BakFormat) iLoadSeg { + return &counterLS{loadSeg{bak: bak, name: "Counters", size: bak.Header.CounterSize}} +} + +func (s *counterLS) newMsg() proto.Message { return &pb.Counters{} } + +func (s *counterLS) decode(msg proto.Message) []interface{} { + fields := getCounterFields(msg.(*pb.Counters)) + counters := make([]interface{}, 0, len(fields)) + for name, field := range fields { + counters = append(counters, &counter{Name: name, Value: *field}) + } + return counters +} + +func (s *counterLS) count(msg proto.Message) int { return 6 } + +type sustainedLS struct { + loadSeg +} + +func newSustainedLS(bak *BakFormat) iLoadSeg { + return &sustainedLS{loadSeg{bak: bak, name: "Sustaineds", size: bak.Header.SustainedSize}} +} + +func (s *sustainedLS) newMsg() proto.Message { return &pb.Sustaineds{} } + +func (s *sustainedLS) decode(msg proto.Message) []interface{} { + sustaineds := msg.(*pb.Sustaineds) + rows := make([]interface{}, 0, len(sustaineds.Sustaineds)) + for _, s := range sustaineds.Sustaineds { + for _, inode := range s.Inodes { + rows = append(rows, &sustained{Sid: s.Sid, Inode: Ino(inode)}) + } + } + return rows +} + +func (s *sustainedLS) count(msg proto.Message) int { return len(msg.(*pb.Sustaineds).Sustaineds) } + +type delFileLS struct { + loadSeg +} + +func newDelFileLS(bak *BakFormat) iLoadSeg { + return &delFileLS{loadSeg{bak: bak, name: "DelFiles", size: bak.Header.DelFileSize}} +} + +func (s *delFileLS) newMsg() proto.Message { return &pb.DelFiles{} } + +func (s *delFileLS) decode(msg proto.Message) []interface{} { + delfiles := msg.(*pb.DelFiles) + rows := make([]interface{}, 0, len(delfiles.Files)) + for _, f := range delfiles.Files { + rows = append(rows, &delfile{Inode: Ino(f.Inode), Length: f.Length, Expire: f.Expire}) + } + return rows +} + +func (s *delFileLS) count(msg proto.Message) int { return len(msg.(*pb.DelFiles).Files) } + +type aclLS struct { + loadSeg +} + +func newAclLS(bak *BakFormat) iLoadSeg { + return &aclLS{loadSeg: loadSeg{bak: bak, name: "Acls", size: bak.Header.AclSize}} +} + +func (s *aclLS) newMsg() proto.Message { return &pb.Acls{} } + +func (s *aclLS) decode(msg proto.Message) []interface{} { + acls := msg.(*pb.Acls) + rows := make([]interface{}, 0, len(acls.Acls)) + for _, a := range acls.Acls { + ba := &acl{} + ba.Id = a.Id + ba.Owner = uint16(a.Owner) + ba.Group = uint16(a.Group) + ba.Mask = uint16(a.Mask) + ba.Other = uint16(a.Other) + + w := utils.NewBuffer(uint32(len(a.Users) * 6)) + for _, u := range a.Users { + w.Put32(u.Id) + w.Put16(uint16(u.Perm)) + } + ba.NamedUsers = w.Bytes() + + w = utils.NewBuffer(uint32(len(a.Groups) * 6)) + for _, g := range a.Groups { + w.Put32(g.Id) + w.Put16(uint16(g.Perm)) + } + ba.NamedGroups = w.Bytes() + rows = append(rows, ba) + } + return rows +} + +func (s *aclLS) count(msg proto.Message) int { return len(msg.(*pb.Acls).Acls) } + +type batchSegReader interface { + read(r io.Reader, idx int, msg proto.Message) error +} + +type iLoadBatchSeg interface { + num() int + String() string + newMsg() proto.Message + batchSegReader + segDecoder +} + +type loadBatchSeg struct { + iLoadBatchSeg + bak *BakFormat + name string + n uint32 + pools []*sync.Pool +} + +func (s *loadBatchSeg) String() string { return s.name } + +func (s *loadBatchSeg) num() int { return int(s.n) } + +func (s *loadBatchSeg) read(r io.Reader, idx int, msg proto.Message) error { + return s.bak.readBatchSeg(r, fmt.Sprintf("%s-%d", s.name, idx), msg) +} + +func (s *loadBatchSeg) release(rows []interface{}) { + for _, row := range rows { + s.pools[0].Put(row) + } +} + +type edgeLBS struct { + loadBatchSeg +} + +func newEdgeLBS(bak *BakFormat) iLoadBatchSeg { + return &edgeLBS{loadBatchSeg{bak: bak, name: "Edges", n: bak.Header.EdgeBatchNum, pools: []*sync.Pool{{New: func() interface{} { return &edge{} }}}}} +} + +func (s *edgeLBS) newMsg() proto.Message { return &pb.EdgeBatch{} } + +func (s *edgeLBS) decode(msg proto.Message) []interface{} { + edges := msg.(*pb.EdgeBatch) + rows := make([]interface{}, 0, len(edges.Batch)) + var pe *edge + for _, e := range edges.Batch { + pe = s.pools[0].Get().(*edge) + pe.Id = 0 + pe.Parent = Ino(e.Parent) + pe.Inode = Ino(e.Inode) + pe.Name = e.Name + pe.Type = uint8(e.Type) + rows = append(rows, pe) + } + return rows +} + +func (s *edgeLBS) count(msg proto.Message) int { return len(msg.(*pb.EdgeBatch).Batch) } + +type nodeLBS struct { + loadBatchSeg +} + +func newNodeLBS(bak *BakFormat) iLoadBatchSeg { + return &nodeLBS{loadBatchSeg{bak: bak, name: "Nodes", n: bak.Header.NodeBatchNum, pools: []*sync.Pool{{New: func() interface{} { return &node{} }}}}} +} + +func (s *nodeLBS) newMsg() proto.Message { return &pb.NodeBatch{} } + +func (s *nodeLBS) decode(msg proto.Message) []interface{} { + nodes := msg.(*pb.NodeBatch) + rows := make([]interface{}, 0, len(nodes.Batch)) + var pn *node + for _, n := range nodes.Batch { + pn = s.pools[0].Get().(*node) + pn.Inode = Ino(n.Inode) + pn.Type = uint8(n.Type) + pn.Flags = uint8(n.Flags) + pn.Mode = uint16(n.Mode) + pn.Uid = n.Uid + pn.Gid = n.Gid + pn.Atime = n.Atime + pn.Mtime = n.Mtime + pn.Ctime = n.Ctime + pn.Atimensec = int16(n.AtimeNsec) + pn.Mtimensec = int16(n.MtimeNsec) + pn.Ctimensec = int16(n.CtimeNsec) + pn.Nlink = n.Nlink + pn.Length = n.Length + pn.Rdev = n.Rdev + pn.Parent = Ino(n.Parent) + pn.AccessACLId = n.AccessAclId + pn.DefaultACLId = n.DefaultAclId + rows = append(rows, pn) + } + return rows +} + +func (s *nodeLBS) count(msg proto.Message) int { return len(msg.(*pb.NodeBatch).Batch) } + +type chunkLBS struct { + loadBatchSeg +} + +func newChunkLBS(bak *BakFormat) iLoadBatchSeg { + return &chunkLBS{ + loadBatchSeg{ + bak: bak, + name: "Chunks", + n: bak.Header.ChunkBatchNum, + pools: []*sync.Pool{{New: func() interface{} { return &chunk{} }}, {New: func() interface{} { return make([]byte, 0) }}}, + }, + } +} + +func (s *chunkLBS) newMsg() proto.Message { return &pb.ChunkBatch{} } + +func (s *chunkLBS) decode(msg proto.Message) []interface{} { + chunks := msg.(*pb.ChunkBatch) + rows := make([]interface{}, 0, len(chunks.Batch)) + var pc *chunk + for _, c := range chunks.Batch { + pc = s.pools[0].Get().(*chunk) + pc.Id = 0 + pc.Inode = Ino(c.Inode) + pc.Indx = c.Index + + n := len(c.Slices) * sliceBytes + pc.Slices = s.pools[1].Get().([]byte)[:0] + if cap(pc.Slices) < n { + pc.Slices = make([]byte, 0, n) + } + for _, s := range c.Slices { + // keep BigEndian order for slices, same as in slice.go + pc.Slices = binary.BigEndian.AppendUint32(pc.Slices, s.Pos) + pc.Slices = binary.BigEndian.AppendUint64(pc.Slices, s.Id) + pc.Slices = binary.BigEndian.AppendUint32(pc.Slices, s.Size) + pc.Slices = binary.BigEndian.AppendUint32(pc.Slices, s.Off) + pc.Slices = binary.BigEndian.AppendUint32(pc.Slices, s.Len) + } + rows = append(rows, pc) + } + return rows +} + +func (s *chunkLBS) count(msg proto.Message) int { return len(msg.(*pb.ChunkBatch).Batch) } + +func (s *chunkLBS) release(rows []interface{}) { + for _, row := range rows { + pc := row.(*chunk) + s.pools[1].Put(pc.Slices) + pc.Slices = nil + s.pools[0].Put(pc) + } +} + +type sliceRefLBS struct { + loadBatchSeg +} + +func newSliceRefLBS(bak *BakFormat) iLoadBatchSeg { + return &sliceRefLBS{loadBatchSeg{bak: bak, name: "SliceRefs", n: bak.Header.SliceRefBatchNum, pools: []*sync.Pool{{New: func() interface{} { return &sliceRef{} }}}}} +} + +func (s *sliceRefLBS) newMsg() proto.Message { return &pb.SliceRefBatch{} } + +func (s *sliceRefLBS) decode(msg proto.Message) []interface{} { + sliceRefs := msg.(*pb.SliceRefBatch) + rows := make([]interface{}, 0, len(sliceRefs.Batch)) + var ps *sliceRef + for _, sr := range sliceRefs.Batch { + ps = s.pools[0].Get().(*sliceRef) + ps.Id = sr.Id + ps.Size = sr.Size + ps.Refs = int(sr.Refs) + rows = append(rows, ps) + } + return rows +} + +func (s *sliceRefLBS) count(msg proto.Message) int { return len(msg.(*pb.SliceRefBatch).Batch) } + +type symlinkLBS struct { + loadBatchSeg +} + +func newSymLinkLBS(bak *BakFormat) iLoadBatchSeg { + return &symlinkLBS{loadBatchSeg{bak: bak, name: "Symlinks", n: bak.Header.SymlinkBatchNum, pools: []*sync.Pool{{New: func() interface{} { return &symlink{} }}}}} +} + +func (s *symlinkLBS) newMsg() proto.Message { return &pb.SymlinkBatch{} } + +func (s *symlinkLBS) decode(msg proto.Message) []interface{} { + symlinks := msg.(*pb.SymlinkBatch) + rows := make([]interface{}, 0, len(symlinks.Batch)) + var ps *symlink + for _, sl := range symlinks.Batch { + ps = s.pools[0].Get().(*symlink) + ps.Inode = Ino(sl.Inode) + ps.Target = sl.Target + rows = append(rows, ps) + } + return rows +} + +func (s *symlinkLBS) count(msg proto.Message) int { return len(msg.(*pb.SymlinkBatch).Batch) } + +type xattrLBS struct { + loadBatchSeg +} + +func newXattrLBS(bak *BakFormat) iLoadBatchSeg { + return &xattrLBS{loadBatchSeg{bak: bak, name: "Xattrs", n: bak.Header.XattrBatchNum, pools: []*sync.Pool{{New: func() interface{} { return &xattr{} }}}}} +} + +func (s *xattrLBS) newMsg() proto.Message { return &pb.XattrBatch{} } + +func (s *xattrLBS) decode(msg proto.Message) []interface{} { + xattrs := msg.(*pb.XattrBatch) + rows := make([]interface{}, 0, len(xattrs.Batch)) + var px *xattr + for _, x := range xattrs.Batch { + px = s.pools[0].Get().(*xattr) + px.Id = 0 + px.Inode = Ino(x.Inode) + px.Name = x.Name + px.Value = x.Value + rows = append(rows, px) + } + return rows +} + +func (s *xattrLBS) count(msg proto.Message) int { return len(msg.(*pb.XattrBatch).Batch) } + +type quotaLBS struct { + loadBatchSeg +} + +func newQuotaLBS(bak *BakFormat) iLoadBatchSeg { + return "aLBS{loadBatchSeg{bak: bak, name: "Quotas", n: bak.Header.QuotaBatchNum, pools: []*sync.Pool{{New: func() interface{} { return &dirQuota{} }}}}} +} + +func (s *quotaLBS) newMsg() proto.Message { return &pb.QuotaBatch{} } + +func (s *quotaLBS) decode(msg proto.Message) []interface{} { + quotas := msg.(*pb.QuotaBatch) + rows := make([]interface{}, 0, len(quotas.Batch)) + var pq *dirQuota + for _, q := range quotas.Batch { + pq = s.pools[0].Get().(*dirQuota) + pq.Inode = Ino(q.Inode) + pq.MaxSpace = q.MaxSpace + pq.MaxInodes = q.MaxInodes + pq.UsedSpace = q.UsedSpace + pq.UsedInodes = q.UsedInodes + rows = append(rows, pq) + } + return rows +} + +func (s *quotaLBS) count(msg proto.Message) int { return len(msg.(*pb.QuotaBatch).Batch) } + +type statLBS struct { + loadBatchSeg +} + +func newStatLBS(bak *BakFormat) iLoadBatchSeg { + return &statLBS{loadBatchSeg{bak: bak, name: "Stats", n: bak.Header.StatBatchNum, pools: []*sync.Pool{{New: func() interface{} { return &dirStats{} }}}}} +} + +func (s *statLBS) newMsg() proto.Message { return &pb.StatBatch{} } + +func (s *statLBS) decode(msg proto.Message) []interface{} { + stats := msg.(*pb.StatBatch) + rows := make([]interface{}, 0, len(stats.Batch)) + var ps *dirStats + for _, st := range stats.Batch { + ps = s.pools[0].Get().(*dirStats) + ps.Inode = Ino(st.Inode) + ps.DataLength = st.DataLength + ps.UsedInodes = st.UsedInodes + ps.UsedSpace = st.UsedSpace + rows = append(rows, ps) + } + return rows +} + +func (s *statLBS) count(msg proto.Message) int { return len(msg.(*pb.StatBatch).Batch) } + +func (m *dbMeta) LoadMetaV2(ctx Context, r io.Reader, opt *LoadOption) error { + opt = opt.check() + if err := m.checkAddr(); err != nil { + return err + } + if err := m.syncAllTables(); err != nil { + return err + } + + bak := newBakFormat() + if err := bak.readHeader(r); err != nil { + return err + } + + insert := func(beans []interface{}) error { + return m.txn(func(s *xorm.Session) error { + n, err := s.Insert(beans...) + if err == nil && int(n) != len(beans) { + err = fmt.Errorf("only %d records inserted", n) + } + return err + }) + } + + type task struct { + msg proto.Message + seg segDecoder + } + + var wg sync.WaitGroup + batch := m.getTxnBatchNum() + simTaskCh := make(chan *task, 6) + batchTaskCh := make(chan *task, 100) + + workerFunc := func(ctx Context, batchSize int, taskCh <-chan *task) { + defer wg.Done() + finished := false + for !finished { + select { + case <-ctx.Done(): + return + case task, ok := <-taskCh: + if !ok { + finished = true + break + } + if task.msg == nil { + continue + } + rows, bs := task.seg.decode(task.msg), batchSize + for len(rows) > 0 { + if len(rows) < bs { + bs = len(rows) + } + if err := insert(rows[:bs]); err != nil { + // TODO hjf + for _, row := range rows[:bs] { + logger.Errorf("Write bean: %+v", row) + } + + logger.Errorf("Write %d beans: %s", len(rows), err) + ctx.Cancel() + return + } + task.seg.release(rows[:bs]) + rows = rows[bs:] + } + } + } + } + + wg.Add(1) + go workerFunc(ctx, batch, simTaskCh) + + for i := 0; i < opt.CoNum; i++ { + wg.Add(1) + go workerFunc(ctx, batch, batchTaskCh) + } + + var msg proto.Message + var err error + for _, seg := range []iLoadSeg{ + newFormatLS(bak), + newCounterLS(bak), + newSustainedLS(bak), + newDelFileLS(bak), + newAclLS(bak), + } { + msg = seg.newMsg() + if err := seg.read(r, seg.newMsg()); err != nil { + ctx.Cancel() + wg.Wait() + return err + } + select { + case <-ctx.Done(): + wg.Wait() + return ctx.Err() + case simTaskCh <- &task{msg, seg}: + } + } + close(simTaskCh) + + for _, seg := range []iLoadBatchSeg{ + newNodeLBS(bak), + newChunkLBS(bak), + newEdgeLBS(bak), + newSliceRefLBS(bak), + newSymLinkLBS(bak), + newXattrLBS(bak), + newQuotaLBS(bak), + newStatLBS(bak), + } { + sum := int64(0) + for i := 0; i < seg.num(); i++ { + msg = seg.newMsg() + if err = seg.read(r, i+1, msg); err != nil { + ctx.Cancel() + wg.Wait() + return err + } + select { + case <-ctx.Done(): + wg.Wait() + return ctx.Err() + case batchTaskCh <- &task{msg, seg}: + } + atomic.AddInt64(&sum, int64(seg.count(msg))) + } + logger.Infof("load %s total num %d", seg, sum) // TODO hjf debug + } + close(batchTaskCh) + + wg.Wait() + return nil +} diff --git a/pkg/meta/tkv.go b/pkg/meta/tkv.go index 078ab70ade27..256d6b8e8e70 100644 --- a/pkg/meta/tkv.go +++ b/pkg/meta/tkv.go @@ -3872,3 +3872,11 @@ func (m *kvMeta) getDirFetcher() dirFetcher { return entries[len(entries)-1].Name, entries, nil } } + +func (m *kvMeta) DumpMetaV2(ctx Context, w io.WriteSeeker, opt *DumpOption) (err error) { + return nil +} + +func (m *kvMeta) LoadMetaV2(ctx Context, r io.Reader, opt *LoadOption) error { + return nil +}