From 9327f226842e8c23fe9c65f3899ad4c5c01a5444 Mon Sep 17 00:00:00 2001 From: Leonard Lyubich Date: Wed, 2 Aug 2023 15:05:26 +0400 Subject: [PATCH] node: Use Peapod instead of Blobovnicza tree as BlobStor sub-storage Recently introduce Peapod component is designed to replace Blobovnicza tree. To facilitate migration to a new storage scheme, it is required to maintain compatibility with the old application configuration, while still moving to a new component. Provide `common.Copy` function which overtakes data from one sub-storage to another. On storage node startup, when Blobovnicza tree is configured for BlobStor, create new Peapod instance on the same level as Blobovnicza and migrate data into it. Filled Peapod is used as sub-storage. Signed-off-by: Leonard Lyubich --- CHANGELOG.md | 8 +++ cmd/neofs-node/config.go | 59 ++++++++++++++--- .../blobstor/common/storage.go | 61 +++++++++++++++++- .../blobstor/common/storage_test.go | 63 +++++++++++++++++++ 4 files changed, 181 insertions(+), 10 deletions(-) create mode 100644 pkg/local_object_storage/blobstor/common/storage_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 155d3d29594..d0b90443c90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,7 @@ Changelog for NeoFS Node - BlobStor tries to store object in any sub-storage with free space (#2450) - SN does not store container estimations in-mem forever (#2472) - CLI `neofs-cli container set-eacl` checks container's ownership (#2436) +- BlobStor stores small objects in new sub-storage Peapod instead of Blobovnicza tree (#2453) ### Updated - `neofs-sdk-go` to `v1.0.0-rc.10` @@ -71,6 +72,13 @@ Docker images now contain a single executable file and SSL certificates only. `neofs-cli control healthcheck` exit code is `0` only for "READY" state. +The local data storage scheme is reorganized automatically: for any shard, the +data from the configured Blobovnicza tree is copied into a created Peapod file +named `peapod.db` in the directory where the tree is located. For example, +`/neofs/data/blobovcniza/*` -> `/neofs/data/peapod.db`. Storage node doesn't +touch existing Blobovnicza tree, but the Peapod is used as `blobovnicza` sub-storage +instead. + ## [0.37.0] - 2023-06-15 - Sogado ### Added diff --git a/cmd/neofs-node/config.go b/cmd/neofs-node/config.go index 5644ef7b205..65598225621 100644 --- a/cmd/neofs-node/config.go +++ b/cmd/neofs-node/config.go @@ -35,7 +35,10 @@ import ( netmapCore "github.com/nspcc-dev/neofs-node/pkg/core/netmap" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/peapod" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama" @@ -707,16 +710,54 @@ func (c *cfg) shardOpts() []shardOptsWithID { for _, sRead := range shCfg.subStorages { switch sRead.typ { case blobovniczatree.Type: + ppdPath := filepath.Join(filepath.Dir(sRead.path), "peapod.db") + ppd := peapod.New(ppdPath, sRead.perm) + + _, err := os.Stat(ppdPath) + if err == nil { + c.log.Info("data was already migrated from Blobovnicza tree to Peapod, continue with it") + ss = append(ss, blobstor.SubStorage{ + Storage: ppd, + Policy: func(_ *objectSDK.Object, data []byte) bool { + return uint64(len(data)) < shCfg.smallSizeObjectLimit + }, + }) + break + } + + if !errors.Is(err, fs.ErrNotExist) { + fatalOnErrDetails("get info about Peapod file", err) + } + + var compressCfg compression.Config + compressCfg.Enabled = shCfg.compress + compressCfg.UncompressableContentTypes = shCfg.uncompressableContentType + + err = compressCfg.Init() + fatalOnErrDetails("init compression config", err) + + bbcz := blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithRootPath(sRead.path), + blobovniczatree.WithPermissions(sRead.perm), + blobovniczatree.WithBlobovniczaSize(sRead.size), + blobovniczatree.WithBlobovniczaShallowDepth(sRead.depth), + blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), + blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), + blobovniczatree.WithLogger(c.log)) + bbcz.SetCompressor(&compressCfg) + + ppd.SetCompressor(&compressCfg) + + c.log.Info("configured Blobovnicza tree is deprecated, overtaking data into Peapod...", + zap.String("from", bbcz.Path()), zap.String("to", ppd.Path())) + + err = common.Copy(ppd, bbcz) + fatalOnErrDetails("overtake data from Blobovcnicza tree to Peapod", err) + + c.log.Info("data was successfully migrated from Blobovnicza tree to Peapod") + ss = append(ss, blobstor.SubStorage{ - Storage: blobovniczatree.NewBlobovniczaTree( - blobovniczatree.WithRootPath(sRead.path), - blobovniczatree.WithPermissions(sRead.perm), - blobovniczatree.WithBlobovniczaSize(sRead.size), - blobovniczatree.WithBlobovniczaShallowDepth(sRead.depth), - blobovniczatree.WithBlobovniczaShallowWidth(sRead.width), - blobovniczatree.WithOpenedCacheSize(sRead.openedCacheSize), - - blobovniczatree.WithLogger(c.log)), + Storage: ppd, Policy: func(_ *objectSDK.Object, data []byte) bool { return uint64(len(data)) < shCfg.smallSizeObjectLimit }, diff --git a/pkg/local_object_storage/blobstor/common/storage.go b/pkg/local_object_storage/blobstor/common/storage.go index b66b4120057..6a6ecd9a910 100644 --- a/pkg/local_object_storage/blobstor/common/storage.go +++ b/pkg/local_object_storage/blobstor/common/storage.go @@ -1,6 +1,10 @@ package common -import "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression" +import ( + "fmt" + + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression" +) // Storage represents key-value object storage. // It is used as a building block for a blobstor of a shard. @@ -23,3 +27,58 @@ type Storage interface { Delete(DeletePrm) (DeleteRes, error) Iterate(IteratePrm) (IterateRes, error) } + +// Copy copies all objects from source Storage into the destination one. If any +// object cannot be stored, Copy immediately fails. +func Copy(dst, src Storage) error { + err := src.Open(true) + if err != nil { + return fmt.Errorf("open source sub-storage: %w", err) + } + + defer func() { _ = src.Close() }() + + err = src.Init() + if err != nil { + return fmt.Errorf("initialize source sub-storage: %w", err) + } + + err = dst.Open(false) + if err != nil { + return fmt.Errorf("open destination sub-storage: %w", err) + } + + defer func() { _ = dst.Close() }() + + err = dst.Init() + if err != nil { + return fmt.Errorf("initialize destination sub-storage: %w", err) + } + + _, err = src.Iterate(IteratePrm{ + Handler: func(el IterationElement) error { + exRes, err := dst.Exists(ExistsPrm{ + Address: el.Address, + }) + if err != nil { + return fmt.Errorf("check presence of object %s in the destination sub-storage: %w", el.Address, err) + } else if exRes.Exists { + return nil + } + + _, err = dst.Put(PutPrm{ + Address: el.Address, + RawData: el.ObjectData, + }) + if err != nil { + return fmt.Errorf("put object %s into destination sub-storage: %w", el.Address, err) + } + return nil + }, + }) + if err != nil { + return fmt.Errorf("iterate over source sub-storage: %w", err) + } + + return nil +} diff --git a/pkg/local_object_storage/blobstor/common/storage_test.go b/pkg/local_object_storage/blobstor/common/storage_test.go new file mode 100644 index 00000000000..793862cbb30 --- /dev/null +++ b/pkg/local_object_storage/blobstor/common/storage_test.go @@ -0,0 +1,63 @@ +package common_test + +import ( + "crypto/rand" + "path/filepath" + "testing" + + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common" + "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/peapod" + oid "github.com/nspcc-dev/neofs-sdk-go/object/id" + oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test" + "github.com/stretchr/testify/require" +) + +func TestCopy(t *testing.T) { + dir := t.TempDir() + const nObjects = 100 + + src := blobovniczatree.NewBlobovniczaTree( + blobovniczatree.WithBlobovniczaShallowWidth(2), + blobovniczatree.WithBlobovniczaShallowDepth(3), + blobovniczatree.WithRootPath(filepath.Join(dir, "blobovnicza")), + ) + + require.NoError(t, src.Open(false)) + require.NoError(t, src.Init()) + + mObjs := make(map[oid.Address][]byte, nObjects) + + for i := 0; i < nObjects; i++ { + addr := oidtest.Address() + data := make([]byte, 32) + rand.Read(data) + mObjs[addr] = data + + _, err := src.Put(common.PutPrm{ + Address: addr, + RawData: data, + }) + require.NoError(t, err) + } + + require.NoError(t, src.Close()) + + dst := peapod.New(filepath.Join(dir, "peapod.db"), 0600) + + err := common.Copy(dst, src) + require.NoError(t, err) + + require.NoError(t, dst.Open(true)) + t.Cleanup(func() { _ = dst.Close() }) + + _, err = dst.Iterate(common.IteratePrm{ + Handler: func(el common.IterationElement) error { + data, ok := mObjs[el.Address] + require.True(t, ok) + require.Equal(t, data, el.ObjectData) + return nil + }, + }) + require.NoError(t, err) +}