Introduce Peapod

CHANGELOG.md

@@ -3,6 +3,9 @@ Changelog for NeoFS Node
 
 ## [Unreleased]
 
+This is the last release to support the Blobovnicza tree. Starting with the next
+minor release, the component will be purged, so be prepared (see `Updating` section).
+
 ### Added
 - Embedded Neo contracts in `contracts` dir (#2391)
 - `dump-names` command for adm
@@ -11,6 +14,8 @@ Changelog for NeoFS Node
 - Stored payload metric per shard (#2023)
 - Histogram metrics for RPC and engine operations (#2351) 
 - SN's version is announced via the attributes automatically but can be overwritten explicitly (#2455)
+- New storage component for small objects named Peapod (#2453)
+- New `blobovnicza-to-peapod` tool providing blobovnicza-to-peapod data migration (#2453)
 
 ### Fixed
 - `neo-go` RPC connection loss handling (#1337)
@@ -71,6 +76,22 @@ Docker images now contain a single executable file and SSL certificates only.
 
 `neofs-cli control healthcheck` exit code is `0` only for "READY" state.
 
+To migrate data from Blobovnicza trees to Peapods:
+```shell
+$ blobovnicza-to-peapod -config </path/to/storage/node/config>
+```
+For any shard, the data from the configured Blobovnicza tree is copied into
+a created Peapod file named `peapod.db` in the directory where the tree is
+located. For example, `/neofs/data/blobovcniza/*` -> `/neofs/data/peapod.db`.
+Notice that existing Blobovnicza trees are untouched. Configuration is also
+updated, for example, `/etc/neofs/config.yaml` -> `/etc/neofs/config_peapod.yaml`.
+WARN: carefully review the updated config before using it in the application!
+
+To store small objects in more effective and simple sub-storage Peapod, replace
+`blobovnicza` sub-storage with the `peapod` one in `blobstor` config section.
+If storage node already stores some data, don't forget to make data migration
+described above.
+
 ## [0.37.0] - 2023-06-15 - Sogado
 
 ### Added

cmd/blobovnicza-to-peapod/main.go

@@ -0,0 +1,262 @@
+package main
+
+import (
+	"errors"
+	"flag"
+	"fmt"
+	"io/fs"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
+	engineconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine"
+	shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard"
+	blobovniczaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/common"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/compression"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/peapod"
+	"gopkg.in/yaml.v3"
+)
+
+func main() {
+	nodeCfgPath := flag.String("config", "", "Path to storage node's YAML configuration file")
+
+	flag.Parse()
+
+	if *nodeCfgPath == "" {
+		log.Fatal("missing storage node config flag")
+	}
+
+	appCfg := config.New(config.Prm{}, config.WithConfigFile(*nodeCfgPath))
+
+	err := engineconfig.IterateShards(appCfg, false, func(sc *shardconfig.Config) error {
+		log.Println("processing shard...")
+
+		var bbcz common.Storage
+		var perm fs.FileMode
+		storagesCfg := sc.BlobStor().Storages()
+
+		for i := range storagesCfg {
+			if storagesCfg[i].Type() == blobovniczatree.Type {
+				bbczCfg := blobovniczaconfig.From((*config.Config)(storagesCfg[i]))
+
+				perm = storagesCfg[i].Perm()
+				bbcz = blobovniczatree.NewBlobovniczaTree(
+					blobovniczatree.WithRootPath(storagesCfg[i].Path()),
+					blobovniczatree.WithPermissions(storagesCfg[i].Perm()),
+					blobovniczatree.WithBlobovniczaSize(bbczCfg.Size()),
+					blobovniczatree.WithBlobovniczaShallowDepth(bbczCfg.ShallowDepth()),
+					blobovniczatree.WithBlobovniczaShallowWidth(bbczCfg.ShallowWidth()),
+					blobovniczatree.WithOpenedCacheSize(bbczCfg.OpenedCacheSize()))
+
+				break
+			}
+		}
+
+		if bbcz == nil {
+			log.Println("Blobovnicza is not configured for the current shard, going to next one...")
+			return nil
+		}
+
+		bbczPath := bbcz.Path()
+		if !filepath.IsAbs(bbczPath) {
+			log.Fatalf("Blobobvnicza tree path '%s' is not absolute, make it like this in the config file first\n", bbczPath)
+		}
+
+		ppdPath := filepath.Join(filepath.Dir(bbcz.Path()), "peapod.db")
+		ppd := peapod.New(ppdPath, perm, 10*time.Millisecond)
+
+		var compressCfg compression.Config
+		compressCfg.Enabled = sc.Compress()
+		compressCfg.UncompressableContentTypes = sc.UncompressableContentTypes()
+
+		err := compressCfg.Init()
+		if err != nil {
+			log.Fatal("init compression config for the current shard: ", err)
+		}
+
+		bbcz.SetCompressor(&compressCfg)
+		ppd.SetCompressor(&compressCfg)
+
+		log.Printf("migrating data from Blobovnicza tree '%s' to Peapod '%s'...\n", bbcz.Path(), ppd.Path())
+
+		err = common.Copy(ppd, bbcz)
+		if err != nil {
+			log.Fatal("migration failed: ", err)
+		}
+
+		log.Println("data successfully migrated in the current shard, going to the next one...")
+
+		return nil
+	})
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	srcPath := *nodeCfgPath
+	ss := strings.Split(srcPath, ".")
+	ss[0] += "_peapod"
+
+	dstPath := strings.Join(ss, ".")
+
+	log.Printf("data successfully migrated in all shards, migrating configuration to '%s' file...\n", dstPath)
+
+	err = migrateConfigToPeapod(dstPath, srcPath)
+	if err != nil {
+		log.Fatal(err)
+	}
+}
+
+func migrateConfigToPeapod(dstPath, srcPath string) error {
+	fData, err := os.ReadFile(srcPath)
+	if err != nil {
+		return fmt.Errorf("read source config config file: %w", err)
+	}
+
+	var mConfig map[any]any
+
+	err = yaml.Unmarshal(fData, &mConfig)
+	if err != nil {
+		return fmt.Errorf("decode config from YAML: %w", err)
+	}
+
+	v, ok := mConfig["storage"]
+	if !ok {
+		return errors.New("missing 'storage' section")
+	}
+
+	mStorage, ok := v.(map[any]any)
+	if !ok {
+		return fmt.Errorf("unexpected 'storage' section type: %T instead of %T", v, mStorage)
+	}
+
+	v, ok = mStorage["shard"]
+	if !ok {
+		return errors.New("missing 'storage.shard' section")
+	}
+
+	mShards, ok := v.(map[any]any)
+	if !ok {
+		return fmt.Errorf("unexpected 'storage.shard' section type: %T instead of %T", v, mShards)
+	}
+
+	replaceBlobovniczaWithPeapod := func(mShard map[any]any, shardDesc any) error {
+		v, ok := mShard["blobstor"]
+		if !ok {
+			return fmt.Errorf("missing 'blobstor' section in shard '%v' config", shardDesc)
+		}
+
+		sBlobStor, ok := v.([]any)
+		if !ok {
+			return fmt.Errorf("unexpected 'blobstor' section type in shard '%v': %T instead of %T", shardDesc, v, sBlobStor)
+		}
+
+		var bbczSubStorage map[any]any
+
+		for i := range sBlobStor {
+			mSubStorage, ok := sBlobStor[i].(map[any]any)
+			if !ok {
+				return fmt.Errorf("unexpected sub-storage #%d type in shard '%v': %T instead of %T", i, shardDesc, v, mStorage)
+			}
+
+			v, ok := mSubStorage["type"]
+			if ok {
+				typ, ok := v.(string)
+				if !ok {
+					return fmt.Errorf("unexpected type of sub-storage name: %T instead of %T", v, typ)
+				}
+
+				if typ == blobovniczatree.Type {
+					bbczSubStorage = mSubStorage
+				}
+
+				continue
+			}
+
+			// in 'default' section 'type' may be missing
+
+			_, withDepth := mSubStorage["depth"]
+			_, withWidth := mSubStorage["width"]
+
+			if withWidth && withDepth {
+				bbczSubStorage = mSubStorage
+			}
+		}
+
+		if bbczSubStorage == nil {
+			log.Printf("blobovnicza tree is not configured for the shard '%s', skip\n", shardDesc)
+			return nil
+		}
+
+		for k := range bbczSubStorage {
+			switch k {
+			default:
+				delete(bbczSubStorage, k)
+			case "type", "path", "perm":
+			}
+		}
+
+		bbczSubStorage["type"] = peapod.Type
+
+		v, ok = bbczSubStorage["path"]
+		if ok {
+			path, ok := v.(string)
+			if !ok {
+				return fmt.Errorf("unexpected sub-storage path type: %T instead of %T", v, path)
+			}
+
+			bbczSubStorage["path"] = filepath.Join(filepath.Dir(path), "peapod.db")
+		}
+
+		return nil
+	}
+
+	v, ok = mShards["default"]
+	if ok {
+		mShard, ok := v.(map[any]any)
+		if !ok {
+			return fmt.Errorf("unexpected 'storage.shard.default' section type: %T instead of %T", v, mShard)
+		}
+
+		err = replaceBlobovniczaWithPeapod(mShard, "default")
+		if err != nil {
+			return err
+		}
+	}
+
+	for i := 0; ; i++ {
+		v, ok = mShards[i]
+		if !ok {
+			if i == 0 {
+				return errors.New("missing numbered shards")
+			}
+			break
+		}
+
+		mShard, ok := v.(map[any]any)
+		if !ok {
+			return fmt.Errorf("unexpected 'storage.shard.%d' section type: %T instead of %T", i, v, mStorage)
+		}
+
+		err = replaceBlobovniczaWithPeapod(mShard, i)
+		if err != nil {
+			return err
+		}
+	}
+
+	data, err := yaml.Marshal(mConfig)
+	if err != nil {
+		return fmt.Errorf("encode modified config into YAML: %w", err)
+	}
+
+	err = os.WriteFile(dstPath, data, 0640)
+	if err != nil {
+		return fmt.Errorf("write resulting config to the destination file: %w", err)
+	}
+
+	return nil
+}

cmd/neofs-node/config.go

@@ -25,6 +25,7 @@ import (
 	shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard"
 	blobovniczaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza"
 	fstreeconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/fstree"
+	peapodconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/peapod"
 	loggerconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/logger"
 	metricsconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/metrics"
 	nodeconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/node"
@@ -36,6 +37,7 @@ import (
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/blobovniczatree"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/fstree"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/peapod"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/engine"
 	meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/pilorama"
@@ -158,16 +160,21 @@ func (c *shardCfg) id() string {
 
 type subStorageCfg struct {
 	// common for all storages
-	typ    string
-	path   string
-	perm   fs.FileMode
+	typ  string
+	path string
+	perm fs.FileMode
+
+	// tree-specific (FS and blobovnicza)
 	depth  uint64
 	noSync bool
 
 	// blobovnicza-specific
 	size            uint64
 	width           uint64
 	openedCacheSize int
+
+	// Peapod-specific
+	flushInterval time.Duration
 }
 
 // readConfig fills applicationConfiguration with raw configuration values
@@ -264,6 +271,9 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error {
 				sub := fstreeconfig.From((*config.Config)(storagesCfg[i]))
 				sCfg.depth = sub.Depth()
 				sCfg.noSync = sub.NoSync()
+			case peapod.Type:
+				peapodCfg := peapodconfig.From((*config.Config)(storagesCfg[i]))
+				sCfg.flushInterval = peapodCfg.FlushInterval()
 			default:
 				return fmt.Errorf("invalid storage type: %s", storagesCfg[i].Type())
 			}
@@ -732,6 +742,13 @@ func (c *cfg) shardOpts() []shardOptsWithID {
 						return true
 					},
 				})
+			case peapod.Type:
+				ss = append(ss, blobstor.SubStorage{
+					Storage: peapod.New(sRead.path, sRead.perm, sRead.flushInterval),
+					Policy: func(_ *objectSDK.Object, data []byte) bool {
+						return uint64(len(data)) < shCfg.smallSizeObjectLimit
+					},
+				})
 			default:
 				// should never happen, that has already
 				// been handled: when the config was read

cmd/neofs-node/config/engine/config_test.go

@@ -10,8 +10,10 @@ import (
 	shardconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard"
 	blobovniczaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/blobovnicza"
 	fstreeconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/fstree"
+	peapodconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/blobstor/peapod"
 	piloramaconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/pilorama"
 	configtest "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/test"
+	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/blobstor/peapod"
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/shard/mode"
 	"github.com/stretchr/testify/require"
 )
@@ -132,13 +134,11 @@ func TestEngineSection(t *testing.T) {
 				require.EqualValues(t, 102400, sc.SmallSizeLimit())
 
 				require.Equal(t, 2, len(ss))
-
-				blz := blobovniczaconfig.From((*config.Config)(ss[0]))
-				require.Equal(t, "tmp/1/blob/blobovnicza", ss[0].Path())
-				require.EqualValues(t, 4194304, blz.Size())
-				require.EqualValues(t, 1, blz.ShallowDepth())
-				require.EqualValues(t, 4, blz.ShallowWidth())
-				require.EqualValues(t, 50, blz.OpenedCacheSize())
+				ppd := peapodconfig.From((*config.Config)(ss[0]))
+				require.Equal(t, "tmp/1/blob/peapod.db", ss[0].Path())
+				require.EqualValues(t, 0644, ss[0].Perm())
+				require.EqualValues(t, peapod.Type, ss[0].Type())
+				require.EqualValues(t, 30*time.Millisecond, ppd.FlushInterval())
 
 				require.Equal(t, "tmp/1/blob", ss[1].Path())
 				require.EqualValues(t, 0644, ss[1].Perm())