writecache: drop BoltDB

Previously, the write-cache has 2 components: bolt database and fstree. BoltDB is absolutely useless for write caching given that the cache uses SSD, so it was decided to drop this database. Drop all code related to BoltDB. Migrate from this database to the main storage, by flushing objects. Remove config parameters for the write-cache: `small_object_size`, `workers_number`, `max_batch_delay`, `max_batch_size`. Update docs. Rewrite lens commands for the current version of the write-cache. Closes #3076. Signed-off-by: Andrey Butusov <[email protected]>
nspcc-dev · Jan 28, 2025 · b9784dd · b9784dd
1 parent dd69e1d
commit b9784dd
Show file tree

Hide file tree

Showing 36 changed files with 167 additions and 789 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,12 +15,17 @@ Changelog for NeoFS Node
 
 ### Changed
 - Number of cuncurrenly handled notifications from the chain was increased from 10 to 300 for IR (#3068)
+- Rewrite lens commands for the current version of the write-cache (#3091)
 
 ### Removed
+- BoltDB from write-cache (#3091)
 
 ### Updated
 
 ### Updating from v0.44.2
+`small_object_size`, `workers_number`, `max_batch_size` and `max_batch_delay`
+paramteters are removed from `writecache` config. These parameters are related
+to the BoltDB part of the write-cache, which is dropped from the code.
 
 ## [0.44.2] - 2024-12-20
 

diff --git a/cmd/neofs-lens/internal/printers.go b/cmd/neofs-lens/internal/printers.go
@@ -95,9 +95,8 @@ func PrintStorageObjectStatus(cmd *cobra.Command, status engine.ObjectStatus) {
 			if shard.Shard.Metabase.Error != nil {
 				cmd.Printf("\t\tMetabase object error:\t%v\n", shard.Shard.Metabase.Error)
 			}
-			if shard.Shard.Writecache.PathDB != "" || shard.Shard.Writecache.PathFSTree != "" {
+			if shard.Shard.Writecache.PathFSTree != "" {
 				cmd.Printf("\tWritecache\n")
-				cmd.Printf("\t\tWritecache DB path:\t%s\n", shard.Shard.Writecache.PathDB)
 				cmd.Printf("\t\tWritecache FSTree path:\t%s\n", shard.Shard.Writecache.PathFSTree)
 			}
 			cmd.Println()

diff --git a/cmd/neofs-lens/internal/storage/root.go b/cmd/neofs-lens/internal/storage/root.go
@@ -82,11 +82,7 @@ func openEngine() (*engine.StorageEngine, error) {
 
 			wc.Enabled = true
 			wc.Path = writeCacheCfg.Path()
-			wc.MaxBatchSize = writeCacheCfg.BoltDB().MaxBatchSize()
-			wc.MaxBatchDelay = writeCacheCfg.BoltDB().MaxBatchDelay()
 			wc.MaxObjSize = writeCacheCfg.MaxObjectSize()
-			wc.SmallObjectSize = writeCacheCfg.SmallObjectSize()
-			wc.FlushWorkerCount = writeCacheCfg.WorkersNumber()
 			wc.SizeLimit = writeCacheCfg.SizeLimit()
 			wc.NoSync = writeCacheCfg.NoSync()
 		}
@@ -166,11 +162,7 @@ func openEngine() (*engine.StorageEngine, error) {
 		if wcRead := shCfg.WritecacheCfg; wcRead.Enabled {
 			writeCacheOpts = append(writeCacheOpts,
 				writecache.WithPath(wcRead.Path),
-				writecache.WithMaxBatchSize(wcRead.MaxBatchSize),
-				writecache.WithMaxBatchDelay(wcRead.MaxBatchDelay),
 				writecache.WithMaxObjectSize(wcRead.MaxObjSize),
-				writecache.WithSmallObjectSize(wcRead.SmallObjectSize),
-				writecache.WithFlushWorkersCount(wcRead.FlushWorkerCount),
 				writecache.WithMaxCacheSize(wcRead.SizeLimit),
 				writecache.WithNoSync(wcRead.NoSync),
 			)

diff --git a/cmd/neofs-lens/internal/writecache/get.go b/cmd/neofs-lens/internal/writecache/get.go
@@ -4,8 +4,7 @@ import (
 	"fmt"
 
 	common "github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
-	"github.com/nspcc-dev/neofs-sdk-go/object"
+	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
 	"github.com/spf13/cobra"
 )
 
@@ -25,23 +24,23 @@ func init() {
 }
 
 func getFunc(cmd *cobra.Command, _ []string) error {
-	db, err := openWC()
+	wc, err := openWC()
 	if err != nil {
 		return err
 	}
-	defer db.Close()
+	defer wc.Close()
 
-	data, err := writecache.Get(db, []byte(vAddress))
+	addr, err := oid.DecodeAddressString(vAddress)
 	if err != nil {
-		return fmt.Errorf("could not fetch object: %w", err)
+		return fmt.Errorf("invalid address: %w", err)
 	}
 
-	var o object.Object
-	if err := o.Unmarshal(data); err != nil {
-		return fmt.Errorf("could not unmarshal object: %w", err)
+	obj, err := wc.Get(addr)
+	if err != nil {
+		return fmt.Errorf("could not fetch object: %w", err)
 	}
 
-	common.PrintObjectHeader(cmd, o)
+	common.PrintObjectHeader(cmd, *obj)
 
-	return common.WriteObjectToFile(cmd, vOut, data, vPayloadOnly)
+	return common.WriteObjectToFile(cmd, vOut, obj.Marshal(), vPayloadOnly)
 }
diff --git a/cmd/neofs-lens/internal/writecache/list.go b/cmd/neofs-lens/internal/writecache/list.go
@@ -5,7 +5,6 @@ import (
 	"io"
 
 	common "github.com/nspcc-dev/neofs-node/cmd/neofs-lens/internal"
-	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
 	oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
 	"github.com/spf13/cobra"
 )
@@ -26,18 +25,18 @@ func listFunc(cmd *cobra.Command, _ []string) error {
 	// other targets can be supported
 	w := cmd.OutOrStderr()
 
-	wAddr := func(addr oid.Address) error {
+	wAddr := func(addr oid.Address, _ []byte) error {
 		_, err := io.WriteString(w, fmt.Sprintf("%s\n", addr))
 		return err
 	}
 
-	db, err := openWC()
+	wc, err := openWC()
 	if err != nil {
 		return err
 	}
-	defer db.Close()
+	defer wc.Close()
 
-	err = writecache.IterateDB(db, wAddr)
+	err = wc.Iterate(wAddr, true)
 	if err != nil {
 		return fmt.Errorf("write-cache iterator failure: %w", err)
 	}

diff --git a/cmd/neofs-lens/internal/writecache/root.go b/cmd/neofs-lens/internal/writecache/root.go
@@ -5,7 +5,6 @@ import (
 
 	"github.com/nspcc-dev/neofs-node/pkg/local_object_storage/writecache"
 	"github.com/spf13/cobra"
-	"go.etcd.io/bbolt"
 )
 
 var (
@@ -26,11 +25,14 @@ func init() {
 	Root.AddCommand(getCMD)
 }
 
-func openWC() (*bbolt.DB, error) {
-	db, err := writecache.OpenDB(vPath, true)
+// openWC opens and returns read-only writecache.Cache located in vPath.
+func openWC() (writecache.Cache, error) {
+	ws := writecache.New(writecache.WithPath(vPath))
+
+	err := ws.Open(true)
 	if err != nil {
-		return nil, fmt.Errorf("could not open write-cache db: %w", err)
+		return nil, fmt.Errorf("could not open write-cache: %w", err)
 	}
 
-	return db, nil
+	return ws, nil
 }
diff --git a/cmd/neofs-node/config.go b/cmd/neofs-node/config.go
@@ -198,11 +198,7 @@ func (a *applicationConfiguration) readConfig(c *config.Config) error {
 
 			wc.Enabled = true
 			wc.Path = writeCacheCfg.Path()
-			wc.MaxBatchSize = writeCacheCfg.BoltDB().MaxBatchSize()
-			wc.MaxBatchDelay = writeCacheCfg.BoltDB().MaxBatchDelay()
 			wc.MaxObjSize = writeCacheCfg.MaxObjectSize()
-			wc.SmallObjectSize = writeCacheCfg.SmallObjectSize()
-			wc.FlushWorkerCount = writeCacheCfg.WorkersNumber()
 			wc.SizeLimit = writeCacheCfg.SizeLimit()
 			wc.NoSync = writeCacheCfg.NoSync()
 		}

diff --git a/cmd/neofs-node/config/engine/config_test.go b/cmd/neofs-node/config/engine/config_test.go
@@ -74,9 +74,7 @@ func TestEngineSection(t *testing.T) {
 				require.Equal(t, true, wc.NoSync())
 
 				require.Equal(t, "tmp/0/cache", wc.Path())
-				require.EqualValues(t, 16384, wc.SmallObjectSize())
 				require.EqualValues(t, 134217728, wc.MaxObjectSize())
-				require.EqualValues(t, 30, wc.WorkersNumber())
 				require.EqualValues(t, 3221225472, wc.SizeLimit())
 
 				require.Equal(t, "tmp/0/meta", meta.Path())
@@ -117,9 +115,7 @@ func TestEngineSection(t *testing.T) {
 				require.Equal(t, false, wc.NoSync())
 
 				require.Equal(t, "tmp/1/cache", wc.Path())
-				require.EqualValues(t, 16384, wc.SmallObjectSize())
 				require.EqualValues(t, 134217728, wc.MaxObjectSize())
-				require.EqualValues(t, 30, wc.WorkersNumber())
 				require.EqualValues(t, 4294967296, wc.SizeLimit())
 
 				require.Equal(t, "tmp/1/meta", meta.Path())

diff --git a/cmd/neofs-node/config/engine/shard/writecache/config.go b/cmd/neofs-node/config/engine/shard/writecache/config.go
@@ -2,23 +2,16 @@ package writecacheconfig
 
 import (
 	"github.com/nspcc-dev/neofs-node/cmd/neofs-node/config"
-	boltdbconfig "github.com/nspcc-dev/neofs-node/cmd/neofs-node/config/engine/shard/boltdb"
 )
 
 // Config is a wrapper over the config section
 // which provides access to WriteCache configurations.
 type Config config.Config
 
 const (
-	// SmallSizeDefault is a default size of small objects.
-	SmallSizeDefault = 32 << 10
-
 	// MaxSizeDefault is a default value of the object payload size limit.
 	MaxSizeDefault = 64 << 20
 
-	// WorkersNumberDefault is a default number of workers.
-	WorkersNumberDefault = 20
-
 	// SizeLimitDefault is a default write-cache size limit.
 	SizeLimitDefault = 1 << 30
 )
@@ -51,22 +44,6 @@ func (x *Config) Path() string {
 	return p
 }
 
-// SmallObjectSize returns the value of "small_object_size" config parameter.
-//
-// Returns SmallSizeDefault if the value is not a positive number.
-func (x *Config) SmallObjectSize() uint64 {
-	s := config.SizeInBytesSafe(
-		(*config.Config)(x),
-		"small_object_size",
-	)
-
-	if s > 0 {
-		return s
-	}
-
-	return SmallSizeDefault
-}
-
 // MaxObjectSize returns the value of "max_object_size" config parameter.
 //
 // Returns MaxSizeDefault if the value is not a positive number.
@@ -83,22 +60,6 @@ func (x *Config) MaxObjectSize() uint64 {
 	return MaxSizeDefault
 }
 
-// WorkersNumber returns the value of "workers_number" config parameter.
-//
-// Returns WorkersNumberDefault if the value is not a positive number.
-func (x *Config) WorkersNumber() int {
-	c := config.IntSafe(
-		(*config.Config)(x),
-		"workers_number",
-	)
-
-	if c > 0 {
-		return int(c)
-	}
-
-	return WorkersNumberDefault
-}
-
 // SizeLimit returns the value of "capacity" config parameter.
 //
 // Returns SizeLimitDefault if the value is not a positive number.
@@ -121,8 +82,3 @@ func (x *Config) SizeLimit() uint64 {
 func (x *Config) NoSync() bool {
 	return config.BoolSafe((*config.Config)(x), "no_sync")
 }
-
-// BoltDB returns config instance for querying bolt db specific parameters.
-func (x *Config) BoltDB() *boltdbconfig.Config {
-	return (*boltdbconfig.Config)(x)
-}
diff --git a/cmd/neofs-node/config/internal/validate/config.go b/cmd/neofs-node/config/internal/validate/config.go
@@ -141,15 +141,11 @@ type shardDetails struct {
 	ResyncMetabase bool   `mapstructure:"resync_metabase"`
 
 	WriteCache struct {
-		Enabled         bool          `mapstructure:"enabled"`
-		Path            string        `mapstructure:"path"`
-		Capacity        string        `mapstructure:"capacity"`
-		NoSync          bool          `mapstructure:"no_sync"`
-		SmallObjectSize string        `mapstructure:"small_object_size"`
-		MaxObjectSize   string        `mapstructure:"max_object_size"`
-		WorkersNumber   int           `mapstructure:"workers_number"`
-		MaxBatchDelay   time.Duration `mapstructure:"max_batch_delay"`
-		MaxBatchSize    int           `mapstructure:"max_batch_size"`
+		Enabled       bool   `mapstructure:"enabled"`
+		Path          string `mapstructure:"path"`
+		Capacity      string `mapstructure:"capacity"`
+		NoSync        bool   `mapstructure:"no_sync"`
+		MaxObjectSize string `mapstructure:"max_object_size"`
 	} `mapstructure:"writecache"`
 
 	Metabase struct {

diff --git a/cmd/neofs-node/storage.go b/cmd/neofs-node/storage.go
@@ -111,11 +111,7 @@ func (c *cfg) shardOpts() []shardOptsWithID {
 		if wcRead := shCfg.WritecacheCfg; wcRead.Enabled {
 			writeCacheOpts = append(writeCacheOpts,
 				writecache.WithPath(wcRead.Path),
-				writecache.WithMaxBatchSize(wcRead.MaxBatchSize),
-				writecache.WithMaxBatchDelay(wcRead.MaxBatchDelay),
 				writecache.WithMaxObjectSize(wcRead.MaxObjSize),
-				writecache.WithSmallObjectSize(wcRead.SmallObjectSize),
-				writecache.WithFlushWorkersCount(wcRead.FlushWorkerCount),
 				writecache.WithMaxCacheSize(wcRead.SizeLimit),
 				writecache.WithNoSync(wcRead.NoSync),
 				writecache.WithLogger(c.log),

diff --git a/config/example/node.env b/config/example/node.env
@@ -101,9 +101,7 @@ NEOFS_STORAGE_SHARD_0_MODE=read-only
 NEOFS_STORAGE_SHARD_0_WRITECACHE_ENABLED=false
 NEOFS_STORAGE_SHARD_0_WRITECACHE_NO_SYNC=true
 NEOFS_STORAGE_SHARD_0_WRITECACHE_PATH=tmp/0/cache
-NEOFS_STORAGE_SHARD_0_WRITECACHE_SMALL_OBJECT_SIZE=16384
 NEOFS_STORAGE_SHARD_0_WRITECACHE_MAX_OBJECT_SIZE=134217728
-NEOFS_STORAGE_SHARD_0_WRITECACHE_WORKERS_NUMBER=30
 NEOFS_STORAGE_SHARD_0_WRITECACHE_CAPACITY=3221225472
 ### Metabase config
 NEOFS_STORAGE_SHARD_0_METABASE_PATH=tmp/0/meta
@@ -142,9 +140,7 @@ NEOFS_STORAGE_SHARD_1_MODE=read-write
 ### Write cache config
 NEOFS_STORAGE_SHARD_1_WRITECACHE_ENABLED=true
 NEOFS_STORAGE_SHARD_1_WRITECACHE_PATH=tmp/1/cache
-NEOFS_STORAGE_SHARD_1_WRITECACHE_SMALL_OBJECT_SIZE=16384
 NEOFS_STORAGE_SHARD_1_WRITECACHE_MAX_OBJECT_SIZE=134217728
-NEOFS_STORAGE_SHARD_1_WRITECACHE_WORKERS_NUMBER=30
 NEOFS_STORAGE_SHARD_1_WRITECACHE_CAPACITY=4294967296
 ### Metabase config
 NEOFS_STORAGE_SHARD_1_METABASE_PATH=tmp/1/meta

diff --git a/config/example/node.json b/config/example/node.json
@@ -128,9 +128,7 @@
           "enabled": false,
           "no_sync": true,
           "path": "tmp/0/cache",
-          "small_object_size": 16384,
           "max_object_size": 134217728,
-          "workers_number": 30,
           "capacity": 3221225472
         },
         "metabase": {
@@ -173,9 +171,7 @@
         "writecache": {
           "enabled": true,
           "path": "tmp/1/cache",
-          "small_object_size": 16384,
           "max_object_size": 134217728,
-          "workers_number": 30,
           "capacity": 4294967296
         },
         "metabase": {

diff --git a/config/example/node.yaml b/config/example/node.yaml
@@ -116,9 +116,7 @@ storage:
 
       writecache:
         enabled: true
-        small_object_size: 16k  # size threshold for "small" objects which are cached in key-value DB, not in FS, bytes
         max_object_size: 134217728  # size threshold for "big" objects which bypass write-cache and go to the storage directly, bytes
-        workers_number: 30  # number of write-cache flusher threads
 
       metabase:
         perm: 0644  # permissions for metabase files(directories: +x for current user and group)

diff --git a/docs/storage-node-configuration.md b/docs/storage-node-configuration.md
@@ -268,9 +268,7 @@ writecache:
   enabled: true
   path: /path/to/writecache
   capacity: 4294967296
-  small_object_size: 16384
   max_object_size: 134217728
-  workers_number: 30
 ```
 
 | Parameter           | Type       | Default value | Description                                                                                                          |
@@ -279,11 +277,7 @@ writecache:
 | `path`              | `string`   |               | Path to the metabase file.                                                                                           |
 | `capacity`          | `size`     | unrestricted  | Approximate maximum size of the writecache. If the writecache is full, objects are written to the blobstor directly. |
 | `no_sync`           | `bool`     | `false`       | Disable write synchronization, makes writes faster, but can lead to data loss.                                       |
-| `small_object_size` | `size`     | `32K`         | Maximum object size for "small" objects. This objects are stored in a key-value database instead of a file-system.   |
 | `max_object_size`   | `size`     | `64M`         | Maximum object size allowed to be stored in the writecache.                                                          |
-| `workers_number`    | `int`      | `20`          | Amount of background workers that move data from the writecache to the blobstor.                                     |
-| `max_batch_size`    | `int`      | `1000`        | Maximum amount of small object `PUT` operations to perform in a single transaction.                                  |
-| `max_batch_delay`   | `duration` | `10ms`        | Maximum delay before a batch starts.                                                                                 |
 
 ### `pilorama` subsection
 

diff --git a/pkg/local_object_storage/shard/dump_test.go b/pkg/local_object_storage/shard/dump_test.go
@@ -49,7 +49,6 @@ func testDump(t *testing.T, objCount int, hasWriteCache bool) {
 	} else {
 		sh = newCustomShard(t, t.TempDir(), true,
 			[]writecache.Option{
-				writecache.WithSmallObjectSize(wcSmallObjectSize),
 				writecache.WithMaxObjectSize(wcBigObjectSize),
 				writecache.WithLogger(zaptest.NewLogger(t)),
 			},
@@ -302,7 +301,6 @@ func TestDumpIgnoreErrors(t *testing.T) {
 	wcPath := filepath.Join(dir, "writecache")
 	wcOpts := []writecache.Option{
 		writecache.WithPath(wcPath),
-		writecache.WithSmallObjectSize(wcSmallObjectSize),
 		writecache.WithMaxObjectSize(wcBigObjectSize),
 	}
 	sh := newCustomShard(t, dir, true, wcOpts, bsOpts(2))