Feat/exchange object meta signatures

CHANGELOG.md

@@ -33,18 +33,25 @@ Changelog for NeoFS Node
 - It was impossible to specify memory amount as "1b" (one byte) in config, default was used instead (#2899)
 - Container session token's lifetime was not checked (#2898)
 - ACL checks for split objects could be forced by a node than might lack access (#2909)
+- Do not search for tombstones when handling their expiration, use local indexes instead (#2929)
 
 ### Changed
 - neofs-cli allows several objects deletion at a time (#2774)
 - `ObjectService.Put` server of in-container node places objects using new `ObjectService.Replicate` RPC (#2802)
 - `ObjectService`'s `Search` and `Replicate` RPC handlers cache up to 1000 lists of container nodes (#2892)
 - Default max_traceable_blocks Morph setting lowered to 17280 from 2102400 (#2897)
 - `ObjectService`'s `Get`/`Head`/`GetRange` RPC handlers cache up to 10K lists of per-object sorted container nodes (#2896)
+- Metabase graveyard scheme (#2929)
 
 ### Updated
 - neofs-contract dependency to 0.20.0 (#2872)
 - NeoGo dependency to 0.106.3 (#2872)
 
+### Updating from v0.42.1
+It is required to resynchronize metabases due to changed metabase scheme; any
+starts with old metabases will fail. See storage node's config documentation
+for details.
+
 ## [0.42.1] - 2024-06-13
 
 A tiny update that adds compatibility with the Neo N3 Domovoi hardfork.

cmd/neofs-lens/internal/meta/list-graveyard.go

@@ -26,9 +26,10 @@
 	gravePrm.SetHandler(
 		func(tsObj meta.TombstonedObject) error {
 			cmd.Printf(
-				"Object: %s\nTS: %s\n",
+				"Object: %s\nTS: %s (TS expiration: %d)\n",
 				tsObj.Address().EncodeToString(),
 				tsObj.Tombstone().EncodeToString(),
+				tsObj.TombstoneExpiration(),
 			)
 
 			return nil

cmd/neofs-node/object.go

@@ -515,7 +515,7 @@
 	return e.engine.IsLocked(address)
 }
 
-func (e storageEngine) Delete(tombstone oid.Address, toDelete []oid.ID) error {
+func (e storageEngine) Delete(tombstone oid.Address, tombExpiration uint64, toDelete []oid.ID) error {
 	var prm engine.InhumePrm
 
 	addrs := make([]oid.Address, len(toDelete))
@@ -524,7 +524,7 @@
 		addrs[i].SetObject(toDelete[i])
 	}
 
-	prm.WithTarget(tombstone, addrs...)
+	prm.WithTombstone(tombstone, tombExpiration, addrs...)
 
 	_, err := e.engine.Inhume(prm)
 	return err

cmd/neofs-node/storage.go

@@ -17,8 +17,6 @@
 	containerEvent "github.com/nspcc-dev/neofs-node/pkg/morph/event/container"
 	"github.com/nspcc-dev/neofs-node/pkg/morph/event/netmap"
 	getsvc "github.com/nspcc-dev/neofs-node/pkg/services/object/get"
-	"github.com/nspcc-dev/neofs-node/pkg/services/object_manager/tombstone"
-	tsourse "github.com/nspcc-dev/neofs-node/pkg/services/object_manager/tombstone/source"
 	"github.com/nspcc-dev/neofs-node/pkg/util"
 	objectSDK "github.com/nspcc-dev/neofs-sdk-go/object"
 	"github.com/panjf2000/ants/v2"
@@ -47,18 +45,9 @@
 	// service will be created later
 	c.cfgObject.getSvc = new(getsvc.Service)
 
-	var tssPrm tsourse.TombstoneSourcePrm
-	tssPrm.SetGetService(c.cfgObject.getSvc)
-	tombstoneSrc := tsourse.NewSource(tssPrm)
-
-	tombstoneSource := tombstone.NewChecker(
-		tombstone.WithLogger(c.log),
-		tombstone.WithTombstoneSource(tombstoneSrc),
-	)
-
 	var shardsAttached int
 	for _, optsWithMeta := range c.shardOpts() {
-		id, err := ls.AddShard(append(optsWithMeta.shOpts, shard.WithTombstoneSource(tombstoneSource))...)
+		id, err := ls.AddShard(optsWithMeta.shOpts...)
 		if err != nil {
 			c.log.Error("failed to attach shard to engine", zap.Error(err))
 		} else {

pkg/local_object_storage/engine/inhume.go

@@ -16,28 +16,30 @@ import (
 
 // InhumePrm encapsulates parameters for inhume operation.
 type InhumePrm struct {
-	tombstone *oid.Address
-	addrs     []oid.Address
+	tombstone      *oid.Address
+	tombExpiration uint64
+	addrs          []oid.Address
 
 	forceRemoval bool
 }
 
 // InhumeRes encapsulates results of inhume operation.
 type InhumeRes struct{}
 
-// WithTarget sets a list of objects that should be inhumed and tombstone address
+// WithTombstone sets a list of objects that should be inhumed and tombstone address
 // as the reason for inhume operation.
 //
-// tombstone should not be nil, addr should not be empty.
+// addrs should not be empty.
 // Should not be called along with MarkAsGarbage.
-func (p *InhumePrm) WithTarget(tombstone oid.Address, addrs ...oid.Address) {
+func (p *InhumePrm) WithTombstone(tombstone oid.Address, tombExpiration uint64, addrs ...oid.Address) {
 	p.addrs = addrs
 	p.tombstone = &tombstone
+	p.tombExpiration = tombExpiration
 }
 
 // MarkAsGarbage marks an object to be physically removed from local storage.
 //
-// Should not be called along with WithTarget.
+// Should not be called along with WithTombstone.
 func (p *InhumePrm) MarkAsGarbage(addrs ...oid.Address) {
 	p.addrs = addrs
 	p.tombstone = nil
@@ -95,7 +97,7 @@ func (e *StorageEngine) inhume(prm InhumePrm) (InhumeRes, error) {
 		}
 
 		if prm.tombstone != nil {
-			shPrm.InhumeByTomb(*prm.tombstone, prm.addrs[i])
+			shPrm.InhumeByTomb(*prm.tombstone, prm.tombExpiration, prm.addrs[i])
 		} else {
 			shPrm.MarkAsGarbage(prm.addrs[i])
 		}
@@ -332,19 +334,6 @@ func (e *StorageEngine) processExpiredObjects(_ context.Context, addrs []oid.Add
 	}
 }
 
-func (e *StorageEngine) processExpiredTombstones(ctx context.Context, addrs []meta.TombstonedObject) {
-	e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) {
-		sh.HandleExpiredTombstones(addrs)
-
-		select {
-		case <-ctx.Done():
-			return true
-		default:
-			return false
-		}
-	})
-}
-
 func (e *StorageEngine) processExpiredLocks(ctx context.Context, lockers []oid.Address) {
 	e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) {
 		sh.HandleExpiredLocks(lockers)

pkg/local_object_storage/engine/inhume_test.go

@@ -46,7 +46,7 @@ func TestStorageEngine_Inhume(t *testing.T) {
 		require.NoError(t, err)
 
 		var inhumePrm InhumePrm
-		inhumePrm.WithTarget(tombstoneID, object.AddressOf(parent))
+		inhumePrm.WithTombstone(tombstoneID, 0, object.AddressOf(parent))
 
 		_, err = e.Inhume(inhumePrm)
 		require.NoError(t, err)
@@ -74,7 +74,7 @@ func TestStorageEngine_Inhume(t *testing.T) {
 		require.NoError(t, err)
 
 		var inhumePrm InhumePrm
-		inhumePrm.WithTarget(tombstoneID, object.AddressOf(parent))
+		inhumePrm.WithTombstone(tombstoneID, 0, object.AddressOf(parent))
 
 		_, err = e.Inhume(inhumePrm)
 		require.NoError(t, err)

pkg/local_object_storage/engine/lock_test.go

@@ -51,7 +51,6 @@ func TestLockUserScenario(t *testing.T) {
 
 			return pool
 		}),
-		shard.WithTombstoneSource(tss{lockerExpiresAfter}),
 	})
 
 	t.Cleanup(func() {
@@ -108,7 +107,7 @@ func TestLockUserScenario(t *testing.T) {
 
 	// 3.
 	var inhumePrm InhumePrm
-	inhumePrm.WithTarget(tombAddr, objAddr)
+	inhumePrm.WithTombstone(tombAddr, 0, objAddr)
 
 	_, err = e.Inhume(inhumePrm)
 	require.ErrorAs(t, err, new(apistatus.ObjectLocked))
@@ -121,7 +120,7 @@ func TestLockUserScenario(t *testing.T) {
 	err = Put(e, tombObj)
 	require.NoError(t, err)
 
-	inhumePrm.WithTarget(tombForLockAddr, lockerAddr)
+	inhumePrm.WithTombstone(tombForLockAddr, 0, lockerAddr)
 
 	_, err = e.Inhume(inhumePrm)
 	require.ErrorIs(t, err, meta.ErrLockObjectRemoval)
@@ -132,7 +131,7 @@ func TestLockUserScenario(t *testing.T) {
 	// delay for GC
 	time.Sleep(time.Second)
 
-	inhumePrm.WithTarget(tombAddr, objAddr)
+	inhumePrm.WithTombstone(tombAddr, 0, objAddr)
 
 	_, err = e.Inhume(inhumePrm)
 	require.NoError(t, err)
@@ -189,7 +188,7 @@ func TestLockExpiration(t *testing.T) {
 	require.NoError(t, err)
 
 	var inhumePrm InhumePrm
-	inhumePrm.WithTarget(objecttest.Address(), objectcore.AddressOf(obj))
+	inhumePrm.WithTombstone(objecttest.Address(), 0, objectcore.AddressOf(obj))
 
 	_, err = e.Inhume(inhumePrm)
 	require.ErrorAs(t, err, new(apistatus.ObjectLocked))
@@ -202,7 +201,7 @@ func TestLockExpiration(t *testing.T) {
 	time.Sleep(time.Second)
 
 	// 4.
-	inhumePrm.WithTarget(objecttest.Address(), objectcore.AddressOf(obj))
+	inhumePrm.WithTombstone(objecttest.Address(), 0, objectcore.AddressOf(obj))
 
 	_, err = e.Inhume(inhumePrm)
 	require.NoError(t, err)
@@ -261,7 +260,7 @@ func TestLockForceRemoval(t *testing.T) {
 	_, err = e.Inhume(inhumePrm)
 	require.ErrorAs(t, err, new(apistatus.ObjectLocked))
 
-	inhumePrm.WithTarget(objecttest.Address(), objectcore.AddressOf(obj))
+	inhumePrm.WithTombstone(objecttest.Address(), 0, objectcore.AddressOf(obj))
 
 	_, err = e.Inhume(inhumePrm)
 	require.ErrorAs(t, err, new(apistatus.ObjectLocked))

pkg/local_object_storage/engine/shards.go

@@ -101,7 +101,6 @@ func (e *StorageEngine) createShard(opts []shard.Option) (*shard.Shard, error) {
 	sh := shard.New(append(opts,
 		shard.WithID(id),
 		shard.WithExpiredObjectsCallback(e.processExpiredObjects),
-		shard.WithExpiredTombstonesCallback(e.processExpiredTombstones),
 		shard.WithExpiredLocksCallback(e.processExpiredLocks),
 		shard.WithDeletedLockCallback(e.processDeletedLocks),
 		shard.WithReportErrorFunc(e.reportShardErrorBackground),

pkg/local_object_storage/metabase/VERSION.md

@@ -4,14 +4,14 @@ This file describes changes between the metabase versions.
 
 ## Current
 
-Numbers stand for a single byte value.
+Numbers stand for a single byte value unless otherwise stated.
 The lowest not used bucket index: 20.
 
 ### Primary buckets
 - Graveyard bucket
   - Name: `0`
   - Key: object address 
-  - Value: tombstone address
+  - Value: tombstone address + little-endian uint64 tombstone expiration epoch
 - Garbage objects bucket
   - Name: `1`
   - Key: object address

pkg/local_object_storage/metabase/counter_test.go

@@ -85,7 +85,7 @@ func TestCounters(t *testing.T) {
 		}
 
 		var prm meta.InhumePrm
-		prm.SetTombstoneAddress(oidtest.Address())
+		prm.SetTombstone(oidtest.Address(), 0)
 		prm.SetAddresses(inhumedObjs...)
 
 		res, err := db.Inhume(prm)
@@ -155,7 +155,7 @@ func TestCounters(t *testing.T) {
 		}
 
 		var prm meta.InhumePrm
-		prm.SetTombstoneAddress(oidtest.Address())
+		prm.SetTombstone(oidtest.Address(), 0)
 		prm.SetAddresses(inhumedObjs...)
 
 		_, err = db.Inhume(prm)

pkg/local_object_storage/metabase/graveyard.go

@@ -2,6 +2,7 @@
 
 import (
 	"bytes"
+	"encoding/binary"
 	"errors"
 	"fmt"
 
@@ -74,8 +75,9 @@
 // TombstonedObject represents descriptor of the
 // object that has been covered with tombstone.
 type TombstonedObject struct {
-	addr oid.Address
-	tomb oid.Address
+	addr           oid.Address
+	tomb           oid.Address
+	tombExpiration uint64
 }
 
 // Address returns tombstoned object address.
@@ -89,6 +91,13 @@
 	return g.tomb
 }
 
+// TombstoneExpiration returns tombstone's expiration. It can be zero if
+// metabase version does not support expiration indexing or if TS does not
+// expire.
+func (g TombstonedObject) TombstoneExpiration() uint64 {
+	return g.tombExpiration
+}
+
 // TombstonedHandler is a TombstonedObject handling function.
 type TombstonedHandler func(object TombstonedObject) error
 
@@ -219,46 +228,67 @@
 }
 
 func graveFromKV(k, v []byte) (res TombstonedObject, err error) {
-	if err = decodeAddressFromKey(&res.addr, k); err != nil {
-		err = fmt.Errorf("decode tombstone target from key: %w", err)
-	} else if err = decodeAddressFromKey(&res.tomb, v); err != nil {
-		err = fmt.Errorf("decode tombstone address from value: %w", err)
+	err = decodeAddressFromKey(&res.addr, k)
+	if err != nil {
+		return res, fmt.Errorf("decode tombstone target from key: %w", err)
+	}
+
+	err = decodeAddressFromKey(&res.tomb, v[:addressKeySize])
+	if err != nil {
+		return res, fmt.Errorf("decode tombstone address from value: %w", err)
+	}
+
+	if len(v) == addressKeySize+8 {
+		res.tombExpiration = binary.LittleEndian.Uint64(v[addressKeySize:])
 	}
 
 	return
 }
 
-// DropGraves deletes tombstoned objects from the
-// graveyard bucket.
-//
-// Returns any error appeared during deletion process.
-func (db *DB) DropGraves(tss []TombstonedObject) error {
+// DropExpiredTSMarks run through the graveyard and drops tombstone marks with
+// tombstones whose expiration is _less_ than provided epoch.
+// Returns number of marks dropped.
+func (db *DB) DropExpiredTSMarks(epoch uint64) (int, error) {
 	db.modeMtx.RLock()
 	defer db.modeMtx.RUnlock()
 
 	if db.mode.NoMetabase() {
-		return ErrDegradedMode
+		return -1, ErrDegradedMode
 	} else if db.mode.ReadOnly() {
-		return ErrReadOnlyMode
+		return -1, ErrReadOnlyMode
 	}
 
-	buf := make([]byte, addressKeySize)
+	var counter int
 
-	return db.boltDB.Update(func(tx *bbolt.Tx) error {
+	err := db.boltDB.Update(func(tx *bbolt.Tx) error {
 		bkt := tx.Bucket(graveyardBucketName)
-		if bkt == nil {
-			return nil
-		}
+		c := bkt.Cursor()
+		k, v := c.First()
 
-		for _, ts := range tss {
-			err := bkt.Delete(addressKey(ts.Address(), buf))
-			if err != nil {
-				return err
+		for k != nil {
+			if binary.LittleEndian.Uint64(v[addressKeySize:]) < epoch {
+				err := c.Delete()
+				if err != nil {
+					return fmt.Errorf("cleared %d TS marks in %d epoch and got error: %w", counter, epoch, err)
+				}
+
+				counter++
+
+				// see https://github.com/etcd-io/bbolt/pull/614; there is not
+				// much we can do with such an unfixed behavior
+				k, v = c.Seek(k)
+			} else {
+				k, v = c.Next()
 			}
 		}
 
 		return nil
 	})
+	if err != nil {
+		return -1, fmt.Errorf("db call: %w", err)
+	}
+
+	return counter, nil
 }
 
 // GetGarbage returns garbage according to the metabase state. Garbage includes