From 99a59ac3c1b33e734e5fbe3c203b89dc6d65c51c Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Thu, 12 Sep 2024 17:51:22 +0300 Subject: [PATCH 1/5] node/meta: store tombstone's expiration in graveyard It takes too much of network/disk to handle tombstones expiration and removal, so it is better to store it when tombstone is being indexed in metabase. Additional little-endian expiration suffix was added to graveyard values. Metabase version was increased as it is a non-compatible change. Relates #2929. Signed-off-by: Pavel Karpy --- CHANGELOG.md | 6 +++++ cmd/neofs-node/object.go | 4 ++-- pkg/local_object_storage/engine/inhume.go | 16 +++++++------ .../engine/inhume_test.go | 4 ++-- pkg/local_object_storage/engine/lock_test.go | 12 +++++----- pkg/local_object_storage/metabase/VERSION.md | 4 ++-- .../metabase/counter_test.go | 4 ++-- .../metabase/graveyard.go | 2 +- .../metabase/graveyard_test.go | 6 ++--- pkg/local_object_storage/metabase/inhume.go | 24 +++++++++++-------- .../metabase/inhume_test.go | 8 +++---- .../metabase/iterators.go | 2 +- .../metabase/iterators_test.go | 2 +- .../metabase/lock_test.go | 4 ++-- pkg/local_object_storage/metabase/version.go | 2 +- pkg/local_object_storage/shard/control.go | 6 ++++- .../shard/control_test.go | 2 +- pkg/local_object_storage/shard/inhume.go | 13 ++++++---- pkg/local_object_storage/shard/inhume_test.go | 2 +- pkg/local_object_storage/shard/lock_test.go | 4 ++-- .../shard/metrics_test.go | 2 +- .../writecache/flush_test.go | 2 +- pkg/services/object/put/local.go | 9 +++++-- 23 files changed, 82 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 16f95c8924..8c3f6248aa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,11 +40,17 @@ Changelog for NeoFS Node - `ObjectService`'s `Search` and `Replicate` RPC handlers cache up to 1000 lists of container nodes (#2892) - Default max_traceable_blocks Morph setting lowered to 17280 from 2102400 (#2897) - `ObjectService`'s `Get`/`Head`/`GetRange` RPC handlers cache up to 10K lists of per-object sorted container nodes (#2896) +- Metabase graveyard scheme (#2929) ### Updated - neofs-contract dependency to 0.20.0 (#2872) - NeoGo dependency to 0.106.3 (#2872) +### Updating from v0.42.1 +It is required to resynchronize metabases due to changed metabase scheme; any +starts with old metabases will fail. See storage node's config documentation +for details. + ## [0.42.1] - 2024-06-13 A tiny update that adds compatibility with the Neo N3 Domovoi hardfork. diff --git a/cmd/neofs-node/object.go b/cmd/neofs-node/object.go index 4fd8b12790..5bd29d9973 100644 --- a/cmd/neofs-node/object.go +++ b/cmd/neofs-node/object.go @@ -515,7 +515,7 @@ func (e storageEngine) IsLocked(address oid.Address) (bool, error) { return e.engine.IsLocked(address) } -func (e storageEngine) Delete(tombstone oid.Address, toDelete []oid.ID) error { +func (e storageEngine) Delete(tombstone oid.Address, tombExpiration uint64, toDelete []oid.ID) error { var prm engine.InhumePrm addrs := make([]oid.Address, len(toDelete)) @@ -524,7 +524,7 @@ func (e storageEngine) Delete(tombstone oid.Address, toDelete []oid.ID) error { addrs[i].SetObject(toDelete[i]) } - prm.WithTarget(tombstone, addrs...) + prm.WithTombstone(tombstone, tombExpiration, addrs...) _, err := e.engine.Inhume(prm) return err diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 5e4eba5e7c..99ed6b7229 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -16,8 +16,9 @@ import ( // InhumePrm encapsulates parameters for inhume operation. type InhumePrm struct { - tombstone *oid.Address - addrs []oid.Address + tombstone *oid.Address + tombExpiration uint64 + addrs []oid.Address forceRemoval bool } @@ -25,19 +26,20 @@ type InhumePrm struct { // InhumeRes encapsulates results of inhume operation. type InhumeRes struct{} -// WithTarget sets a list of objects that should be inhumed and tombstone address +// WithTombstone sets a list of objects that should be inhumed and tombstone address // as the reason for inhume operation. // -// tombstone should not be nil, addr should not be empty. +// addrs should not be empty. // Should not be called along with MarkAsGarbage. -func (p *InhumePrm) WithTarget(tombstone oid.Address, addrs ...oid.Address) { +func (p *InhumePrm) WithTombstone(tombstone oid.Address, tombExpiration uint64, addrs ...oid.Address) { p.addrs = addrs p.tombstone = &tombstone + p.tombExpiration = tombExpiration } // MarkAsGarbage marks an object to be physically removed from local storage. // -// Should not be called along with WithTarget. +// Should not be called along with WithTombstone. func (p *InhumePrm) MarkAsGarbage(addrs ...oid.Address) { p.addrs = addrs p.tombstone = nil @@ -95,7 +97,7 @@ func (e *StorageEngine) inhume(prm InhumePrm) (InhumeRes, error) { } if prm.tombstone != nil { - shPrm.InhumeByTomb(*prm.tombstone, prm.addrs[i]) + shPrm.InhumeByTomb(*prm.tombstone, prm.tombExpiration, prm.addrs[i]) } else { shPrm.MarkAsGarbage(prm.addrs[i]) } diff --git a/pkg/local_object_storage/engine/inhume_test.go b/pkg/local_object_storage/engine/inhume_test.go index dc033ce743..b50e2a0d38 100644 --- a/pkg/local_object_storage/engine/inhume_test.go +++ b/pkg/local_object_storage/engine/inhume_test.go @@ -46,7 +46,7 @@ func TestStorageEngine_Inhume(t *testing.T) { require.NoError(t, err) var inhumePrm InhumePrm - inhumePrm.WithTarget(tombstoneID, object.AddressOf(parent)) + inhumePrm.WithTombstone(tombstoneID, 0, object.AddressOf(parent)) _, err = e.Inhume(inhumePrm) require.NoError(t, err) @@ -74,7 +74,7 @@ func TestStorageEngine_Inhume(t *testing.T) { require.NoError(t, err) var inhumePrm InhumePrm - inhumePrm.WithTarget(tombstoneID, object.AddressOf(parent)) + inhumePrm.WithTombstone(tombstoneID, 0, object.AddressOf(parent)) _, err = e.Inhume(inhumePrm) require.NoError(t, err) diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index a2cf509290..5cd24a3f34 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -108,7 +108,7 @@ func TestLockUserScenario(t *testing.T) { // 3. var inhumePrm InhumePrm - inhumePrm.WithTarget(tombAddr, objAddr) + inhumePrm.WithTombstone(tombAddr, 0, objAddr) _, err = e.Inhume(inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) @@ -121,7 +121,7 @@ func TestLockUserScenario(t *testing.T) { err = Put(e, tombObj) require.NoError(t, err) - inhumePrm.WithTarget(tombForLockAddr, lockerAddr) + inhumePrm.WithTombstone(tombForLockAddr, 0, lockerAddr) _, err = e.Inhume(inhumePrm) require.ErrorIs(t, err, meta.ErrLockObjectRemoval) @@ -132,7 +132,7 @@ func TestLockUserScenario(t *testing.T) { // delay for GC time.Sleep(time.Second) - inhumePrm.WithTarget(tombAddr, objAddr) + inhumePrm.WithTombstone(tombAddr, 0, objAddr) _, err = e.Inhume(inhumePrm) require.NoError(t, err) @@ -189,7 +189,7 @@ func TestLockExpiration(t *testing.T) { require.NoError(t, err) var inhumePrm InhumePrm - inhumePrm.WithTarget(objecttest.Address(), objectcore.AddressOf(obj)) + inhumePrm.WithTombstone(objecttest.Address(), 0, objectcore.AddressOf(obj)) _, err = e.Inhume(inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) @@ -202,7 +202,7 @@ func TestLockExpiration(t *testing.T) { time.Sleep(time.Second) // 4. - inhumePrm.WithTarget(objecttest.Address(), objectcore.AddressOf(obj)) + inhumePrm.WithTombstone(objecttest.Address(), 0, objectcore.AddressOf(obj)) _, err = e.Inhume(inhumePrm) require.NoError(t, err) @@ -261,7 +261,7 @@ func TestLockForceRemoval(t *testing.T) { _, err = e.Inhume(inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) - inhumePrm.WithTarget(objecttest.Address(), objectcore.AddressOf(obj)) + inhumePrm.WithTombstone(objecttest.Address(), 0, objectcore.AddressOf(obj)) _, err = e.Inhume(inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) diff --git a/pkg/local_object_storage/metabase/VERSION.md b/pkg/local_object_storage/metabase/VERSION.md index b80a735470..62ad0dc08b 100644 --- a/pkg/local_object_storage/metabase/VERSION.md +++ b/pkg/local_object_storage/metabase/VERSION.md @@ -4,14 +4,14 @@ This file describes changes between the metabase versions. ## Current -Numbers stand for a single byte value. +Numbers stand for a single byte value unless otherwise stated. The lowest not used bucket index: 20. ### Primary buckets - Graveyard bucket - Name: `0` - Key: object address - - Value: tombstone address + - Value: tombstone address + little-endian uint64 tombstone expiration epoch - Garbage objects bucket - Name: `1` - Key: object address diff --git a/pkg/local_object_storage/metabase/counter_test.go b/pkg/local_object_storage/metabase/counter_test.go index 75c7ad0f34..85d9273483 100644 --- a/pkg/local_object_storage/metabase/counter_test.go +++ b/pkg/local_object_storage/metabase/counter_test.go @@ -85,7 +85,7 @@ func TestCounters(t *testing.T) { } var prm meta.InhumePrm - prm.SetTombstoneAddress(oidtest.Address()) + prm.SetTombstone(oidtest.Address(), 0) prm.SetAddresses(inhumedObjs...) res, err := db.Inhume(prm) @@ -155,7 +155,7 @@ func TestCounters(t *testing.T) { } var prm meta.InhumePrm - prm.SetTombstoneAddress(oidtest.Address()) + prm.SetTombstone(oidtest.Address(), 0) prm.SetAddresses(inhumedObjs...) _, err = db.Inhume(prm) diff --git a/pkg/local_object_storage/metabase/graveyard.go b/pkg/local_object_storage/metabase/graveyard.go index 8ef329fe33..551c7d5667 100644 --- a/pkg/local_object_storage/metabase/graveyard.go +++ b/pkg/local_object_storage/metabase/graveyard.go @@ -221,7 +221,7 @@ func garbageFromKV(k []byte) (res GarbageObject, err error) { func graveFromKV(k, v []byte) (res TombstonedObject, err error) { if err = decodeAddressFromKey(&res.addr, k); err != nil { err = fmt.Errorf("decode tombstone target from key: %w", err) - } else if err = decodeAddressFromKey(&res.tomb, v); err != nil { + } else if err = decodeAddressFromKey(&res.tomb, v[:addressKeySize]); err != nil { err = fmt.Errorf("decode tombstone address from value: %w", err) } diff --git a/pkg/local_object_storage/metabase/graveyard_test.go b/pkg/local_object_storage/metabase/graveyard_test.go index 747e512ac2..d45decf6fd 100644 --- a/pkg/local_object_storage/metabase/graveyard_test.go +++ b/pkg/local_object_storage/metabase/graveyard_test.go @@ -138,7 +138,7 @@ func TestDB_IterateDeletedObjects(t *testing.T) { addrTombstone := oidtest.Address() inhumePrm.SetAddresses(object.AddressOf(obj1), object.AddressOf(obj2)) - inhumePrm.SetTombstoneAddress(addrTombstone) + inhumePrm.SetTombstone(addrTombstone, 0) _, err = db.Inhume(inhumePrm) require.NoError(t, err) @@ -225,7 +225,7 @@ func TestDB_IterateOverGraveyard_Offset(t *testing.T) { inhumePrm.SetAddresses( object.AddressOf(obj1), object.AddressOf(obj2), object.AddressOf(obj3), object.AddressOf(obj4)) - inhumePrm.SetTombstoneAddress(addrTombstone) + inhumePrm.SetTombstone(addrTombstone, 0) _, err = db.Inhume(inhumePrm) require.NoError(t, err) @@ -404,7 +404,7 @@ func TestDB_DropGraves(t *testing.T) { var inhumePrm meta.InhumePrm inhumePrm.SetAddresses(object.AddressOf(obj1), object.AddressOf(obj2)) - inhumePrm.SetTombstoneAddress(addrTombstone) + inhumePrm.SetTombstone(addrTombstone, 0) _, err = db.Inhume(inhumePrm) require.NoError(t, err) diff --git a/pkg/local_object_storage/metabase/inhume.go b/pkg/local_object_storage/metabase/inhume.go index 2f00e44079..fcb5f7bf13 100644 --- a/pkg/local_object_storage/metabase/inhume.go +++ b/pkg/local_object_storage/metabase/inhume.go @@ -2,6 +2,7 @@ package meta import ( "bytes" + "encoding/binary" "errors" "fmt" @@ -15,7 +16,8 @@ import ( // InhumePrm encapsulates parameters for Inhume operation. type InhumePrm struct { - tomb *oid.Address + tomb *oid.Address + tombExpiration uint64 target []oid.Address @@ -48,17 +50,19 @@ func (p *InhumePrm) SetAddresses(addrs ...oid.Address) { p.target = addrs } -// SetTombstoneAddress sets tombstone address as the reason for inhume operation. +// SetTombstone sets tombstone address as the reason for inhume operation +// and tombstone's expiration. // // addr should not be nil. // Should not be called along with SetGCMark. -func (p *InhumePrm) SetTombstoneAddress(addr oid.Address) { +func (p *InhumePrm) SetTombstone(addr oid.Address, epoch uint64) { p.tomb = &addr + p.tombExpiration = epoch } // SetGCMark marks the object to be physically removed. // -// Should not be called along with SetTombstoneAddress. +// Should not be called along with SetTombstone. func (p *InhumePrm) SetGCMark() { p.tomb = nil } @@ -114,15 +118,15 @@ func (db *DB) Inhume(prm InhumePrm) (res InhumeRes, err error) { // 2. Garbage if Inhume was called with a GC mark bkt *bbolt.Bucket // value that will be put in the bucket, one of the: - // 1. tombstone address if Inhume was called with - // a Tombstone + // 1. tombstone address + tomb expiration epoch if Inhume was called + // with a Tombstone // 2. zeroValue if Inhume was called with a GC mark value []byte ) if prm.tomb != nil { bkt = graveyardBKT - tombKey := addressKey(*prm.tomb, make([]byte, addressKeySize)) + tombKey := addressKey(*prm.tomb, make([]byte, addressKeySize+8)) // it is forbidden to have a tomb-on-tomb in NeoFS, // so graveyard keys must not be addresses of tombstones @@ -134,7 +138,7 @@ func (db *DB) Inhume(prm InhumePrm) (res InhumeRes, err error) { } } - value = tombKey + value = binary.LittleEndian.AppendUint64(tombKey, prm.tombExpiration) } else { bkt = garbageObjectsBKT value = zeroValue @@ -187,10 +191,10 @@ func (db *DB) Inhume(prm InhumePrm) (res InhumeRes, err error) { // iterate over graveyard and check if target address // is the address of tombstone in graveyard. - err = bkt.ForEach(func(k, v []byte) error { + err = graveyardBKT.ForEach(func(k, v []byte) error { // check if graveyard has record with key corresponding // to tombstone address (at least one) - targetIsTomb = bytes.Equal(v, targetKey) + targetIsTomb = bytes.Equal(v[:addressKeySize], targetKey) if targetIsTomb { // break bucket iterator diff --git a/pkg/local_object_storage/metabase/inhume_test.go b/pkg/local_object_storage/metabase/inhume_test.go index 3a38a21991..6c7f04da91 100644 --- a/pkg/local_object_storage/metabase/inhume_test.go +++ b/pkg/local_object_storage/metabase/inhume_test.go @@ -49,7 +49,7 @@ func TestInhumeTombOnTomb(t *testing.T) { ) inhumePrm.SetAddresses(addr1) - inhumePrm.SetTombstoneAddress(addr2) + inhumePrm.SetTombstone(addr2, 0) // inhume addr1 via addr2 _, err = db.Inhume(inhumePrm) @@ -62,7 +62,7 @@ func TestInhumeTombOnTomb(t *testing.T) { require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) inhumePrm.SetAddresses(addr3) - inhumePrm.SetTombstoneAddress(addr1) + inhumePrm.SetTombstone(addr1, 0) // try to inhume addr3 via addr1 _, err = db.Inhume(inhumePrm) @@ -82,7 +82,7 @@ func TestInhumeTombOnTomb(t *testing.T) { require.ErrorAs(t, err, new(apistatus.ObjectAlreadyRemoved)) inhumePrm.SetAddresses(addr1) - inhumePrm.SetTombstoneAddress(oidtest.Address()) + inhumePrm.SetTombstone(oidtest.Address(), 0) // try to inhume addr1 (which is already a tombstone in graveyard) _, err = db.Inhume(inhumePrm) @@ -162,7 +162,7 @@ func TestInhumeContainer(t *testing.T) { func metaInhume(db *meta.DB, target, tomb oid.Address) error { var inhumePrm meta.InhumePrm inhumePrm.SetAddresses(target) - inhumePrm.SetTombstoneAddress(tomb) + inhumePrm.SetTombstone(tomb, 0) _, err := db.Inhume(inhumePrm) return err diff --git a/pkg/local_object_storage/metabase/iterators.go b/pkg/local_object_storage/metabase/iterators.go index 120bef72a6..c158b00868 100644 --- a/pkg/local_object_storage/metabase/iterators.go +++ b/pkg/local_object_storage/metabase/iterators.go @@ -146,7 +146,7 @@ func (db *DB) iterateCoveredByTombstones(tx *bbolt.Tx, tss map[string]oid.Addres err := bktGraveyard.ForEach(func(k, v []byte) error { var addr oid.Address - if err := decodeAddressFromKey(&addr, v); err != nil { + if err := decodeAddressFromKey(&addr, v[:addressKeySize]); err != nil { return err } if _, ok := tss[addr.EncodeToString()]; ok { diff --git a/pkg/local_object_storage/metabase/iterators_test.go b/pkg/local_object_storage/metabase/iterators_test.go index 2f0f8eb203..8c604aeee0 100644 --- a/pkg/local_object_storage/metabase/iterators_test.go +++ b/pkg/local_object_storage/metabase/iterators_test.go @@ -77,7 +77,7 @@ func TestDB_IterateCoveredByTombstones(t *testing.T) { var err error prm.SetAddresses(protected1, protected2, protectedLocked) - prm.SetTombstoneAddress(ts) + prm.SetTombstone(ts, 0) _, err = db.Inhume(prm) require.NoError(t, err) diff --git a/pkg/local_object_storage/metabase/lock_test.go b/pkg/local_object_storage/metabase/lock_test.go index 2b09453afc..85d00ec3c5 100644 --- a/pkg/local_object_storage/metabase/lock_test.go +++ b/pkg/local_object_storage/metabase/lock_test.go @@ -67,7 +67,7 @@ func TestDB_Lock(t *testing.T) { _, err := db.Inhume(inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) - inhumePrm.SetTombstoneAddress(oidtest.Address()) + inhumePrm.SetTombstone(oidtest.Address(), 0) _, err = db.Inhume(inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) @@ -83,7 +83,7 @@ func TestDB_Lock(t *testing.T) { _, err = db.Inhume(inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) - inhumePrm.SetTombstoneAddress(oidtest.Address()) + inhumePrm.SetTombstone(oidtest.Address(), 0) _, err = db.Inhume(inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) }) diff --git a/pkg/local_object_storage/metabase/version.go b/pkg/local_object_storage/metabase/version.go index 23cd2c705a..2af3c35e16 100644 --- a/pkg/local_object_storage/metabase/version.go +++ b/pkg/local_object_storage/metabase/version.go @@ -9,7 +9,7 @@ import ( ) // version contains current metabase version. -const version = 2 +const version = 3 var versionKey = []byte("version") diff --git a/pkg/local_object_storage/shard/control.go b/pkg/local_object_storage/shard/control.go index c144b66b70..e38d7c3d25 100644 --- a/pkg/local_object_storage/shard/control.go +++ b/pkg/local_object_storage/shard/control.go @@ -176,6 +176,10 @@ func (s *Shard) refillMetabase() error { switch obj.Type() { case objectSDK.TypeTombstone: tombstone := objectSDK.NewTombstone() + exp, err := object.Expiration(*obj) + if err != nil && !errors.Is(err, object.ErrNoExpiration) { + return fmt.Errorf("tombstone's expiration: %w", err) + } if err := tombstone.Unmarshal(obj.Payload()); err != nil { return fmt.Errorf("could not unmarshal tombstone content: %w", err) @@ -194,7 +198,7 @@ func (s *Shard) refillMetabase() error { var inhumePrm meta.InhumePrm - inhumePrm.SetTombstoneAddress(tombAddr) + inhumePrm.SetTombstone(tombAddr, exp) inhumePrm.SetAddresses(tombMembers...) _, err = s.metaBase.Inhume(inhumePrm) diff --git a/pkg/local_object_storage/shard/control_test.go b/pkg/local_object_storage/shard/control_test.go index 9e0490c58b..ed2979f5c9 100644 --- a/pkg/local_object_storage/shard/control_test.go +++ b/pkg/local_object_storage/shard/control_test.go @@ -244,7 +244,7 @@ func TestRefillMetabase(t *testing.T) { require.NoError(t, sh.Lock(cnrLocked, lockID, locked)) var inhumePrm InhumePrm - inhumePrm.InhumeByTomb(object.AddressOf(&tombObj), tombMembers...) + inhumePrm.InhumeByTomb(object.AddressOf(&tombObj), 0, tombMembers...) _, err = sh.Inhume(inhumePrm) require.NoError(t, err) diff --git a/pkg/local_object_storage/shard/inhume.go b/pkg/local_object_storage/shard/inhume.go index 1c272cc0e9..9890e83338 100644 --- a/pkg/local_object_storage/shard/inhume.go +++ b/pkg/local_object_storage/shard/inhume.go @@ -12,9 +12,10 @@ import ( // InhumePrm encapsulates parameters for inhume operation. type InhumePrm struct { - target []oid.Address - tombstone *oid.Address - forceRemoval bool + target []oid.Address + tombstone *oid.Address + tombstoneExpiration uint64 + forceRemoval bool } // InhumeRes encapsulates results of inhume operation. @@ -25,10 +26,11 @@ type InhumeRes struct{} // // tombstone should not be nil, addr should not be empty. // Should not be called along with MarkAsGarbage. -func (p *InhumePrm) InhumeByTomb(tombstone oid.Address, addrs ...oid.Address) { +func (p *InhumePrm) InhumeByTomb(tombstone oid.Address, tombExpiration uint64, addrs ...oid.Address) { if p != nil { p.target = addrs p.tombstone = &tombstone + p.tombstoneExpiration = tombExpiration } } @@ -39,6 +41,7 @@ func (p *InhumePrm) MarkAsGarbage(addr ...oid.Address) { if p != nil { p.target = addr p.tombstone = nil + p.tombstoneExpiration = 0 } } @@ -91,7 +94,7 @@ func (s *Shard) Inhume(prm InhumePrm) (InhumeRes, error) { metaPrm.SetLockObjectHandling() if prm.tombstone != nil { - metaPrm.SetTombstoneAddress(*prm.tombstone) + metaPrm.SetTombstone(*prm.tombstone, prm.tombstoneExpiration) } else { metaPrm.SetGCMark() } diff --git a/pkg/local_object_storage/shard/inhume_test.go b/pkg/local_object_storage/shard/inhume_test.go index 011b0266fe..ab81c6355c 100644 --- a/pkg/local_object_storage/shard/inhume_test.go +++ b/pkg/local_object_storage/shard/inhume_test.go @@ -35,7 +35,7 @@ func testShardInhume(t *testing.T, hasWriteCache bool) { putPrm.SetObject(obj) var inhPrm shard.InhumePrm - inhPrm.InhumeByTomb(object.AddressOf(ts), object.AddressOf(obj)) + inhPrm.InhumeByTomb(object.AddressOf(ts), 0, object.AddressOf(obj)) var getPrm shard.GetPrm getPrm.SetAddress(object.AddressOf(obj)) diff --git a/pkg/local_object_storage/shard/lock_test.go b/pkg/local_object_storage/shard/lock_test.go index bc5826a89b..abf92d4226 100644 --- a/pkg/local_object_storage/shard/lock_test.go +++ b/pkg/local_object_storage/shard/lock_test.go @@ -86,7 +86,7 @@ func TestShard_Lock(t *testing.T) { ts := generateObjectWithCID(t, cnr) var inhumePrm shard.InhumePrm - inhumePrm.InhumeByTomb(objectcore.AddressOf(ts), objectcore.AddressOf(obj)) + inhumePrm.InhumeByTomb(objectcore.AddressOf(ts), 0, objectcore.AddressOf(obj)) _, err = sh.Inhume(inhumePrm) require.ErrorAs(t, err, new(apistatus.ObjectLocked)) @@ -100,7 +100,7 @@ func TestShard_Lock(t *testing.T) { ts := generateObjectWithCID(t, cnr) var inhumePrm shard.InhumePrm - inhumePrm.InhumeByTomb(objectcore.AddressOf(ts), objectcore.AddressOf(lock)) + inhumePrm.InhumeByTomb(objectcore.AddressOf(ts), 0, objectcore.AddressOf(lock)) _, err = sh.Inhume(inhumePrm) require.Error(t, err) diff --git a/pkg/local_object_storage/shard/metrics_test.go b/pkg/local_object_storage/shard/metrics_test.go index 7c4574b9a9..9b4b808046 100644 --- a/pkg/local_object_storage/shard/metrics_test.go +++ b/pkg/local_object_storage/shard/metrics_test.go @@ -145,7 +145,7 @@ func TestCounters(t *testing.T) { logic := mm.objectCounters[logical] inhumedNumber := int(phy / 4) - prm.InhumeByTomb(ts, addrFromObjs(oo[:inhumedNumber])...) + prm.InhumeByTomb(ts, 0, addrFromObjs(oo[:inhumedNumber])...) _, err := sh.Inhume(prm) require.NoError(t, err) diff --git a/pkg/local_object_storage/writecache/flush_test.go b/pkg/local_object_storage/writecache/flush_test.go index 79c53a4786..f45962e67b 100644 --- a/pkg/local_object_storage/writecache/flush_test.go +++ b/pkg/local_object_storage/writecache/flush_test.go @@ -224,7 +224,7 @@ func TestFlush(t *testing.T) { var inhumePrm meta.InhumePrm inhumePrm.SetAddresses(objects[0].addr, objects[1].addr) - inhumePrm.SetTombstoneAddress(oidtest.Address()) + inhumePrm.SetTombstone(oidtest.Address(), 0) _, err := mb.Inhume(inhumePrm) require.NoError(t, err) diff --git a/pkg/services/object/put/local.go b/pkg/services/object/put/local.go index 755274e738..cb65806930 100644 --- a/pkg/services/object/put/local.go +++ b/pkg/services/object/put/local.go @@ -24,7 +24,7 @@ type ObjectStorage interface { Put(obj *object.Object, objBin []byte, hdrLen int) error // Delete must delete passed objects // and return any appeared error. - Delete(tombstone oid.Address, toDelete []oid.ID) error + Delete(tombstone oid.Address, tombExpiration uint64, toDelete []oid.ID) error // Lock must lock passed objects // and return any appeared error. Lock(locker oid.Address, toLock []oid.ID) error @@ -62,7 +62,12 @@ func (t *localTarget) Close() (oid.ID, *neofscrypto.Signature, error) { func putObjectLocally(storage ObjectStorage, obj *object.Object, meta objectCore.ContentMeta, enc *encodedObject) error { switch meta.Type() { case object.TypeTombstone: - err := storage.Delete(objectCore.AddressOf(obj), meta.Objects()) + exp, err := objectCore.Expiration(*obj) + if err != nil && !errors.Is(err, objectCore.ErrNoExpiration) { + return fmt.Errorf("reading tombstone expiration: %w", err) + } + + err = storage.Delete(objectCore.AddressOf(obj), exp, meta.Objects()) if err != nil { return fmt.Errorf("could not delete objects from tombstone locally: %w", err) } From e741d1b6ace53756059bc319bdd6cac97b99e6e0 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Thu, 12 Sep 2024 19:01:23 +0300 Subject: [PATCH 2/5] node/shard: allow tombstones expire naturally They do not differ from the other objects (e.g. locks do). Initial logic has changed much, graveyard now allows to handle expired tombstones marks (do not confuse it with the lists of regular indexes) independently, while disk can be cleared with the other types of object. Also, add tests for it. Signed-off-by: Pavel Karpy --- pkg/local_object_storage/shard/gc.go | 2 +- pkg/local_object_storage/shard/gc_test.go | 83 +++++++++++++++++++++++ 2 files changed, 84 insertions(+), 1 deletion(-) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 81b0daae43..cae0bf2be7 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -236,7 +236,7 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { log.Debug("started expired objects handling") expired, err := s.getExpiredObjects(ctx, e.(newEpoch).epoch, func(typ object.Type) bool { - return typ != object.TypeTombstone && typ != object.TypeLock + return typ != object.TypeLock }) if err != nil || len(expired) == 0 { if err != nil { diff --git a/pkg/local_object_storage/shard/gc_test.go b/pkg/local_object_storage/shard/gc_test.go index 798fb0bc63..8ff64e1a93 100644 --- a/pkg/local_object_storage/shard/gc_test.go +++ b/pkg/local_object_storage/shard/gc_test.go @@ -4,7 +4,9 @@ import ( "context" "errors" "fmt" + "math" "path/filepath" + "strconv" "testing" "time" @@ -18,6 +20,7 @@ import ( "github.com/nspcc-dev/neofs-node/pkg/util" apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test" + neofscryptotest "github.com/nspcc-dev/neofs-sdk-go/crypto/test" "github.com/nspcc-dev/neofs-sdk-go/object" objectSDK "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" @@ -181,3 +184,83 @@ func TestGC_ContainerCleanup(t *testing.T) { return len(res.Containers()) == 0 }, time.Second, 100*time.Millisecond) } + +func TestExpiration(t *testing.T) { + rootPath := t.TempDir() + var sh *shard.Shard + + opts := []shard.Option{ + shard.WithLogger(zap.NewNop()), + shard.WithBlobStorOptions( + blobstor.WithStorages([]blobstor.SubStorage{ + { + Storage: fstree.New( + fstree.WithPath(filepath.Join(rootPath, "blob"))), + Policy: func(_ *objectSDK.Object, _ []byte) bool { return true }, + }, + }), + ), + shard.WithMetaBaseOptions( + meta.WithPath(filepath.Join(rootPath, "meta")), + meta.WithEpochState(epochState{Value: math.MaxUint64 / 2}), + ), + shard.WithExpiredObjectsCallback( + func(_ context.Context, addresses []oid.Address) { + var p shard.InhumePrm + p.MarkAsGarbage(addresses...) + _, err := sh.Inhume(p) + require.NoError(t, err) + }, + ), + shard.WithGCWorkerPoolInitializer(func(sz int) util.WorkerPool { + pool, err := ants.NewPool(sz) + require.NoError(t, err) + + return pool + }), + } + + sh = shard.New(opts...) + require.NoError(t, sh.Open()) + require.NoError(t, sh.Init()) + + t.Cleanup(func() { + releaseShard(sh, t) + }) + ch := sh.NotificationChannel() + + var expAttr objectSDK.Attribute + expAttr.SetKey(objectV2.SysAttributeExpEpoch) + + obj := generateObject(t) + + for i, typ := range []object.Type{object.TypeRegular, object.TypeTombstone, object.TypeLink, objectSDK.TypeStorageGroup} { + t.Run(fmt.Sprintf("type: %s", typ), func(t *testing.T) { + exp := uint64(i * 10) + + expAttr.SetValue(strconv.FormatUint(exp, 10)) + obj.SetAttributes(expAttr) + obj.SetType(typ) + require.NoError(t, obj.SetIDWithSignature(neofscryptotest.Signer())) + + var putPrm shard.PutPrm + putPrm.SetObject(obj) + + _, err := sh.Put(putPrm) + require.NoError(t, err) + + var getPrm shard.GetPrm + getPrm.SetAddress(objectCore.AddressOf(obj)) + + _, err = sh.Get(getPrm) + require.NoError(t, err) + + ch <- shard.EventNewEpoch(exp + 1) + + require.Eventually(t, func() bool { + _, err = sh.Get(getPrm) + return shard.IsErrNotFound(err) + }, 3*time.Second, 100*time.Millisecond, "lock expiration should free object removal") + }) + } +} From cd7d5fa589641f0971bc23398a8c7b1652b06716 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Thu, 12 Sep 2024 20:18:26 +0300 Subject: [PATCH 3/5] node/meta,shard: use tombstone expiration index instead of network Graveyard now has tombstone expiration marks in epochs, there is no need to use any network requests, just drop records if an epoch is big enough. Closes #2929. Signed-off-by: Pavel Karpy --- CHANGELOG.md | 1 + .../metabase/graveyard.go | 47 ++++++++++++++ .../metabase/graveyard_test.go | 54 ++++++++++++++++ pkg/local_object_storage/shard/gc.go | 62 ++----------------- 4 files changed, 108 insertions(+), 56 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c3f6248aa..0f8ae0125a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,7 @@ Changelog for NeoFS Node - It was impossible to specify memory amount as "1b" (one byte) in config, default was used instead (#2899) - Container session token's lifetime was not checked (#2898) - ACL checks for split objects could be forced by a node than might lack access (#2909) +- Do not search for tombstones when handling their expiration, use local indexes instead (#2929) ### Changed - neofs-cli allows several objects deletion at a time (#2774) diff --git a/pkg/local_object_storage/metabase/graveyard.go b/pkg/local_object_storage/metabase/graveyard.go index 551c7d5667..8d84c89df3 100644 --- a/pkg/local_object_storage/metabase/graveyard.go +++ b/pkg/local_object_storage/metabase/graveyard.go @@ -2,6 +2,7 @@ package meta import ( "bytes" + "encoding/binary" "errors" "fmt" @@ -261,6 +262,52 @@ func (db *DB) DropGraves(tss []TombstonedObject) error { }) } +// DropExpiredTSMarks run through the graveyard and drops tombstone marks with +// tombstones whose expiration is _less_ than provided epoch. +// Returns number of marks dropped. +func (db *DB) DropExpiredTSMarks(epoch uint64) (int, error) { + db.modeMtx.RLock() + defer db.modeMtx.RUnlock() + + if db.mode.NoMetabase() { + return -1, ErrDegradedMode + } else if db.mode.ReadOnly() { + return -1, ErrReadOnlyMode + } + + var counter int + + err := db.boltDB.Update(func(tx *bbolt.Tx) error { + bkt := tx.Bucket(graveyardBucketName) + c := bkt.Cursor() + k, v := c.First() + + for k != nil { + if binary.LittleEndian.Uint64(v[addressKeySize:]) < epoch { + err := c.Delete() + if err != nil { + return fmt.Errorf("cleared %d TS marks in %d epoch and got error: %w", counter, epoch, err) + } + + counter++ + + // see https://github.com/etcd-io/bbolt/pull/614; there is not + // much we can do with such an unfixed behavior + k, v = c.Seek(k) + } else { + k, v = c.Next() + } + } + + return nil + }) + if err != nil { + return -1, fmt.Errorf("db call: %w", err) + } + + return counter, nil +} + // GetGarbage returns garbage according to the metabase state. Garbage includes // objects marked with GC mark (expired, tombstoned but not deleted from disk, // extra replicated, etc.) and removed containers. diff --git a/pkg/local_object_storage/metabase/graveyard_test.go b/pkg/local_object_storage/metabase/graveyard_test.go index d45decf6fd..7ecbd9b8ee 100644 --- a/pkg/local_object_storage/metabase/graveyard_test.go +++ b/pkg/local_object_storage/metabase/graveyard_test.go @@ -1,11 +1,13 @@ package meta_test import ( + "math" "strconv" "testing" "github.com/nspcc-dev/neofs-node/pkg/core/object" meta "github.com/nspcc-dev/neofs-node/pkg/local_object_storage/metabase" + apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test" objectsdk "github.com/nspcc-dev/neofs-sdk-go/object" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" @@ -484,3 +486,55 @@ func TestDB_GetGarbage(t *testing.T) { require.Len(t, garbageContainers, 1) // but container can be deleted now require.Equal(t, garbageContainers[0], cID) } + +func TestDropExpiredTSMarks(t *testing.T) { + epoch := uint64(math.MaxUint64 / 2) + db := newDB(t) + droppedObjects := oidtest.Addresses(1024) + tombstone := oidtest.Address() + + var pInh meta.InhumePrm + pInh.SetTombstone(tombstone, epoch) + pInh.SetAddresses(droppedObjects[:len(droppedObjects)/2]...) + _, err := db.Inhume(pInh) + require.NoError(t, err) + + pInh.SetTombstone(tombstone, epoch+1) + pInh.SetAddresses(droppedObjects[len(droppedObjects)/2:]...) + _, err = db.Inhume(pInh) + require.NoError(t, err) + + for _, o := range droppedObjects { + var pGet meta.GetPrm + pGet.SetAddress(o) + _, err = db.Get(pGet) + require.ErrorIs(t, err, apistatus.ErrObjectAlreadyRemoved) + } + + res, err := db.DropExpiredTSMarks(epoch + 1) + require.NoError(t, err) + require.EqualValues(t, len(droppedObjects)/2, res) // first half with epoch + 1 expiration + + for i, o := range droppedObjects { + var pGet meta.GetPrm + pGet.SetAddress(o) + + _, err = db.Get(pGet) + if i < len(droppedObjects)/2 { + require.ErrorIs(t, err, apistatus.ErrObjectNotFound) + } else { + require.ErrorIs(t, err, apistatus.ErrObjectAlreadyRemoved) + } + } + + res, err = db.DropExpiredTSMarks(epoch + 2) + require.NoError(t, err) + require.EqualValues(t, len(droppedObjects)/2, res) // second half with epoch + 2 expiration + + for _, o := range droppedObjects { + var pGet meta.GetPrm + pGet.SetAddress(o) + _, err = db.Get(pGet) + require.ErrorIs(t, err, apistatus.ErrObjectNotFound) + } +} diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index cae0bf2be7..8aaeebb4eb 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -252,69 +252,19 @@ func (s *Shard) collectExpiredObjects(ctx context.Context, e Event) { log.Debug("finished expired objects handling") } -func (s *Shard) collectExpiredTombstones(ctx context.Context, e Event) { +func (s *Shard) collectExpiredTombstones(_ context.Context, e Event) { epoch := e.(newEpoch).epoch log := s.log.With(zap.Uint64("epoch", epoch)) log.Debug("started expired tombstones handling") - const tssDeleteBatch = 50 - tss := make([]meta.TombstonedObject, 0, tssDeleteBatch) - tssExp := make([]meta.TombstonedObject, 0, tssDeleteBatch) - - var iterPrm meta.GraveyardIterationPrm - iterPrm.SetHandler(func(deletedObject meta.TombstonedObject) error { - tss = append(tss, deletedObject) - - if len(tss) == tssDeleteBatch { - return meta.ErrInterruptIterator - } - - return nil - }) - - for { - log.Debug("iterating tombstones") - - s.m.RLock() - - if s.info.Mode.NoMetabase() { - s.log.Debug("shard is in a degraded mode, skip collecting expired tombstones") - s.m.RUnlock() - - return - } - - err := s.metaBase.IterateOverGraveyard(iterPrm) - if err != nil { - log.Error("iterator over graveyard failed", zap.Error(err)) - s.m.RUnlock() - - return - } - - s.m.RUnlock() - - tssLen := len(tss) - if tssLen == 0 { - break - } - - for _, ts := range tss { - if !s.tsSource.IsTombstoneAvailable(ctx, ts.Tombstone(), epoch) { - tssExp = append(tssExp, ts) - } - } - - log.Debug("handling expired tombstones batch", zap.Int("number", len(tssExp))) - s.expiredTombstonesCallback(ctx, tssExp) - - iterPrm.SetOffset(tss[tssLen-1].Address()) - tss = tss[:0] - tssExp = tssExp[:0] + dropped, err := s.metaBase.DropExpiredTSMarks(epoch) + if err != nil { + log.Error("cleaning graveyard up failed", zap.Error(err)) + return } - log.Debug("finished expired tombstones handling") + log.Debug("finished expired tombstones handling", zap.Int("dropped marks", dropped)) } func (s *Shard) collectExpiredLocks(ctx context.Context, e Event) { From 90d5cfed972f93a29e93e254a6ee6b8845a8a0c3 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Thu, 12 Sep 2024 20:37:58 +0300 Subject: [PATCH 4/5] *: drop unused code related to the old tombstones handling Signed-off-by: Pavel Karpy --- cmd/neofs-node/storage.go | 13 +-- pkg/local_object_storage/engine/inhume.go | 13 --- pkg/local_object_storage/engine/lock_test.go | 1 - pkg/local_object_storage/engine/shards.go | 1 - .../metabase/graveyard.go | 33 ------- .../metabase/graveyard_test.go | 53 ----------- pkg/local_object_storage/shard/gc.go | 49 ---------- pkg/local_object_storage/shard/shard.go | 22 ----- .../object_manager/tombstone/checker.go | 92 ------------------- .../object_manager/tombstone/constructor.go | 85 ----------------- .../object_manager/tombstone/source/source.go | 83 ----------------- 11 files changed, 1 insertion(+), 444 deletions(-) delete mode 100644 pkg/services/object_manager/tombstone/checker.go delete mode 100644 pkg/services/object_manager/tombstone/constructor.go delete mode 100644 pkg/services/object_manager/tombstone/source/source.go diff --git a/cmd/neofs-node/storage.go b/cmd/neofs-node/storage.go index 13714fbfbb..eef9aff2b4 100644 --- a/cmd/neofs-node/storage.go +++ b/cmd/neofs-node/storage.go @@ -17,8 +17,6 @@ import ( containerEvent "github.com/nspcc-dev/neofs-node/pkg/morph/event/container" "github.com/nspcc-dev/neofs-node/pkg/morph/event/netmap" getsvc "github.com/nspcc-dev/neofs-node/pkg/services/object/get" - "github.com/nspcc-dev/neofs-node/pkg/services/object_manager/tombstone" - tsourse "github.com/nspcc-dev/neofs-node/pkg/services/object_manager/tombstone/source" "github.com/nspcc-dev/neofs-node/pkg/util" objectSDK "github.com/nspcc-dev/neofs-sdk-go/object" "github.com/panjf2000/ants/v2" @@ -47,18 +45,9 @@ func initLocalStorage(c *cfg) { // service will be created later c.cfgObject.getSvc = new(getsvc.Service) - var tssPrm tsourse.TombstoneSourcePrm - tssPrm.SetGetService(c.cfgObject.getSvc) - tombstoneSrc := tsourse.NewSource(tssPrm) - - tombstoneSource := tombstone.NewChecker( - tombstone.WithLogger(c.log), - tombstone.WithTombstoneSource(tombstoneSrc), - ) - var shardsAttached int for _, optsWithMeta := range c.shardOpts() { - id, err := ls.AddShard(append(optsWithMeta.shOpts, shard.WithTombstoneSource(tombstoneSource))...) + id, err := ls.AddShard(optsWithMeta.shOpts...) if err != nil { c.log.Error("failed to attach shard to engine", zap.Error(err)) } else { diff --git a/pkg/local_object_storage/engine/inhume.go b/pkg/local_object_storage/engine/inhume.go index 99ed6b7229..770c6b97f4 100644 --- a/pkg/local_object_storage/engine/inhume.go +++ b/pkg/local_object_storage/engine/inhume.go @@ -334,19 +334,6 @@ func (e *StorageEngine) processExpiredObjects(_ context.Context, addrs []oid.Add } } -func (e *StorageEngine) processExpiredTombstones(ctx context.Context, addrs []meta.TombstonedObject) { - e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { - sh.HandleExpiredTombstones(addrs) - - select { - case <-ctx.Done(): - return true - default: - return false - } - }) -} - func (e *StorageEngine) processExpiredLocks(ctx context.Context, lockers []oid.Address) { e.iterateOverUnsortedShards(func(sh hashedShard) (stop bool) { sh.HandleExpiredLocks(lockers) diff --git a/pkg/local_object_storage/engine/lock_test.go b/pkg/local_object_storage/engine/lock_test.go index 5cd24a3f34..f4b42ec6d7 100644 --- a/pkg/local_object_storage/engine/lock_test.go +++ b/pkg/local_object_storage/engine/lock_test.go @@ -51,7 +51,6 @@ func TestLockUserScenario(t *testing.T) { return pool }), - shard.WithTombstoneSource(tss{lockerExpiresAfter}), }) t.Cleanup(func() { diff --git a/pkg/local_object_storage/engine/shards.go b/pkg/local_object_storage/engine/shards.go index 8d07157c17..42254c4bf9 100644 --- a/pkg/local_object_storage/engine/shards.go +++ b/pkg/local_object_storage/engine/shards.go @@ -101,7 +101,6 @@ func (e *StorageEngine) createShard(opts []shard.Option) (*shard.Shard, error) { sh := shard.New(append(opts, shard.WithID(id), shard.WithExpiredObjectsCallback(e.processExpiredObjects), - shard.WithExpiredTombstonesCallback(e.processExpiredTombstones), shard.WithExpiredLocksCallback(e.processExpiredLocks), shard.WithDeletedLockCallback(e.processDeletedLocks), shard.WithReportErrorFunc(e.reportShardErrorBackground), diff --git a/pkg/local_object_storage/metabase/graveyard.go b/pkg/local_object_storage/metabase/graveyard.go index 8d84c89df3..228736ddb3 100644 --- a/pkg/local_object_storage/metabase/graveyard.go +++ b/pkg/local_object_storage/metabase/graveyard.go @@ -229,39 +229,6 @@ func graveFromKV(k, v []byte) (res TombstonedObject, err error) { return } -// DropGraves deletes tombstoned objects from the -// graveyard bucket. -// -// Returns any error appeared during deletion process. -func (db *DB) DropGraves(tss []TombstonedObject) error { - db.modeMtx.RLock() - defer db.modeMtx.RUnlock() - - if db.mode.NoMetabase() { - return ErrDegradedMode - } else if db.mode.ReadOnly() { - return ErrReadOnlyMode - } - - buf := make([]byte, addressKeySize) - - return db.boltDB.Update(func(tx *bbolt.Tx) error { - bkt := tx.Bucket(graveyardBucketName) - if bkt == nil { - return nil - } - - for _, ts := range tss { - err := bkt.Delete(addressKey(ts.Address(), buf)) - if err != nil { - return err - } - } - - return nil - }) -} - // DropExpiredTSMarks run through the graveyard and drops tombstone marks with // tombstones whose expiration is _less_ than provided epoch. // Returns number of marks dropped. diff --git a/pkg/local_object_storage/metabase/graveyard_test.go b/pkg/local_object_storage/metabase/graveyard_test.go index 7ecbd9b8ee..50ca87849a 100644 --- a/pkg/local_object_storage/metabase/graveyard_test.go +++ b/pkg/local_object_storage/metabase/graveyard_test.go @@ -386,59 +386,6 @@ func TestDB_IterateOverGarbage_Offset(t *testing.T) { require.False(t, iWasCalled) } -func TestDB_DropGraves(t *testing.T) { - db := newDB(t) - - // generate and put 2 objects - obj1 := generateObject(t) - obj2 := generateObject(t) - - var err error - - err = putBig(db, obj1) - require.NoError(t, err) - - err = putBig(db, obj2) - require.NoError(t, err) - - // inhume with tombstone - addrTombstone := oidtest.Address() - - var inhumePrm meta.InhumePrm - inhumePrm.SetAddresses(object.AddressOf(obj1), object.AddressOf(obj2)) - inhumePrm.SetTombstone(addrTombstone, 0) - - _, err = db.Inhume(inhumePrm) - require.NoError(t, err) - - buriedTS := make([]meta.TombstonedObject, 0) - var iterGravePRM meta.GraveyardIterationPrm - var counter int - iterGravePRM.SetHandler(func(tomstoned meta.TombstonedObject) error { - buriedTS = append(buriedTS, tomstoned) - counter++ - - return nil - }) - - err = db.IterateOverGraveyard(iterGravePRM) - require.NoError(t, err) - require.Equal(t, 2, counter) - - err = db.DropGraves(buriedTS) - require.NoError(t, err) - - counter = 0 - iterGravePRM.SetHandler(func(_ meta.TombstonedObject) error { - counter++ - return nil - }) - - err = db.IterateOverGraveyard(iterGravePRM) - require.NoError(t, err) - require.Zero(t, counter) -} - func TestDB_GetGarbage(t *testing.T) { db := newDB(t) diff --git a/pkg/local_object_storage/shard/gc.go b/pkg/local_object_storage/shard/gc.go index 8aaeebb4eb..e013bbd0ff 100644 --- a/pkg/local_object_storage/shard/gc.go +++ b/pkg/local_object_storage/shard/gc.go @@ -13,15 +13,6 @@ import ( "go.uber.org/zap" ) -// TombstoneSource is an interface that checks -// tombstone status in the NeoFS network. -type TombstoneSource interface { - // IsTombstoneAvailable must return boolean value that means - // provided tombstone's presence in the NeoFS network at the - // time of the passed epoch. - IsTombstoneAvailable(ctx context.Context, addr oid.Address, epoch uint64) bool -} - // Event represents class of external events. type Event interface { typ() eventType @@ -308,46 +299,6 @@ func (s *Shard) getExpiredObjects(ctx context.Context, epoch uint64, typeCond fu return expired, ctx.Err() } -// HandleExpiredTombstones marks tombstones themselves as garbage -// and clears up corresponding graveyard records. -// -// Does not modify tss. -func (s *Shard) HandleExpiredTombstones(tss []meta.TombstonedObject) { - if s.GetMode().NoMetabase() { - return - } - - // Mark tombstones as garbage. - var pInhume meta.InhumePrm - - tsAddrs := make([]oid.Address, 0, len(tss)) - for _, ts := range tss { - tsAddrs = append(tsAddrs, ts.Tombstone()) - } - - pInhume.SetGCMark() - pInhume.SetAddresses(tsAddrs...) - - // inhume tombstones - res, err := s.metaBase.Inhume(pInhume) - if err != nil { - s.log.Warn("could not mark tombstones as garbage", - zap.String("error", err.Error()), - ) - - return - } - - s.decObjectCounterBy(logical, res.AvailableInhumed()) - - // drop just processed expired tombstones - // from graveyard - err = s.metaBase.DropGraves(tss) - if err != nil { - s.log.Warn("could not drop expired grave records", zap.Error(err)) - } -} - // HandleExpiredLocks unlocks all objects which were locked by lockers. // If successful, marks lockers themselves as garbage. Also, marks as // garbage every object that becomes free-to-remove and just removed diff --git a/pkg/local_object_storage/shard/shard.go b/pkg/local_object_storage/shard/shard.go index ae6d27ab2a..7817947106 100644 --- a/pkg/local_object_storage/shard/shard.go +++ b/pkg/local_object_storage/shard/shard.go @@ -28,8 +28,6 @@ type Shard struct { pilorama pilorama.ForestStorage metaBase *meta.DB - - tsSource TombstoneSource } // Option represents Shard's constructor option. @@ -96,14 +94,10 @@ type cfg struct { expiredObjectsCallback ExpiredObjectsCallback - expiredTombstonesCallback ExpiredTombstonesCallback - expiredLocksCallback ExpiredObjectsCallback deletedLockCallBack DeletedLockCallback - tsSource TombstoneSource - metricsWriter MetricsWriter reportErrorFunc func(selfID string, message string, err error) @@ -133,7 +127,6 @@ func New(opts ...Option) *Shard { cfg: c, blobStor: bs, metaBase: mb, - tsSource: c.tsSource, } reportFunc := func(msg string, err error) { @@ -251,14 +244,6 @@ func WithExpiredObjectsCallback(cb ExpiredObjectsCallback) Option { } } -// WithExpiredTombstonesCallback returns option to specify callback -// of the expired tombstones handler. -func WithExpiredTombstonesCallback(cb ExpiredTombstonesCallback) Option { - return func(c *cfg) { - c.expiredTombstonesCallback = cb - } -} - // WithExpiredLocksCallback returns option to specify callback // of the expired LOCK objects handler. func WithExpiredLocksCallback(cb ExpiredObjectsCallback) Option { @@ -283,13 +268,6 @@ func WithMode(v mode.Mode) Option { } } -// WithTombstoneSource returns option to set TombstoneSource. -func WithTombstoneSource(v TombstoneSource) Option { - return func(c *cfg) { - c.tsSource = v - } -} - // WithDeletedLockCallback returns option to specify callback // of the deleted LOCK objects handler. func WithDeletedLockCallback(v DeletedLockCallback) Option { diff --git a/pkg/services/object_manager/tombstone/checker.go b/pkg/services/object_manager/tombstone/checker.go deleted file mode 100644 index 91b1478b2d..0000000000 --- a/pkg/services/object_manager/tombstone/checker.go +++ /dev/null @@ -1,92 +0,0 @@ -package tombstone - -import ( - "context" - "strconv" - - lru "github.com/hashicorp/golang-lru/v2" - "github.com/nspcc-dev/neofs-sdk-go/object" - oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - "go.uber.org/zap" -) - -// Source is a tombstone source interface. -type Source interface { - // Tombstone must return tombstone from the source it was - // configured to fetch from and any error that appeared during - // fetching process. - // - // Tombstone MUST return (nil, nil) if requested tombstone is - // missing in the storage for the provided epoch. - Tombstone(ctx context.Context, a oid.Address, epoch uint64) (*object.Object, error) -} - -// ExpirationChecker is a tombstone source wrapper. -// It checks tombstones presence via tombstone -// source, caches it checks its expiration. -// -// Must be created via NewChecker function. `var` and -// `ExpirationChecker{}` declarations leads to undefined behaviour -// and may lead to panics. -type ExpirationChecker struct { - cache *lru.Cache[oid.Address, uint64] - - log *zap.Logger - - tsSource Source -} - -// IsTombstoneAvailable checks the tombstone presence in the system in the -// following order: -// - 1. Local LRU cache; -// - 2. Tombstone source. -// -// If a tombstone was successfully fetched (regardless of its expiration) -// it is cached in the LRU cache. -func (g *ExpirationChecker) IsTombstoneAvailable(ctx context.Context, a oid.Address, epoch uint64) bool { - addrStr := a.EncodeToString() - log := g.log.With(zap.String("address", addrStr)) - - expEpoch, ok := g.cache.Get(a) - if ok { - return expEpoch > epoch - } - - ts, err := g.tsSource.Tombstone(ctx, a, epoch) - if err != nil { - log.Warn( - "tombstone getter: could not get the tombstone the source", - zap.Error(err), - ) - } else { - if ts != nil { - return g.handleTS(a, ts, epoch) - } - } - - // requested tombstone not - // found in the NeoFS network - return false -} - -func (g *ExpirationChecker) handleTS(addr oid.Address, ts *object.Object, reqEpoch uint64) bool { - for _, atr := range ts.Attributes() { - if atr.Key() == object.AttributeExpirationEpoch { - epoch, err := strconv.ParseUint(atr.Value(), 10, 64) - if err != nil { - g.log.Warn( - "tombstone getter: could not parse tombstone expiration epoch", - zap.Error(err), - ) - - return false - } - - g.cache.Add(addr, epoch) - return epoch >= reqEpoch - } - } - - // unexpected tombstone without expiration epoch - return false -} diff --git a/pkg/services/object_manager/tombstone/constructor.go b/pkg/services/object_manager/tombstone/constructor.go deleted file mode 100644 index 6f876cf0af..0000000000 --- a/pkg/services/object_manager/tombstone/constructor.go +++ /dev/null @@ -1,85 +0,0 @@ -package tombstone - -import ( - "fmt" - - lru "github.com/hashicorp/golang-lru/v2" - oid "github.com/nspcc-dev/neofs-sdk-go/object/id" - "go.uber.org/zap" -) - -const defaultLRUCacheSize = 100 - -type cfg struct { - log *zap.Logger - - cacheSize int - - tsSource Source -} - -// Option is an option of ExpirationChecker's constructor. -type Option func(*cfg) - -func defaultCfg() *cfg { - return &cfg{ - log: zap.NewNop(), - cacheSize: defaultLRUCacheSize, - } -} - -// NewChecker creates, initializes and returns tombstone ExpirationChecker. -// The returned structure is ready to use. -// -// Panics if any of the provided options does not allow -// constructing a valid tombstone ExpirationChecker. -func NewChecker(oo ...Option) *ExpirationChecker { - cfg := defaultCfg() - - for _, o := range oo { - o(cfg) - } - - panicOnNil := func(v any, name string) { - if v == nil { - panic(fmt.Sprintf("tombstone getter constructor: %s is nil", name)) - } - } - - panicOnNil(cfg.tsSource, "Tombstone source") - - cache, err := lru.New[oid.Address, uint64](cfg.cacheSize) - if err != nil { - panic(fmt.Errorf("could not create LRU cache with %d size: %w", cfg.cacheSize, err)) - } - - return &ExpirationChecker{ - cache: cache, - log: cfg.log, - tsSource: cfg.tsSource, - } -} - -// WithLogger returns an option to specify -// logger. -func WithLogger(v *zap.Logger) Option { - return func(c *cfg) { - c.log = v - } -} - -// WithCacheSize returns an option to specify -// LRU cache size. -func WithCacheSize(v int) Option { - return func(c *cfg) { - c.cacheSize = v - } -} - -// WithTombstoneSource returns an option -// to specify tombstone source. -func WithTombstoneSource(v Source) Option { - return func(c *cfg) { - c.tsSource = v - } -} diff --git a/pkg/services/object_manager/tombstone/source/source.go b/pkg/services/object_manager/tombstone/source/source.go deleted file mode 100644 index 8444e6b60d..0000000000 --- a/pkg/services/object_manager/tombstone/source/source.go +++ /dev/null @@ -1,83 +0,0 @@ -package tsourse - -import ( - "context" - "errors" - "fmt" - - getsvc "github.com/nspcc-dev/neofs-node/pkg/services/object/get" - "github.com/nspcc-dev/neofs-node/pkg/services/object/util" - apistatus "github.com/nspcc-dev/neofs-sdk-go/client/status" - "github.com/nspcc-dev/neofs-sdk-go/object" - oid "github.com/nspcc-dev/neofs-sdk-go/object/id" -) - -// Source represents wrapper over the object service that -// allows checking if a tombstone is available in NeoFS -// network. -// -// Must be created via NewSource function. `var` and `Source{}` -// declarations leads to undefined behaviour and may lead -// to panics. -type Source struct { - s *getsvc.Service -} - -// TombstoneSourcePrm groups required parameters for Source creation. -type TombstoneSourcePrm struct { - s *getsvc.Service -} - -// SetGetService sets object service. -func (s *TombstoneSourcePrm) SetGetService(v *getsvc.Service) { - s.s = v -} - -// NewSource creates, initialize and returns local tombstone Source. -// The returned structure is ready to use. -// -// Panics if any of the provided options does not allow -// constructing a valid tombstone local Source. -func NewSource(p TombstoneSourcePrm) Source { - if p.s == nil { - panic("Tombstone source: nil object service") - } - - return Source(p) -} - -type headerWriter struct { - o *object.Object -} - -func (h *headerWriter) WriteHeader(o *object.Object) error { - h.o = o - return nil -} - -// Tombstone checks if the engine stores tombstone. -// Returns nil, nil if the tombstone has been removed -// or marked for removal. -func (s Source) Tombstone(ctx context.Context, a oid.Address, _ uint64) (*object.Object, error) { - var hr headerWriter - - var headPrm getsvc.HeadPrm - headPrm.WithAddress(a) - headPrm.SetHeaderWriter(&hr) - headPrm.SetCommonParameters(&util.CommonPrm{}) // default values are ok for that operation - - err := s.s.Head(ctx, headPrm) - switch { - case errors.As(err, new(apistatus.ObjectNotFound)) || errors.As(err, new(apistatus.ObjectAlreadyRemoved)): - return nil, nil - case err != nil: - return nil, fmt.Errorf("could not get tombstone from the source: %w", err) - default: - } - - if hr.o.Type() != object.TypeTombstone { - return nil, fmt.Errorf("returned %s object is not a tombstone", a) - } - - return hr.o, nil -} From 28fd1e7a03f2d2f9c2e4deded3f141fabf6bf772 Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Thu, 12 Sep 2024 20:45:33 +0300 Subject: [PATCH 5/5] lens, meta: support both graveyard version It may or may not index tombstone's expiration in graveyard. Signed-off-by: Pavel Karpy --- .../internal/meta/list-graveyard.go | 3 +- .../metabase/graveyard.go | 28 +++++++++++++++---- 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/cmd/neofs-lens/internal/meta/list-graveyard.go b/cmd/neofs-lens/internal/meta/list-graveyard.go index 09457dfb3f..a73833e62e 100644 --- a/cmd/neofs-lens/internal/meta/list-graveyard.go +++ b/cmd/neofs-lens/internal/meta/list-graveyard.go @@ -26,9 +26,10 @@ func listGraveyardFunc(cmd *cobra.Command, _ []string) { gravePrm.SetHandler( func(tsObj meta.TombstonedObject) error { cmd.Printf( - "Object: %s\nTS: %s\n", + "Object: %s\nTS: %s (TS expiration: %d)\n", tsObj.Address().EncodeToString(), tsObj.Tombstone().EncodeToString(), + tsObj.TombstoneExpiration(), ) return nil diff --git a/pkg/local_object_storage/metabase/graveyard.go b/pkg/local_object_storage/metabase/graveyard.go index 228736ddb3..685fd95eaa 100644 --- a/pkg/local_object_storage/metabase/graveyard.go +++ b/pkg/local_object_storage/metabase/graveyard.go @@ -75,8 +75,9 @@ func (db *DB) IterateOverGarbage(p GarbageIterationPrm) error { // TombstonedObject represents descriptor of the // object that has been covered with tombstone. type TombstonedObject struct { - addr oid.Address - tomb oid.Address + addr oid.Address + tomb oid.Address + tombExpiration uint64 } // Address returns tombstoned object address. @@ -90,6 +91,13 @@ func (g TombstonedObject) Tombstone() oid.Address { return g.tomb } +// TombstoneExpiration returns tombstone's expiration. It can be zero if +// metabase version does not support expiration indexing or if TS does not +// expire. +func (g TombstonedObject) TombstoneExpiration() uint64 { + return g.tombExpiration +} + // TombstonedHandler is a TombstonedObject handling function. type TombstonedHandler func(object TombstonedObject) error @@ -220,10 +228,18 @@ func garbageFromKV(k []byte) (res GarbageObject, err error) { } func graveFromKV(k, v []byte) (res TombstonedObject, err error) { - if err = decodeAddressFromKey(&res.addr, k); err != nil { - err = fmt.Errorf("decode tombstone target from key: %w", err) - } else if err = decodeAddressFromKey(&res.tomb, v[:addressKeySize]); err != nil { - err = fmt.Errorf("decode tombstone address from value: %w", err) + err = decodeAddressFromKey(&res.addr, k) + if err != nil { + return res, fmt.Errorf("decode tombstone target from key: %w", err) + } + + err = decodeAddressFromKey(&res.tomb, v[:addressKeySize]) + if err != nil { + return res, fmt.Errorf("decode tombstone address from value: %w", err) + } + + if len(v) == addressKeySize+8 { + res.tombExpiration = binary.LittleEndian.Uint64(v[addressKeySize:]) } return