From c89f16dfa75f8d4e351ac19fd9c8e613f535913e Mon Sep 17 00:00:00 2001 From: Pavel Karpy Date: Wed, 25 Dec 2024 19:18:55 +0300 Subject: [PATCH] node/object/put: push first and previous parts to chain A first object has its own OID to simplify detection of such objects by indexed meta-information. Closes #2935. Signed-off-by: Pavel Karpy --- pkg/core/object/replicate.go | 16 +++++++++++--- pkg/core/object/replicate_test.go | 22 ++++++++++++++----- .../transport/object/grpc/replication.go | 13 ++++++++--- .../transport/object/grpc/replication_test.go | 3 ++- pkg/services/object/put/distributed.go | 9 +++++++- 5 files changed, 50 insertions(+), 13 deletions(-) diff --git a/pkg/core/object/replicate.go b/pkg/core/object/replicate.go index 0d0f3ee955..c6b44b7c2d 100644 --- a/pkg/core/object/replicate.go +++ b/pkg/core/object/replicate.go @@ -17,8 +17,10 @@ const ( networkMagicKey = "network" // optional fields. - deletedKey = "deleted" - lockedKey = "locked" + firstPartKey = "firstPart" + previousPartKey = "previousPart" + deletedKey = "deleted" + lockedKey = "locked" ) // EncodeReplicationMetaInfo uses NEO's map (strict order) serialized format as a raw @@ -31,9 +33,11 @@ const ( // "size": payload size // "validUntil": last valid block number for meta information // "network": network magic +// "firstPart": [OPTIONAL] _raw_ object ID (32 bytes) +// "previousPart": [OPTIONAL] _raw_ object ID (32 bytes) // "deleted": [OPTIONAL] array of _raw_ object IDs // "locked": [OPTIONAL] array of _raw_ object IDs -func EncodeReplicationMetaInfo(cID cid.ID, oID oid.ID, pSize uint64, +func EncodeReplicationMetaInfo(cID cid.ID, oID, firstPart, previousPart oid.ID, pSize uint64, deleted, locked []oid.ID, vub uint64, magicNumber uint32) []byte { kvs := []stackitem.MapElement{ kv(cidKey, cID[:]), @@ -43,6 +47,12 @@ func EncodeReplicationMetaInfo(cID cid.ID, oID oid.ID, pSize uint64, kv(networkMagicKey, magicNumber), } + if !firstPart.IsZero() { + kvs = append(kvs, kv(firstPartKey, firstPart[:])) + } + if !previousPart.IsZero() { + kvs = append(kvs, kv(previousPartKey, previousPart[:])) + } if len(deleted) > 0 { kvs = append(kvs, oidsKV(deletedKey, deleted)) } diff --git a/pkg/core/object/replicate_test.go b/pkg/core/object/replicate_test.go index 0bf9351bf4..fb98e748cf 100644 --- a/pkg/core/object/replicate_test.go +++ b/pkg/core/object/replicate_test.go @@ -20,6 +20,8 @@ type m struct { vub uint64 magic uint32 + first oid.ID + prev oid.ID deleted []oid.ID locked []oid.ID } @@ -31,6 +33,8 @@ func TestMetaInfo(t *testing.T) { size: rand.Uint64(), vub: rand.Uint64(), magic: rand.Uint32(), + first: oidtest.ID(), + prev: oidtest.ID(), deleted: oidtest.IDs(10), locked: oidtest.IDs(10), } @@ -40,6 +44,8 @@ func TestMetaInfo(t *testing.T) { }) t.Run("no optional", func(t *testing.T) { + meta.first = oid.ID{} + meta.prev = oid.ID{} meta.deleted = nil meta.deleted = nil meta.locked = nil @@ -49,7 +55,7 @@ func TestMetaInfo(t *testing.T) { } func testMeta(t *testing.T, m m, full bool) { - raw := EncodeReplicationMetaInfo(m.cID, m.oID, m.size, m.deleted, m.locked, m.vub, m.magic) + raw := EncodeReplicationMetaInfo(m.cID, m.oID, m.first, m.prev, m.size, m.deleted, m.locked, m.vub, m.magic) item, err := stackitem.Deserialize(raw) require.NoError(t, err) @@ -77,11 +83,17 @@ func testMeta(t *testing.T, m m, full bool) { return } - require.Equal(t, deletedKey, string(mm[5].Key.Value().([]byte))) - require.Equal(t, m.deleted, stackItemToOIDs(t, mm[5].Value)) + require.Equal(t, firstPartKey, string(mm[5].Key.Value().([]byte))) + require.Equal(t, m.first[:], mm[5].Value.Value().([]byte)) - require.Equal(t, lockedKey, string(mm[6].Key.Value().([]byte))) - require.Equal(t, m.locked, stackItemToOIDs(t, mm[6].Value)) + require.Equal(t, previousPartKey, string(mm[6].Key.Value().([]byte))) + require.Equal(t, m.prev[:], mm[6].Value.Value().([]byte)) + + require.Equal(t, deletedKey, string(mm[7].Key.Value().([]byte))) + require.Equal(t, m.deleted, stackItemToOIDs(t, mm[7].Value)) + + require.Equal(t, lockedKey, string(mm[8].Key.Value().([]byte))) + require.Equal(t, m.locked, stackItemToOIDs(t, mm[8].Value)) } func stackItemToOIDs(t *testing.T, value stackitem.Item) []oid.ID { diff --git a/pkg/network/transport/object/grpc/replication.go b/pkg/network/transport/object/grpc/replication.go index 4c99e5cf89..5bd1398458 100644 --- a/pkg/network/transport/object/grpc/replication.go +++ b/pkg/network/transport/object/grpc/replication.go @@ -206,6 +206,13 @@ func objectFromMessage(gMsg *objectGRPC.Object) (*object.Object, error) { } func (s *Server) metaInfoSignature(o object.Object) ([]byte, error) { + firstObj := o.GetFirstID() + if o.HasParent() && firstObj.IsZero() { + // object itself is the first one + firstObj = o.GetID() + } + prevObj := o.GetPreviousID() + var deleted []oid.ID var locked []oid.ID switch o.Type() { @@ -235,9 +242,9 @@ func (s *Server) metaInfoSignature(o object.Object) ([]byte, error) { secondBlock := firstBlock + currentEpochDuration thirdBlock := secondBlock + currentEpochDuration - firstMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), o.PayloadSize(), deleted, locked, firstBlock, s.mNumber) - secondMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), o.PayloadSize(), deleted, locked, secondBlock, s.mNumber) - thirdMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), o.PayloadSize(), deleted, locked, thirdBlock, s.mNumber) + firstMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), firstObj, prevObj, o.PayloadSize(), deleted, locked, firstBlock, s.mNumber) + secondMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), firstObj, prevObj, o.PayloadSize(), deleted, locked, secondBlock, s.mNumber) + thirdMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), firstObj, prevObj, o.PayloadSize(), deleted, locked, thirdBlock, s.mNumber) var firstSig neofscrypto.Signature var secondSig neofscrypto.Signature diff --git a/pkg/network/transport/object/grpc/replication_test.go b/pkg/network/transport/object/grpc/replication_test.go index 6dbd1dbae7..660ca5d962 100644 --- a/pkg/network/transport/object/grpc/replication_test.go +++ b/pkg/network/transport/object/grpc/replication_test.go @@ -135,6 +135,7 @@ func anyValidRequest(tb testing.TB, signer neofscrypto.Signer, cnr cid.ID, objID obj.SetType(object.TypeRegular) obj.SetContainerID(cnr) obj.SetID(objID) + obj.SetFirstID(oidtest.ID()) sig, err := signer.Sign(objID[:]) require.NoError(tb, err) @@ -408,7 +409,7 @@ func TestServer_Replicate(t *testing.T) { require.Equal(t, signer.PublicKeyBytes, sig.PublicKeyBytes()) require.True(t, sig.Verify(objectcore.EncodeReplicationMetaInfo( - o.GetContainerID(), o.GetID(), o.PayloadSize(), nil, nil, + o.GetContainerID(), o.GetID(), o.GetFirstID(), o.GetPreviousID(), o.PayloadSize(), nil, nil, uint64((123+1+i)*240), mNumber))) sigsRaw = sigsRaw[:4+l] diff --git a/pkg/services/object/put/distributed.go b/pkg/services/object/put/distributed.go index ce5d26f3ae..52bd2ce47e 100644 --- a/pkg/services/object/put/distributed.go +++ b/pkg/services/object/put/distributed.go @@ -121,6 +121,13 @@ func (t *distributedTarget) Close() (oid.ID, error) { t.encodedObject.b = nil }() + firstObj := t.obj.GetFirstID() + if t.obj.HasParent() && firstObj.IsZero() { + // object itself is the first one + firstObj = t.obj.GetID() + } + prevObj := t.obj.GetPreviousID() + t.obj.SetPayload(t.encodedObject.b[t.encodedObject.pldOff:]) tombOrLink := t.obj.Type() == objectSDK.TypeLink || t.obj.Type() == objectSDK.TypeTombstone @@ -152,7 +159,7 @@ func (t *distributedTarget) Close() (oid.ID, error) { } expectedVUB := (uint64(t.currentBlock)/t.currentEpochDuration + 2) * t.currentEpochDuration - t.objSharedMeta = object.EncodeReplicationMetaInfo(t.obj.GetContainerID(), t.obj.GetID(), t.obj.PayloadSize(), deletedObjs, + t.objSharedMeta = object.EncodeReplicationMetaInfo(t.obj.GetContainerID(), t.obj.GetID(), firstObj, prevObj, t.obj.PayloadSize(), deletedObjs, lockedObjs, expectedVUB, t.networkMagicNumber) id := t.obj.GetID() err := t.placementIterator.iterateNodesForObject(id, t.sendObject)