Skip to content

Commit

Permalink
Feat/add big objects support for meta on chain (#3063)
Browse files Browse the repository at this point in the history
  • Loading branch information
@roman-khimov
roman-khimov authored Dec 28, 2024
2 parents 6749956 + 8ebd425 commit b39e7b8
Show file tree
Hide file tree
Showing 5 changed files with 137 additions and 50 deletions.
50 changes: 34 additions & 16 deletions pkg/core/object/replicate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,47 +5,65 @@ import (

"github.com/nspcc-dev/neo-go/pkg/vm/stackitem"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
objectsdk "github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
)

const (
currentVersion = 7 // it is also a number of fields
)

const (
networkMagicKey = "network"
// required fields.
cidKey = "cid"
oidKey = "oid"
sizeKey = "size"
validUntilKey = "validUntil"
networkMagicKey = "network"

// optional fields.
firstPartKey = "firstPart"
previousPartKey = "previousPart"
deletedKey = "deleted"
lockedKey = "locked"
validUntilKey = "validuntil"
typeKey = "type"
)

// EncodeReplicationMetaInfo uses NEO's map (strict order) serialized format as a raw
// representation of object's meta information.
//
// This (ordered) format is used (keys are strings):
//
// "network": network magic
// "cid": _raw_ container ID (32 bytes)
// "oid": _raw_ object ID (32 bytes)
// "size": payload size
// "deleted": array of _raw_ object IDs
// "locked": array of _raw_ object IDs
// "validuntil": last valid block number for meta information
//
// Last valid epoch is object's creation epoch + 10.
func EncodeReplicationMetaInfo(cID cid.ID, oID oid.ID, pSize uint64,
// "validUntil": last valid block number for meta information
// "network": network magic
// "firstPart": [OPTIONAL] _raw_ object ID (32 bytes)
// "previousPart": [OPTIONAL] _raw_ object ID (32 bytes)
// "deleted": [OPTIONAL] array of _raw_ object IDs
// "locked": [OPTIONAL] array of _raw_ object IDs
// "type": [OPTIONAL] object type enumeration
func EncodeReplicationMetaInfo(cID cid.ID, oID, firstPart, previousPart oid.ID, pSize uint64, typ objectsdk.Type,
deleted, locked []oid.ID, vub uint64, magicNumber uint32) []byte {
kvs := []stackitem.MapElement{
kv(networkMagicKey, magicNumber),
kv(cidKey, cID[:]),
kv(oidKey, oID[:]),
kv(sizeKey, pSize),
oidsKV(deletedKey, deleted),
oidsKV(lockedKey, locked),
kv(validUntilKey, vub),
kv(networkMagicKey, magicNumber),
}

if !firstPart.IsZero() {
kvs = append(kvs, kv(firstPartKey, firstPart[:]))
}
if !previousPart.IsZero() {
kvs = append(kvs, kv(previousPartKey, previousPart[:]))
}
if len(deleted) > 0 {
kvs = append(kvs, oidsKV(deletedKey, deleted))
}
if len(locked) > 0 {
kvs = append(kvs, oidsKV(lockedKey, locked))
}
if typ != objectsdk.TypeRegular {
kvs = append(kvs, kv(typeKey, uint32(typ)))
}

result, err := stackitem.Serialize(stackitem.NewMapWithValue(kvs))
Expand Down
98 changes: 74 additions & 24 deletions pkg/core/object/replicate_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,51 +6,101 @@ import (
"testing"

"github.com/nspcc-dev/neo-go/pkg/vm/stackitem"
cid "github.com/nspcc-dev/neofs-sdk-go/container/id"
cidtest "github.com/nspcc-dev/neofs-sdk-go/container/id/test"
"github.com/nspcc-dev/neofs-sdk-go/object"
oid "github.com/nspcc-dev/neofs-sdk-go/object/id"
oidtest "github.com/nspcc-dev/neofs-sdk-go/object/id/test"
"github.com/stretchr/testify/require"
)

type m struct {
cID cid.ID
oID oid.ID
size uint64
vub uint64
magic uint32

first oid.ID
prev oid.ID
deleted []oid.ID
locked []oid.ID
typ object.Type
}

func TestMetaInfo(t *testing.T) {
network := rand.Uint32()
oID := oidtest.ID()
cID := cidtest.ID()
size := rand.Uint64()
deleted := oidtest.IDs(10)
locked := oidtest.IDs(10)
validUntil := rand.Uint64()

raw := EncodeReplicationMetaInfo(cID, oID, size, deleted, locked, validUntil, network)
meta := m{
cID: cidtest.ID(),
oID: oidtest.ID(),
size: rand.Uint64(),
vub: rand.Uint64(),
magic: rand.Uint32(),
first: oidtest.ID(),
prev: oidtest.ID(),
deleted: oidtest.IDs(10),
locked: oidtest.IDs(10),
typ: object.TypeTombstone,
}

t.Run("full", func(t *testing.T) {
testMeta(t, meta, true)
})

t.Run("no optional", func(t *testing.T) {
meta.first = oid.ID{}
meta.prev = oid.ID{}
meta.deleted = nil
meta.deleted = nil
meta.locked = nil
meta.typ = object.TypeRegular

testMeta(t, meta, false)
})
}

func testMeta(t *testing.T, m m, full bool) {
raw := EncodeReplicationMetaInfo(m.cID, m.oID, m.first, m.prev, m.size, m.typ, m.deleted, m.locked, m.vub, m.magic)
item, err := stackitem.Deserialize(raw)
require.NoError(t, err)

require.Equal(t, stackitem.MapT, item.Type())
mm, ok := item.Value().([]stackitem.MapElement)
require.True(t, ok)

require.Len(t, mm, currentVersion)
require.Equal(t, cidKey, string(mm[0].Key.Value().([]byte)))
require.Equal(t, m.cID[:], mm[0].Value.Value().([]byte))

require.Equal(t, oidKey, string(mm[1].Key.Value().([]byte)))
require.Equal(t, m.oID[:], mm[1].Value.Value().([]byte))

require.Equal(t, networkMagicKey, string(mm[0].Key.Value().([]byte)))
require.Equal(t, network, uint32(mm[0].Value.Value().(*big.Int).Uint64()))
require.Equal(t, sizeKey, string(mm[2].Key.Value().([]byte)))
require.Equal(t, m.size, mm[2].Value.Value().(*big.Int).Uint64())

require.Equal(t, cidKey, string(mm[1].Key.Value().([]byte)))
require.Equal(t, cID[:], mm[1].Value.Value().([]byte))
require.Equal(t, validUntilKey, string(mm[3].Key.Value().([]byte)))
require.Equal(t, m.vub, mm[3].Value.Value().(*big.Int).Uint64())

require.Equal(t, networkMagicKey, string(mm[4].Key.Value().([]byte)))
require.Equal(t, m.magic, uint32(mm[4].Value.Value().(*big.Int).Uint64()))

if !full {
require.Len(t, mm, 5)
return
}

require.Equal(t, oidKey, string(mm[2].Key.Value().([]byte)))
require.Equal(t, oID[:], mm[2].Value.Value().([]byte))
require.Equal(t, firstPartKey, string(mm[5].Key.Value().([]byte)))
require.Equal(t, m.first[:], mm[5].Value.Value().([]byte))

require.Equal(t, sizeKey, string(mm[3].Key.Value().([]byte)))
require.Equal(t, size, mm[3].Value.Value().(*big.Int).Uint64())
require.Equal(t, previousPartKey, string(mm[6].Key.Value().([]byte)))
require.Equal(t, m.prev[:], mm[6].Value.Value().([]byte))

require.Equal(t, deletedKey, string(mm[4].Key.Value().([]byte)))
require.Equal(t, deleted, stackItemToOIDs(t, mm[4].Value))
require.Equal(t, deletedKey, string(mm[7].Key.Value().([]byte)))
require.Equal(t, m.deleted, stackItemToOIDs(t, mm[7].Value))

require.Equal(t, lockedKey, string(mm[5].Key.Value().([]byte)))
require.Equal(t, locked, stackItemToOIDs(t, mm[5].Value))
require.Equal(t, lockedKey, string(mm[8].Key.Value().([]byte)))
require.Equal(t, m.locked, stackItemToOIDs(t, mm[8].Value))

require.Equal(t, validUntilKey, string(mm[6].Key.Value().([]byte)))
require.Equal(t, validUntil, mm[6].Value.Value().(*big.Int).Uint64())
require.Equal(t, typeKey, string(mm[9].Key.Value().([]byte)))
require.Equal(t, int(m.typ), int(mm[9].Value.Value().(*big.Int).Uint64()))
}

func stackItemToOIDs(t *testing.T, value stackitem.Item) []oid.ID {
Expand Down
16 changes: 12 additions & 4 deletions pkg/network/transport/object/grpc/replication.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -206,9 +206,17 @@ func objectFromMessage(gMsg *objectGRPC.Object) (*object.Object, error) {
}

func (s *Server) metaInfoSignature(o object.Object) ([]byte, error) {
firstObj := o.GetFirstID()
if o.HasParent() && firstObj.IsZero() {
// object itself is the first one
firstObj = o.GetID()
}
prevObj := o.GetPreviousID()

var deleted []oid.ID
var locked []oid.ID
switch o.Type() {
typ := o.Type()
switch typ {
case object.TypeTombstone:
var t object.Tombstone
err := t.Unmarshal(o.Payload())
Expand All @@ -235,9 +243,9 @@ func (s *Server) metaInfoSignature(o object.Object) ([]byte, error) {
secondBlock := firstBlock + currentEpochDuration
thirdBlock := secondBlock + currentEpochDuration

firstMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), o.PayloadSize(), deleted, locked, firstBlock, s.mNumber)
secondMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), o.PayloadSize(), deleted, locked, secondBlock, s.mNumber)
thirdMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), o.PayloadSize(), deleted, locked, thirdBlock, s.mNumber)
firstMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), firstObj, prevObj, o.PayloadSize(), typ, deleted, locked, firstBlock, s.mNumber)
secondMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), firstObj, prevObj, o.PayloadSize(), typ, deleted, locked, secondBlock, s.mNumber)
thirdMeta := objectcore.EncodeReplicationMetaInfo(o.GetContainerID(), o.GetID(), firstObj, prevObj, o.PayloadSize(), typ, deleted, locked, thirdBlock, s.mNumber)

var firstSig neofscrypto.Signature
var secondSig neofscrypto.Signature
Expand Down
11 changes: 7 additions & 4 deletions pkg/network/transport/object/grpc/replication_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"crypto/rand"
"encoding/binary"
"errors"
"fmt"
"testing"

objectV2 "github.com/nspcc-dev/neofs-api-go/v2/object"
Expand Down Expand Up @@ -135,6 +136,8 @@ func anyValidRequest(tb testing.TB, signer neofscrypto.Signer, cnr cid.ID, objID
obj.SetType(object.TypeRegular)
obj.SetContainerID(cnr)
obj.SetID(objID)
obj.SetFirstID(oidtest.ID())
obj.SetPreviousID(oidtest.ID())

sig, err := signer.Sign(objID[:])
require.NoError(tb, err)
Expand Down Expand Up @@ -397,7 +400,7 @@ func TestServer_Replicate(t *testing.T) {

sigsRaw := resp.GetObjectSignature()

for i := range 1 {
for i := range 3 {
var sigV2 refsv2.Signature
l := binary.LittleEndian.Uint32(sigsRaw)

Expand All @@ -408,10 +411,10 @@ func TestServer_Replicate(t *testing.T) {

require.Equal(t, signer.PublicKeyBytes, sig.PublicKeyBytes())
require.True(t, sig.Verify(objectcore.EncodeReplicationMetaInfo(
o.GetContainerID(), o.GetID(), o.PayloadSize(), nil, nil,
uint64((123+1+i)*240), mNumber)))
o.GetContainerID(), o.GetID(), o.GetFirstID(), o.GetPreviousID(), o.PayloadSize(), o.Type(), nil, nil,
uint64((123+1+i)*240), mNumber)), fmt.Sprintf("wrong %d signature", i+1))

sigsRaw = sigsRaw[:4+l]
sigsRaw = sigsRaw[4+l:]
}
})
})
Expand Down
12 changes: 10 additions & 2 deletions pkg/services/object/put/distributed.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,13 @@ func (t *distributedTarget) Close() (oid.ID, error) {
t.encodedObject.b = nil
}()

firstObj := t.obj.GetFirstID()
if t.obj.HasParent() && firstObj.IsZero() {
// object itself is the first one
firstObj = t.obj.GetID()
}
prevObj := t.obj.GetPreviousID()

t.obj.SetPayload(t.encodedObject.b[t.encodedObject.pldOff:])

tombOrLink := t.obj.Type() == objectSDK.TypeLink || t.obj.Type() == objectSDK.TypeTombstone
Expand All @@ -143,7 +150,8 @@ func (t *distributedTarget) Close() (oid.ID, error) {

var deletedObjs []oid.ID
var lockedObjs []oid.ID
switch t.objMeta.Type() {
typ := t.objMeta.Type()
switch typ {
case objectSDK.TypeTombstone:
deletedObjs = t.objMeta.Objects()
case objectSDK.TypeLock:
Expand All @@ -152,7 +160,7 @@ func (t *distributedTarget) Close() (oid.ID, error) {
}

expectedVUB := (uint64(t.currentBlock)/t.currentEpochDuration + 2) * t.currentEpochDuration
t.objSharedMeta = object.EncodeReplicationMetaInfo(t.obj.GetContainerID(), t.obj.GetID(), t.obj.PayloadSize(), deletedObjs,
t.objSharedMeta = object.EncodeReplicationMetaInfo(t.obj.GetContainerID(), t.obj.GetID(), firstObj, prevObj, t.obj.PayloadSize(), typ, deletedObjs,
lockedObjs, expectedVUB, t.networkMagicNumber)
id := t.obj.GetID()
err := t.placementIterator.iterateNodesForObject(id, t.sendObject)
Expand Down

0 comments on commit b39e7b8

Please sign in to comment.