From 2708db9398711b0a022964195234d06ef41c7dac Mon Sep 17 00:00:00 2001 From: Evgenii Baidakov Date: Thu, 6 Jul 2023 13:39:38 +0400 Subject: [PATCH] slicer: Added split info to the first child closes #448 Signed-off-by: Evgenii Baidakov --- object/slicer/slicer.go | 4 +- object/slicer/slicer_test.go | 86 ++++++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/object/slicer/slicer.go b/object/slicer/slicer.go index f2cecfb5..c8f34cef 100644 --- a/object/slicer/slicer.go +++ b/object/slicer/slicer.go @@ -355,6 +355,9 @@ func (x *PayloadWriter) Write(chunk []byte) (int, error) { if !x.withSplit { x.splitID = object.NewSplitID() + // note: don't move next row, the value of this flag will be used inside writeIntermediateChild + // to fill splitInfo in all child objects. + x.withSplit = true err = x.writeIntermediateChild(x.rootMeta) if err != nil { @@ -362,7 +365,6 @@ func (x *PayloadWriter) Write(chunk []byte) (int, error) { } x.currentWriter.reset(io.MultiWriter(&x.buf, &x.rootMeta, &x.childMeta)) - x.withSplit = true } else { err = x.writeIntermediateChild(x.childMeta) if err != nil { diff --git a/object/slicer/slicer_test.go b/object/slicer/slicer_test.go index 32d2609e..ff43fdc3 100644 --- a/object/slicer/slicer_test.go +++ b/object/slicer/slicer_test.go @@ -505,3 +505,89 @@ func (x *chainCollector) verify(in input, rootID oid.ID) { require.Equal(x.tb, in.payload, rootObj.Payload()) require.NoError(x.tb, rootObj.VerifyPayloadChecksum(), "payload checksum must be correctly set") } + +type memoryWriter struct { + headers []object.Object + splitID *object.SplitID +} + +func (w *memoryWriter) InitDataStream(hdr object.Object) (io.Writer, error) { + w.headers = append(w.headers, hdr) + if w.splitID == nil && hdr.SplitID() != nil { + w.splitID = hdr.SplitID() + } + + return &memoryPayload{}, nil +} + +type memoryPayload struct { +} + +func (p *memoryPayload) Write(data []byte) (int, error) { + return len(data), nil +} + +func TestSlicedObjectsHaveSplitID(t *testing.T) { + maxObjectSize := uint64(10) + overheadAmount := uint64(3) + + var containerID cid.ID + id := make([]byte, sha256.Size) + _, err := rand.Read(id) + require.NoError(t, err) + containerID.Encode(id) + + var ownerID user.ID + signer := test.RandomSignerRFC6979(t) + require.NoError(t, user.IDFromSigner(&ownerID, signer)) + + opts := slicer.Options{} + opts.SetObjectPayloadLimit(maxObjectSize) + opts.SetCurrentNeoFSEpoch(10) + + t.Run("slice", func(t *testing.T) { + writer := &memoryWriter{} + sl := slicer.New(signer, containerID, ownerID, writer, opts) + + payload := make([]byte, maxObjectSize*overheadAmount) + _, err = rand.Read(payload) + require.NoError(t, err) + + _, err = sl.Slice(bytes.NewBuffer(payload)) + require.NoError(t, err) + + require.Equal(t, overheadAmount+1, uint64(len(writer.headers))) + + for _, h := range writer.headers { + splitID := h.SplitID() + require.NotNil(t, splitID) + require.Equal(t, writer.splitID.ToV2(), splitID.ToV2()) + } + }) + + t.Run("InitPayloadStream", func(t *testing.T) { + writer := &memoryWriter{} + sl := slicer.New(signer, containerID, ownerID, writer, opts) + + payloadWriter, err := sl.InitPayloadStream() + require.NoError(t, err) + + for i := uint64(0); i < overheadAmount; i++ { + payload := make([]byte, maxObjectSize) + _, err = rand.Read(payload) + require.NoError(t, err) + + _, err := payloadWriter.Write(payload) + require.NoError(t, err) + } + + require.NoError(t, payloadWriter.Close()) + require.Equal(t, overheadAmount+1, uint64(len(writer.headers))) + + for _, h := range writer.headers { + splitID := h.SplitID() + require.NotNil(t, splitID) + require.Equal(t, writer.splitID.ToV2(), splitID.ToV2()) + } + }) +}