From a13249c1a91abb606ef746a3a8d974306c39727c Mon Sep 17 00:00:00 2001 From: Evgenii Baidakov Date: Thu, 27 Jul 2023 15:57:52 +0400 Subject: [PATCH] slicer: Reuse stub object MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Updated after benchmark. During object slicing we create object for each part, we may reuse it, just resetting some fields. │ old.txt │ new.txt │ │ sec/op │ sec/op vs base │ SliceDataIntoObjects/slice_1-128/reader-16 62.90µ ± 0% 62.77µ ± 0% ~ (p=0.589 n=6) SliceDataIntoObjects/slice_1-128/writer-16 65.30µ ± 1% 65.82µ ± 0% +0.80% (p=0.002 n=6) SliceDataIntoObjects/slice_4-128/reader-16 61.19µ ± 1% 67.40µ ± 1% +10.16% (p=0.002 n=6) SliceDataIntoObjects/slice_4-128/writer-16 63.62µ ± 1% 70.42µ ± 2% +10.69% (p=0.002 n=6) SliceDataIntoObjects/slice_16-128/reader-16 63.15µ ± 1% 65.74µ ± 1% +4.10% (p=0.002 n=6) SliceDataIntoObjects/slice_16-128/writer-16 64.99µ ± 1% 68.16µ ± 1% +4.87% (p=0.002 n=6) SliceDataIntoObjects/slice_64-128/reader-16 66.61µ ± 1% 66.09µ ± 1% ~ (p=0.093 n=6) SliceDataIntoObjects/slice_64-128/writer-16 71.96µ ± 2% 77.10µ ± 7% +7.14% (p=0.041 n=6) SliceDataIntoObjects/slice_256-128/reader-16 65.92µ ± 1% 70.01µ ± 10% ~ (p=0.394 n=6) SliceDataIntoObjects/slice_256-128/writer-16 252.7µ ± 1% 234.0µ ± 1% -7.42% (p=0.002 n=6) SliceDataIntoObjects/slice_1024-128/reader-16 64.30µ ± 2% 65.18µ ± 6% ~ (p=0.240 n=6) SliceDataIntoObjects/slice_1024-128/writer-16 611.3µ ± 2% 570.0µ ± 5% -6.76% (p=0.002 n=6) SliceDataIntoObjects/slice_4096-128/reader-16 67.79µ ± 1% 69.01µ ± 3% +1.80% (p=0.002 n=6) SliceDataIntoObjects/slice_4096-128/writer-16 2.073m ± 1% 2.024m ± 3% -2.39% (p=0.026 n=6) SliceDataIntoObjects/slice_16384-128/reader-16 64.50µ ± 1% 64.07µ ± 0% -0.66% (p=0.026 n=6) SliceDataIntoObjects/slice_16384-128/writer-16 7.067m ± 1% 6.792m ± 8% ~ (p=0.065 n=6) SliceDataIntoObjects/slice_65536-128/reader-16 65.98µ ± 1% 69.42µ ± 1% +5.22% (p=0.002 n=6) SliceDataIntoObjects/slice_65536-128/writer-16 31.06m ± 1% 30.11m ± 1% -3.07% (p=0.002 n=6) SliceDataIntoObjects/slice_262144-128/reader-16 73.88µ ± 1% 76.69µ ± 3% +3.81% (p=0.002 n=6) SliceDataIntoObjects/slice_262144-128/writer-16 112.6m ± 0% 123.6m ± 4% +9.76% (p=0.002 n=6) SliceDataIntoObjects/slice_1048576-128/reader-16 125.1µ ± 3% 106.8µ ± 12% -14.61% (p=0.002 n=6) SliceDataIntoObjects/slice_1048576-128/writer-16 510.0m ± 1% 444.4m ± 1% -12.85% (p=0.002 n=6) geomean 321.5µ 322.7µ +0.37% │ old.txt │ new.txt │ │ B/op │ B/op vs base │ SliceDataIntoObjects/slice_1-128/reader-16 9.547Ki ± 0% 9.547Ki ± 0% ~ (p=1.000 n=6) SliceDataIntoObjects/slice_1-128/writer-16 9.438Ki ± 0% 9.421Ki ± 0% -0.18% (p=0.002 n=6) SliceDataIntoObjects/slice_4-128/reader-16 9.077Ki ± 0% 11.298Ki ± 0% +24.46% (p=0.002 n=6) SliceDataIntoObjects/slice_4-128/writer-16 8.968Ki ± 0% 11.173Ki ± 0% +24.59% (p=0.002 n=6) SliceDataIntoObjects/slice_16-128/reader-16 9.547Ki ± 0% 10.548Ki ± 0% +10.48% (p=0.002 n=6) SliceDataIntoObjects/slice_16-128/writer-16 9.438Ki ± 0% 10.423Ki ± 0% +10.44% (p=0.002 n=6) SliceDataIntoObjects/slice_64-128/reader-16 11.17Ki ± 0% 10.55Ki ± 0% -5.59% (p=0.002 n=6) SliceDataIntoObjects/slice_64-128/writer-16 11.06Ki ± 0% 10.42Ki ± 0% -5.79% (p=0.002 n=6) SliceDataIntoObjects/slice_256-128/reader-16 10.893Ki ± 0% 9.204Ki ± 0% -15.50% (p=0.002 n=6) SliceDataIntoObjects/slice_256-128/writer-16 38.08Ki ± 0% 29.42Ki ± 0% -22.73% (p=0.002 n=6) SliceDataIntoObjects/slice_1024-128/reader-16 9.425Ki ± 0% 9.675Ki ± 0% +2.65% (p=0.002 n=6) SliceDataIntoObjects/slice_1024-128/writer-16 75.66Ki ± 0% 69.60Ki ± 0% -8.01% (p=0.002 n=6) SliceDataIntoObjects/slice_4096-128/reader-16 11.19Ki ± 0% 11.31Ki ± 0% +1.11% (p=0.002 n=6) SliceDataIntoObjects/slice_4096-128/writer-16 260.8Ki ± 0% 235.7Ki ± 0% -9.65% (p=0.002 n=6) SliceDataIntoObjects/slice_16384-128/reader-16 10.004Ki ± 0% 9.716Ki ± 0% -2.88% (p=0.002 n=6) SliceDataIntoObjects/slice_16384-128/writer-16 936.3Ki ± 0% 831.3Ki ± 0% -11.21% (p=0.002 n=6) SliceDataIntoObjects/slice_65536-128/reader-16 10.10Ki ± 0% 11.50Ki ± 0% +13.79% (p=0.002 n=6) SliceDataIntoObjects/slice_65536-128/writer-16 3.708Mi ± 0% 3.307Mi ± 0% -10.80% (p=0.002 n=6) SliceDataIntoObjects/slice_262144-128/reader-16 11.92Ki ± 0% 12.32Ki ± 0% +3.36% (p=0.002 n=6) SliceDataIntoObjects/slice_262144-128/writer-16 14.45Mi ± 0% 13.26Mi ± 0% -8.21% (p=0.002 n=6) SliceDataIntoObjects/slice_1048576-128/reader-16 18.17Ki ± 3% 15.16Ki ± 6% -16.53% (p=0.002 n=6) SliceDataIntoObjects/slice_1048576-128/writer-16 59.48Mi ± 0% 51.22Mi ± 0% -13.89% (p=0.002 n=6) geomean 46.55Ki 45.35Ki -2.56% │ old.txt │ new.txt │ │ allocs/op │ allocs/op vs base │ SliceDataIntoObjects/slice_1-128/reader-16 149.0 ± 0% 149.0 ± 0% ~ (p=1.000 n=6) ¹ SliceDataIntoObjects/slice_1-128/writer-16 148.0 ± 0% 148.0 ± 0% ~ (p=1.000 n=6) ¹ SliceDataIntoObjects/slice_4-128/reader-16 142.0 ± 0% 174.0 ± 0% +22.54% (p=0.002 n=6) SliceDataIntoObjects/slice_4-128/writer-16 141.0 ± 0% 173.0 ± 0% +22.70% (p=0.002 n=6) SliceDataIntoObjects/slice_16-128/reader-16 149.0 ± 0% 163.0 ± 0% +9.40% (p=0.002 n=6) SliceDataIntoObjects/slice_16-128/writer-16 148.0 ± 0% 162.0 ± 0% +9.46% (p=0.002 n=6) SliceDataIntoObjects/slice_64-128/reader-16 172.0 ± 0% 163.0 ± 0% -5.23% (p=0.002 n=6) SliceDataIntoObjects/slice_64-128/writer-16 171.0 ± 0% 162.0 ± 0% -5.26% (p=0.002 n=6) SliceDataIntoObjects/slice_256-128/reader-16 167.0 ± 0% 144.0 ± 0% -13.77% (p=0.002 n=6) SliceDataIntoObjects/slice_256-128/writer-16 583.0 ± 0% 437.0 ± 0% -25.04% (p=0.002 n=6) SliceDataIntoObjects/slice_1024-128/reader-16 147.0 ± 0% 151.0 ± 0% +2.72% (p=0.002 n=6) SliceDataIntoObjects/slice_1024-128/writer-16 1.155k ± 0% 1.030k ± 0% -10.82% (p=0.002 n=6) SliceDataIntoObjects/slice_4096-128/reader-16 172.0 ± 0% 174.0 ± 0% +1.16% (p=0.002 n=6) SliceDataIntoObjects/slice_4096-128/writer-16 4.021k ± 0% 3.464k ± 0% -13.85% (p=0.002 n=6) SliceDataIntoObjects/slice_16384-128/reader-16 155.0 ± 0% 151.0 ± 0% -2.58% (p=0.002 n=6) SliceDataIntoObjects/slice_16384-128/writer-16 14.51k ± 0% 12.20k ± 0% -15.96% (p=0.002 n=6) SliceDataIntoObjects/slice_65536-128/reader-16 157.0 ± 0% 176.0 ± 0% +12.10% (p=0.002 n=6) SliceDataIntoObjects/slice_65536-128/writer-16 58.66k ± 0% 49.56k ± 0% -15.52% (p=0.002 n=6) SliceDataIntoObjects/slice_262144-128/reader-16 182.0 ± 0% 188.0 ± 1% +3.30% (p=0.002 n=6) SliceDataIntoObjects/slice_262144-128/writer-16 227.8k ± 0% 197.1k ± 0% -13.48% (p=0.002 n=6) SliceDataIntoObjects/slice_1048576-128/reader-16 279.0 ± 3% 230.5 ± 6% -17.38% (p=0.002 n=6) SliceDataIntoObjects/slice_1048576-128/writer-16 934.5k ± 0% 762.4k ± 0% -18.42% (p=0.002 n=6) geomean 720.3 690.1 -4.19% Signed-off-by: Evgenii Baidakov --- object/object.go | 17 +++++++++++++++++ object/slicer/slicer.go | 32 ++++++++++++++++++++------------ 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/object/object.go b/object/object.go index c956bad6..db0fe1e0 100644 --- a/object/object.go +++ b/object/object.go @@ -111,6 +111,14 @@ func (o *Object) SetID(v oid.ID) { SetObjectID(&v2) } +// ResetID removes object identifier. +// +// See also [Object.SetID]. +func (o *Object) ResetID() { + (*object.Object)(o). + SetObjectID(nil) +} + // Signature returns signature of the object identifier. // // See also [Object.SetSignature]. @@ -536,6 +544,15 @@ func (o *Object) SetParentID(v oid.ID) { }) } +// ResetParentID removes identifier of the parent object. +// +// See also [Object.SetParentID]. +func (o *Object) ResetParentID() { + o.setSplitFields(func(split *object.SplitHeader) { + split.SetParent(nil) + }) +} + // Parent returns parent object w/o payload. // // See also [Object.SetParent]. diff --git a/object/slicer/slicer.go b/object/slicer/slicer.go index 4e176811..8b63ab49 100644 --- a/object/slicer/slicer.go +++ b/object/slicer/slicer.go @@ -232,6 +232,14 @@ func initPayloadStream(ctx context.Context, ow ObjectWriter, header object.Objec currentVersion := version.Current() header.SetVersion(¤tVersion) + var stubObject object.Object + stubObject.SetVersion(¤tVersion) + stubObject.SetContainerID(containerID) + stubObject.SetCreationEpoch(opts.currentNeoFSEpoch) + stubObject.SetType(object.TypeRegular) + stubObject.SetOwnerID(&owner) + stubObject.SetSessionToken(opts.sessionToken) + res := &PayloadWriter{ ctx: ctx, isHeaderWriteStep: true, @@ -245,6 +253,7 @@ func initPayloadStream(ctx context.Context, ow ObjectWriter, header object.Objec rootMeta: newDynamicObjectMetadata(opts.withHomoChecksum), childMeta: newDynamicObjectMetadata(opts.withHomoChecksum), prmObjectPutInit: prm, + stubObject: &stubObject, } maxObjSize := childPayloadSizeLimit(opts) @@ -283,6 +292,7 @@ type PayloadWriter struct { writtenChildren []oid.ID prmObjectPutInit client.PrmObjectPutInit + stubObject *object.Object } // Write writes next chunk of the object data. Concatenation of all chunks forms @@ -364,19 +374,14 @@ func (x *PayloadWriter) writeLastChild(ctx context.Context, meta dynamicObjectMe } func (x *PayloadWriter) _writeChild(ctx context.Context, meta dynamicObjectMetadata, last bool, rootIDHandler func(id oid.ID)) error { - currentVersion := version.Current() - - fCommon := func(obj *object.Object) { - obj.SetVersion(¤tVersion) - obj.SetContainerID(x.container) - obj.SetCreationEpoch(x.currentEpoch) - obj.SetType(object.TypeRegular) - obj.SetOwnerID(&x.owner) - obj.SetSessionToken(x.sessionToken) - } + obj := *x.stubObject + obj.SetSplitID(nil) + obj.ResetPreviousID() + obj.SetParent(nil) + obj.ResetParentID() + obj.SetSignature(nil) + obj.ResetID() - var obj object.Object - fCommon(&obj) if x.withSplit { obj.SetSplitID(x.splitID) } @@ -420,6 +425,9 @@ func (x *PayloadWriter) _writeChild(ctx context.Context, meta dynamicObjectMetad meta.reset() obj.ResetPreviousID() obj.SetChildren(x.writtenChildren...) + // we reuse already written object, we should reset these fields, to eval them one more time in writeInMemObject. + obj.ResetID() + obj.SetSignature(nil) _, err = writeInMemObject(ctx, x.signer, x.stream, obj, nil, meta, x.prmObjectPutInit) if err != nil {