From 7c18f46be3166824e0b3e0ab70eb0c0a9fac0619 Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Sun, 12 Jan 2025 03:34:03 -0800 Subject: [PATCH 1/2] perf(tm2/pkg/amino): reduce RAM heavy-handedness by *bytesBuffer pooled reuse MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This change comes from an analysis of a bunch of RAM and CPU profiles and noticing that realm storage needs to invoke amino.MustMarshalAny but that in the profile for TestStdlibs, it was consuming 1.28GB. ```shell ROUTINE ======================== github.com/gnolang/gno/tm2/pkg/amino.MustMarshalAny in /Users/emmanuelodeke/go/src/github.com/gnolang/gno/tm2/pkg/amino/amino.go 0 1.28GB (flat, cum) 0.61% of Total . . 80:func MustMarshalAny(o interface{}) []byte { . 1.28GB 81: return gcdc.MustMarshalAny(o) . . 82:} . . 83: . . 84:func MarshalAnySized(o interface{}) ([]byte, error) { . . 85: return gcdc.MarshalAnySized(o) . . 86:} ``` and ```shell focus=MarshalAny Showing nodes accounting for 1303.02MB, 0.6% of 217023.96MB total Dropped 13 nodes (cum <= 1085.12MB) ----------------------------------------------------------+------------- flat flat% sum% cum cum% calls calls% + context ----------------------------------------------------------+------------- 539.49MB 100% | bytes.(*Buffer).grow 539.49MB 0.25% 0.25% 539.49MB 0.25% | bytes.growSlice ----------------------------------------------------------+------------- 706.50MB 100% | bytes.(*Buffer).Write 167.01MB 0.077% 0.33% 706.50MB 0.33% | bytes.(*Buffer).grow 539.49MB 76.36% | bytes.growSlice ----------------------------------------------------------+------------- 93MB 58.68% | github.com/gnolang/gno/tm2/pkg/amino.(*Codec).encodeReflectBinaryInterface (inline) 56.50MB 35.65% | github.com/gnolang/gno/tm2/pkg/amino.(*Codec).encodeReflectBinaryStruct (inline) 9MB 5.68% | github.com/gnolang/gno/tm2/pkg/amino.(*Codec).encodeReflectBinaryList (inline) 158.51MB 0.073% 0.4% 158.51MB 0.073% | bytes.NewBuffer ----------------------------------------------------------+------------- 145.01MB 57.77% | github.com/gnolang/gno/tm2/pkg/amino.(*Codec).writeFieldIfNotEmpty 86MB 34.26% | github.com/gnolang/gno/tm2/pkg/amino.(*Codec).encodeReflectBinaryInterface 20MB 7.97% | github.com/gnolang/gno/tm2/pkg/amino.(*Codec).encodeReflectBinaryList 85.50MB 0.039% 0.44% 251.01MB 0.12% | github.com/gnolang/gno/tm2/pkg/amino.encodeFieldNumberAndTyp3 165.51MB 65.94% | bytes.(*Buffer).Write ----------------------------------------------------------+------------- 77.01MB 100% | github.com/gnolang/gno/tm2/pkg/amino.EncodeByteSlice 61.50MB 0.028% 0.47% 77.01MB 0.035% | github.com/gnolang/gno/tm2/pkg/amino.EncodeUvarint 15.51MB 20.14% | bytes.(*Buffer).Write ----------------------------------------------------------+------------- ``` but after this change, we see more than 560MB shaved off ```shell ROUTINE ======================== github.com/gnolang/gno/tm2/pkg/amino.MustMarshalAny in /Users/emmanuelodeke/go/src/github.com/gnolang/gno/tm2/pkg/amino/amino.go 0 560.95MB (flat, cum) 0.26% of Total . . 80:func MustMarshalAny(o interface{}) []byte { . 560.95MB 81: return gcdc.MustMarshalAny(o) . . 82:} . . 83: . . 84:func MarshalAnySized(o interface{}) ([]byte, error) { . . 85: return gcdc.MarshalAnySized(o) . . 86:} ``` and ```shell ----------------------------------------------------------+------------- 16.35MB 52.46% | github.com/gnolang/gno/tm2/pkg/amino.EncodeByteSlice 14.81MB 47.54% | github.com/gnolang/gno/tm2/pkg/amino.writeMaybeBare 0 0% 0.26% 31.16MB 0.014% | bytes.(*Buffer).Write 31.16MB 100% | bytes.(*Buffer).grow ----------------------------------------------------------+------------- 31.16MB 100% | bytes.(*Buffer).Write 0 0% 0.26% 31.16MB 0.014% | bytes.(*Buffer).grow 31.16MB 100% | bytes.growSlice ----------------------------------------------------------+------------- ``` and even more after the change on ensuring that tm2/pkg/amino benchmarks could run we have quite good improvements! Running out of RAM is much worse than a couple of microseconds so we can tolerate an increase in some CPU time benchmarks. ```shell name old time/op new time/op delta Binary/EmptyStruct:encode-8 3.86µs ± 5% 3.92µs ± 5% ~ (p=0.548 n=5+5) Binary/EmptyStruct:decode-8 3.79µs ± 5% 3.79µs ± 6% ~ (p=0.690 n=5+5) Binary/PrimitivesStruct:encode-8 35.5µs ± 2% 36.5µs ± 5% ~ (p=0.151 n=5+5) Binary/PrimitivesStruct:decode-8 35.0µs ± 2% 38.6µs ±11% +10.17% (p=0.016 n=5+5) Binary/ShortArraysStruct:encode-8 5.91µs ± 6% 6.36µs ± 8% +7.61% (p=0.032 n=5+5) Binary/ShortArraysStruct:decode-8 6.07µs ±21% 6.39µs ± 8% ~ (p=0.151 n=5+5) Binary/ArraysStruct:encode-8 95.1µs ± 8% 100.6µs ± 7% ~ (p=0.222 n=5+5) Binary/ArraysStruct:decode-8 91.3µs ± 5% 98.5µs ±12% ~ (p=0.222 n=5+5) Binary/ArraysArraysStruct:encode-8 131µs ± 3% 132µs ± 6% ~ (p=0.841 n=5+5) Binary/ArraysArraysStruct:decode-8 136µs ± 9% 134µs ± 3% ~ (p=0.548 n=5+5) Binary/SlicesStruct:encode-8 85.4µs ± 1% 92.3µs ± 9% +8.15% (p=0.008 n=5+5) Binary/SlicesStruct:decode-8 87.1µs ± 8% 94.8µs ± 7% ~ (p=0.056 n=5+5) Binary/SlicesSlicesStruct:encode-8 506µs ± 2% 545µs ± 9% ~ (p=0.151 n=5+5) Binary/SlicesSlicesStruct:decode-8 506µs ± 3% 523µs ± 3% ~ (p=0.095 n=5+5) Binary/PointersStruct:encode-8 56.8µs ± 4% 65.5µs ±20% +15.43% (p=0.016 n=5+5) Binary/PointersStruct:decode-8 57.5µs ± 3% 55.9µs ± 3% ~ (p=0.095 n=5+5) Binary/PointerSlicesStruct:encode-8 162µs ± 4% 172µs ±21% ~ (p=0.841 n=5+5) Binary/PointerSlicesStruct:decode-8 163µs ± 5% 185µs ±13% ~ (p=0.095 n=5+5) Binary/ComplexSt:encode-8 314µs ± 3% 354µs ±11% +12.90% (p=0.008 n=5+5) Binary/ComplexSt:decode-8 319µs ± 2% 338µs ± 4% +5.87% (p=0.008 n=5+5) Binary/EmbeddedSt1:encode-8 39.8µs ± 7% 39.3µs ± 8% ~ (p=1.000 n=5+5) Binary/EmbeddedSt1:decode-8 37.0µs ± 4% 37.8µs ± 6% ~ (p=0.690 n=5+5) Binary/EmbeddedSt2:encode-8 316µs ± 7% 307µs ± 3% ~ (p=0.222 n=5+5) Binary/EmbeddedSt2:decode-8 316µs ± 3% 306µs ± 2% ~ (p=0.095 n=5+5) Binary/EmbeddedSt3:encode-8 217µs ± 7% 201µs ± 1% -7.26% (p=0.008 n=5+5) Binary/EmbeddedSt3:decode-8 222µs ±10% 204µs ± 2% -8.50% (p=0.032 n=5+5) Binary/EmbeddedSt4:encode-8 332µs ± 4% 325µs ± 3% ~ (p=0.421 n=5+5) Binary/EmbeddedSt4:decode-8 332µs ± 4% 324µs ± 5% ~ (p=0.095 n=5+5) Binary/EmbeddedSt5:encode-8 218µs ± 2% 212µs ± 3% ~ (p=0.056 n=5+5) Binary/EmbeddedSt5:decode-8 224µs ± 8% 209µs ± 1% -6.85% (p=0.008 n=5+5) Binary/AminoMarshalerStruct1:encode-8 9.03µs ± 6% 8.97µs ±12% ~ (p=0.841 n=5+5) Binary/AminoMarshalerStruct1:decode-8 8.91µs ± 5% 8.81µs ± 4% ~ (p=0.841 n=5+5) Binary/AminoMarshalerStruct2:encode-8 13.2µs ±10% 12.2µs ± 2% -7.26% (p=0.008 n=5+5) Binary/AminoMarshalerStruct2:decode-8 13.2µs ± 6% 12.5µs ± 5% ~ (p=0.095 n=5+5) Binary/AminoMarshalerStruct3:encode-8 7.17µs ± 3% 7.50µs ± 8% ~ (p=0.548 n=5+5) Binary/AminoMarshalerStruct3:decode-8 7.12µs ± 4% 7.84µs ±10% +10.12% (p=0.016 n=5+5) Binary/AminoMarshalerInt4:encode-8 6.60µs ± 5% 6.96µs ±11% ~ (p=0.421 n=5+5) Binary/AminoMarshalerInt4:decode-8 6.79µs ±12% 7.04µs ±15% ~ (p=0.690 n=5+5) Binary/AminoMarshalerInt5:encode-8 6.64µs ± 4% 6.92µs ± 5% +4.09% (p=0.032 n=5+5) Binary/AminoMarshalerInt5:decode-8 6.55µs ± 3% 7.76µs ±10% +18.44% (p=0.008 n=5+5) Binary/AminoMarshalerStruct6:encode-8 11.7µs ± 5% 13.2µs ±10% +13.09% (p=0.008 n=5+5) Binary/AminoMarshalerStruct6:decode-8 11.4µs ± 3% 11.6µs ± 2% ~ (p=0.222 n=5+5) Binary/AminoMarshalerStruct7:encode-8 9.86µs ± 1% 10.10µs ±19% ~ (p=0.310 n=5+5) Binary/AminoMarshalerStruct7:decode-8 9.55µs ± 3% 9.75µs ±10% ~ (p=0.690 n=5+5) name old alloc/op new alloc/op delta Binary/EmptyStruct:encode-8 1.50kB ± 0% 1.41kB ± 0% -6.32% (p=0.008 n=5+5) Binary/EmptyStruct:decode-8 1.50kB ± 0% 1.41kB ± 0% -6.32% (p=0.008 n=5+5) Binary/PrimitivesStruct:encode-8 10.4kB ± 0% 9.6kB ± 0% -7.82% (p=0.008 n=5+5) Binary/PrimitivesStruct:decode-8 10.4kB ± 0% 9.6kB ± 0% -7.82% (p=0.000 n=4+5) Binary/ShortArraysStruct:encode-8 2.11kB ± 0% 1.92kB ± 0% -9.04% (p=0.008 n=5+5) Binary/ShortArraysStruct:decode-8 2.11kB ± 0% 1.92kB ± 0% -9.04% (p=0.008 n=5+5) Binary/ArraysStruct:encode-8 25.9kB ± 0% 22.0kB ± 0% -15.04% (p=0.008 n=5+5) Binary/ArraysStruct:decode-8 25.9kB ± 0% 22.0kB ± 0% -15.04% (p=0.008 n=5+5) Binary/ArraysArraysStruct:encode-8 37.7kB ± 0% 25.3kB ± 0% -33.07% (p=0.008 n=5+5) Binary/ArraysArraysStruct:decode-8 37.7kB ± 0% 25.3kB ± 0% -33.07% (p=0.008 n=5+5) Binary/SlicesStruct:encode-8 28.2kB ± 0% 25.1kB ± 0% -10.96% (p=0.008 n=5+5) Binary/SlicesStruct:decode-8 28.2kB ± 0% 25.1kB ± 0% -10.97% (p=0.008 n=5+5) Binary/SlicesSlicesStruct:encode-8 183kB ± 0% 147kB ± 0% -19.92% (p=0.008 n=5+5) Binary/SlicesSlicesStruct:decode-8 183kB ± 0% 147kB ± 0% -19.92% (p=0.008 n=5+5) Binary/PointersStruct:encode-8 14.4kB ± 0% 13.6kB ± 0% -5.64% (p=0.008 n=5+5) Binary/PointersStruct:decode-8 14.4kB ± 0% 13.6kB ± 0% -5.64% (p=0.008 n=5+5) Binary/PointerSlicesStruct:encode-8 43.9kB ± 0% 40.2kB ± 0% -8.49% (p=0.008 n=5+5) Binary/PointerSlicesStruct:decode-8 43.9kB ± 0% 40.2kB ± 0% -8.49% (p=0.008 n=5+5) Binary/ComplexSt:encode-8 95.3kB ± 0% 78.2kB ± 0% -17.97% (p=0.008 n=5+5) Binary/ComplexSt:decode-8 95.3kB ± 0% 78.2kB ± 0% -17.97% (p=0.008 n=5+5) Binary/EmbeddedSt1:encode-8 11.3kB ± 0% 10.2kB ± 0% -9.62% (p=0.000 n=5+4) Binary/EmbeddedSt1:decode-8 11.3kB ± 0% 10.2kB ± 0% -9.61% (p=0.000 n=5+4) Binary/EmbeddedSt2:encode-8 95.5kB ± 0% 78.3kB ± 0% -17.96% (p=0.008 n=5+5) Binary/EmbeddedSt2:decode-8 95.5kB ± 0% 78.4kB ± 0% -17.94% (p=0.008 n=5+5) Binary/EmbeddedSt3:encode-8 68.3kB ± 0% 56.6kB ± 0% -17.22% (p=0.008 n=5+5) Binary/EmbeddedSt3:decode-8 68.3kB ± 0% 56.6kB ± 0% -17.21% (p=0.008 n=5+5) Binary/EmbeddedSt4:encode-8 97.2kB ± 0% 82.3kB ± 0% -15.32% (p=0.008 n=5+5) Binary/EmbeddedSt4:decode-8 97.2kB ± 0% 82.3kB ± 0% -15.31% (p=0.008 n=5+5) Binary/EmbeddedSt5:encode-8 65.9kB ± 0% 55.3kB ± 0% -16.19% (p=0.008 n=5+5) Binary/EmbeddedSt5:decode-8 66.0kB ± 0% 55.3kB ± 0% -16.18% (p=0.008 n=5+5) Binary/AminoMarshalerStruct1:encode-8 2.87kB ± 0% 2.66kB ± 0% -7.23% (p=0.008 n=5+5) Binary/AminoMarshalerStruct1:decode-8 2.87kB ± 0% 2.66kB ± 0% -7.23% (p=0.008 n=5+5) Binary/AminoMarshalerStruct2:encode-8 4.58kB ± 0% 3.62kB ± 0% -20.95% (p=0.008 n=5+5) Binary/AminoMarshalerStruct2:decode-8 4.58kB ± 0% 3.62kB ± 0% -20.95% (p=0.008 n=5+5) Binary/AminoMarshalerStruct3:encode-8 2.42kB ± 0% 2.31kB ± 0% -4.62% (p=0.008 n=5+5) Binary/AminoMarshalerStruct3:decode-8 2.42kB ± 0% 2.31kB ± 0% -4.62% (p=0.008 n=5+5) Binary/AminoMarshalerInt4:encode-8 2.38kB ± 0% 2.15kB ± 0% -9.38% (p=0.008 n=5+5) Binary/AminoMarshalerInt4:decode-8 2.38kB ± 0% 2.15kB ± 0% -9.38% (p=0.008 n=5+5) Binary/AminoMarshalerInt5:encode-8 2.36kB ± 0% 2.27kB ± 0% -4.07% (p=0.008 n=5+5) Binary/AminoMarshalerInt5:decode-8 2.36kB ± 0% 2.27kB ± 0% -4.07% (p=0.008 n=5+5) Binary/AminoMarshalerStruct6:encode-8 3.51kB ± 0% 3.19kB ± 0% -9.05% (p=0.008 n=5+5) Binary/AminoMarshalerStruct6:decode-8 3.51kB ± 0% 3.19kB ± 0% -9.05% (p=0.008 n=5+5) Binary/AminoMarshalerStruct7:encode-8 2.89kB ± 0% 2.67kB ± 0% -7.72% (p=0.008 n=5+5) Binary/AminoMarshalerStruct7:decode-8 2.89kB ± 0% 2.67kB ± 0% -7.72% (p=0.008 n=5+5) name old allocs/op new allocs/op delta Binary/EmptyStruct:encode-8 38.0 ± 0% 36.0 ± 0% -5.26% (p=0.008 n=5+5) Binary/EmptyStruct:decode-8 38.0 ± 0% 36.0 ± 0% -5.26% (p=0.008 n=5+5) Binary/PrimitivesStruct:encode-8 439 ± 0% 429 ± 0% -2.28% (p=0.008 n=5+5) Binary/PrimitivesStruct:decode-8 439 ± 0% 429 ± 0% -2.28% (p=0.008 n=5+5) Binary/ShortArraysStruct:encode-8 56.0 ± 0% 52.0 ± 0% -7.14% (p=0.008 n=5+5) Binary/ShortArraysStruct:decode-8 56.0 ± 0% 52.0 ± 0% -7.14% (p=0.008 n=5+5) Binary/ArraysStruct:encode-8 977 ± 0% 919 ± 0% -5.94% (p=0.008 n=5+5) Binary/ArraysStruct:decode-8 977 ± 0% 919 ± 0% -5.94% (p=0.008 n=5+5) Binary/ArraysArraysStruct:encode-8 1.28k ± 0% 1.08k ± 0% -15.05% (p=0.008 n=5+5) Binary/ArraysArraysStruct:decode-8 1.28k ± 0% 1.08k ± 0% -15.05% (p=0.008 n=5+5) Binary/SlicesStruct:encode-8 1.01k ± 0% 0.97k ± 0% -3.77% (p=0.008 n=5+5) Binary/SlicesStruct:decode-8 1.01k ± 0% 0.97k ± 0% -3.77% (p=0.008 n=5+5) Binary/SlicesSlicesStruct:encode-8 6.33k ± 0% 5.95k ± 0% -5.90% (p=0.008 n=5+5) Binary/SlicesSlicesStruct:decode-8 6.33k ± 0% 5.95k ± 0% -5.90% (p=0.008 n=5+5) Binary/PointersStruct:encode-8 637 ± 0% 627 ± 0% -1.57% (p=0.008 n=5+5) Binary/PointersStruct:decode-8 637 ± 0% 627 ± 0% -1.57% (p=0.008 n=5+5) Binary/PointerSlicesStruct:encode-8 1.62k ± 0% 1.56k ± 0% -3.28% (p=0.008 n=5+5) Binary/PointerSlicesStruct:decode-8 1.62k ± 0% 1.56k ± 0% -3.28% (p=0.008 n=5+5) Binary/ComplexSt:encode-8 3.37k ± 0% 3.22k ± 0% -4.62% (p=0.008 n=5+5) Binary/ComplexSt:decode-8 3.37k ± 0% 3.22k ± 0% -4.62% (p=0.008 n=5+5) Binary/EmbeddedSt1:encode-8 453 ± 0% 440 ± 0% -2.87% (p=0.008 n=5+5) Binary/EmbeddedSt1:decode-8 453 ± 0% 440 ± 0% -2.87% (p=0.008 n=5+5) Binary/EmbeddedSt2:encode-8 3.37k ± 0% 3.22k ± 0% -4.62% (p=0.008 n=5+5) Binary/EmbeddedSt2:decode-8 3.37k ± 0% 3.22k ± 0% -4.62% (p=0.008 n=5+5) Binary/EmbeddedSt3:encode-8 2.32k ± 0% 2.20k ± 0% -5.38% (p=0.008 n=5+5) Binary/EmbeddedSt3:decode-8 2.32k ± 0% 2.20k ± 0% -5.38% (p=0.008 n=5+5) Binary/EmbeddedSt4:encode-8 3.67k ± 0% 3.54k ± 0% -3.73% (p=0.008 n=5+5) Binary/EmbeddedSt4:decode-8 3.67k ± 0% 3.54k ± 0% -3.73% (p=0.008 n=5+5) Binary/EmbeddedSt5:encode-8 2.32k ± 0% 2.20k ± 0% -5.00% (p=0.008 n=5+5) Binary/EmbeddedSt5:decode-8 2.32k ± 0% 2.20k ± 0% -5.00% (p=0.008 n=5+5) Binary/AminoMarshalerStruct1:encode-8 97.0 ± 0% 94.0 ± 0% -3.09% (p=0.008 n=5+5) Binary/AminoMarshalerStruct1:decode-8 97.0 ± 0% 94.0 ± 0% -3.09% (p=0.008 n=5+5) Binary/AminoMarshalerStruct2:encode-8 149 ± 0% 133 ± 0% -10.74% (p=0.008 n=5+5) Binary/AminoMarshalerStruct2:decode-8 149 ± 0% 133 ± 0% -10.74% (p=0.008 n=5+5) Binary/AminoMarshalerStruct3:encode-8 77.0 ± 0% 76.0 ± 0% -1.30% (p=0.008 n=5+5) Binary/AminoMarshalerStruct3:decode-8 77.0 ± 0% 76.0 ± 0% -1.30% (p=0.008 n=5+5) Binary/AminoMarshalerInt4:encode-8 71.0 ± 0% 68.0 ± 0% -4.23% (p=0.008 n=5+5) Binary/AminoMarshalerInt4:decode-8 71.0 ± 0% 68.0 ± 0% -4.23% (p=0.008 n=5+5) Binary/AminoMarshalerInt5:encode-8 74.0 ± 0% 73.0 ± 0% -1.35% (p=0.008 n=5+5) Binary/AminoMarshalerInt5:decode-8 74.0 ± 0% 73.0 ± 0% -1.35% (p=0.008 n=5+5) Binary/AminoMarshalerStruct6:encode-8 122 ± 0% 117 ± 0% -4.10% (p=0.008 n=5+5) Binary/AminoMarshalerStruct6:decode-8 122 ± 0% 117 ± 0% -4.10% (p=0.008 n=5+5) Binary/AminoMarshalerStruct7:encode-8 101 ± 0% 98 ± 0% -2.97% (p=0.008 n=5+5) Binary/AminoMarshalerStruct7:decode-8 101 ± 0% 98 ± 0% -2.97% (p=0.008 n=5+5) ``` Fixes #3488 --- contribs/gnodev/go.mod | 1 + contribs/gnodev/go.sum | 2 ++ contribs/gnofaucet/go.mod | 1 + contribs/gnofaucet/go.sum | 2 ++ contribs/gnogenesis/go.mod | 1 + contribs/gnogenesis/go.sum | 2 ++ contribs/gnohealth/go.mod | 1 + contribs/gnohealth/go.sum | 2 ++ contribs/gnokeykc/go.mod | 1 + contribs/gnokeykc/go.sum | 2 ++ contribs/gnomigrate/go.mod | 1 + contribs/gnomigrate/go.sum | 2 ++ go.mod | 1 + go.sum | 2 ++ misc/autocounterd/go.mod | 1 + misc/autocounterd/go.sum | 2 ++ misc/loop/go.mod | 1 + misc/loop/go.sum | 2 ++ tm2/pkg/amino/amino.go | 65 +++++++++++++++++++++++++--------- tm2/pkg/amino/binary_encode.go | 52 ++++++++++++++++++++++----- tm2/pkg/amino/codec.go | 8 +++-- tm2/pkg/amino/json_encode.go | 8 +++-- tm2/pkg/amino/wellknown.go | 8 +++-- 23 files changed, 138 insertions(+), 30 deletions(-) diff --git a/contribs/gnodev/go.mod b/contribs/gnodev/go.mod index a4c106a24ee..0ad16ba9bb3 100644 --- a/contribs/gnodev/go.mod +++ b/contribs/gnodev/go.mod @@ -79,6 +79,7 @@ require ( github.com/rs/xid v1.6.0 // indirect github.com/sig-0/insertion-queue v0.0.0-20241004125609-6b3ca841346b // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect github.com/yuin/goldmark v1.7.8 // indirect github.com/yuin/goldmark-emoji v1.0.2 // indirect diff --git a/contribs/gnodev/go.sum b/contribs/gnodev/go.sum index e87c2de6441..f4bf32aafd5 100644 --- a/contribs/gnodev/go.sum +++ b/contribs/gnodev/go.sum @@ -226,6 +226,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= github.com/yuin/goldmark v1.3.7/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= diff --git a/contribs/gnofaucet/go.mod b/contribs/gnofaucet/go.mod index 32d2e322098..88c05e0d778 100644 --- a/contribs/gnofaucet/go.mod +++ b/contribs/gnofaucet/go.mod @@ -32,6 +32,7 @@ require ( github.com/rs/cors v1.11.1 // indirect github.com/rs/xid v1.6.0 // indirect github.com/sig-0/insertion-queue v0.0.0-20241004125609-6b3ca841346b // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/otel v1.34.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.34.0 // indirect diff --git a/contribs/gnofaucet/go.sum b/contribs/gnofaucet/go.sum index 5b3cfdc3289..e6743b75960 100644 --- a/contribs/gnofaucet/go.sum +++ b/contribs/gnofaucet/go.sum @@ -126,6 +126,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= diff --git a/contribs/gnogenesis/go.mod b/contribs/gnogenesis/go.mod index 5b28c8774c8..8af370f8169 100644 --- a/contribs/gnogenesis/go.mod +++ b/contribs/gnogenesis/go.mod @@ -36,6 +36,7 @@ require ( github.com/rs/xid v1.6.0 // indirect github.com/sig-0/insertion-queue v0.0.0-20241004125609-6b3ca841346b // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/zondax/hid v0.9.2 // indirect github.com/zondax/ledger-go v0.14.3 // indirect go.etcd.io/bbolt v1.3.11 // indirect diff --git a/contribs/gnogenesis/go.sum b/contribs/gnogenesis/go.sum index dcd853e9148..e3462f9c431 100644 --- a/contribs/gnogenesis/go.sum +++ b/contribs/gnogenesis/go.sum @@ -133,6 +133,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/zondax/hid v0.9.2 h1:WCJFnEDMiqGF64nlZz28E9qLVZ0KSJ7xpc5DLEyma2U= github.com/zondax/hid v0.9.2/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= github.com/zondax/ledger-go v0.14.3 h1:wEpJt2CEcBJ428md/5MgSLsXLBos98sBOyxNmCjfUCw= diff --git a/contribs/gnohealth/go.mod b/contribs/gnohealth/go.mod index 203dac360b7..76d7cd9c437 100644 --- a/contribs/gnohealth/go.mod +++ b/contribs/gnohealth/go.mod @@ -23,6 +23,7 @@ require ( github.com/rs/xid v1.6.0 // indirect github.com/sig-0/insertion-queue v0.0.0-20241004125609-6b3ca841346b // indirect github.com/stretchr/testify v1.10.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/otel v1.34.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.34.0 // indirect diff --git a/contribs/gnohealth/go.sum b/contribs/gnohealth/go.sum index e51cadf1564..3c8b5de45f2 100644 --- a/contribs/gnohealth/go.sum +++ b/contribs/gnohealth/go.sum @@ -118,6 +118,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= go.etcd.io/bbolt v1.3.11 h1:yGEzV1wPz2yVCLsD8ZAiGHhHVlczyC9d1rP43/VCRJ0= go.etcd.io/bbolt v1.3.11/go.mod h1:dksAq7YMXoljX0xu6VF5DMZGbhYYoLUalEiSySYAS4I= go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= diff --git a/contribs/gnokeykc/go.mod b/contribs/gnokeykc/go.mod index 73e51f6b25e..3abcf3d834f 100644 --- a/contribs/gnokeykc/go.mod +++ b/contribs/gnokeykc/go.mod @@ -38,6 +38,7 @@ require ( github.com/sig-0/insertion-queue v0.0.0-20241004125609-6b3ca841346b // indirect github.com/stretchr/testify v1.10.0 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/zondax/hid v0.9.2 // indirect github.com/zondax/ledger-go v0.14.3 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect diff --git a/contribs/gnokeykc/go.sum b/contribs/gnokeykc/go.sum index 7a058c85750..6b4f81dfcf5 100644 --- a/contribs/gnokeykc/go.sum +++ b/contribs/gnokeykc/go.sum @@ -139,6 +139,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/zalando/go-keyring v0.2.3 h1:v9CUu9phlABObO4LPWycf+zwMG7nlbb3t/B5wa97yms= github.com/zalando/go-keyring v0.2.3/go.mod h1:HL4k+OXQfJUWaMnqyuSOc0drfGPX2b51Du6K+MRgZMk= github.com/zondax/hid v0.9.2 h1:WCJFnEDMiqGF64nlZz28E9qLVZ0KSJ7xpc5DLEyma2U= diff --git a/contribs/gnomigrate/go.mod b/contribs/gnomigrate/go.mod index 83d88c354e7..96f6dc9bdc6 100644 --- a/contribs/gnomigrate/go.mod +++ b/contribs/gnomigrate/go.mod @@ -33,6 +33,7 @@ require ( github.com/rs/xid v1.6.0 // indirect github.com/sig-0/insertion-queue v0.0.0-20241004125609-6b3ca841346b // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect go.etcd.io/bbolt v1.3.11 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/otel v1.34.0 // indirect diff --git a/contribs/gnomigrate/go.sum b/contribs/gnomigrate/go.sum index dcd853e9148..e3462f9c431 100644 --- a/contribs/gnomigrate/go.sum +++ b/contribs/gnomigrate/go.sum @@ -133,6 +133,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/zondax/hid v0.9.2 h1:WCJFnEDMiqGF64nlZz28E9qLVZ0KSJ7xpc5DLEyma2U= github.com/zondax/hid v0.9.2/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= github.com/zondax/ledger-go v0.14.3 h1:wEpJt2CEcBJ428md/5MgSLsXLBos98sBOyxNmCjfUCw= diff --git a/go.mod b/go.mod index ce58b8f7998..027ba6359bc 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/sig-0/insertion-queue v0.0.0-20241004125609-6b3ca841346b github.com/stretchr/testify v1.10.0 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 + github.com/valyala/bytebufferpool v1.0.0 github.com/yuin/goldmark v1.7.8 github.com/yuin/goldmark-highlighting/v2 v2.0.0-20230729083705-37449abec8cc go.etcd.io/bbolt v1.3.11 diff --git a/go.sum b/go.sum index 046d9c8c75a..5fd4cddd627 100644 --- a/go.sum +++ b/go.sum @@ -148,6 +148,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/yuin/goldmark v1.4.15/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic= github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= diff --git a/misc/autocounterd/go.mod b/misc/autocounterd/go.mod index 730a3d901b7..972975d4fb0 100644 --- a/misc/autocounterd/go.mod +++ b/misc/autocounterd/go.mod @@ -29,6 +29,7 @@ require ( github.com/sig-0/insertion-queue v0.0.0-20241004125609-6b3ca841346b // indirect github.com/stretchr/testify v1.10.0 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect github.com/zondax/hid v0.9.2 // indirect github.com/zondax/ledger-go v0.14.3 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect diff --git a/misc/autocounterd/go.sum b/misc/autocounterd/go.sum index 3d0eae7661b..6d6d87fa01a 100644 --- a/misc/autocounterd/go.sum +++ b/misc/autocounterd/go.sum @@ -133,6 +133,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/zondax/hid v0.9.2 h1:WCJFnEDMiqGF64nlZz28E9qLVZ0KSJ7xpc5DLEyma2U= github.com/zondax/hid v0.9.2/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= github.com/zondax/ledger-go v0.14.3 h1:wEpJt2CEcBJ428md/5MgSLsXLBos98sBOyxNmCjfUCw= diff --git a/misc/loop/go.mod b/misc/loop/go.mod index c72101c7c1e..f3cb892030a 100644 --- a/misc/loop/go.mod +++ b/misc/loop/go.mod @@ -56,6 +56,7 @@ require ( github.com/sig-0/insertion-queue v0.0.0-20241004125609-6b3ca841346b // indirect github.com/stretchr/testify v1.10.0 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect go.etcd.io/bbolt v1.3.11 // indirect go.opentelemetry.io/auto/sdk v1.1.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 // indirect diff --git a/misc/loop/go.sum b/misc/loop/go.sum index 634dbdac082..0bd87d8bf43 100644 --- a/misc/loop/go.sum +++ b/misc/loop/go.sum @@ -179,6 +179,8 @@ github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOf github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/zondax/hid v0.9.2 h1:WCJFnEDMiqGF64nlZz28E9qLVZ0KSJ7xpc5DLEyma2U= diff --git a/tm2/pkg/amino/amino.go b/tm2/pkg/amino/amino.go index 262f5d9a54e..5d77a41a306 100644 --- a/tm2/pkg/amino/amino.go +++ b/tm2/pkg/amino/amino.go @@ -219,7 +219,11 @@ func (cdc *Codec) MarshalSized(o interface{}) ([]byte, error) { cdc.doAutoseal() // Write the bytes here. - buf := new(bytes.Buffer) + buf := poolBytesBuffer.Get() + defer func() { + buf.Reset() + poolBytesBuffer.Put(buf) + }() // Write the bz without length-prefixing. bz, err := cdc.Marshal(o) @@ -239,7 +243,7 @@ func (cdc *Codec) MarshalSized(o interface{}) ([]byte, error) { return nil, err } - return buf.Bytes(), nil + return copyBytes(buf.Bytes()), nil } // MarshalSizedWriter writes the bytes as would be returned from @@ -271,7 +275,11 @@ func (cdc *Codec) MarshalAnySized(o interface{}) ([]byte, error) { cdc.doAutoseal() // Write the bytes here. - buf := new(bytes.Buffer) + buf := poolBytesBuffer.Get() + defer func() { + buf.Reset() + poolBytesBuffer.Put(buf) + }() // Write the bz without length-prefixing. bz, err := cdc.MarshalAny(o) @@ -291,7 +299,7 @@ func (cdc *Codec) MarshalAnySized(o interface{}) ([]byte, error) { return nil, err } - return buf.Bytes(), nil + return copyBytes(buf.Bytes()), nil } func (cdc *Codec) MustMarshalAnySized(o interface{}) []byte { @@ -357,7 +365,12 @@ func (cdc *Codec) MarshalReflect(o interface{}) ([]byte, error) { // Encode Amino:binary bytes. var bz []byte - buf := new(bytes.Buffer) + buf := poolBytesBuffer.Get() + defer func() { + buf.Reset() + poolBytesBuffer.Put(buf) + }() + rt := rv.Type() info, err := cdc.getTypeInfoWLock(rt) if err != nil { @@ -377,7 +390,7 @@ func (cdc *Codec) MarshalReflect(o interface{}) ([]byte, error) { if err = cdc.writeFieldIfNotEmpty(buf, 1, info, FieldOptions{}, FieldOptions{}, rv, writeEmpty); err != nil { return nil, err } - bz = buf.Bytes() + bz = copyBytes(buf.Bytes()) } else { // The passed in BinFieldNum is only relevant for when the type is to // be encoded unpacked (elements are Typ3_ByteLength). In that case, @@ -387,7 +400,7 @@ func (cdc *Codec) MarshalReflect(o interface{}) ([]byte, error) { if err != nil { return nil, err } - bz = buf.Bytes() + bz = copyBytes(buf.Bytes()) } // If bz is empty, prefer nil. if len(bz) == 0 { @@ -443,16 +456,26 @@ func (cdc *Codec) MarshalAny(o interface{}) ([]byte, error) { } // Encode as interface. - buf := new(bytes.Buffer) + buf := poolBytesBuffer.Get() + defer func() { + buf.Reset() + poolBytesBuffer.Put(buf) + }() err = cdc.encodeReflectBinaryInterface(buf, iinfo, reflect.ValueOf(&ivar).Elem(), FieldOptions{}, true) if err != nil { return nil, err } - bz := buf.Bytes() + bz := copyBytes(buf.Bytes()) return bz, nil } +func copyBytes(bz []byte) []byte { + cp := make([]byte, len(bz)) + copy(cp, bz) + return cp +} + // Panics if error. func (cdc *Codec) MustMarshalAny(o interface{}) []byte { bz, err := cdc.MarshalAny(o) @@ -764,7 +787,11 @@ func (cdc *Codec) JSONMarshal(o interface{}) ([]byte, error) { return []byte("null"), nil } rt := rv.Type() - w := new(bytes.Buffer) + w := poolBytesBuffer.Get() + defer func() { + w.Reset() + poolBytesBuffer.Put(w) + }() info, err := cdc.getTypeInfoWLock(rt) if err != nil { return nil, err @@ -772,7 +799,8 @@ func (cdc *Codec) JSONMarshal(o interface{}) ([]byte, error) { if err = cdc.encodeReflectJSON(w, info, rv, FieldOptions{}); err != nil { return nil, err } - return w.Bytes(), nil + + return copyBytes(w.Bytes()), nil } func (cdc *Codec) MarshalJSONAny(o interface{}) ([]byte, error) { @@ -802,12 +830,17 @@ func (cdc *Codec) MarshalJSONAny(o interface{}) ([]byte, error) { } // Encode as interface. - buf := new(bytes.Buffer) + buf := poolBytesBuffer.Get() + defer func() { + buf.Reset() + poolBytesBuffer.Put(buf) + }() + err = cdc.encodeReflectJSONInterface(buf, iinfo, reflect.ValueOf(&ivar).Elem(), FieldOptions{}) if err != nil { return nil, err } - bz := buf.Bytes() + bz := copyBytes(buf.Bytes()) return bz, nil } @@ -863,12 +896,12 @@ func (cdc *Codec) MarshalJSONIndent(o interface{}, prefix, indent string) ([]byt if err != nil { return nil, err } + var out bytes.Buffer - err = json.Indent(&out, bz, prefix, indent) - if err != nil { + if err := json.Indent(&out, bz, prefix, indent); err != nil { return nil, err } - return out.Bytes(), nil + return copyBytes(out.Bytes()), nil } // ---------------------------------------- diff --git a/tm2/pkg/amino/binary_encode.go b/tm2/pkg/amino/binary_encode.go index 426cc520604..676f441d863 100644 --- a/tm2/pkg/amino/binary_encode.go +++ b/tm2/pkg/amino/binary_encode.go @@ -1,12 +1,13 @@ package amino import ( - "bytes" "encoding/binary" "errors" "fmt" "io" "reflect" + + "github.com/valyala/bytebufferpool" ) const beOptionByte = 0x01 @@ -209,6 +210,8 @@ func (cdc *Codec) encodeReflectBinary(w io.Writer, info *TypeInfo, rv reflect.Va return err } +var poolBytesBuffer = new(bytebufferpool.Pool) + func (cdc *Codec) encodeReflectBinaryInterface(w io.Writer, iinfo *TypeInfo, rv reflect.Value, fopts FieldOptions, bare bool, ) (err error) { @@ -250,7 +253,12 @@ func (cdc *Codec) encodeReflectBinaryInterface(w io.Writer, iinfo *TypeInfo, rv // For Proto3 compatibility, encode interfaces as google.protobuf.Any // Write field #1, TypeURL - buf := bytes.NewBuffer(nil) + buf := poolBytesBuffer.Get() + defer func() { + buf.Reset() + poolBytesBuffer.Put(buf) + }() + { fnum := uint32(1) err = encodeFieldNumberAndTyp3(buf, fnum, Typ3ByteLength) @@ -269,7 +277,12 @@ func (cdc *Codec) encodeReflectBinaryInterface(w io.Writer, iinfo *TypeInfo, rv { // google.protobuf.Any values must be a struct, or an unpacked list which // is indistinguishable from a struct. - buf2 := bytes.NewBuffer(nil) + buf2 := poolBytesBuffer.Get() + defer func() { + buf2.Reset() + poolBytesBuffer.Put(buf2) + }() + if !cinfo.IsStructOrUnpacked(fopts) { writeEmpty := false // Encode with an implicit struct, with a single field with number 1. @@ -356,7 +369,11 @@ func (cdc *Codec) encodeReflectBinaryList(w io.Writer, info *TypeInfo, rv reflec // Proto3 byte-length prefixing incurs alloc cost on the encoder. // Here we incur it for unpacked form for ease of dev. - buf := bytes.NewBuffer(nil) + buf := poolBytesBuffer.Get() + defer func() { + buf.Reset() + poolBytesBuffer.Put(buf) + }() // If elem is not already a ByteLength type, write in packed form. // This is a Proto wart due to Proto backwards compatibility issues. @@ -431,9 +448,15 @@ func (cdc *Codec) encodeReflectBinaryList(w io.Writer, info *TypeInfo, rv reflec // form) are represented as lists of implicit structs. if writeImplicit { // Write field key for Value field of implicit struct. - buf2 := new(bytes.Buffer) + buf2 := poolBytesBuffer.Get() + buf2Done := func() { + buf2.Reset() + poolBytesBuffer.Put(buf2) + } + err = encodeFieldNumberAndTyp3(buf2, 1, Typ3ByteLength) if err != nil { + buf2Done() return } // Write field value of implicit struct to buf2. @@ -441,10 +464,12 @@ func (cdc *Codec) encodeReflectBinaryList(w io.Writer, info *TypeInfo, rv reflec efopts.BinFieldNum = 0 // dontcare err = cdc.encodeReflectBinary(buf2, einfo, derv, efopts, false, 0) if err != nil { + buf2Done() return } // Write implicit struct to buf. err = EncodeByteSlice(buf, buf2.Bytes()) + buf2Done() if err != nil { return } @@ -497,7 +522,11 @@ func (cdc *Codec) encodeReflectBinaryStruct(w io.Writer, info *TypeInfo, rv refl // Proto3 incurs a cost in writing non-root structs. // Here we incur it for root structs as well for ease of dev. - buf := bytes.NewBuffer(nil) + buf := poolBytesBuffer.Get() + defer func() { + buf.Reset() + poolBytesBuffer.Put(buf) + }() for _, field := range info.Fields { // Get type info for field. @@ -552,8 +581,15 @@ func encodeFieldNumberAndTyp3(w io.Writer, num uint32, typ Typ3) (err error) { return } +type bufLike interface { + io.Writer + Len() int + Bytes() []byte + Set([]byte) +} + func (cdc *Codec) writeFieldIfNotEmpty( - buf *bytes.Buffer, + buf bufLike, fieldNum uint32, finfo *TypeInfo, structsFopts FieldOptions, // the wrapping struct's FieldOptions if any @@ -579,7 +615,7 @@ func (cdc *Codec) writeFieldIfNotEmpty( if !isWriteEmpty && lBeforeValue == lAfterValue-1 && buf.Bytes()[buf.Len()-1] == 0x00 { // rollback typ3/fieldnum and last byte if // not a pointer and empty: - buf.Truncate(lBeforeKey) + buf.Set(buf.Bytes()[:lBeforeKey]) } return nil } diff --git a/tm2/pkg/amino/codec.go b/tm2/pkg/amino/codec.go index 3fa7634e3ad..882d68de6b6 100644 --- a/tm2/pkg/amino/codec.go +++ b/tm2/pkg/amino/codec.go @@ -1,7 +1,6 @@ package amino import ( - "bytes" "fmt" "io" "reflect" @@ -113,7 +112,12 @@ func (info *TypeInfo) String() string { // before it's fully populated. return "" } - buf := new(bytes.Buffer) + buf := poolBytesBuffer.Get() + defer func() { + buf.Reset() + poolBytesBuffer.Put(buf) + }() + buf.Write([]byte("TypeInfo{")) buf.Write([]byte(fmt.Sprintf("Type:%v,", info.Type))) if info.ConcreteInfo.Registered { diff --git a/tm2/pkg/amino/json_encode.go b/tm2/pkg/amino/json_encode.go index 113c3486565..e1f5562846f 100644 --- a/tm2/pkg/amino/json_encode.go +++ b/tm2/pkg/amino/json_encode.go @@ -1,7 +1,6 @@ package amino import ( - "bytes" "encoding/json" "fmt" "io" @@ -156,7 +155,12 @@ func (cdc *Codec) encodeReflectJSONInterface(w io.Writer, iinfo *TypeInfo, rv re } // Write Value to buffer - buf := new(bytes.Buffer) + buf := poolBytesBuffer.Get() + defer func() { + buf.Reset() + poolBytesBuffer.Put(buf) + }() + cdc.encodeReflectJSON(buf, cinfo, crv, fopts) value := buf.Bytes() if len(value) == 0 { diff --git a/tm2/pkg/amino/wellknown.go b/tm2/pkg/amino/wellknown.go index 7720c2894d9..bdba530463e 100644 --- a/tm2/pkg/amino/wellknown.go +++ b/tm2/pkg/amino/wellknown.go @@ -3,7 +3,6 @@ package amino // NOTE: We must not depend on protubuf libraries for serialization. import ( - "bytes" "fmt" "io" "reflect" @@ -342,7 +341,12 @@ func encodeReflectBinaryWellKnown(w io.Writer, info *TypeInfo, rv reflect.Value, } // Maybe recurse with length-prefixing. if !bare { - buf := bytes.NewBuffer(nil) + buf := poolBytesBuffer.Get() + defer func() { + buf.Reset() + poolBytesBuffer.Put(buf) + }() + ok, err = encodeReflectBinaryWellKnown(buf, info, rv, fopts, true) if err != nil { return false, err From 3104eef75b3a0091dc03bcb9d72d9744264567d0 Mon Sep 17 00:00:00 2001 From: Morgan Bazalgette Date: Thu, 6 Feb 2025 18:43:27 +0100 Subject: [PATCH 2/2] don't reset before put, optimize repeated fields --- tm2/pkg/amino/amino.go | 31 +++++----------------- tm2/pkg/amino/binary_encode.go | 47 +++++++++------------------------- tm2/pkg/amino/codec.go | 5 +--- tm2/pkg/amino/json_encode.go | 5 +--- tm2/pkg/amino/wellknown.go | 5 +--- 5 files changed, 21 insertions(+), 72 deletions(-) diff --git a/tm2/pkg/amino/amino.go b/tm2/pkg/amino/amino.go index 5d77a41a306..b8942c49029 100644 --- a/tm2/pkg/amino/amino.go +++ b/tm2/pkg/amino/amino.go @@ -220,10 +220,7 @@ func (cdc *Codec) MarshalSized(o interface{}) ([]byte, error) { // Write the bytes here. buf := poolBytesBuffer.Get() - defer func() { - buf.Reset() - poolBytesBuffer.Put(buf) - }() + defer poolBytesBuffer.Put(buf) // Write the bz without length-prefixing. bz, err := cdc.Marshal(o) @@ -276,11 +273,7 @@ func (cdc *Codec) MarshalAnySized(o interface{}) ([]byte, error) { // Write the bytes here. buf := poolBytesBuffer.Get() - defer func() { - buf.Reset() - poolBytesBuffer.Put(buf) - }() - + defer poolBytesBuffer.Put(buf) // Write the bz without length-prefixing. bz, err := cdc.MarshalAny(o) if err != nil { @@ -366,10 +359,7 @@ func (cdc *Codec) MarshalReflect(o interface{}) ([]byte, error) { // Encode Amino:binary bytes. var bz []byte buf := poolBytesBuffer.Get() - defer func() { - buf.Reset() - poolBytesBuffer.Put(buf) - }() + defer poolBytesBuffer.Put(buf) rt := rv.Type() info, err := cdc.getTypeInfoWLock(rt) @@ -457,10 +447,7 @@ func (cdc *Codec) MarshalAny(o interface{}) ([]byte, error) { // Encode as interface. buf := poolBytesBuffer.Get() - defer func() { - buf.Reset() - poolBytesBuffer.Put(buf) - }() + defer poolBytesBuffer.Put(buf) err = cdc.encodeReflectBinaryInterface(buf, iinfo, reflect.ValueOf(&ivar).Elem(), FieldOptions{}, true) if err != nil { return nil, err @@ -788,10 +775,7 @@ func (cdc *Codec) JSONMarshal(o interface{}) ([]byte, error) { } rt := rv.Type() w := poolBytesBuffer.Get() - defer func() { - w.Reset() - poolBytesBuffer.Put(w) - }() + defer poolBytesBuffer.Put(w) info, err := cdc.getTypeInfoWLock(rt) if err != nil { return nil, err @@ -831,10 +815,7 @@ func (cdc *Codec) MarshalJSONAny(o interface{}) ([]byte, error) { // Encode as interface. buf := poolBytesBuffer.Get() - defer func() { - buf.Reset() - poolBytesBuffer.Put(buf) - }() + defer poolBytesBuffer.Put(buf) err = cdc.encodeReflectJSONInterface(buf, iinfo, reflect.ValueOf(&ivar).Elem(), FieldOptions{}) if err != nil { diff --git a/tm2/pkg/amino/binary_encode.go b/tm2/pkg/amino/binary_encode.go index 676f441d863..45758329284 100644 --- a/tm2/pkg/amino/binary_encode.go +++ b/tm2/pkg/amino/binary_encode.go @@ -254,10 +254,7 @@ func (cdc *Codec) encodeReflectBinaryInterface(w io.Writer, iinfo *TypeInfo, rv // For Proto3 compatibility, encode interfaces as google.protobuf.Any // Write field #1, TypeURL buf := poolBytesBuffer.Get() - defer func() { - buf.Reset() - poolBytesBuffer.Put(buf) - }() + defer poolBytesBuffer.Put(buf) { fnum := uint32(1) @@ -278,10 +275,7 @@ func (cdc *Codec) encodeReflectBinaryInterface(w io.Writer, iinfo *TypeInfo, rv // google.protobuf.Any values must be a struct, or an unpacked list which // is indistinguishable from a struct. buf2 := poolBytesBuffer.Get() - defer func() { - buf2.Reset() - poolBytesBuffer.Put(buf2) - }() + defer poolBytesBuffer.Put(buf2) if !cinfo.IsStructOrUnpacked(fopts) { writeEmpty := false @@ -370,10 +364,7 @@ func (cdc *Codec) encodeReflectBinaryList(w io.Writer, info *TypeInfo, rv reflec // Proto3 byte-length prefixing incurs alloc cost on the encoder. // Here we incur it for unpacked form for ease of dev. buf := poolBytesBuffer.Get() - defer func() { - buf.Reset() - poolBytesBuffer.Put(buf) - }() + defer poolBytesBuffer.Put(buf) // If elem is not already a ByteLength type, write in packed form. // This is a Proto wart due to Proto backwards compatibility issues. @@ -410,6 +401,9 @@ func (cdc *Codec) encodeReflectBinaryList(w io.Writer, info *TypeInfo, rv reflec einfo.Elem.ReprType.Type.Kind() != reflect.Uint8 && einfo.Elem.ReprType.GetTyp3(fopts) != Typ3ByteLength + elemBuf := poolBytesBuffer.Get() + defer poolBytesBuffer.Put(elemBuf) + // Write elems in unpacked form. for i := 0; i < rv.Len(); i++ { // Write elements as repeated fields of the parent struct. @@ -448,28 +442,21 @@ func (cdc *Codec) encodeReflectBinaryList(w io.Writer, info *TypeInfo, rv reflec // form) are represented as lists of implicit structs. if writeImplicit { // Write field key for Value field of implicit struct. - buf2 := poolBytesBuffer.Get() - buf2Done := func() { - buf2.Reset() - poolBytesBuffer.Put(buf2) - } - err = encodeFieldNumberAndTyp3(buf2, 1, Typ3ByteLength) + err = encodeFieldNumberAndTyp3(elemBuf, 1, Typ3ByteLength) if err != nil { - buf2Done() return } // Write field value of implicit struct to buf2. efopts := fopts efopts.BinFieldNum = 0 // dontcare - err = cdc.encodeReflectBinary(buf2, einfo, derv, efopts, false, 0) + err = cdc.encodeReflectBinary(elemBuf, einfo, derv, efopts, false, 0) if err != nil { - buf2Done() return } // Write implicit struct to buf. - err = EncodeByteSlice(buf, buf2.Bytes()) - buf2Done() + err = EncodeByteSlice(buf, elemBuf.Bytes()) + elemBuf.Reset() if err != nil { return } @@ -523,10 +510,7 @@ func (cdc *Codec) encodeReflectBinaryStruct(w io.Writer, info *TypeInfo, rv refl // Proto3 incurs a cost in writing non-root structs. // Here we incur it for root structs as well for ease of dev. buf := poolBytesBuffer.Get() - defer func() { - buf.Reset() - poolBytesBuffer.Put(buf) - }() + defer poolBytesBuffer.Put(buf) for _, field := range info.Fields { // Get type info for field. @@ -581,15 +565,8 @@ func encodeFieldNumberAndTyp3(w io.Writer, num uint32, typ Typ3) (err error) { return } -type bufLike interface { - io.Writer - Len() int - Bytes() []byte - Set([]byte) -} - func (cdc *Codec) writeFieldIfNotEmpty( - buf bufLike, + buf *bytebufferpool.ByteBuffer, fieldNum uint32, finfo *TypeInfo, structsFopts FieldOptions, // the wrapping struct's FieldOptions if any diff --git a/tm2/pkg/amino/codec.go b/tm2/pkg/amino/codec.go index 882d68de6b6..ba24f49a808 100644 --- a/tm2/pkg/amino/codec.go +++ b/tm2/pkg/amino/codec.go @@ -113,10 +113,7 @@ func (info *TypeInfo) String() string { return "" } buf := poolBytesBuffer.Get() - defer func() { - buf.Reset() - poolBytesBuffer.Put(buf) - }() + defer poolBytesBuffer.Put(buf) buf.Write([]byte("TypeInfo{")) buf.Write([]byte(fmt.Sprintf("Type:%v,", info.Type))) diff --git a/tm2/pkg/amino/json_encode.go b/tm2/pkg/amino/json_encode.go index e1f5562846f..99e1b445917 100644 --- a/tm2/pkg/amino/json_encode.go +++ b/tm2/pkg/amino/json_encode.go @@ -156,10 +156,7 @@ func (cdc *Codec) encodeReflectJSONInterface(w io.Writer, iinfo *TypeInfo, rv re // Write Value to buffer buf := poolBytesBuffer.Get() - defer func() { - buf.Reset() - poolBytesBuffer.Put(buf) - }() + defer poolBytesBuffer.Put(buf) cdc.encodeReflectJSON(buf, cinfo, crv, fopts) value := buf.Bytes() diff --git a/tm2/pkg/amino/wellknown.go b/tm2/pkg/amino/wellknown.go index bdba530463e..4053c23e893 100644 --- a/tm2/pkg/amino/wellknown.go +++ b/tm2/pkg/amino/wellknown.go @@ -342,10 +342,7 @@ func encodeReflectBinaryWellKnown(w io.Writer, info *TypeInfo, rv reflect.Value, // Maybe recurse with length-prefixing. if !bare { buf := poolBytesBuffer.Get() - defer func() { - buf.Reset() - poolBytesBuffer.Put(buf) - }() + defer poolBytesBuffer.Put(buf) ok, err = encodeReflectBinaryWellKnown(buf, info, rv, fopts, true) if err != nil {