From 8dbebcd378ee874993e4c970cce875f9b1b92301 Mon Sep 17 00:00:00 2001 From: Calvin Kim Date: Thu, 18 Feb 2021 01:46:01 +0900 Subject: [PATCH 1/3] common: Add freeBytes pool in a new package 'common' Adds a new pool for bytes to help with gc slowdowns. Common package is needed to avoid import cycles when putting freeBytes into util package --- common/common.go | 119 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 common/common.go diff --git a/common/common.go b/common/common.go new file mode 100644 index 00000000..675206f0 --- /dev/null +++ b/common/common.go @@ -0,0 +1,119 @@ +package common + +import ( + "encoding/binary" + "io" + "sync" +) + +// FreeBytes is a wrapper around bytes +type FreeBytes struct { + Bytes []byte +} + +// FreeBytes the bytes to the FreeBytes pool +func (fb *FreeBytes) Free() { + fb.Bytes = fb.Bytes[:0] + FreeBytesPool.Put(fb) +} + +// NewFreeBytes returns a parentHashBytes from the pool. Will allocate if the +// Pool returns parentHashBytes that doesn't have bytes allocated +func NewFreeBytes() *FreeBytes { + fb := FreeBytesPool.Get().(*FreeBytes) + + if fb.Bytes == nil { + // set minimum to 64 since that's what parentHash() + // requires and parentHash() is called very frequently + fb.Bytes = make([]byte, 0, 64) + } + + return fb +} + +// FreeBytesPool is the pool of bytes to recycle&relieve gc pressure. +var FreeBytesPool = sync.Pool{ + New: func() interface{} { return new(FreeBytes) }, +} + +// Uint8 reads a single byte from the provided reader using a buffer from the +// free list and returns it as a uint8. +func (fb *FreeBytes) Uint8(r io.Reader) (uint8, error) { + buf := fb.Bytes[:1] + if _, err := io.ReadFull(r, buf); err != nil { + return 0, err + } + return buf[0], nil +} + +// Uint16 reads two bytes from the provided reader using a buffer from the +// free list, converts it to a number using the provided byte order, and returns +// the resulting uint16. +func (fb *FreeBytes) Uint16(r io.Reader, byteOrder binary.ByteOrder) (uint16, error) { + buf := fb.Bytes[:2] + if _, err := io.ReadFull(r, buf); err != nil { + return 0, err + } + return byteOrder.Uint16(buf), nil +} + +// Uint32 reads four bytes from the provided reader using a buffer from the +// free list, converts it to a number using the provided byte order, and returns +// the resulting uint32. +func (fb *FreeBytes) Uint32(r io.Reader, byteOrder binary.ByteOrder) (uint32, error) { + buf := fb.Bytes[:4] + if _, err := io.ReadFull(r, buf); err != nil { + return 0, err + } + return byteOrder.Uint32(buf), nil +} + +// Uint64 reads eight bytes from the provided reader using a buffer from the +// free list, converts it to a number using the provided byte order, and returns +// the resulting uint64. +func (fb *FreeBytes) Uint64(r io.Reader, byteOrder binary.ByteOrder) (uint64, error) { + buf := fb.Bytes[:8] + if _, err := io.ReadFull(r, buf); err != nil { + return 0, err + } + return byteOrder.Uint64(buf), nil +} + +// PutUint8 copies the provided uint8 into a buffer from the free list and +// writes the resulting byte to the given writer. +func (fb *FreeBytes) PutUint8(w io.Writer, val uint8) error { + buf := fb.Bytes[:1] + buf[0] = val + _, err := w.Write(buf) + return err +} + +// PutUint16 serializes the provided uint16 using the given byte order into a +// buffer from the free list and writes the resulting two bytes to the given +// writer. +func (fb *FreeBytes) PutUint16(w io.Writer, byteOrder binary.ByteOrder, val uint16) error { + buf := fb.Bytes[:2] + byteOrder.PutUint16(buf, val) + _, err := w.Write(buf) + return err +} + +// PutUint32 serializes the provided uint32 using the given byte order into a +// buffer from the free list and writes the resulting four bytes to the given +// writer. +func (fb *FreeBytes) PutUint32(w io.Writer, byteOrder binary.ByteOrder, val uint32) error { + buf := fb.Bytes[:4] + byteOrder.PutUint32(buf, val) + _, err := w.Write(buf) + return err +} + +// PutUint64 serializes the provided uint64 using the given byte order into a +// buffer from the free list and writes the resulting eight bytes to the given +// writer. +func (fb *FreeBytes) PutUint64(w io.Writer, byteOrder binary.ByteOrder, val uint64) error { + buf := fb.Bytes[:8] + byteOrder.PutUint64(buf, val) + _, err := w.Write(buf) + return err +} From d1001ad1682dd90d64d1cf802f17ea1577cff015 Mon Sep 17 00:00:00 2001 From: Calvin Kim Date: Thu, 18 Feb 2021 02:02:39 +0900 Subject: [PATCH 2/3] accumulator/btcacc: Use sync.Pool for byte slices in package common There are functions in these packages that allocate and de-allocate byte slices numerous times per function call. Using a pool for byte slices help reduce this allocation and speeds up the code. --- accumulator/types.go | 28 +++++++++++++++-------- btcacc/leaf.go | 21 ++++++++++++----- btcacc/udata.go | 54 +++++++++++++++----------------------------- 3 files changed, 51 insertions(+), 52 deletions(-) diff --git a/accumulator/types.go b/accumulator/types.go index b6d7cf95..484c3abc 100644 --- a/accumulator/types.go +++ b/accumulator/types.go @@ -5,9 +5,11 @@ import ( "crypto/sha512" "fmt" "math/rand" + + "github.com/mit-dci/utreexo/common" ) -// Hash : +// Hash is the 32 bytes of a sha256 hash type Hash [32]byte // Prefix for printfs @@ -15,16 +17,16 @@ func (h Hash) Prefix() []byte { return h[:4] } -// Mini : +// Mini takes the first 12 slices of a hash and outputs a MiniHash func (h Hash) Mini() (m MiniHash) { copy(m[:], h[:12]) return } -// MiniHash : +// MiniHash is the first 12 bytes of a sha256 hash type MiniHash [12]byte -// HashFromString : +// HashFromString takes a string and hashes with sha256 func HashFromString(s string) Hash { return sha256.Sum256([]byte(s)) } @@ -35,7 +37,8 @@ type arrow struct { collapse bool } -// Node : +// node is an element in the utreexo tree and is represented by a position +// and a hash type node struct { Pos uint64 Val Hash @@ -53,14 +56,18 @@ type simLeaf struct { duration int32 } -// Parent gets you the merkle parent. So far no committing to height. -// if the left child is zero it should crash... +// parentHash gets you the merkle parent of two children hashes. +// TODO So far no committing to height. func parentHash(l, r Hash) Hash { var empty Hash if l == empty || r == empty { panic("got an empty leaf here. ") } - return sha512.Sum512_256(append(l[:], r[:]...)) + buf := common.NewFreeBytes() + defer buf.Free() + buf.Bytes = append(buf.Bytes, l[:]...) + buf.Bytes = append(buf.Bytes, r[:]...) + return sha512.Sum512_256(buf.Bytes) } // SimChain is for testing; it spits out "blocks" of adds and deletes @@ -73,7 +80,7 @@ type SimChain struct { lookahead int32 } -// NewSimChain : +// NewSimChain initializes and returns a Simchain func NewSimChain(duration uint32) *SimChain { var s SimChain s.blockHeight = -1 @@ -119,7 +126,8 @@ func (s *SimChain) ttlString() string { return x } -// NextBlock : +// NextBlock outputs a new simulation block given the additions for the block +// to be outputed func (s *SimChain) NextBlock(numAdds uint32) ([]Leaf, []int32, []Hash) { s.blockHeight++ fmt.Printf("blockHeight %d\n", s.blockHeight) diff --git a/btcacc/leaf.go b/btcacc/leaf.go index c76728c8..4fdbb43a 100644 --- a/btcacc/leaf.go +++ b/btcacc/leaf.go @@ -8,6 +8,8 @@ import ( "fmt" "io" "strconv" + + "github.com/mit-dci/utreexo/common" ) const HashSize = 32 @@ -71,14 +73,19 @@ func (l *LeafData) Serialize(w io.Writer) (err error) { _, err = w.Write(l.BlockHash[:]) _, err = w.Write(l.TxHash[:]) - err = binary.Write(w, binary.BigEndian, l.Index) - err = binary.Write(w, binary.BigEndian, hcb) - err = binary.Write(w, binary.BigEndian, l.Amt) + + freeBytes := common.NewFreeBytes() + defer freeBytes.Free() + + err = freeBytes.PutUint32(w, binary.BigEndian, l.Index) + err = freeBytes.PutUint32(w, binary.BigEndian, uint32(hcb)) + err = freeBytes.PutUint64(w, binary.BigEndian, uint64(l.Amt)) + if len(l.PkScript) > 10000 { err = fmt.Errorf("pksize too long") return } - err = binary.Write(w, binary.BigEndian, uint16(len(l.PkScript))) + err = freeBytes.PutUint16(w, binary.BigEndian, uint16(len(l.PkScript))) _, err = w.Write(l.PkScript) return } @@ -121,7 +128,9 @@ func (l *LeafData) Deserialize(r io.Reader) (err error) { // LeafHash turns a LeafData into a LeafHash func (l *LeafData) LeafHash() [32]byte { - var buf bytes.Buffer - l.Serialize(&buf) + freeBytes := common.NewFreeBytes() + defer freeBytes.Free() + buf := bytes.NewBuffer(freeBytes.Bytes) + l.Serialize(buf) return sha512.Sum512_256(buf.Bytes()) } diff --git a/btcacc/udata.go b/btcacc/udata.go index cd58b4b6..e33fd028 100644 --- a/btcacc/udata.go +++ b/btcacc/udata.go @@ -7,8 +7,11 @@ import ( "io" "github.com/mit-dci/utreexo/accumulator" + "github.com/mit-dci/utreexo/common" ) +// UData is all the data needed to verify the utreexo accumulator proof +// for a given block type UData struct { Height int32 AccProof accumulator.BatchProof @@ -52,8 +55,7 @@ func (ud *UData) ProofSanity(nl uint64, h uint8) bool { return false } } - // return to presorted target list - // ud.AccProof.Targets = presort + return true } @@ -61,13 +63,12 @@ func (ud *UData) ProofSanity(nl uint64, h uint8) bool { // aaff aaff 0000 0014 0000 0001 0000 0001 0000 0000 0000 0000 0000 0000 // magic | size | height | numttls | ttl0 | numTgts | ???? -// ToBytes serializes UData into bytes. +// Serialize serializes UData into bytes. // First, height, 4 bytes. // Then, number of TTL values (4 bytes, even though we only need 2) -// Then a bunch of TTL values, (4B each) one for each txo in the associated block -// batch proof -// Bunch of LeafDatas - +// Then a bunch of TTL values, (4B each) one for each txo in the +// associated block batch proof +// And the rest is a bunch of LeafDatas func (ud *UData) Serialize(w io.Writer) (err error) { err = binary.Write(w, binary.BigEndian, ud.Height) if err != nil { // ^ 4B block height @@ -89,43 +90,33 @@ func (ud *UData) Serialize(w io.Writer) (err error) { return } - // fmt.Printf("accproof %d bytes\n", ud.AccProof.SerializeSize()) - // write all the leafdatas for _, ld := range ud.Stxos { - // fmt.Printf("writing ld %d %s\n", i, ld.ToString()) err = ld.Serialize(w) if err != nil { return } - // fmt.Printf("h %d leaf %d %s len %d\n", - // ud.Height, i, ld.Outpoint.String(), len(ld.PkScript)) } return } -// +// SerializeSize outputs the size of the udata when it is serialized func (ud *UData) SerializeSize() int { var ldsize int - var b bytes.Buffer + buf := common.NewFreeBytes() + bufWriter := bytes.NewBuffer(buf.Bytes) - // TODO this is slow, can remove double checking once it works reliably + // Grab the size of all the stxos for _, l := range ud.Stxos { ldsize += l.SerializeSize() - b.Reset() - l.Serialize(&b) - if b.Len() != l.SerializeSize() { - fmt.Printf(" b.Len() %d, l.SerializeSize() %d\n", - b.Len(), l.SerializeSize()) - } } - b.Reset() - ud.AccProof.Serialize(&b) - if b.Len() != ud.AccProof.SerializeSize() { + bufWriter.Reset() + ud.AccProof.Serialize(bufWriter) + if bufWriter.Len() != ud.AccProof.SerializeSize() { fmt.Printf(" b.Len() %d, AccProof.SerializeSize() %d\n", - b.Len(), ud.AccProof.SerializeSize()) + bufWriter.Len(), ud.AccProof.SerializeSize()) } guess := 8 + (4 * len(ud.TxoTTLs)) + ud.AccProof.SerializeSize() + ldsize @@ -134,14 +125,13 @@ func (ud *UData) SerializeSize() int { return guess } +// Deserialize reads from the reader and deserializes the udata func (ud *UData) Deserialize(r io.Reader) (err error) { - err = binary.Read(r, binary.BigEndian, &ud.Height) if err != nil { // ^ 4B block height fmt.Printf("ud deser Height err %s\n", err.Error()) return } - // fmt.Printf("read height %d\n", ud.Height) var numTTLs uint32 err = binary.Read(r, binary.BigEndian, &numTTLs) @@ -149,8 +139,6 @@ func (ud *UData) Deserialize(r io.Reader) (err error) { fmt.Printf("ud deser numTTLs err %s\n", err.Error()) return } - // fmt.Printf("read ttls %d\n", numTTLs) - // fmt.Printf("UData deser read h %d - %d ttls ", ud.Height, numTTLs) ud.TxoTTLs = make([]int32, numTTLs) for i, _ := range ud.TxoTTLs { // write all ttls @@ -159,7 +147,6 @@ func (ud *UData) Deserialize(r io.Reader) (err error) { fmt.Printf("ud deser LeafTTLs[%d] err %s\n", i, err.Error()) return } - // fmt.Printf("read ttl[%d] %d\n", i, ud.TxoTTLs[i]) } err = ud.AccProof.Deserialize(r) @@ -168,8 +155,6 @@ func (ud *UData) Deserialize(r io.Reader) (err error) { return } - // fmt.Printf("%d byte accproof, read %d targets\n", - // ud.AccProof.SerializeSize(), len(ud.AccProof.Targets)) // we've already gotten targets. 1 leafdata per target ud.Stxos = make([]LeafData, len(ud.AccProof.Targets)) for i, _ := range ud.Stxos { @@ -180,9 +165,6 @@ func (ud *UData) Deserialize(r io.Reader) (err error) { ud.Height, numTTLs, len(ud.AccProof.Targets), i, err.Error()) return } - // fmt.Printf("h %d leaf %d %s len %d\n", - // ud.Height, i, ud.Stxos[i].Outpoint.String(), len(ud.Stxos[i].PkScript)) - } return @@ -212,6 +194,7 @@ func GenUData(delLeaves []LeafData, forest *accumulator.Forest, height int32) ( ud.Height = height ud.Stxos = delLeaves + // make slice of hashes from leafdata delHashes := make([]accumulator.Hash, len(ud.Stxos)) for i, _ := range ud.Stxos { @@ -233,6 +216,5 @@ func GenUData(delLeaves []LeafData, forest *accumulator.Forest, height int32) ( return } - // fmt.Printf(ud.AccProof.ToString()) return } From 4d73eb7667749832b6df7b408e19006265dab651 Mon Sep 17 00:00:00 2001 From: Calvin Kim Date: Thu, 18 Feb 2021 15:01:30 +0900 Subject: [PATCH 3/3] btcacc/leaf_test: Add test for serializing and de-serializing leafdata --- btcacc/leaf_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 btcacc/leaf_test.go diff --git a/btcacc/leaf_test.go b/btcacc/leaf_test.go new file mode 100644 index 00000000..e4fdb53c --- /dev/null +++ b/btcacc/leaf_test.go @@ -0,0 +1,38 @@ +package btcacc + +import ( + "bytes" + "fmt" + "testing" +) + +func TestLeafDataSerialize(t *testing.T) { + ld := LeafData{ + TxHash: Hash{1, 2, 3, 4}, + Index: 0, + Height: 2, + Coinbase: false, + Amt: 3000, + PkScript: []byte{1, 2, 3, 4, 5, 6}, + } + + // Before + writer := &bytes.Buffer{} + ld.Serialize(writer) + beforeBytes := writer.Bytes() + + // After + checkLeaf := LeafData{} + checkLeaf.Deserialize(writer) + + afterWriter := &bytes.Buffer{} + checkLeaf.Serialize(afterWriter) + afterBytes := afterWriter.Bytes() + + if !bytes.Equal(beforeBytes, afterBytes) { + err := fmt.Errorf("Serialize/Deserialize LeafData fail\n"+ + "beforeBytes len: %v\n, afterBytes len:%v\n", + len(beforeBytes), len(afterBytes)) + t.Fatal(err) + } +}