From 3d6ac0cca37773bcb73da36eaeefa376e3f1ce2a Mon Sep 17 00:00:00 2001 From: Pavel Kalinnikov Date: Fri, 1 Apr 2022 06:11:37 -0400 Subject: [PATCH] Implement get-compact-range using RFC 6962 methods This is a proof of concept change demonstrating that it is possible to obtain arbitrary compact ranges from a Merkle tree log that restricts itself only to endpoints represented in RFC 6962, in constant time interaction complexity. Specifically, it is possible to obtain comact range [begin, end) by calling "get consistency proof" endpoints <= 2 times for carefully crafted tree sizes. In a few cases where it is impossible to get certain hashes, this approach falls back to calling the "get entries" endpoint 1 time to obtain between 1-3 entries and reconstruct the compact range. Overall, the interaction with the log is limited by 2 calls, and each call is limited in size. --- exp/README.md | 7 ++ exp/get_compact_range.go | 139 ++++++++++++++++++++++++++++++++++ exp/get_compact_range_test.go | 116 ++++++++++++++++++++++++++++ exp/go.mod | 5 ++ exp/go.sum | 4 + 5 files changed, 271 insertions(+) create mode 100644 exp/README.md create mode 100644 exp/get_compact_range.go create mode 100644 exp/get_compact_range_test.go create mode 100644 exp/go.mod create mode 100644 exp/go.sum diff --git a/exp/README.md b/exp/README.md new file mode 100644 index 0000000..fd5cfb1 --- /dev/null +++ b/exp/README.md @@ -0,0 +1,7 @@ +Experimental +------------ + +This directory contains a Go module with experimental features not included into +the main Go module of this repository. These must be used with caution. + +The idea of this module is similar to Go's https://pkg.go.dev/golang.org/x/exp. diff --git a/exp/get_compact_range.go b/exp/get_compact_range.go new file mode 100644 index 0000000..7e3e331 --- /dev/null +++ b/exp/get_compact_range.go @@ -0,0 +1,139 @@ +package merkle + +import ( + "fmt" + + "github.com/transparency-dev/merkle/compact" + "github.com/transparency-dev/merkle/proof" +) + +type HashGetter interface { + GetConsistencyProof(first, second uint64) ([][]byte, error) + GetLeafHashes(begin, end uint64) ([][]byte, error) +} + +func GetCompactRange(rf *compact.RangeFactory, begin, end, size uint64, hg HashGetter) (*compact.Range, error) { + if begin > size || end > size { + return nil, fmt.Errorf("[%d, %d) out of range in %d", begin, end, size) + } + if begin >= end { + return rf.NewEmptyRange(begin), nil + } + + if size <= 3 || end == 1 { + hashes, err := hg.GetLeafHashes(begin, end) + if err != nil { + return nil, fmt.Errorf("GetLeafHashes(%d, %d): %v", begin, end, err) + } + if got, want := uint64(len(hashes)), end-begin; got != want { + return nil, fmt.Errorf("GetLeafHashes(%d, %d): %d hashes, want %d", begin, end, got, want) + } + r := rf.NewEmptyRange(begin) + for _, h := range hashes { + if err := r.Append(h, nil); err != nil { + return nil, fmt.Errorf("Append: %v", err) + } + } + return r, nil + } + // size >= 4 && end >= 2 + + known := make(map[compact.NodeID][]byte) + + store := func(nodes proof.Nodes, hashes [][]byte) error { + _, b, e := nodes.Ephem() + wantSize := len(nodes.IDs) - (e - b) + if b != e { + wantSize++ + } + if got := len(hashes); got != wantSize { + return fmt.Errorf("proof size mismatch: got %d, want %d", got, wantSize) + } + + idx := 0 + for _, hash := range hashes { + if idx == b && b+1 < e { + idx = e - 1 + continue + } + known[nodes.IDs[idx]] = hash + idx++ + } + return nil + } + + newRange := func(begin, end uint64) (*compact.Range, error) { + size := compact.RangeSize(begin, end) + ids := compact.RangeNodes(begin, end, make([]compact.NodeID, 0, size)) + hashes := make([][]byte, 0, len(ids)) + for _, id := range ids { + if hash, ok := known[id]; ok { + hashes = append(hashes, hash) + } else { + return nil, fmt.Errorf("hash not known: %+v", id) + } + } + return rf.NewRange(begin, end, hashes) + } + + fetch := func(first, second uint64) error { + nodes, err := proof.Consistency(first, second) + if err != nil { + return fmt.Errorf("proof.Consistency: %v", err) + } + hashes, err := hg.GetConsistencyProof(first, second) + if err != nil { + return fmt.Errorf("GetConsistencyProof(%d, %d): %v", first, second, err) + } + store(nodes, hashes) + return nil + } + + mid, _ := compact.Decompose(begin, end) + mid += begin + if err := fetch(begin, mid); err != nil { + return nil, err + } + + if begin == 0 && end == 2 || end == 3 { + if err := fetch(3, 4); err != nil { + return nil, err + } + } + if end <= 3 { + return newRange(begin, end) + } + // end >= 4 + + if (end-1)&(end-2) != 0 { // end-1 is not a power of 2. + if err := fetch(end-1, end); err != nil { + return nil, err + } + r, err := newRange(begin, end-1) + if err != nil { + return nil, err + } + if err := r.Append(known[compact.NewNodeID(0, end-1)], nil); err != nil { + return nil, fmt.Errorf("Append: %v", err) + } + return r, nil + } + + // At this point: end >= 4, end-1 is a power of 2; thus, end-2 is not a power of 2. + if err := fetch(end-2, end); err != nil { + return nil, err + } + r := rf.NewEmptyRange(begin) + if end-2 > begin { + var err error + if r, err = newRange(begin, end-2); err != nil { + return nil, err + } + } + for index := r.End(); index < end; index++ { + if err := r.Append(known[compact.NewNodeID(0, index)], nil); err != nil { + return nil, fmt.Errorf("Append: %v", err) + } + } + return r, nil +} diff --git a/exp/get_compact_range_test.go b/exp/get_compact_range_test.go new file mode 100644 index 0000000..baa7951 --- /dev/null +++ b/exp/get_compact_range_test.go @@ -0,0 +1,116 @@ +package merkle_test + +import ( + "fmt" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/transparency-dev/merkle" + "github.com/transparency-dev/merkle/compact" + "github.com/transparency-dev/merkle/proof" +) + +func TestGetCompactRange(t *testing.T) { + rf := compact.RangeFactory{Hash: func(left, right []byte) []byte { + return append(append(make([]byte, 0, len(left)+len(right)), left...), right...) + }} + tr := newTree(t, 256, &rf) + + test := func(begin, end, size uint64) { + t.Run(fmt.Sprintf("%d:%d_%d", size, begin, end), func(t *testing.T) { + got, err := merkle.GetCompactRange(&rf, begin, end, size, tr) + if err != nil { + t.Fatalf("GetCompactRange: %v", err) + } + want, err := tr.getCompactRange(begin, end) + if err != nil { + t.Fatalf("GetCompactRange: %v", err) + } + if diff := cmp.Diff(got, want); diff != "" { + t.Fatalf("Diff: %s", diff) + } + }) + } + + for begin := uint64(0); begin <= tr.size; begin++ { + for end := begin; end <= tr.size; end++ { + for size := end; size < end+5 && size < tr.size; size++ { + test(begin, end, size) + } + test(begin, end, tr.size) + } + } +} + +type tree struct { + rf *compact.RangeFactory + size uint64 + nodes map[compact.NodeID][]byte +} + +func newTree(t *testing.T, size uint64, rf *compact.RangeFactory) *tree { + hash := func(leaf uint64) []byte { + if leaf >= 256 { + t.Fatalf("leaf %d not supported in this test", leaf) + } + return []byte{byte(leaf)} + } + + nodes := make(map[compact.NodeID][]byte, size*2-1) + r := rf.NewEmptyRange(0) + for i := uint64(0); i < size; i++ { + nodes[compact.NewNodeID(0, i)] = hash(i) + if err := r.Append(hash(i), func(id compact.NodeID, hash []byte) { + nodes[id] = hash + }); err != nil { + t.Fatalf("Append: %v", err) + } + } + return &tree{rf: rf, size: size, nodes: nodes} +} + +func (t *tree) GetConsistencyProof(first, second uint64) ([][]byte, error) { + if first > t.size || second > t.size { + return nil, fmt.Errorf("%d or %d is beyond %d", first, second, t.size) + } + nodes, err := proof.Consistency(first, second) + if err != nil { + return nil, err + } + hashes, err := t.getNodes(nodes.IDs) + if err != nil { + return nil, err + } + return nodes.Rehash(hashes, t.rf.Hash) +} + +func (t *tree) GetLeafHashes(begin, end uint64) ([][]byte, error) { + if begin >= end { + return nil, nil + } + ids := make([]compact.NodeID, 0, end-begin) + for i := begin; i < end; i++ { + ids = append(ids, compact.NewNodeID(0, i)) + } + return t.getNodes(ids) +} + +func (t *tree) getCompactRange(begin, end uint64) (*compact.Range, error) { + hashes, err := t.getNodes(compact.RangeNodes(begin, end)) + if err != nil { + return nil, err + } + return t.rf.NewRange(begin, end, hashes) +} + +func (t *tree) getNodes(ids []compact.NodeID) ([][]byte, error) { + hashes := make([][]byte, len(ids)) + for i, id := range ids { + if hash, ok := t.nodes[id]; ok { + hashes[i] = hash + } else { + return nil, fmt.Errorf("node %+v not found", id) + } + } + return hashes, nil +} diff --git a/exp/go.mod b/exp/go.mod new file mode 100644 index 0000000..78c02a5 --- /dev/null +++ b/exp/go.mod @@ -0,0 +1,5 @@ +module github.com/transparency-dev/merkle/exp + +go 1.16 + +require github.com/transparency-dev/merkle v0.0.0-20220425113829-c120179f55ad // indirect diff --git a/exp/go.sum b/exp/go.sum new file mode 100644 index 0000000..2b129cc --- /dev/null +++ b/exp/go.sum @@ -0,0 +1,4 @@ +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/transparency-dev/merkle v0.0.0-20220425113829-c120179f55ad h1:82yvTO+VijfWulMsMQvqQSZ0zNEAgmEUeBG+ArrO9Js= +github.com/transparency-dev/merkle v0.0.0-20220425113829-c120179f55ad/go.mod h1:B8FIw5LTq6DaULoHsVFRzYIUDkl8yuSwCdZnOZGKL/A= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=