diff --git a/accumulator/forest.go b/accumulator/forest.go index 12cba1ee..f03ff9ce 100644 --- a/accumulator/forest.go +++ b/accumulator/forest.go @@ -7,6 +7,8 @@ import ( "os" "sort" "time" + + "golang.org/x/exp/slices" ) const ( @@ -186,6 +188,428 @@ func (f *Forest) removev4(dels []uint64) error { return nil } +func (f *Forest) removeSwapless(dels []uint64) error { + // Check that all dels are there. + for _, dpos := range dels { + if dpos > maxPosition(f.rows) { + return fmt.Errorf( + "Trying to delete leaf at %d, beyond max %d", dpos, maxPosition(f.rows)) + } + } + + sortUint64s(dels) + + //fmt.Println("dels", dels) + for _, del := range dels { + delete(f.positionMap, f.data.read(del).Mini()) + //f.data.write(del, empty) + + //fmt.Println("DEL", del) + //fmt.Println(f.SubTreeToString(del)) + } + + deTwinedDels := make([]uint64, len(dels)) + copy(deTwinedDels, dels) + deTwin(&deTwinedDels, f.rows) + + for _, deTwined := range deTwinedDels { + if f.isLeaf(deTwined, detectRow(deTwined, f.rows)) { + f.data.write(deTwined, empty) + } else { + f.deleteSubTree(deTwined) + } + } + + // Calculate which leaves should go where when the deletion happens. + moveRows := Transform(dels, f.numLeaves, f.rows) + //fmt.Println("moves", moveRows) + + // Go through all the moves from bottom to top. + for i := 0; i < len(moveRows); i++ { + moveRow := moveRows[i] + + for _, move := range moveRow { + // If from and to are the same, it means that the entire subtree + // is being deleted. + if move.from == move.to { + f.deleteSubTree(move.from) + } else { + f.movePositionUp2(move.from, move.to) + } + } + } + + // Calculate which nodes need to be hashed again. + dirtyRows := calcDirtyNodes2(moveRows, f.numLeaves, f.rows) + + //fmt.Println("dirtyRows", dirtyRows) + + for currentRow, dirtyRow := range dirtyRows { + for _, dirtyPos := range dirtyRow { + leftChild := child(dirtyPos, f.rows) + rightChild := rightSib(leftChild) + + if f.data.read(leftChild) == empty { + fmt.Println("currentRow", currentRow) + + fmt.Println("f.rows", f.rows) + fmt.Println("numleaves", f.numLeaves) + fmt.Println(f.SubTreeToString(leftChild)) + //fmt.Println(f.ToString()) + return fmt.Errorf("removeSwapless error: couldn't hash dirty "+ + "position at %d as the leftChild of %d was empty", + dirtyPos, leftChild) + } + + if f.data.read(rightChild) == empty { + fmt.Println("HERE") + fmt.Println(f.SubTreeToString(rightChild)) + //fmt.Println(f.ToString()) + return fmt.Errorf("removeSwapless error: couldn't hash dirty "+ + "position at %d as the rightChild of %d was empty", + dirtyPos, rightChild) + } + + hash := parentHash(f.data.read(leftChild), f.data.read(rightChild)) + f.data.write(dirtyPos, hash) + + // If the dirty position has a parent, then that is also dirty. + if currentRow < int(logicalTreeRows(f.numLeaves)) && + !isRootPosition(dirtyPos, f.numLeaves, f.rows) { + + parentPos := parent(dirtyPos, f.rows) + parentRow := detectRow(parentPos, f.rows) + + // Insert in order. + insertSort(&dirtyRows[parentRow], parentPos) + + // If it's already there, remove it. + dirtyRows[parentRow] = removeDuplicateInt(dirtyRows[parentRow]) + } + } + } + + return nil +} + +// deleteSubTree deletes the position and the entire subtree below it. +func (f *Forest) deleteSubTree(position uint64) { + //fmt.Println("deleteSubTree", position) + fromRow := int(detectRow(position, f.rows)) + + positions := []uint64{position} + + for currentRow := fromRow; currentRow >= 0; currentRow-- { + nextPositions := []uint64{} + + for _, position := range positions { + // Check children and add to the list of dels. + leftChild := child(position, f.rows) + rightChild := rightSib(leftChild) + + if currentRow != 0 && f.data.read(leftChild) != empty { + nextPositions = append(nextPositions, leftChild) + } + + if currentRow != 0 && f.data.read(rightChild) != empty { + nextPositions = append(nextPositions, rightChild) + } + + f.data.write(position, empty) + } + + positions = nextPositions + } +} + +func (f *Forest) deleteRows(position uint64, endRow uint8) { + leftChild := child(position, f.rows) + rightChild := rightSib(leftChild) + positions := []uint64{leftChild, rightChild} + + currentRow := int(detectRow(position, f.rows)) - 1 + for ; currentRow >= int(endRow); currentRow-- { + nextPositions := []uint64{} + + for _, position := range positions { + // Check children and add to the list of dels. + leftChild := child(position, f.rows) + rightChild := rightSib(leftChild) + + if currentRow != 0 && f.data.read(leftChild) != empty { + nextPositions = append(nextPositions, leftChild) + } + + if currentRow != 0 && f.data.read(rightChild) != empty { + nextPositions = append(nextPositions, rightChild) + } + + f.data.write(position, empty) + } + + positions = nextPositions + } +} + +// movePositionUp2 moves the position and all of its children up to the desired position. +// It writes empty hashes to the positions that are deleted/won't have other hashes replacing +// it. +func (f *Forest) movePositionUp2(from, to uint64) { + fromPositions := []uint64{from} + toPositions := []uint64{to} + + // Go top down, starting from the row where the from position is at. + fromRow := int(detectRow(from, f.rows)) + for currentRow := fromRow; currentRow >= 0; currentRow-- { + nextFromPositions := []uint64{} + nextToPositions := []uint64{} + + // Process all the froms for this row. + for i, fromPosition := range fromPositions { + toPosition := toPositions[i] + + // Grab and write the from-hash to the poition to move to. + fromHash := f.data.read(fromPosition) + f.data.write(toPosition, fromHash) + + // If the node we're on is a leaf, then remap the position map as well. + // Otherwise, append the children to be processed next. + if f.isLeaf(fromPosition, uint8(currentRow)) { + // The node that we're on is a leaf. Update the + // positionMap entry. + _, found := f.positionMap[fromHash.Mini()] + if !found { + //fmt.Println(f.SubTreeToString(fromPosition)) + //fmt.Printf("currentRow %d, fromPos %d\n", currentRow, fromPosition) + panic("couldn't find pos") + } + + //fmt.Printf("Remapping hash %s. From %d, to %d\n", + // hex.EncodeToString(fromHash[:]), origPos, toPosition) + f.positionMap[fromHash.Mini()] = toPosition + } else { + // Left child. + nextFromPositions = append(nextFromPositions, child(fromPosition, f.rows)) + nextToPositions = append(nextToPositions, child(toPosition, f.rows)) + + // Right child. + nextFromPositions = append(nextFromPositions, rightSib(child(fromPosition, f.rows))) + nextToPositions = append(nextToPositions, rightSib(child(toPosition, f.rows))) + } + } + + //fmt.Println(f.ToString()) + + // Zero out all the positions that won't have another hash replacing it. + for i, fromPosition := range fromPositions { + if f.isLeaf(fromPosition, uint8(currentRow)) { + // Empty left child. + f.data.write(child(toPositions[i], f.rows), empty) + + // Empty right child. + f.data.write(rightSib(child(toPositions[i], f.rows)), empty) + + // Empty myself. + f.data.write(fromPosition, empty) + + // Delete all descendents of the from-leaf. Start from the row where to-position + // is all the way down to the row where from-position is. + f.deleteRows(toPositions[i], uint8(detectRow(fromPosition, f.rows))) + + f.deleteRows(fromPosition, 0) + } else { + // Write empty if nothing is replacing where this node was. + if !slices.Contains(nextToPositions, fromPosition) { + //fmt.Println("Writing empty to ", fromPosition) + f.data.write(fromPosition, empty) + } + } + } + + fromPositions = nextFromPositions + toPositions = nextToPositions + } +} + +// isLeaf returns if the position is a leaf or a node. +func (f *Forest) isLeaf(position uint64, currentRow uint8) bool { + if currentRow == 0 { + return true + } + return f.data.read(child(position, f.rows)) == empty +} + +//// movePositionUp moves the position and all of its children up to the desired position. +//// It writes empty hashes to the positions that are deleted/won't have other hashes replacing +//// it. +//func (f *Forest) movePositionUp(from, to uint64) { +// fromRow := int(detectRow(from, f.rows)) +// +// fromPositions := []uint64{from} +// toPositions := []uint64{to} +// +// for currentRow := fromRow; currentRow >= 0; currentRow-- { +// fmt.Println("from: ", fromPositions, currentRow) +// fmt.Println("to: ", toPositions) +// +// nextFromPositions := []uint64{} +// nextToPositions := []uint64{} +// +// for i, fromPosition := range fromPositions { +// toPosition := toPositions[i] +// +// // Grab and write the from-hash to the poition to move to. +// fromHash := f.data.read(fromPosition) +// f.data.write(toPosition, fromHash) +// +// //if int(detectRow(toPosition, f.rows))+1 < int(f.rows) { +// // var leftKid, rightKid uint64 +// // if toPosition&1 != 1 { +// // leftKid = toPosition +// // rightKid = rightSib(leftKid) +// // } else { +// // rightKid = toPosition +// // leftKid = leftSib(leftKid) +// // } +// +// // fmt.Printf("toPos %d leftkid %d, rightKid %d\n", toPosition, leftKid, rightKid) +// +// // newParentHash := parentHash(f.data.read(leftKid), f.data.read(rightKid)) +// // f.data.write(parent(toPosition, f.rows), newParentHash) +// //} +// +// fmt.Printf("FromPos %d, read %s and wrote to %d\n", +// fromPosition, hex.EncodeToString(fromHash[:]), toPosition) +// +// // Check children and add to the list of to-s and from-s if +// // they are not empty. +// leftChild := child(fromPosition, f.rows) +// rightChild := rightSib(leftChild) +// +// // If one child isn't empty, that means that this node isn't a +// // leaf and there's more nodes below. +// if currentRow != 0 && f.data.read(leftChild) != empty && f.data.read(rightChild) != empty { +// // Left child. +// nextFromPositions = append(nextFromPositions, leftChild) +// nextToPositions = append(nextToPositions, +// child(toPosition, f.rows)) +// +// // Right child. +// nextFromPositions = append(nextFromPositions, rightChild) +// nextToPositions = append(nextToPositions, +// rightSib(child(toPosition, f.rows))) +// } else { +// origPos, found := f.positionMap[fromHash.Mini()] +// if !found { +// fmt.Println(f.SubTreeToString(fromPosition)) +// fmt.Printf("currentRow %d, fromPos %d\n", currentRow, fromPosition) +// panic("couldn't find pos") +// } +// // The node that we're on is a leaf. Update the +// // positionMap entry. +// f.positionMap[fromHash.Mini()] = toPosition +// +// fmt.Printf("Remapping hash %s. From %d, to %d\n", +// hex.EncodeToString(fromHash[:]), origPos, toPosition) +// } +// } +// +// // Zero out all the positions that won't have another hash replacing it. +// for i, fromPosition := range fromPositions { +// //isLeaf := currentRow == 0 +// +// //if f.data.read(child(fromPosition, f.rows)) == empty { +// // isLeaf = true +// //} +// +// //if isLeaf { +// // leftChild := child(toPositions[i], f.rows) +// // rightChild := leftChild | 1 +// +// // f.deleteSubTree(leftChild) +// // f.deleteSubTree(rightChild) +// +// // f.data.write(fromPosition, empty) +// // continue +// //} else { +// // //f.data.write(fromPosition, empty) +// // // Write empty if nothing is replacing where this node was. +// // if !slices.Contains(nextToPositions, fromPosition) { +// // fmt.Println("Writing empty to ", fromPosition) +// // f.data.write(fromPosition, empty) +// // } +// //} +// +// //// Write empty if nothing is replacing where this node was. +// //if !slices.Contains(nextToPositions, fromPosition) { +// // fmt.Println("Writing empty to ", fromPosition) +// // f.data.write(fromPosition, empty) +// //} +// +// if currentRow != 0 { +// // Check if I'm a leaf. If I am, then empty all of the children +// // from my to-position. +// if f.data.read(child(fromPosition, f.rows)) == empty { +// leftChild := child(toPositions[i], f.rows) +// rightChild := leftChild | 1 +// +// fmt.Printf("At row 0, frompos %d, deleting all children of %d and %d\n", +// fromPosition, leftChild, rightChild) +// +// f.deleteRows(toPositions[i], +// uint8(detectRow(fromPosition, f.rows))) +// +// f.data.write(leftChild, empty) +// f.data.write(rightChild, empty) +// } else { +// // Write empty if nothing is replacing where this node was. +// if !slices.Contains(nextToPositions, fromPosition) { +// fmt.Println("Writing empty to ", fromPosition) +// f.data.write(fromPosition, empty) +// } +// } +// } else { +// // If we're moving from row 0, there's only leaves here. Empty +// // all the children of the to-position. +// if detectRow(fromPosition, f.rows) == 0 { +// leftChild := child(toPositions[i], f.rows) +// rightChild := leftChild | 1 +// +// fmt.Printf("At row 0, frompos %d, deleting all children of %d and %d\n", +// fromPosition, leftChild, rightChild) +// fmt.Println(f.ToString()) +// +// f.deleteRows(toPositions[i], +// uint8(detectRow(fromPosition, f.rows))) +// f.data.write(fromPosition, empty) +// } else { +// f.data.write(fromPosition, empty) +// } +// } +// } +// +// fromPositions = nextFromPositions +// toPositions = nextToPositions +// } +//} + +func (f *Forest) childrenExists(from uint64) (bool, bool) { + leftChild := child(from, f.rows) + rightChild := leftChild ^ 1 + + left, right := false, false + + if f.data.read(leftChild) != empty { + left = true + } + + if f.data.read(rightChild) != empty { + right = true + } + + return left, right +} + func updateDirt(hashDirt []uint64, swapRow []arrow, numLeaves uint64, rows uint8) (nextHashDirt []uint64) { var prevHash uint64 hashDirt = dedupeSwapDirt(hashDirt, swapRow) @@ -402,6 +826,96 @@ func (f *Forest) addv2(adds []Leaf) { } } +func (f *Forest) addSwapless(adds []Leaf) { + // Allocate the positionList first. This helps with garbage collection. + positionList := NewPositionList() + defer positionList.Free() + + for _, add := range adds { + // Reset positionList after each loop so we can recycle the list. + positionList.list = positionList.list[:0] + + // Add the hash to the map. + f.positionMap[add.Mini()] = f.numLeaves + + // Grab the roots. + getRootsForwards(f.numLeaves, f.rows, &positionList.list) + + // Current position. + position := f.numLeaves + + //fmt.Println("all root pos: ", positionList.list) + + node := add.Hash + f.data.write(position, node) + add.Hash = empty + + // We can tell where the roots are by looking at the binary representation + // of the numLeaves. Wherever there's a 1, there's a root. + // + // numLeaves of 8 will be '1000' in binary, so there will be one root at + // row 3. numLeaves of 3 will be '11' in binary, so there's two roots. One at + // row 0 and one at row 1. + // + // In this loop below, we're looking for these roots by checking if there's + // a '1'. If there is a '1', we'll hash the root being added with that root + // until we hit a '0'. + for h := uint8(0); (f.numLeaves>>h)&1 == 1; h++ { + rootIdx := len(positionList.list) - int(h+1) + root := f.data.read(positionList.list[rootIdx]) + + // If the root that we're gonna hash with is empty, move the current + // node up to the position of the parent. + // + // Example: + // + // 12 + // |-------\ + // 08 09 + // |---\ |---\ + // 00 01 02 03 -- + // + // When we add 05 to this tree, 04 is empty so we move 05 to 10. + // The resulting tree looks like below. The hash at position 10 + // is not hash(04 || 05) but just the hash of 05. + // + // 12 + // |-------\ + // 08 09 10 + // |---\ |---\ |---\ + // 00 01 02 03 -- -- + if root == empty { + //fmt.Println("root empty at pos", positionList.list[rootIdx]) + + arw := arrow{ + from: position, + to: parent(uint64(positionList.list[rootIdx]), f.rows), + } + + f.movePositionUp2(arw.from, arw.to) + + position = arw.to + continue + } + + // Calculate the hash of the new root. + node = parentHash(root, node) + + // Calculate the position of the new root. + position = parent(position, f.rows) + + // Write the hash at the calculated position. + f.data.write(position, node) + + //fmt.Printf("add: wriiting %s at pos %d\n", + // hex.EncodeToString(node[:]), position) + } + + // Increment as we added a leaf. + f.numLeaves++ + } +} + // Modify changes the forest, adding and deleting leaves and updating internal nodes. // Note that this does not modify in place! All deletes occur simultaneous with // adds, which show up on the right. @@ -452,6 +966,99 @@ func (f *Forest) Modify(adds []Leaf, delsUn []uint64) (*UndoBlock, error) { return ub, err } +// 243:70c +// | +// 230:f76 +// |-------------------------------\ +// 204:300 205:2eb +// |---------------\ |---------------\ +// 154:5b5 155:370 +// |-------\ |-------\ |-------\ |-------\ +// 52:3400 53:3500 + +func (f *Forest) ModifySwapless(adds []Leaf, dels []uint64) (*UndoBlock, error) { + // Check for empty leaves. + for _, a := range adds { + if a.Hash == empty { + return nil, fmt.Errorf("Can't add empty (all 0s) leaf to accumulator") + } + } + + // Remove first since the remap can alter positions of leaves. + err := f.removeSwapless(dels) + if err != nil { + return nil, err + } + + // Remap to expand the forest if needed. + for int64(f.numLeaves)+int64(len(adds)) > int64(1< f.rows+1 || (f.rows > 0 && destRows < f.rows-1) { + // return fmt.Errorf("changing by more than 1 not programmed yet") + //} + + //if verbose { + // fmt.Printf("remap forest %d rows -> %d rows\n", f.rows, destRows) + //} + + //// for row reduction + //if destRows < f.rows { + // return fmt.Errorf("row reduction not implemented") + //} + + for hash, position := range f.positionMap { + row := detectRow(position, destRows-1) + if row != 0 { + beforeOffset := getRowOffset(row, destRows-1) + afterOffset := getRowOffset(row, destRows) + + delta := position - beforeOffset + f.positionMap[hash] = afterOffset + delta + } + } + + return nil +} + // reMap changes the rows in the forest func (f *Forest) reMap(destRows uint8) error { @@ -545,6 +1152,17 @@ func (f *Forest) PosMapSanity() error { return nil } +func (f *Forest) PosMapSanitySwapless() error { + for hash, pos := range f.positionMap { + gotHash := f.data.read(pos).Mini() + if gotHash != hash { + return fmt.Errorf("For position %d, positionMap has %s but forest has %s", + pos, hex.EncodeToString(hash[:]), hex.EncodeToString(gotHash[:])) + } + } + return nil +} + // RestoreForest restores the forest on restart. Needed when resuming after exiting. // miscForestFile is where numLeaves and rows is stored func RestoreForest( @@ -731,6 +1349,13 @@ func (f *Forest) ToString() string { val := f.data.read(uint64(pos)) if val != empty { valstring = fmt.Sprintf("%x", val[:2]) + + if pos >= 100 { + valstring = fmt.Sprintf("%x", val[:2]) + valstring = valstring[:len(valstring)-1] + } else { + valstring = fmt.Sprintf("%x", val[:2]) + } } } if valstring != "" { @@ -765,6 +1390,172 @@ func (f *Forest) ToString() string { } +func getRootPosition(position uint64, numLeaves uint64, forestRows uint8) uint64 { + returnPos := position + + // Since we'll be comparing against the parent of the given position, + // start one row above. + h := detectRow(position, forestRows) + + for ; h <= forestRows; h++ { + //// Check if there's a root here. + //rootPresent := numLeaves&(1< 7 { + s := fmt.Sprintf("can't print subtree with rows %d. roots:\n", subTreeRow) + //roots := f.GetRoots() + //for i, r := range roots { + // s += fmt.Sprintf("\t%d %x\n", i, r.Mini()) + //} + return s + } + + //rootPresent := f.numLeaves&(1<= 0; h-- { + //fmt.Println("h", h) + nextPositions := []uint64{} + + for _, position := range positionsToRead { + var readHashString string + readHash := f.data.read(position) + + if readHash != empty { + if position > 65535 { + readHashString = fmt.Sprintf("%x", readHash[:2]) + readHashString = readHashString[:len(readHashString)-3] + } else if position > 4095 { + readHashString = fmt.Sprintf("%x", readHash[:2]) + readHashString = readHashString[:len(readHashString)-2] + } else if position > 255 { + readHashString = fmt.Sprintf("%x", readHash[:2]) + readHashString = readHashString[:len(readHashString)-1] + } else { + readHashString = fmt.Sprintf("%x", readHash[:2]) + } + //if position >= 10000 { + // readHashString = fmt.Sprintf("%x", readHash[:2]) + // readHashString = readHashString[:len(readHashString)-3] + //} else if position >= 1000 { + // readHashString = fmt.Sprintf("%x", readHash[:2]) + // readHashString = readHashString[:len(readHashString)-2] + //} else if position >= 100 { + // readHashString = fmt.Sprintf("%x", readHash[:2]) + // readHashString = readHashString[:len(readHashString)-1] + //} else { + // readHashString = fmt.Sprintf("%x", readHash[:2]) + //} + } + + leftChild := child(position, f.rows) + rightChild := rightSib(leftChild) + //if f.data.read(leftChild) != empty { + //fmt.Printf("f.rows %d, child of %d is %d\n", + // f.rows, position, leftChild) + nextPositions = append(nextPositions, leftChild) + nextPositions = append(nextPositions, rightChild) + + if readHashString != "" { + //output[h*2] += fmt.Sprintf("%02d:%s ", position, readHashString) + output[h*2] += fmt.Sprintf("%02x:%s ", position, readHashString) + + // Add spaces. + totalSpaceLen := baseLen + for x := 0; x < h; x++ { + totalSpaceLen *= 2 + } + totalSpaceLen -= baseSpaceLen + for j := 1; j < totalSpaceLen; j++ { + output[h*2] += " " + } + + //fmt.Println("total space len at h", totalSpaceLen, h) + } else { + output[h*2] += " " + + // Add spaces. + totalSpaceLen := baseLen + for x := 0; x < h; x++ { + totalSpaceLen *= 2 + } + totalSpaceLen -= baseSpaceLen + for j := 1; j < totalSpaceLen; j++ { + output[h*2] += " " + } + } + + // Add connections. + if h > 0 { + totalLen := baseLen + for x := 0; x < h-1; x++ { + totalLen *= 2 + } + + //fmt.Println("totallen at h", totalLen, h) + + for j := 1; j < totalLen; j++ { + if j == 1 { + output[(h*2)-1] += "|" + } + output[(h*2)-1] += "-" + } + output[(h*2)-1] += "\\" + + for j := 0; j < totalLen-1; j++ { + output[(h*2)-1] += " " + } + } + } + + positionsToRead = nextPositions + } + + var s string + for z := len(output) - 1; z >= 0; z-- { + s += output[z] + "\n" + } + + return s +} + // FindLeaf finds a leave from the positionMap and returns a bool func (f *Forest) FindLeaf(leaf Hash) bool { _, found := f.positionMap[leaf.Mini()] diff --git a/accumulator/forest_test.go b/accumulator/forest_test.go index 28ee04df..17c4626f 100644 --- a/accumulator/forest_test.go +++ b/accumulator/forest_test.go @@ -1,6 +1,7 @@ package accumulator import ( + "encoding/hex" "fmt" "math/rand" "os" @@ -10,6 +11,876 @@ import ( "testing/quick" ) +func TestForestSwaplessSimple(t *testing.T) { + var tests = []struct { + startLeaves []Leaf + dels []uint64 + expected map[uint64]Hash + }{ + // 14 + // |---------------\ + // 12 13 + // |-------\ |-------\ + // 08 09 10 11 + // |---\ |---\ |---\ |---\ + // 00* 01* 02* 03* 04* 05 06* 07* + { + []Leaf{ + {Hash: Hash{1}}, + {Hash: Hash{2}}, + {Hash: Hash{3}}, + {Hash: Hash{4}}, + {Hash: Hash{5}}, + {Hash: Hash{6}}, + {Hash: Hash{7}}, + {Hash: Hash{8}}, + }, + []uint64{0, 1, 2, 3, 4, 6, 7}, + map[uint64]Hash{ + 0: empty, + 1: empty, + 2: empty, + 3: empty, + 4: empty, + 5: empty, + 6: empty, + 7: empty, + 8: empty, + 9: empty, + 10: empty, + 11: empty, + 12: empty, + 13: empty, + 14: {6}, + }, + }, + + // 14 + // |---------------\ + // 12 13 + // |-------\ |-------\ + // 08 09 10 11 + // |---\ |---\ |---\ |---\ + // 00* 01* 02* 03* 04 05 06 07 + { + []Leaf{ + {Hash: Hash{1}}, + {Hash: Hash{2}}, + {Hash: Hash{3}}, + {Hash: Hash{4}}, + {Hash: Hash{5}}, + {Hash: Hash{6}}, + {Hash: Hash{7}}, + {Hash: Hash{8}}, + }, + []uint64{0, 1, 2, 3}, + map[uint64]Hash{ + 0: empty, + 1: empty, + 2: empty, + 3: empty, + 4: empty, + 5: empty, + 6: empty, + 7: empty, + 8: {5}, + 9: {6}, + 10: {7}, + 11: {8}, + 12: parentHash(Hash{5}, Hash{6}), + 13: parentHash(Hash{7}, Hash{8}), + 14: parentHash( + parentHash(Hash{5}, Hash{6}), + parentHash(Hash{7}, Hash{8})), + }, + }, + + // 14 + // |---------------\ + // 12 13 + // |-------\ |-------\ + // 08 09 10 11 + // |---\ |---\ |---\ |---\ + // 00* 01* 02 03* 04 05 06 07 + { + []Leaf{ + {Hash: Hash{1}}, + {Hash: Hash{2}}, + {Hash: Hash{3}}, + {Hash: Hash{4}}, + {Hash: Hash{5}}, + {Hash: Hash{6}}, + {Hash: Hash{7}}, + {Hash: Hash{8}}, + }, + []uint64{0, 1, 3}, + map[uint64]Hash{ + 0: empty, + 1: empty, + 2: empty, + 3: empty, + 4: {5}, + 5: {6}, + 6: {7}, + 7: {8}, + 8: empty, + 9: empty, + 10: parentHash(Hash{5}, Hash{6}), + 11: parentHash(Hash{7}, Hash{8}), + 12: {3}, + 13: parentHash( + parentHash(Hash{5}, Hash{6}), + parentHash(Hash{7}, Hash{8})), + 14: parentHash( + Hash{3}, + parentHash(parentHash(Hash{5}, Hash{6}), + parentHash(Hash{7}, Hash{8})), + ), + }, + }, + + { + []Leaf{ + {Hash: Hash{1}}, + {Hash: Hash{2}}, + {Hash: Hash{3}}, + {Hash: Hash{4}}, + {Hash: Hash{5}}, + {Hash: Hash{6}}, + {Hash: Hash{7}}, + {Hash: Hash{8}}, + {Hash: Hash{9}}, + }, + []uint64{0, 1, 2}, + map[uint64]Hash{ + 0: empty, + 1: empty, + 2: empty, + 3: empty, + 4: {5}, + 5: {6}, + 6: {7}, + 7: {8}, + 8: {9}, + 16: empty, + 17: empty, + 18: parentHash(Hash{5}, Hash{6}), + 19: parentHash(Hash{7}, Hash{8}), + 25: parentHash( + parentHash(Hash{5}, Hash{6}), + parentHash(Hash{7}, Hash{8})), + 28: parentHash( + Hash{4}, + parentHash( + parentHash(Hash{5}, Hash{6}), + parentHash(Hash{7}, Hash{8})), + ), + }, + }, + } + + for i, test := range tests { + //if i != 1 { + // continue + //} + + f := NewForest(RamForest, nil, "", 0) + + _, err := f.Modify(test.startLeaves, nil) + if err != nil { + t.Fatal(err) + } + + //fmt.Println(f.ToString()) + + err = f.removeSwapless(test.dels) + if err != nil { + t.Fatal(err) + } + + //fmt.Println(f.ToString()) + + for pos, expectedHash := range test.expected { + hash := f.data.read(pos) + + if hash != expectedHash { + t.Errorf("TestForestSwaplessSimple Fail: test %d failed "+ + "for position %d, expected %s got %s", + i, pos, hex.EncodeToString(expectedHash[:]), + hex.EncodeToString(hash[:])) + } + } + + err = f.checkForestHashes() + if err != nil { + t.Fatalf("TestForestSwaplessSimple Fail: test %d failed. err: %v", i, err) + } + } +} + +type modification struct { + add []Leaf + del []uint64 + + expected map[MiniHash]uint64 + expectedEmpty []uint64 + expectedRoots []Hash +} + +func stringToHash(str string) Hash { + hash, err := hex.DecodeString(str) + if err != nil { + // Ok to panic since this function is only used for testing. + // If it panics, it means that the hardcoded string is wrong. + panic(err) + } + + return *(*Hash)(hash) +} + +func TestSwaplessModify(t *testing.T) { + var tests = []struct { + name string + modifies []modification + }{ + { + "Only adds", + []modification{ + // Only adds block 1 + { + add: []Leaf{ + {Hash: Hash{1}}, + {Hash: Hash{2}}, + {Hash: Hash{3}}, + {Hash: Hash{4}}, + {Hash: Hash{5}}, + {Hash: Hash{6}}, + {Hash: Hash{7}}, + {Hash: Hash{8}}, + {Hash: Hash{9}}, + }, + del: nil, + expected: map[MiniHash]uint64{ + Hash{1}.Mini(): 0, + Hash{2}.Mini(): 1, + Hash{3}.Mini(): 2, + Hash{4}.Mini(): 3, + Hash{5}.Mini(): 4, + Hash{6}.Mini(): 5, + Hash{7}.Mini(): 6, + Hash{8}.Mini(): 7, + Hash{9}.Mini(): 8, + }, + expectedEmpty: nil, + expectedRoots: []Hash{ + stringToHash("ee4c7313527e3ee54ee97793cd35e8df" + + "4f7fcf3b0012bec7e7cfdb9ace0cd3fd"), + stringToHash("09000000000000000000000000000000" + + "00000000000000000000000000000000"), + }, + }, + + // Only adds block 2 + { + add: []Leaf{ + {Hash: Hash{10}}, + {Hash: Hash{11}}, + {Hash: Hash{12}}, + {Hash: Hash{13}}, + {Hash: Hash{14}}, + {Hash: Hash{15}}, + {Hash: Hash{16}}, + {Hash: Hash{17}}, + {Hash: Hash{18}}, + {Hash: Hash{19}}, + {Hash: Hash{20}}, + }, + del: nil, + expected: map[MiniHash]uint64{ + Hash{1}.Mini(): 0, + Hash{2}.Mini(): 1, + Hash{3}.Mini(): 2, + Hash{4}.Mini(): 3, + Hash{5}.Mini(): 4, + Hash{6}.Mini(): 5, + Hash{7}.Mini(): 6, + Hash{8}.Mini(): 7, + Hash{9}.Mini(): 8, + Hash{10}.Mini(): 9, + Hash{11}.Mini(): 10, + Hash{12}.Mini(): 11, + Hash{13}.Mini(): 12, + Hash{14}.Mini(): 13, + Hash{15}.Mini(): 14, + Hash{16}.Mini(): 15, + Hash{17}.Mini(): 16, + Hash{18}.Mini(): 17, + Hash{19}.Mini(): 18, + Hash{20}.Mini(): 19, + }, + expectedEmpty: nil, + expectedRoots: []Hash{ + stringToHash("2c1ecb81b164c6dff4a6d89c19fcddf1" + + "5356f37e5a1f5e82f505c5b9ef856e25"), + stringToHash("8b34c7baf39f2216fd352a86616adaa5" + + "1f394103b0524d9dc2430045de50d116"), + }, + }, + }, + }, + + { + "Delete once", + []modification{ + // Delete once block 1 + { + add: []Leaf{ + {Hash: Hash{1}}, + {Hash: Hash{2}}, + {Hash: Hash{3}}, + {Hash: Hash{4}}, + {Hash: Hash{5}}, + {Hash: Hash{6}}, + {Hash: Hash{7}}, + {Hash: Hash{8}}, + {Hash: Hash{9}}, + {Hash: Hash{10}}, + {Hash: Hash{11}}, + {Hash: Hash{12}}, + {Hash: Hash{13}}, + {Hash: Hash{14}}, + {Hash: Hash{15}}, + {Hash: Hash{16}}, + }, + del: nil, + expected: map[MiniHash]uint64{ + Hash{1}.Mini(): 0, + Hash{2}.Mini(): 1, + Hash{3}.Mini(): 2, + Hash{4}.Mini(): 3, + Hash{5}.Mini(): 4, + Hash{6}.Mini(): 5, + Hash{7}.Mini(): 6, + Hash{8}.Mini(): 7, + Hash{9}.Mini(): 8, + Hash{10}.Mini(): 9, + Hash{11}.Mini(): 10, + Hash{12}.Mini(): 11, + Hash{13}.Mini(): 12, + Hash{14}.Mini(): 13, + Hash{15}.Mini(): 14, + Hash{16}.Mini(): 15, + }, + expectedEmpty: nil, + expectedRoots: []Hash{ + stringToHash("2c1ecb81b164c6dff4a6d89c19fcddf1" + + "5356f37e5a1f5e82f505c5b9ef856e25"), + }, + }, + + // Delete once block 2 + { + add: nil, + del: []uint64{0, 2, 3, 4, 5, 6, 7}, + expected: map[MiniHash]uint64{ + Hash{2}.Mini(): 28, + Hash{9}.Mini(): 8, + Hash{10}.Mini(): 9, + Hash{11}.Mini(): 10, + Hash{12}.Mini(): 11, + Hash{13}.Mini(): 12, + Hash{14}.Mini(): 13, + Hash{15}.Mini(): 14, + Hash{16}.Mini(): 15, + }, + expectedEmpty: []uint64{0, 1, 2, 3, 4, 5, 6, 7, 16, 17, + 18, 19, 24, 25}, + expectedRoots: []Hash{ + stringToHash("e018100dadcc58df80b4e955fffcc3fd" + + "a1b3c9831b86bb6b7a80f824c046c360"), + }, + }, + }, + }, + + { + "edge case", + []modification{ + // 1st block. + { + add: []Leaf{ + {Hash: Hash{1}}, + {Hash: Hash{2}}, + {Hash: Hash{3}}, + {Hash: Hash{4}}, + {Hash: Hash{5}}, + {Hash: Hash{6}}, + {Hash: Hash{7}}, + {Hash: Hash{8}}, + }, + del: nil, + expected: map[MiniHash]uint64{ + Hash{1}.Mini(): 0, + Hash{2}.Mini(): 1, + Hash{3}.Mini(): 2, + Hash{4}.Mini(): 3, + Hash{5}.Mini(): 4, + Hash{6}.Mini(): 5, + Hash{7}.Mini(): 6, + Hash{8}.Mini(): 7, + }, + expectedEmpty: nil, + expectedRoots: []Hash{ + stringToHash("ee4c7313527e3ee54ee97793cd35e8df" + + "4f7fcf3b0012bec7e7cfdb9ace0cd3fd"), + }, + }, + + { + add: nil, + del: []uint64{0, 1, 2, 4}, + expectedEmpty: []uint64{0, 1, 2, 3, 4, 5, 8, 9}, + expectedRoots: []Hash{ + stringToHash("d10790b861666df44c6dfb52aa700ae3" + + "c469ebf4e06de7e2c55da7dae6fd93f0"), + }, + }, + + { + add: nil, + del: []uint64{6, 12}, + expectedEmpty: []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + expectedRoots: []Hash{ + stringToHash("17778a33dcae949b32cec24c8d06971f" + + "b4802f0bf463ec279e6efa323a09fbd9"), + }, + }, + }, + }, + + { + "edge case 2", + []modification{ + // 1st block. + { + add: []Leaf{ + {Hash: Hash{1}}, + {Hash: Hash{2}}, + {Hash: Hash{3}}, + {Hash: Hash{4}}, + {Hash: Hash{5}}, + {Hash: Hash{6}}, + {Hash: Hash{7}}, + {Hash: Hash{8}}, + }, + del: nil, + expected: map[MiniHash]uint64{ + Hash{1}.Mini(): 0, + Hash{2}.Mini(): 1, + Hash{3}.Mini(): 2, + Hash{4}.Mini(): 3, + Hash{5}.Mini(): 4, + Hash{6}.Mini(): 5, + Hash{7}.Mini(): 6, + Hash{8}.Mini(): 7, + }, + expectedEmpty: nil, + expectedRoots: []Hash{ + stringToHash("ee4c7313527e3ee54ee97793cd35e8df" + + "4f7fcf3b0012bec7e7cfdb9ace0cd3fd"), + }, + }, + + { + add: nil, + del: []uint64{0, 1, 2, 4}, + expectedEmpty: []uint64{0, 1, 2, 3, 4, 5, 8, 9}, + expectedRoots: []Hash{ + stringToHash("d10790b861666df44c6dfb52aa700ae3" + + "c469ebf4e06de7e2c55da7dae6fd93f0"), + }, + }, + + { + add: nil, + del: []uint64{6, 12}, + expectedEmpty: []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11}, + expectedRoots: []Hash{ + stringToHash("17778a33dcae949b32cec24c8d06971f" + + "b4802f0bf463ec279e6efa323a09fbd9"), + }, + }, + }, + }, + + { + "test empty deletions", + []modification{ + // 1st block. + { + add: []Leaf{ + {Hash: Hash{1}}, + {Hash: Hash{2}}, + {Hash: Hash{3}}, + {Hash: Hash{4}}, + {Hash: Hash{5}}, + {Hash: Hash{6}}, + {Hash: Hash{7}}, + {Hash: Hash{8}}, + {Hash: Hash{9}}, + {Hash: Hash{10}}, + {Hash: Hash{11}}, + {Hash: Hash{12}}, + {Hash: Hash{13}}, + {Hash: Hash{14}}, + {Hash: Hash{15}}, + {Hash: Hash{16}}, + }, + del: nil, + expected: map[MiniHash]uint64{ + Hash{1}.Mini(): 0, + Hash{2}.Mini(): 1, + Hash{3}.Mini(): 2, + Hash{4}.Mini(): 3, + Hash{5}.Mini(): 4, + Hash{6}.Mini(): 5, + Hash{7}.Mini(): 6, + Hash{8}.Mini(): 7, + Hash{9}.Mini(): 8, + Hash{10}.Mini(): 9, + Hash{11}.Mini(): 10, + Hash{12}.Mini(): 11, + Hash{13}.Mini(): 12, + Hash{14}.Mini(): 13, + Hash{15}.Mini(): 14, + Hash{16}.Mini(): 15, + }, + expectedEmpty: nil, + expectedRoots: []Hash{ + stringToHash("2c1ecb81b164c6dff4a6d89c19fcddf1" + + "5356f37e5a1f5e82f505c5b9ef856e25"), + }, + }, + + { + add: nil, + del: []uint64{1, 2, 3, 4, 5, 6, 7, 8, 14}, + expectedEmpty: []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 14, 15, + 16, 17, 18, 19, 24, 25}, + expectedRoots: []Hash{ + stringToHash("4010aedd4e90764f805667487be4b48c" + + "9ad3f81a11a827455cf61ddc55f12f05"), + }, + }, + + { + add: nil, + del: []uint64{11, 12, 13, 28}, + expectedEmpty: []uint64{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, + 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 26, 27}, + expectedRoots: []Hash{ + stringToHash("4b963007f029831e0759ed5bf0646b4d" + + "77acc60e50aaec212b6add8e20bb0776"), + }, + }, + }, + }, + + { + "4 blocks. Adds and dels", + []modification{ + // 1st block. + { + add: []Leaf{ + {Hash: Hash{1}}, + {Hash: Hash{2}}, + {Hash: Hash{3}}, + {Hash: Hash{4}}, + {Hash: Hash{5}}, + {Hash: Hash{6}}, + {Hash: Hash{7}}, + {Hash: Hash{8}}, + {Hash: Hash{9}}, + }, + del: nil, + expected: map[MiniHash]uint64{ + Hash{1}.Mini(): 0, + Hash{2}.Mini(): 1, + Hash{3}.Mini(): 2, + Hash{4}.Mini(): 3, + Hash{5}.Mini(): 4, + Hash{6}.Mini(): 5, + Hash{7}.Mini(): 6, + Hash{8}.Mini(): 7, + Hash{9}.Mini(): 8, + }, + expectedEmpty: nil, + expectedRoots: []Hash{ + stringToHash("ee4c7313527e3ee54ee97793cd35e8df" + + "4f7fcf3b0012bec7e7cfdb9ace0cd3fd"), + stringToHash("09000000000000000000000000000000" + + "00000000000000000000000000000000"), + }, + }, + + // 2nd block. + { + add: []Leaf{ + {Hash: Hash{10}}, + {Hash: Hash{11}}, + {Hash: Hash{12}}, + }, + expected: map[MiniHash]uint64{ + Hash{3}.Mini(): 16, + Hash{4}.Mini(): 17, + Hash{6}.Mini(): 18, + Hash{7}.Mini(): 6, + Hash{8}.Mini(): 7, + Hash{9}.Mini(): 8, + Hash{10}.Mini(): 9, + Hash{11}.Mini(): 10, + Hash{12}.Mini(): 11, + }, + del: []uint64{0, 1, 4}, + expectedEmpty: []uint64{0, 1, 2, 3, 4, 5}, + expectedRoots: []Hash{ + stringToHash("536cd74e712f63e10ecec98f084024ed" + + "6ab16db91f24847206e08c9c2af7f339"), + stringToHash("f7a8b895af8b0261718021e0cfcabd9e" + + "0cf17bfd6c8e2755ee6461c3c85b986e"), + }, + }, + + // 3rd block. + { + add: []Leaf{ + {Hash: Hash{13}}, + {Hash: Hash{14}}, + {Hash: Hash{15}}, + {Hash: Hash{16}}, + {Hash: Hash{17}}, + }, + expected: map[MiniHash]uint64{ + Hash{4}.Mini(): 56, + Hash{9}.Mini(): 8, + Hash{10}.Mini(): 9, + Hash{11}.Mini(): 10, + Hash{12}.Mini(): 11, + Hash{13}.Mini(): 12, + Hash{14}.Mini(): 13, + Hash{15}.Mini(): 14, + Hash{16}.Mini(): 15, + Hash{17}.Mini(): 16, + }, + del: []uint64{6, 7, 16, 18}, + expectedEmpty: []uint64{0, 1, 2, 3, 4, 5, 6, 7, + 32, 33, 34, 35, 48, 49}, + expectedRoots: []Hash{ + stringToHash("ab567775cdbd9373c465cb68e79ac367" + + "b0cbf385c19aacd2ae0787bad39b0957"), + stringToHash("11000000000000000000000000000000" + + "00000000000000000000000000000000"), + }, + }, + + // 4th block. + { + add: []Leaf{ + {Hash: Hash{18}}, + {Hash: Hash{19}}, + }, + expected: map[MiniHash]uint64{ + Hash{4}.Mini(): 56, + Hash{9}.Mini(): 36, + Hash{10}.Mini(): 37, + Hash{11}.Mini(): 38, + Hash{12}.Mini(): 39, + Hash{17}.Mini(): 16, + Hash{18}.Mini(): 17, + Hash{19}.Mini(): 18, + }, + del: []uint64{12, 13, 14, 15}, + expectedEmpty: []uint64{0, 1, 2, 3, 4, 5, 6, 7, + 12, 13, 14, 15, 32, 33, 34, 35, 48, 49}, + expectedRoots: []Hash{ + stringToHash("c0fce95da61de813a7f043e23ac13dd0" + + "7c47f52b2565315d5a43d89d9bef0904"), + stringToHash("a3cdd313912ce74d5335dbddee5b8682" + + "567ad6ac6ca4d33ac8dc20ea864989a3"), + stringToHash("13000000000000000000000000000000" + + "00000000000000000000000000000000"), + }, + }, + }, + }, + } + + for _, test := range tests { + //if i != 2 { + // continue + //} + //if test.name != "test empty deletions" { + // continue + //} + + forest := NewForest(RamForest, nil, "", 0) + + for blockHeight, modify := range test.modifies { + // Perform the modify. + _, err := forest.ModifySwapless(modify.add, modify.del) + if err != nil { + t.Fatalf("TestSwaplessModify test \"%s\" fail at block %d. Modify err:%v", + test.name, blockHeight, err) + } + + //fmt.Println(forest.ToString()) + //for _, root := range forest.GetRoots() { + // fmt.Println(hex.EncodeToString(root[:])) + //} + //fmt.Println(forest.positionMap) + + // Check that all the leaves in the map are there and that the + // values are the same. + for miniHash, position := range modify.expected { + gotPosition, found := forest.positionMap[miniHash] + if !found { + t.Fatalf("TestSwaplessModify test \"%s\" fail at block %d. "+ + "Couldn't find expected minihash of %s", + test.name, blockHeight, hex.EncodeToString(miniHash[:])) + } + + if gotPosition != position { + t.Fatalf("TestSwaplessModify test \"%s\" fail at block %d. "+ + "For minihash %s, expected %d but got %d", + test.name, blockHeight, hex.EncodeToString(miniHash[:]), + position, gotPosition) + } + } + + // Check that the length of the roots match up. + roots := forest.GetRoots() + if len(roots) != len(modify.expectedRoots) { + t.Fatalf("TestSwaplessModify test \"%s\" fail at block %d. "+ + "Expected %d roots but got %d", test.name, blockHeight, + len(modify.expectedRoots), len(roots)) + } + + // Check that the roots match up. + for i, expectedRoot := range modify.expectedRoots { + if expectedRoot != roots[i] { + t.Fatalf("TestSwaplessModify test \"%s\" fail at block %d. "+ + "Root %d mismatch. Expected %s, got %s", test.name, + blockHeight, i, + hex.EncodeToString(expectedRoot[:]), + hex.EncodeToString(roots[i][:])) + } + } + + // Check that the empty positions are indeed empty. + for _, expectedEmpty := range modify.expectedEmpty { + readHash := forest.data.read(expectedEmpty) + + if readHash != empty { + t.Fatalf("TestSwaplessModify test \"%s\" fail at block %d. "+ + "Position %d was expected to be empty but read %s", + test.name, blockHeight, expectedEmpty, + hex.EncodeToString(readHash[:])) + } + } + err = forest.checkForestHashes() + if err != nil { + t.Fatalf("TestSwaplessModify Fail: test %s failed. err: %v", test.name, err) + } + } + } +} + +func TestSwapLessAdd(t *testing.T) { + f := NewForest(RamForest, nil, "", 0) + leaves := []Leaf{ + {Hash: Hash{1}}, + {Hash: Hash{2}}, + {Hash: Hash{3}}, + {Hash: Hash{4}}, + {Hash: Hash{5}}, + {Hash: Hash{6}}, + {Hash: Hash{7}}, + {Hash: Hash{8}}, + {Hash: Hash{9}}, + } + + // remap to expand the forest if needed + numdels, numadds := 0, len(leaves) + delta := int64(numadds - numdels) // watch 32/64 bit + for int64(f.numLeaves)+delta > int64(1<>h)&1 == 1 { + // pos := rootPosition(f.numLeaves, h, f.rows) + // fmt.Println() + // fmt.Printf("pos %d, numLeaves %d, f.rows %d, root at row %d\n", + // pos, f.numLeaves, f.rows, h) + // fmt.Println(f.SubTreeToString(pos)) + // fmt.Println() + // } + //} + adds, _, delHashes := sc.NextBlock(numAdds) + + bp, err := f.ProveBatch(delHashes) + if err != nil { + t.Fatalf("TestSwapLessAddDel fail at block %d. Error: %v", b, err) + } + + for _, del := range bp.Targets { + //_, found := allDelPos[del] + //if found { + // t.Fatalf("TestSwapLessAddDel fail. Re-deleting position %d", del) + //} + //allDelPos[del] = nil + + readHash := f.data.read(del) + if readHash == empty { + for h := uint8(0); h < f.rows; h++ { + if (f.numLeaves>>h)&1 == 1 { + pos := rootPosition(f.numLeaves, h, f.rows) + fmt.Println() + fmt.Printf("pos %d, numLeaves %d, f.rows %d, root at row %d\n", + pos, f.numLeaves, f.rows, h) + fmt.Println(f.SubTreeToString(pos)) + fmt.Println() + } + } + t.Fatalf("Trying to delete empty at pos %d, block %d", del, b) + } + } + + _, err = f.ModifySwapless(adds, bp.Targets) + if err != nil { + t.Fatalf("TestSwapLessAddDel fail at block %d. Error: %v", b, err) + } + + err = f.PosMapSanitySwapless() + if err != nil { + for h := uint8(0); h < f.rows; h++ { + if (f.numLeaves>>h)&1 == 1 { + pos := rootPosition(f.numLeaves, h, f.rows) + fmt.Println() + fmt.Println("bp targets", bp.Targets) + fmt.Printf("pos %d, numLeaves %d, f.rows %d, root at row %d\n", + pos, f.numLeaves, f.rows, h) + fmt.Println(f.SubTreeToString(pos)) + fmt.Println() + } + } + t.Fatalf("TestSwapLessAddDel fail at block %d. Error: %v", b, err) + } + + roots := f.GetRoots() + for i, root := range roots { + if root == empty { + fmt.Println(f.ToString()) + for h := uint8(0); h < f.rows; h++ { + if (f.numLeaves>>h)&1 == 1 { + pos := rootPosition(f.numLeaves, h, f.rows) + fmt.Println() + fmt.Printf("pos %d, numLeaves %d, f.rows %d, root at row %d\n", + pos, f.numLeaves, f.rows, h) + fmt.Println(f.SubTreeToString(pos)) + fmt.Println() + } + } + fmt.Printf("root %d is empty\n", i) + //t.Fatalf("TestSwapLessAddDel fail: root %d is empty", i) + } + } + + ////fmt.Println("ROOTS") + ////for i, root := range roots { + //// fmt.Printf("root %d: %s\n", i, hex.EncodeToString(root[:])) + ////} + ////fmt.Println("ROOTS END") + + //for h := uint8(0); h < f.rows; h++ { + // if (f.numLeaves>>h)&1 == 1 { + // pos := rootPosition(f.numLeaves, h, f.rows) + // fmt.Println() + // fmt.Printf("pos %d, numLeaves %d, f.rows %d, root at row %d\n", + // pos, f.numLeaves, f.rows, h) + // fmt.Println(f.SubTreeToString(pos)) + // fmt.Println() + // } + //} + + ////fmt.Printf("nl %d %s", f.numLeaves, f.ToString()) + + for _, add := range adds { + allLeaves[add.Hash] = nil + } + + for _, del := range delHashes { + delete(allLeaves, del) + } + + for hash := range allLeaves { + pos, found := f.positionMap[hash.Mini()] + if !found { + err := fmt.Errorf("Hash %s not present in the position map", + hex.EncodeToString(hash[:])) + t.Fatalf("TestSwapLessAddDel fail at block %d. Error: %v", b, err) + } + + gotHash := f.data.read(pos) + if gotHash != hash { + err := fmt.Errorf("At position %d, expected %s, got %s", + pos, hex.EncodeToString(hash[:]), hex.EncodeToString(gotHash[:])) + t.Fatalf("TestSwapLessAddDel fail at block %d. Error: %v", b, err) + } + } + + allProof += len(bp.Proof) + + for hash, pos := range f.positionMap { + readHash := f.data.read(pos) + if hash != readHash.Mini() { + err := fmt.Errorf("At position %d, position map had %s, read %s", + pos, hex.EncodeToString(hash[:]), hex.EncodeToString(readHash[:])) + t.Fatalf("TestSwapLessAddDel fail at block %d. Error: %v", b, err) + } + + err := f.checkPosBelowAreEmpty(pos) + if err != nil { + t.Fatalf("TestSwapLessAddDel fail at block %d. Error: %v", b, err) + } + } + + err = f.checkForestHashes() + if err != nil { + t.Fatalf("TestSwapLessAddDel Fail: err: %v", err) + } + } + + //fmt.Println("swapless allproof", allProof) +} + +func (f *Forest) checkPosBelowAreEmpty(origPosition uint64) error { + fromRow := int(detectRow(origPosition, f.rows)) + + positions := []uint64{origPosition} + + for currentRow := fromRow; currentRow >= 0; currentRow-- { + nextPositions := []uint64{} + + for _, position := range positions { + // Check children and add to the list of dels. + leftChild := child(position, f.rows) + rightChild := rightSib(leftChild) + + nextPositions = append(nextPositions, leftChild, rightChild) + + leftChildHash := f.data.read(leftChild) + if currentRow != 0 && leftChildHash != empty { + err := fmt.Errorf("Descendent %d of %d position not empty, read %s", + origPosition, leftChild, hex.EncodeToString(leftChildHash[:])) + return err + } + + rightChildHash := f.data.read(rightChild) + if currentRow != 0 && rightChildHash != empty { + err := fmt.Errorf("Descendent %d of %d position not empty, read %s", + origPosition, rightChild, hex.EncodeToString(rightChildHash[:])) + return err + } + } + + positions = nextPositions + } + + return nil +} + +func TestGetRootPos(t *testing.T) { + fmt.Println(getRootPosition(0, 15, 4)) + fmt.Println(getRootPosition(0, 31, 5)) +} + +func TestSubTreeToString(t *testing.T) { + f := NewForest(RamForest, nil, "", 0) + + leaves := []Leaf{ + {Hash: Hash{1}}, + {Hash: Hash{2}}, + {Hash: Hash{3}}, + {Hash: Hash{4}}, + {Hash: Hash{5}}, + {Hash: Hash{6}}, + {Hash: Hash{7}}, + {Hash: Hash{8}}, + {Hash: Hash{9}}, + {Hash: Hash{9}}, + {Hash: Hash{10}}, + {Hash: Hash{11}}, + {Hash: Hash{12}}, + {Hash: Hash{13}}, + {Hash: Hash{14}}, + {Hash: Hash{15}}, + {Hash: Hash{16}}, + {Hash: Hash{17}}, + {Hash: Hash{18}}, + {Hash: Hash{19}}, + {Hash: Hash{20}}, + {Hash: Hash{21}}, + } + + _, err := f.ModifySwapless(leaves, nil) + if err != nil { + t.Fatal(err) + } + + fmt.Println(f.ToString()) + + for h := uint8(0); h < f.rows; h++ { + if (f.numLeaves>>h)&1 == 1 { + pos := rootPosition(f.numLeaves, h, f.rows) + fmt.Println() + fmt.Println("pos ", pos) + fmt.Println(f.SubTreeToString(pos)) + fmt.Println() + } + } + + _, err = f.ModifySwapless(nil, []uint64{2, 3, 4, 5, 6, 7}) + if err != nil { + t.Fatal(err) + } + + for h := uint8(0); h < f.rows; h++ { + if (f.numLeaves>>h)&1 == 1 { + pos := rootPosition(f.numLeaves, h, f.rows) + fmt.Println() + fmt.Println("pos ", pos) + fmt.Println(f.SubTreeToString(pos)) + fmt.Println() + } } } @@ -387,3 +1588,78 @@ func TestSmallRandomForests(t *testing.T) { } } } + +func (f *Forest) checkForestHashes() error { + roots := NewPositionList() + defer roots.Free() + + getRootsForwards(f.numLeaves, f.rows, &roots.list) + + for _, root := range roots.list { + nodesToCheck := []uint64{root} + topRow := detectRow(root, f.rows) + + for row := int(topRow); row >= 0; row-- { + nextNodes := []uint64{} + + for _, node := range nodesToCheck { + leftChild := child(node, f.rows) + rightChild := rightSib(leftChild) + + leftHash := f.data.read(leftChild) + rightHash := f.data.read(rightChild) + + if isRootPosition(node, f.numLeaves, f.rows) { + if row == 0 || leftHash == empty && rightHash == empty { + continue + } + } + + //if leftHash == empty || rightHash == empty { + //return fmt.Errorf("One of the children of %d is empty, left %s, right %s", + // node, hex.EncodeToString(leftHash[:]), hex.EncodeToString(rightHash[:])) + //} + + calculated := parentHash(leftHash, rightHash) + read := f.data.read(node) + + if calculated != read { + return fmt.Errorf("Position %d, calculated %s but read %s", node, + hex.EncodeToString(calculated[:]), hex.EncodeToString(read[:])) + } + + if row != 0 { + // Children of the leftChild. + //if f.data.read(child(leftChild, f.rows)) != empty { + //if !f.isLeaf(child(leftChild, f.rows), uint8(row)) { + if !f.isLeaf(leftChild, uint8(row-1)) { + nextNodes = append(nextNodes, leftChild) + //nextNodes = append(nextNodes, child(leftChild, f.rows)) + //nextNodes = append(nextNodes, rightSib(child(leftChild, f.rows))) + } + //if !f.isLeaf(rightSib(child(leftChild, f.rows)), uint8(row)) { + // nextNodes = append(nextNodes, rightSib(child(leftChild, f.rows))) + //} + + // Children of the rightChild. + //if f.data.read(child(rightChild, f.rows)) != empty { + //if !f.isLeaf(child(rightChild, f.rows), f.rows) { + if !f.isLeaf(rightChild, uint8(row-1)) { + nextNodes = append(nextNodes, rightChild) + //nextNodes = append(nextNodes, child(rightChild, f.rows)) + //nextNodes = append(nextNodes, rightSib(child(rightChild, f.rows))) + } + //if f.data.read(rightSib(child(rightChild, f.rows))) != empty { + //if !f.isLeaf(rightSib(child(rightChild, f.rows)), f.rows) { + // nextNodes = append(nextNodes, rightSib(child(rightChild, f.rows))) + //} + } + } + + //fmt.Println("llop", nextNodes, nodesToCheck) + nodesToCheck = nextNodes + } + } + + return nil +} diff --git a/accumulator/forestproofs.go b/accumulator/forestproofs.go index b6157344..95732489 100644 --- a/accumulator/forestproofs.go +++ b/accumulator/forestproofs.go @@ -160,15 +160,15 @@ func (f *Forest) ProveBatch(hs []Hash) (BatchProof, error) { return bp, fmt.Errorf("hash %x not found", wanted) } - // should never happen - if pos > f.numLeaves { - for m, p := range f.positionMap { - fmt.Printf("%x @%d\t", m[:4], p) - } - return bp, fmt.Errorf( - "ProveBatch: got leaf position %d but only %d leaves exist", - pos, f.numLeaves) - } + //// should never happen + //if pos > f.numLeaves { + // for m, p := range f.positionMap { + // fmt.Printf("%x @%d\t", m[:4], p) + // } + // return bp, fmt.Errorf( + // "ProveBatch: got leaf position %d but only %d leaves exist", + // pos, f.numLeaves) + //} bp.Targets[i] = pos } // targets need to be sorted because the proof hashes are sorted @@ -188,6 +188,11 @@ func (f *Forest) ProveBatch(hs []Hash) (BatchProof, error) { bp.Proof = make([]Hash, len(proofPositions.list)) for i, proofPos := range proofPositions.list { bp.Proof[i] = f.data.read(proofPos) + + if bp.Proof[i] == empty { + return bp, fmt.Errorf("Read empty for position %d", + proofPos) + } } if verbose { diff --git a/accumulator/pollard.go b/accumulator/pollard.go index 8370f39c..fab470a3 100644 --- a/accumulator/pollard.go +++ b/accumulator/pollard.go @@ -1,6 +1,7 @@ package accumulator import ( + "encoding/hex" "fmt" ) @@ -20,6 +21,8 @@ type Pollard struct { // number of leaves in the pollard forest numLeaves uint64 + numDels uint64 + // roots are the slice of the tree roots ordered in big to small // (right to left of the forest). // @@ -45,6 +48,12 @@ type Pollard struct { // It is only used for fullPollard. positionMap map[MiniHash]uint64 + // nodeMap maps a hash to a polNode. Useful for during proving. + NodeMap map[MiniHash]*polNode + + // full indicates that the pollard will store every leaf. + full bool + // Below are for keeping statistics. // hashesEver is all the hashes that have ever been performed. // rememberEver is all the nodes that have ever been cached. @@ -169,16 +178,17 @@ func (p *Pollard) addOne(add Hash, remember bool) error { n.remember = true } - // if add is forgetable, forget all the new nodes made - // loop until we find a zero; destroy roots until you make one + // if add is forgetable, forget all the new nodes made loop until we find a zero; destroy roots until you make one var h uint8 for ; (p.numLeaves>>h)&1 == 1; h++ { // grab, pop, swap, hash, new - leftRoot := p.roots[len(p.roots)-1] // grab - p.roots = p.roots[:len(p.roots)-1] // pop - leftRoot.niece, n.niece = n.niece, leftRoot.niece // swap - nHash := parentHash(leftRoot.data, n.data) // hash - n = &polNode{data: nHash, niece: [2]*polNode{leftRoot, n}} // new + leftRoot := p.roots[len(p.roots)-1] // grab + p.roots = p.roots[:len(p.roots)-1] // pop + leftRoot.leftNiece, leftRoot.rightNiece, n.leftNiece, n.rightNiece = + n.leftNiece, n.rightNiece, leftRoot.leftNiece, leftRoot.rightNiece // swap + + nHash := parentHash(leftRoot.data, n.data) // hash + n = &polNode{data: nHash, leftNiece: leftRoot, rightNiece: n} // new n.remember = remember p.hashesEver++ @@ -194,6 +204,327 @@ func (p *Pollard) addOne(add Hash, remember bool) error { return nil } +func (p *Pollard) move(del uint64) (*polNode, error) { + from, to := sibling(del), parent(del, treeRows(p.numLeaves)) + fromNode, fromNodeSib, _, err := p.readPos(from) + if err != nil { + return nil, err + } + + toNode, toSib, _, err := p.readPos(to) + if err != nil { + return nil, err + } + + toNode.data = fromNode.data + toSib.leftNiece, toSib.rightNiece = fromNodeSib.leftNiece, fromNodeSib.rightNiece + + updateAunt(toSib) + delNode(fromNode) + + delHash := fromNodeSib.data + if p.NodeMap != nil { + p.NodeMap[fromNode.data.Mini()] = toNode + delete(p.NodeMap, delHash.Mini()) + } + + var dirty *polNode + if isRootPosition(to, p.numLeaves, treeRows(p.numLeaves)) { + toNode.aunt = nil + } else { + parent, parentSib, _, err := p.readPos(parent(to, treeRows(p.numLeaves))) + if err != nil { + return nil, err + } + dirty = parent + + toNode.aunt = parentSib + } + + return dirty, nil +} + +//func (p *Pollard) moveAndHash(from, to uint64) error { +// //fmt.Printf("Moving from %d, to %d\n", from, to) +// fromNode, fromNodeSib, _, err := p.readPos(from) +// if err != nil { +// return err +// } +// +// toNode, toSib, _, err := p.readPos(to) +// if err != nil { +// return err +// } +// +// toNode.data = fromNode.data +// toSib.niece = fromNodeSib.niece +// +// updateAunt(toSib) +// +// if p.NodeMap != nil { +// //fmt.Println("toNode data", hex.EncodeToString(toNode.data[:])) +// //mini := fromNode.data.Mini() +// //fmt.Println("fromNode.data.mINi", hex.EncodeToString(mini[:])) +// //fmt.Println("tonode.aunt.data", hex.EncodeToString(toNode.aunt.data[:])) +// //fmt.Println("tonode.aunt.niece[0]", hex.EncodeToString(toNode.aunt.niece[0].data[:])) +// //fmt.Println("tonode.aunt.niece[1]", hex.EncodeToString(toNode.aunt.niece[1].data[:])) +// p.NodeMap[fromNode.data.Mini()] = toNode +// } +// +// //fmt.Printf("move.to %d, nieces %v, aunt %v\n", to, toNode.niece, toNode.aunt) +// +// // If to position is a root, there's no parent hash to be calculated. +// if isRootPosition(to, p.numLeaves, treeRows(p.numLeaves)) { +// toNode.aunt = nil +// return nil +// } +// +// // TODO have the readPos also return the parent polnode. We can avoid this +// // extra read here. +// parentNode, parentSib, _, err := p.readPos(parent(to, treeRows(p.numLeaves))) +// if err != nil { +// return err +// } +// +// toNode.aunt = parentSib +// +// //fmt.Printf("node %d, parentdata %s, parentsibdata %s\n", +// // to, hex.EncodeToString(parent.data[:]), hex.EncodeToString(parentSib.data[:])) +// +// var pHash Hash +// if isLeftChild(to) { +// pHash = parentHash(toNode.data, toSib.data) +// } else { +// pHash = parentHash(toSib.data, toNode.data) +// } +// parentNode.data = pHash +// +// return nil +//} + +func removeDuplicatePolNode(polNodes []*polNode) []*polNode { + allKeys := make(map[*polNode]bool) + list := []*polNode{} + for _, item := range polNodes { + if _, value := allKeys[item]; !value { + allKeys[item] = true + list = append(list, item) + } + } + return list +} + +func (n *polNode) getSibling() (*polNode, error) { + aunt := n.aunt + + // I'm a root so I have no sibling. + if aunt == nil { + return nil, nil + } + + // Get my sibling which is pointing to my children. + var sibling *polNode + if n == aunt.leftNiece { + sibling = aunt.rightNiece + } else if n == aunt.rightNiece { + sibling = aunt.leftNiece + } else { + return nil, fmt.Errorf("Node with hash %s has an incorrect aunt pointer "+ + "or the aunt with hash %s has incorrect pointer to its nieces", + hex.EncodeToString(n.data[:]), hex.EncodeToString(aunt.data[:])) + } + + return sibling, nil +} + +func (n *polNode) getParent() (*polNode, error) { + aunt := n.aunt + + // I'm a root so there's no parent. + if aunt == nil { + return nil, nil + } + + // My aunt is a root so my aunt is my parent. This is because roots point to their children. + if aunt.aunt == nil { + return aunt, nil + } + + var parent *polNode + if aunt.aunt.leftNiece == aunt { + parent = aunt.aunt.rightNiece + } else if aunt.aunt.rightNiece == aunt { + parent = aunt.aunt.leftNiece + } else { + return nil, fmt.Errorf("Node with hash %s has an incorrect aunt pointer "+ + "or the aunt with hash %s has incorrect pointer to its nieces", + hex.EncodeToString(aunt.data[:]), hex.EncodeToString(aunt.aunt.data[:])) + } + + return parent, nil +} + +func (n *polNode) getChildren() (*polNode, *polNode, error) { + aunt := n.aunt + + // No aunt means that this node is a root. Roots point to their children. + if aunt == nil { + return n.leftNiece, n.rightNiece, nil + } + + // Get my sibling which is pointing to my children. + sibling, err := n.getSibling() + if err != nil { + return nil, nil, err + } + + if sibling == nil { + return nil, nil, fmt.Errorf("Node with hash %s isn't a root but doens't have a sibling", + hex.EncodeToString(n.data[:])) + } + + return sibling.leftNiece, sibling.rightNiece, nil +} + +func (p *Pollard) removeSwaplessParallel(dels []uint64) error { + sortUint64s(dels) + totalRows := treeRows(p.numLeaves) + + moveRows := Transform(dels, p.numLeaves, totalRows) + deTwin(&dels, totalRows) + + p.numDels += uint64(len(dels)) + + //dirtyNodes := make([]*polNode, 0, len(dels)) + for _, del := range dels { + // If a root is being deleted, then we mark it and all the leaves below + // it to be deleted. + if isRootPosition(del, p.numLeaves, totalRows) { + node, _, _, err := p.grabPos(del) + if err != nil { + return err + } + + if p.NodeMap != nil { + delete(p.NodeMap, node.data.Mini()) + } + + idx := -1 + for i, root := range p.roots { + if root.data == node.data { + idx = i + } + } + + if p.roots[idx].leftNiece != nil { + p.roots[idx].leftNiece.aunt = nil + } + if p.roots[idx].rightNiece != nil { + p.roots[idx].rightNiece.aunt = nil + } + p.roots[idx].chop() + p.roots[idx].aunt = nil + p.roots[idx].data = empty + } else { + _, err := p.move(del) + if err != nil { + return err + } + + //if dirty != nil { + // dirtyNodes = append(dirtyNodes, dirty) + //} + } + } + + //for len(dirtyNodes) > 0 { + // // Pop front. + // var dirtyNode *polNode + // dirtyNode, dirtyNodes = dirtyNodes[0], dirtyNodes[1:] + + // leftChild, rightChild, err := dirtyNode.getChildren() + // if err != nil { + // return err + // } + + // if leftChild == nil || rightChild == nil { + // pos := dirtyNode.calculatePosition(p.numLeaves, p.roots) + // fmt.Printf("pos %d child empty\n", pos) + // fmt.Println(p.SubTreeString(pos)) + // continue + // } + + // fmt.Printf("for node %s, got leftchild %s, rightchild %s\n", + // hex.EncodeToString(dirtyNode.data[:]), + // hex.EncodeToString(leftChild.data[:]), + // hex.EncodeToString(rightChild.data[:])) + + // fmt.Printf("dirtyPos %d, leftChild %d, rightChild %d\n", + // dirtyNode.calculatePosition(p.numLeaves, p.roots), + // leftChild.calculatePosition(p.numLeaves, p.roots), + // rightChild.calculatePosition(p.numLeaves, p.roots), + // ) + + // dirtyNode.data = parentHash(leftChild.data, rightChild.data) + + // //if !isRootPosition(dirtyNode.calculatePosition(p.numLeaves, p.roots), p.numLeaves, totalRows) { + // parent, err := dirtyNode.getParent() + // if err != nil { + // return err + // } + // if parent != nil { + // dirtyNodes = append(dirtyNodes, parent) + // dirtyNodes = removeDuplicatePolNode(dirtyNodes) + // } + //} + + // Calculate which nodes need to be hashed again. + dirtyRows := calcDirtyNodes2(moveRows, p.numLeaves, totalRows) + + for currentRow, dirtyRow := range dirtyRows { + for _, dirtyPos := range dirtyRow { + leftChild := child(dirtyPos, totalRows) + rightChild := rightSib(leftChild) + + node, sibling, par, err := p.grabPos(leftChild) + if err != nil { + return err + } + + if node == nil { + return fmt.Errorf("removeSwapless error: couldn't hash dirty "+ + "position at %d as the leftChild of %d was empty", + dirtyPos, leftChild) + } + + if sibling == nil { + return fmt.Errorf("removeSwapless error: couldn't hash dirty "+ + "position at %d as the rightChild of %d was empty", + dirtyPos, rightChild) + } + + hash := parentHash(node.data, sibling.data) + par.dest.data = hash + + // If the dirty position has a parent, then that is also dirty. + if currentRow < int(logicalTreeRows(p.numLeaves)) && + !isRootPosition(dirtyPos, p.numLeaves, totalRows) { + + parentPos := parent(dirtyPos, totalRows) + parentRow := detectRow(parentPos, totalRows) + + // Insert in order. + insertSort(&dirtyRows[parentRow], parentPos) + + // If it's already there, remove it. + dirtyRows[parentRow] = removeDuplicateInt(dirtyRows[parentRow]) + } + } + } + + return nil +} + // rem2 deletes the passed in dels from the pollard func (p *Pollard) rem2(dels []uint64) error { // Hash and swap. "grabPos" in rowdirt / hashdirt is inefficient because you @@ -242,7 +573,8 @@ func (p *Pollard) rem2(dels []uint64) error { // This likely does nothing since the leaf nieces are never set. // Just putting it here since the cost of putting this in is // basically nothing. - n.niece[0], n.niece[1] = nil, nil + //n.niece[0], n.niece[1] = nil, nil + n.leftNiece, n.rightNiece = nil, nil } // get all the swaps, then apply them all @@ -301,8 +633,10 @@ func (p *Pollard) rem2(dels []uint64) error { // do all the hashes at once at the end for _, hn := range hnslice { // skip hashes we can't compute - if hn.sib.niece[0] == nil || hn.sib.niece[1] == nil || - hn.sib.niece[0].data == empty || hn.sib.niece[1].data == empty { + //if hn.sib.niece[0] == nil || hn.sib.niece[1] == nil || + // hn.sib.niece[0].data == empty || hn.sib.niece[1].data == empty { + if hn.sib.leftNiece == nil || hn.sib.rightNiece == nil || + hn.sib.leftNiece.data == empty || hn.sib.rightNiece.data == empty { // TODO when is hn nil? is this OK? // it'd be better to avoid this and not create hns that aren't // supposed to exist. @@ -333,7 +667,8 @@ func (p *Pollard) rem2(dels []uint64) error { // so should become it's sibling's nieces. nt.chop() } else { - nt.niece = ntsib.niece + nt.leftNiece, nt.rightNiece = ntsib.leftNiece, ntsib.rightNiece + //nt.niece = ntsib.niece } nextRoots[i] = nt } @@ -398,7 +733,7 @@ func (p *Pollard) swapNodes(s arrow, row uint8) (*hashableNode, error) { if err != nil { return nil, err } - if bhn.sib.niece[0].data == empty || bhn.sib.niece[1].data == empty { + if bhn.sib.leftNiece.data == empty || bhn.sib.rightNiece.data == empty { bhn = nil // we can't perform this hash as we don't know the children } return bhn, nil @@ -406,8 +741,7 @@ func (p *Pollard) swapNodes(s arrow, row uint8) (*hashableNode, error) { // readPos returns a pointer to the node at the requested position. readPos does not // mutate the Pollard in any way. -func (p *Pollard) readPos(pos uint64) ( - n, nsib *polNode, hn *hashableNode, err error) { +func (p *Pollard) readPos(pos uint64) (n, nsib *polNode, hn *hashableNode, err error) { // Grab the tree that the position is at tree, branchLen, bits := detectOffset(pos, p.numLeaves) if tree >= uint8(len(p.roots)) { @@ -422,21 +756,36 @@ func (p *Pollard) readPos(pos uint64) ( } for h := branchLen - 1; h != 0; h-- { // go through branch - lr := uint8(bits>>h) & 1 - // grab the sibling of lr - lrSib := lr ^ 1 + niece := uint8(bits>>h) & 1 + + if isLeftChild(uint64(niece)) { + n, nsib = n.leftNiece, n.rightNiece + } else { + n, nsib = n.rightNiece, n.leftNiece + } - n, nsib = n.niece[lr], n.niece[lrSib] if n == nil { return nil, nil, nil, err } + + ////// grab the sibling of lr + ////lrSib := lr ^ 1 + + //n, nsib = n.niece[lr], n.niece[lrSib] } - lr := uint8(bits) & 1 - // grab the sibling of lr - lrSib := lr ^ 1 + //lr := uint8(bits) & 1 + niece := uint8(bits) & 1 + //// grab the sibling of lr + //lrSib := lr ^ 1 + + // Switch siblings here. + if isLeftChild(uint64(niece)) { + n, nsib = n.rightNiece, n.leftNiece + } else { + n, nsib = n.leftNiece, n.rightNiece + } - n, nsib = n.niece[lrSib], n.niece[lr] return // only happens when returning a root } @@ -464,15 +813,29 @@ func (p *Pollard) grabPos( } for h := branchLen - 1; h != 0; h-- { // go through branch - lr := uint8(bits>>h) & 1 - // grab the sibling of lr - lrSib := lr ^ 1 + //lr := uint8(bits>>h) & 1 + //// grab the sibling of lr + //lrSib := lr ^ 1 - // if a sib doesn't exist, need to create it and hook it in - if n.niece[lrSib] == nil { - n.niece[lrSib] = &polNode{} + //// if a sib doesn't exist, need to create it and hook it in + //if n.niece[lrSib] == nil { + // n.niece[lrSib] = &polNode{} + //} + //n, nsib = n.niece[lr], n.niece[lrSib] + + niece := uint8(bits>>h) & 1 + + if isLeftChild(uint64(niece)) { + if n.rightNiece == nil { + n.rightNiece = new(polNode) + } + n, nsib = n.leftNiece, n.rightNiece + } else { + if n.leftNiece == nil { + n.leftNiece = new(polNode) + } + n, nsib = n.rightNiece, n.leftNiece } - n, nsib = n.niece[lr], n.niece[lrSib] if n == nil { // if a node doesn't exist, crash // no niece in this case @@ -482,13 +845,24 @@ func (p *Pollard) grabPos( } } - lr := uint8(bits) & 1 - // grab the sibling of lr - lrSib := lr ^ 1 + //lr := uint8(bits) & 1 + //// grab the sibling of lr + //lrSib := lr ^ 1 + + //hn.dest = nsib // this is kind of confusing eh? + //hn.sib = n // but yeah, switch siblingness + //n, nsib = n.niece[lrSib], n.niece[lr] + + hn.dest = nsib + hn.sib = n + + niece := uint8(bits) & 1 + if isLeftChild(uint64(niece)) { + n, nsib = n.rightNiece, n.leftNiece + } else { + n, nsib = n.leftNiece, n.rightNiece + } - hn.dest = nsib // this is kind of confusing eh? - hn.sib = n // but yeah, switch siblingness - n, nsib = n.niece[lrSib], n.niece[lr] return // only happens when returning a root } @@ -544,6 +918,14 @@ func (p *Pollard) ToString() string { return f.ToString() } +func (p *Pollard) SubTreeString(position uint64) string { + f, err := p.toFull() + if err != nil { + return err.Error() + } + return f.SubTreeToString(position) +} + // equalToForest checks if the pollard has the same leaves as the forest. // doesn't check roots and stuff func (p *Pollard) equalToForest(f *Forest) bool { diff --git a/accumulator/pollard_test.go b/accumulator/pollard_test.go index 0acca051..67b8e3bc 100644 --- a/accumulator/pollard_test.go +++ b/accumulator/pollard_test.go @@ -7,6 +7,327 @@ import ( "testing" ) +// checkHashes moves down the tree and calculates the parent hash from the children. +// It errors if the calculated hash doesn't match the hash found in the pollard. +func checkHashes(node, sibling *polNode, p *Pollard) error { + // If node has a niece, then we can calculate the hash of the sibling because + // every tree is a perfect binary tree. + if node.leftNiece != nil { + calculated := parentHash(node.leftNiece.data, node.rightNiece.data) + if sibling.data != calculated { + return fmt.Errorf("For position %d, calculated %s from left %s, right %s but read %s", + sibling.calculatePosition(p.numLeaves, p.roots), + hex.EncodeToString(calculated[:]), + hex.EncodeToString(node.leftNiece.data[:]), hex.EncodeToString(node.rightNiece.data[:]), + hex.EncodeToString(sibling.data[:])) + } + + err := checkHashes(node.leftNiece, node.rightNiece, p) + if err != nil { + return err + } + } + + if sibling.leftNiece != nil { + calculated := parentHash(sibling.leftNiece.data, sibling.rightNiece.data) + if node.data != calculated { + return fmt.Errorf("For position %d, calculated %s from left %s, right %s but read %s", + node.calculatePosition(p.numLeaves, p.roots), + hex.EncodeToString(calculated[:]), + hex.EncodeToString(sibling.leftNiece.data[:]), hex.EncodeToString(sibling.rightNiece.data[:]), + hex.EncodeToString(node.data[:])) + } + + err := checkHashes(sibling.leftNiece, sibling.rightNiece, p) + if err != nil { + return err + } + } + + return nil +} + +func TestPollardAdd(t *testing.T) { + // simulate blocks with simchain + numAdds := uint32(300) + sc := newSimChain(0x07) + + var p Pollard + p.MakeFull() + + for b := 0; b < 2000; b++ { + fmt.Println("on block", b) + adds, _, _ := sc.NextBlock(numAdds) + + err := p.ModifySwapless(adds, nil) + if err != nil { + t.Fatalf("TestSwapLessAddDel fail at block %d. Error: %v", b, err) + } + + if b%100 == 0 { + for _, root := range p.roots { + if root.leftNiece != nil && root.rightNiece != nil { + err = checkHashes(root.leftNiece, root.rightNiece, &p) + if err != nil { + t.Fatal(err) + } + } + } + } + } +} + +func TestPollardAddDel(t *testing.T) { + // simulate blocks with simchain + numAdds := uint32(150) + sc := newSimChain(0x07) + + var p Pollard + p.MakeFull() + + for b := 0; b < 2000; b++ { + fmt.Println("on block", b) + adds, _, delHashes := sc.NextBlock(numAdds) + + bp, err := p.ProveBatchSwapless(delHashes) + if err != nil { + t.Fatalf("TestSwapLessAddDel fail at block %d. Error: %v", b, err) + } + + err = p.Verify(delHashes, bp) + if err != nil { + t.Fatal(err) + } + + //err = p.VerifyCached(delHashes, bp) + //if err != nil { + // t.Fatal(err) + //} + + //fmt.Println(bp.ToString()) + + for _, target := range bp.Targets { + n, _, _, err := p.readPos(target) + if err != nil { + t.Fatalf("TestSwapLessAddDel fail at block %d. Error: %v", b, err) + } + if n == nil { + fmt.Println(bp.ToString()) + t.Fatalf("TestSwapLessAddDel fail to read %d at block %d.", target, b) + } + //fmt.Printf("read %s at pos %d\n", hex.EncodeToString(n.data[:]), target) + } + + err = p.ModifySwapless(adds, bp.Targets) + if err != nil { + t.Fatalf("TestSwapLessAddDel fail at block %d. Error: %v", b, err) + } + + for _, root := range p.roots { + if root.leftNiece != nil && root.rightNiece != nil { + err = checkHashes(root.leftNiece, root.rightNiece, &p) + if err != nil { + t.Fatal(err) + } + } + } + + //fmt.Println(p.ToString()) + } +} + +func TestPollardProveSwapless(t *testing.T) { + var tests = []struct { + leaves []Leaf + dels []Hash + expected BatchProof + }{ + { + []Leaf{ + {Hash{1}, false}, + {Hash{2}, false}, + {Hash{3}, false}, + {Hash{4}, false}, + {Hash{5}, true}, + {Hash{6}, false}, + {Hash{7}, false}, + {Hash{8}, false}, + {Hash{9}, false}, + {Hash{10}, false}, + {Hash{11}, false}, + {Hash{12}, false}, + {Hash{13}, false}, + {Hash{14}, false}, + {Hash{15}, false}, + {Hash{16}, false}, + {Hash{17}, false}, + }, + []Hash{{5}, {6}, {7}, {9}}, + BatchProof{}, + }, + } + + for _, test := range tests { + var p, fullP Pollard + fullP.MakeFull() + p.NodeMap = make(map[MiniHash]*polNode) + + err := p.ModifySwapless(test.leaves, nil) + if err != nil { + t.Fatal(err) + } + + err = fullP.ModifySwapless(test.leaves, nil) + if err != nil { + t.Fatal(err) + } + + fmt.Println("pol", p.ToString()) + + fmt.Println("fullpol", fullP.ToString()) + + bp, err := fullP.ProveBatchSwapless(test.dels) + if err != nil { + t.Fatal(err) + } + + err = p.VerifyCached(test.dels, bp) + if err != nil { + t.Fatal(err) + } + + fmt.Println("bp", bp.ToString()) + bp.Proof[2] = Hash{1} + fmt.Println("modified bp", bp.ToString()) + + err = p.VerifyCached(test.dels, bp) + if err == nil { + err := fmt.Errorf("Modified Proof passed the Verify check") + t.Fatal(err) + } + + bp, err = fullP.ProveBatchSwapless([]Hash{{17}}) + if err != nil { + t.Fatal(err) + } + + fmt.Println(bp.ToString()) + + err = p.Verify([]Hash{{17}}, bp) + if err != nil { + t.Fatal(err) + } + } + +} + +func TestPollardAddSwapless(t *testing.T) { + var p Pollard + p.NodeMap = make(map[MiniHash]*polNode) + //p.MakeFull() + + // Create the starting off pollard. + adds := make([]Leaf, 6) + for i := 0; i < len(adds); i++ { + adds[i].Hash[0] = uint8(i) + adds[i].Hash[20] = 0xff + adds[i].Remember = true + } + + p.addSwapless(adds) + + fmt.Println(p.ToString()) + + //err := p.removeSwapless([]uint64{4, 5, 6, 8, 9, 10, 11, 12, 13, 14}) + //if err != nil { + // t.Fatal(err) + //} + + //fmt.Println(p.ToString()) + + //bp, err := p.ProveBatch([]Hash{{1}, {2}}) + //if err != nil { + // t.Fatal(err) + //} + + //fmt.Println(bp.ToString()) + + n, _, _, err := p.readPos(5) + if err != nil { + t.Fatal(err) + } + + //fmt.Printf("n %s\n", hex.EncodeToString(n.data[:])) + //fmt.Printf("aunt %s\n", hex.EncodeToString(n.aunt.data[:])) + + //fmt.Printf("niece[0] %s\n", hex.EncodeToString(n.aunt.niece[0].data[:])) + //fmt.Printf("niece[1] %s\n", hex.EncodeToString(n.aunt.niece[1].data[:])) + + //fmt.Printf("n.aunt.aunt %s\n", hex.EncodeToString(n.aunt.aunt.data[:])) + //fmt.Printf("n.aunt.aunt.aunt %s\n", hex.EncodeToString(n.aunt.aunt.aunt.data[:])) + //fmt.Printf("n.aunt.aunt.aunt.aunt %s\n", hex.EncodeToString(n.aunt.aunt.aunt.aunt.data[:])) + + pos := n.calculatePosition(p.numLeaves, p.roots) + fmt.Println("got pos", pos) + + //p.addSwapless([]Leaf{{Hash{7}, false}}) + + //fmt.Println(p.ToString()) + + fmt.Println("node map len", len(p.NodeMap)) + + err = p.removeSwapless([]uint64{5}) + if err != nil { + t.Fatal(err) + } + + fmt.Println(p.ToString()) + + for key, value := range p.NodeMap { + fmt.Println("hi") + fmt.Printf("hash %s, node pos %d\n", hex.EncodeToString(key[:]), + value.calculatePosition(p.numLeaves, p.roots)) + } + + node, ok := p.NodeMap[Hash{4}.Mini()] + if !ok { + hash := Hash{4} + t.Fatalf("couldn't find %s", hex.EncodeToString(hash[:])) + } + + fmt.Println(node.data) + + //p.addSwapless([]Leaf{{Hash{8}, true}, {Hash{9}, true}}) + + //fmt.Println(p.ToString()) + + //err = p.removeSwapless([]uint64{9}) + //if err != nil { + // t.Fatal(err) + //} + + p.addSwapless([]Leaf{ + {Hash{6}, true}, + {Hash{7}, true}, + //{Hash{6}, true}, + //{Hash{7}, true}, + }) + + fmt.Println(p.ToString()) + + p.removeSwapless([]uint64{0}) + fmt.Println(p.ToString()) + + for _, root := range p.roots { + if root.leftNiece != nil && root.rightNiece != nil { + err = checkHashes(root.leftNiece, root.rightNiece, &p) + if err != nil { + t.Fatal(err) + } + } + } +} + func TestPollardNoSiblingFound(t *testing.T) { var p Pollard diff --git a/accumulator/pollardfull.go b/accumulator/pollardfull.go index 19cbadf0..3783c066 100644 --- a/accumulator/pollardfull.go +++ b/accumulator/pollardfull.go @@ -1,9 +1,110 @@ package accumulator import ( + "encoding/hex" "fmt" ) +func (p *polNode) calculatePosition(numLeaves uint64, roots []*polNode) uint64 { + // Tells whether to follow the left child or the right child when going + // down the tree. 0 means left, 1 means right. + leftRightIndicator := uint64(0) + + polNode := p + + //auntHash := "nil" + //if polNode.aunt != nil { + // auntHash = hex.EncodeToString(polNode.aunt.data[:]) + //} + + //fmt.Printf("starting with polnode with hash %s, and aunthash %s\n", + // hex.EncodeToString(polNode.data[:]), auntHash) + + rowsToTop := 0 + for polNode.aunt != nil { + //fmt.Printf("aunt %s\n", hex.EncodeToString(polNode.aunt.data[:])) + //fmt.Printf("aunt.niece[0] %s\n", hex.EncodeToString(polNode.aunt.niece[0].data[:])) + //if polNode.aunt.niece[0] == polNode { + if polNode.aunt.leftNiece == polNode { + leftRightIndicator <<= 1 + //fmt.Println("left ", strconv.FormatUint(leftRightIndicator, 2)) + } else { + leftRightIndicator <<= 1 + leftRightIndicator |= 1 + + //fmt.Println("right", strconv.FormatUint(leftRightIndicator, 2)) + } + + polNode = polNode.aunt + + // Ugly but need to flip the bit that we set in this loop as the roots + // don't point their children instead of their nieces. + if rowsToTop == 0 { + leftRightIndicator ^= 1 + } + rowsToTop++ + } + //fmt.Printf("leftRightIndicator %s, leftRightIndicator^1 %s\n", strconv.FormatUint(leftRightIndicator, 2), strconv.FormatUint(leftRightIndicator^1, 2)) + + forestRows := treeRows(numLeaves) + + // Calculate which row the root is on. + rootRow := 0 + // Start from the lowest root. + rootIdx := len(roots) - 1 + for h := 0; h <= int(forestRows); h++ { + //fmt.Println("h", h) + + // Because every root represents a perfect tree of every leaf + // we ever added, each root position will be a power of 2. + // + // Go through the bits of numLeaves. Every bit that is on + // represents a root. + if (numLeaves>>h)&1 == 1 { + // If we found the root, save the row to rootRow + // and return. + if roots[rootIdx] == polNode { + rootRow = h + break + } + + // Check the next higher root. + rootIdx-- + } + } + + //fmt.Println("leftRightIndicator: ", strconv.FormatUint(leftRightIndicator, 2)) + //fmt.Println("rows: ", rows) + //fmt.Println("rootrow: ", rootRow) + //fmt.Println("numLeaves: ", numLeaves) + //fmt.Println("polNode: ", hex.EncodeToString(polNode.data[:])) + //fmt.Println("rootPos: ", rootPosition(numLeaves, uint8(rootRow), treeRows(numLeaves))) + + // Start from the root and work our way down the position that we want. + retPos := rootPosition(numLeaves, uint8(rootRow), forestRows) + + for i := 0; i < rowsToTop; i++ { + isRight := uint64(1) << i + if leftRightIndicator&isRight == isRight { + //fmt.Printf("current pos %d, going to %d\n", + // retPos, child(retPos, treeRows(numLeaves))^1) + + // Grab the sibling since the pollard nodes point to their + // niece. My sibling's nieces are my children. + retPos = sibling(rightChild(retPos, forestRows)) + } else { + //fmt.Printf("current pos %d, going to %d\n", + // retPos, rightChild(retPos, treeRows(numLeaves))^1) + + // Grab the sibling since the pollard nodes point to their + // niece. My sibling's nieces are my children. + retPos = sibling(child(retPos, forestRows)) + } + } + + return retPos +} + // read is just like forestData read but for pollard func (p *Pollard) read(pos uint64) Hash { n, _, _, err := p.grabPos(pos) @@ -100,3 +201,69 @@ func (p *Pollard) ProveBatch(hs []Hash) (BatchProof, error) { return bp, nil } + +func (p *Pollard) ProveBatchSwapless(hashes []Hash) (BatchProof, error) { + var bp BatchProof + // skip everything if empty (should this be an error? + if len(hashes) == 0 { + return bp, nil + } + if p.numLeaves < 2 { + return bp, nil + } + + // first get all the leaf positions + // there shouldn't be any duplicates in hs, but if there are I guess + // it's not an error. + bp.Targets = make([]uint64, len(hashes)) + + for i, wanted := range hashes { + node, ok := p.NodeMap[wanted.Mini()] + if !ok { + fmt.Print(p.ToString()) + return bp, fmt.Errorf("ProveBatchSwapless hash %x not found", wanted) + } + + pos := node.calculatePosition(p.numLeaves, p.roots) + + n, _, _, err := p.readPos(pos) + if err != nil { + return bp, err + } + if n == nil { + return bp, fmt.Errorf("failed to read %d, wanted %s", + pos, hex.EncodeToString(wanted[:])) + } + if n.data != wanted { + return bp, fmt.Errorf("pos %d: wanted %s, got %s", + pos, hex.EncodeToString(wanted[:]), hex.EncodeToString(n.data[:])) + } + + bp.Targets[i] = pos + } + + // targets need to be sorted because the proof hashes are sorted + // NOTE that this is a big deal -- we lose in-block positional information + // because of this sorting. Does that hurt locality or performance? My + // guess is no, but that's untested. + sortedTargets := make([]uint64, len(bp.Targets)) + copy(sortedTargets, bp.Targets) + sortUint64s(sortedTargets) + + proofPositions := NewPositionList() + defer proofPositions.Free() + + // Get the positions of all the hashes that are needed to prove the targets + ProofPositions(sortedTargets, p.numLeaves, p.rows(), &proofPositions.list) + + bp.Proof = make([]Hash, len(proofPositions.list)) + for i, proofPos := range proofPositions.list { + bp.Proof[i] = p.read(proofPos) + } + + if verbose { + fmt.Printf("blockproof targets: %v\n", bp.Targets) + } + + return bp, nil +} diff --git a/accumulator/pollardproof.go b/accumulator/pollardproof.go index 5f2c760f..dfaf3445 100644 --- a/accumulator/pollardproof.go +++ b/accumulator/pollardproof.go @@ -268,21 +268,21 @@ func nextNodes(curBranch, rows uint8, curNodes []*polNodeAndPos, trees []miniTre nextNodesIdx++ // Should never happen but keep the check in there // for now - if curNode.node.niece[0] == nil { - curNode.node.niece[0] = &polNode{} + if curNode.node.leftNiece == nil { + curNode.node.leftNiece = &polNode{} } nextCurNodes = append(nextCurNodes, - &polNodeAndPos{curNode.node.niece[0], lNiecePos}) + &polNodeAndPos{curNode.node.leftNiece, lNiecePos}) } if rExists { nextNodesIdx++ // Should never happen but keep the check in there // for now. - if curNode.node.niece[1] == nil { - curNode.node.niece[1] = &polNode{} + if curNode.node.rightNiece == nil { + curNode.node.rightNiece = &polNode{} } nextCurNodes = append(nextCurNodes, - &polNodeAndPos{curNode.node.niece[1], rNiecePos}) + &polNodeAndPos{curNode.node.rightNiece, rNiecePos}) } } @@ -304,21 +304,21 @@ func nextNodes(curBranch, rows uint8, curNodes []*polNodeAndPos, trees []miniTre func populateOne(tree miniTree, node *polNode, rememberAll bool) int { nodesAllocated := 0 - if node.niece[0] == nil { - node.niece[0] = &polNode{} + if node.leftNiece == nil { + node.leftNiece = &polNode{} nodesAllocated++ } - node.niece[0].data = tree.leftChild.Val + node.leftNiece.data = tree.leftChild.Val - if node.niece[1] == nil { - node.niece[1] = &polNode{} + if node.rightNiece == nil { + node.rightNiece = &polNode{} nodesAllocated++ } - node.niece[1].data = tree.rightChild.Val + node.rightNiece.data = tree.rightChild.Val if rememberAll { - node.niece[0].remember = rememberAll - node.niece[1].remember = rememberAll + node.leftNiece.remember = rememberAll + node.rightNiece.remember = rememberAll node.remember = rememberAll } @@ -351,8 +351,8 @@ func populate(rows uint8, pos uint64, root *polNode, trees *[]miniTree, remember // Append the root's children to curNodes. We start populating from the root's children curNodes := make([]*polNodeAndPos, 0, 2) - curNodes = append(curNodes, &polNodeAndPos{root.niece[0], child(pos, rows)}) - curNodes = append(curNodes, &polNodeAndPos{root.niece[1], child(pos, rows) | 1}) + curNodes = append(curNodes, &polNodeAndPos{root.leftNiece, child(pos, rows)}) + curNodes = append(curNodes, &polNodeAndPos{root.rightNiece, child(pos, rows) | 1}) // populate all the trees passed in. for len(*trees) > 0 { diff --git a/accumulator/pollardutil.go b/accumulator/pollardutil.go index 9aec2d42..04383b77 100644 --- a/accumulator/pollardutil.go +++ b/accumulator/pollardutil.go @@ -10,19 +10,22 @@ import ( // PolNode is a node in the pollard forest type polNode struct { - data Hash - niece [2]*polNode + data Hash + leftNiece *polNode + rightNiece *polNode + + aunt *polNode remember bool } // auntOp returns the hash of a nodes nieces. crashes if you call on nil nieces. func (n *polNode) auntOp() Hash { - return parentHash(n.niece[0].data, n.niece[1].data) + return parentHash(n.leftNiece.data, n.rightNiece.data) } // auntable tells you if you can call auntOp on a node func (n *polNode) auntable() bool { - return n.niece[0] != nil && n.niece[1] != nil + return n.leftNiece != nil && n.rightNiece != nil } // deadEnd returns true if both nieces are nil @@ -32,13 +35,13 @@ func (n *polNode) deadEnd() bool { // fmt.Printf("nil deadend\n") // return true // } - return n.niece[0] == nil && n.niece[1] == nil + return n.leftNiece == nil && n.rightNiece == nil } // chop turns a node into a deadEnd by setting both nieces to nil. func (n *polNode) chop() { - n.niece[0] = nil - n.niece[1] = nil + n.leftNiece = nil + n.rightNiece = nil } // printout printfs the node @@ -48,15 +51,15 @@ func (n *polNode) printout() { return } fmt.Printf("Node %x ", n.data[:4]) - if n.niece[0] == nil { + if n.leftNiece == nil { fmt.Printf("l nil ") } else { - fmt.Printf("l %x ", n.niece[0].data[:4]) + fmt.Printf("l %x ", n.leftNiece.data[:4]) } - if n.niece[1] == nil { + if n.rightNiece == nil { fmt.Printf("r nil\n") } else { - fmt.Printf("r %x\n", n.niece[1].data[:4]) + fmt.Printf("r %x\n", n.rightNiece.data[:4]) } } @@ -72,15 +75,19 @@ func (p *Pollard) NumLeaves() uint64 { return p.numLeaves } +func (p *Pollard) NumDels() uint64 { + return p.numDels +} + // prune prunes deadend children. // don't prune at the bottom; use leaf prune instead at row 1 func (n *polNode) prune() { - remember := n.niece[0].remember || n.niece[1].remember - if n.niece[0].deadEnd() && !remember { - n.niece[0] = nil + remember := n.leftNiece.remember || n.rightNiece.remember + if n.leftNiece.deadEnd() && !remember { + n.leftNiece = nil } - if n.niece[1].deadEnd() && !remember { - n.niece[1] = nil + if n.rightNiece.deadEnd() && !remember { + n.rightNiece = nil } } @@ -89,7 +96,7 @@ func getCount(n *polNode) int64 { if n == nil { return 0 } - return (getCount(n.niece[0]) + 1 + getCount(n.niece[1])) + return (getCount(n.leftNiece) + 1 + getCount(n.rightNiece)) } // polSwap swaps the contents of two polNodes & leaves pointers to them intact @@ -100,7 +107,9 @@ func polSwap(a, asib, b, bsib *polNode) error { return fmt.Errorf("polSwap given nil node") } a.data, b.data = b.data, a.data - asib.niece, bsib.niece = bsib.niece, asib.niece + //asib.niece, bsib.niece = bsib.niece, asib.niece + asib.leftNiece, bsib.leftNiece = bsib.leftNiece, asib.leftNiece + asib.rightNiece, bsib.rightNiece = bsib.rightNiece, asib.rightNiece a.remember, b.remember = b.remember, a.remember return nil } diff --git a/accumulator/swapless.go b/accumulator/swapless.go new file mode 100644 index 00000000..b392895e --- /dev/null +++ b/accumulator/swapless.go @@ -0,0 +1,701 @@ +package accumulator + +import ( + "encoding/hex" + "fmt" + "sort" +) + +func (p *Pollard) addSwapless(adds []Leaf) { + for _, add := range adds { + node := new(polNode) + node.data = add.Hash + + if p.full { + add.Remember = true + } + node.remember = add.Remember + + // Add the hash to the map. + if p.NodeMap != nil && add.Remember { + p.NodeMap[add.Mini()] = node + } + + // We can tell where the roots are by looking at the binary representation + // of the numLeaves. Wherever there's a 1, there's a root. + // + // numLeaves of 8 will be '1000' in binary, so there will be one root at + // row 3. numLeaves of 3 will be '11' in binary, so there's two roots. One at + // row 0 and one at row 1. + // + // In this loop below, we're looking for these roots by checking if there's + // a '1'. If there is a '1', we'll hash the root being added with that root + // until we hit a '0'. + for h := uint8(0); (p.numLeaves>>h)&1 == 1; h++ { + // Grab and pop off the root that will become a node. + root := p.roots[len(p.roots)-1] + popSlice(&p.roots) + + // If the root that we're gonna hash with is empty, move the current + // node up to the position of the parent. + // + // Example: + // + // 12 + // |-------\ + // 08 09 + // |---\ |---\ + // 00 01 02 03 -- + // + // When we add 05 to this tree, 04 is empty so we move 05 to 10. + // The resulting tree looks like below. The hash at position 10 + // is not hash(04 || 05) but just the hash of 05. + // + // 12 + // |-------\ + // 08 09 10 + // |---\ |---\ |---\ + // 00 01 02 03 -- -- + if root.data == empty { + continue + } + + // Roots point to their children. Those children become nieces here. + //root.niece, node.niece = node.niece, root.niece + root.leftNiece, root.rightNiece, node.leftNiece, node.rightNiece = node.leftNiece, node.rightNiece, + root.leftNiece, root.rightNiece + + // Calculate the hash of the new root. + nHash := parentHash(root.data, node.data) + + //node = &polNode{data: nHash, niece: [2]*polNode{root, node}} + newRoot := &polNode{data: nHash, leftNiece: root, rightNiece: node} + if p.full { + newRoot.remember = true + } + + // Set aunt. + updateAunt(newRoot) + newRoot.prune() + node = newRoot + } + + p.roots = append(p.roots, node) + + // Increment as we added a leaf. + p.numLeaves++ + } +} + +// updateAunt works its way down, updating the aunts for all the nieces until it +// encounters the first niece that has the correct aunt. +func updateAunt(n *polNode) { + if n.leftNiece != nil { + // If the aunt is correct, we can return now as all nieces + // of this niece will have the correct aunt. + if n.leftNiece.aunt == n { + return + } else { + // Update the aunt for this niece and check the nieces of this niece. + n.leftNiece.aunt = n + updateAunt(n.leftNiece) + } + } + + // Do the same for the other niece. + if n.rightNiece != nil { + if n.rightNiece.aunt == n { + return + } else { + n.rightNiece.aunt = n + updateAunt(n.rightNiece) + } + } +} + +func delNode(node *polNode) { + // Stop pointing to my aunt and make my aunt stop pointing at me. + if node.aunt != nil { + // Figure out if this node is the left or right niece and make that nil. + if node.aunt.rightNiece == node { + node.aunt.rightNiece = nil + } else if node.aunt.leftNiece == node { + node.aunt.leftNiece = nil + } else { + // Purposely left empty. It's ok if my aunt is not pointing + // at me because that means it's already been updated. + } + } + node.aunt = nil + + // Stop pointing to my leftNiece and make my leftNiece stop pointing at me. + if node.leftNiece != nil { + node.leftNiece.aunt = nil + } + node.leftNiece = nil + + // Same for right niece. + if node.rightNiece != nil { + node.rightNiece.aunt = nil + } + node.rightNiece = nil + + // Make myself nil. + node = nil +} + +func (p *Pollard) deleteNode(del uint64) error { + //fmt.Printf("deleting %d\n", del) + totalRows := treeRows(p.numLeaves) + + from := sibling(del) + to := parent(del, totalRows) + + fromNode, fromNodeSib, _, err := p.readPosition(from) + if err != nil { + return err + } + + toNode, toSib, parentNode, err := p.readPosition(to) + if err != nil { + return err + } + + if p.NodeMap != nil { + delete(p.NodeMap, toNode.data.Mini()) + } + + //fmt.Printf("tonode %v, fromnode %v\n", toNode, fromNode) + toNode.data = fromNode.data + toSib.leftNiece, toSib.rightNiece = fromNodeSib.leftNiece, fromNodeSib.rightNiece + + //fromNode.leftNiece, fromNode.rightNiece = toNode.leftNiece, toNode.rightNiece + //aunt := toNode.aunt + //if aunt != nil { + // if aunt.leftNiece == toNode { + // aunt.leftNiece = fromNode + // } else if aunt.rightNiece == toNode { + // aunt.rightNiece = fromNode + // } else { + // return fmt.Errorf("Node with hash %s has an incorrect aunt pointer "+ + // "or the aunt with hash %s has incorrect pointer to its nieces", + // hex.EncodeToString(toNode.data[:]), hex.EncodeToString(aunt.data[:])) + // } + //} + //fromNode.aunt = aunt + + delHash := fromNodeSib.data + // For GC. + delNode(fromNode) + delNode(fromNodeSib) + + if p.NodeMap != nil { + p.NodeMap[toNode.data.Mini()] = toNode + delete(p.NodeMap, delHash.Mini()) + } + + updateAunt(toSib) + + // If to position is a root, there's no parent hash to be calculated. + if isRootPosition(to, p.numLeaves, totalRows) { + toNode.aunt = nil + return nil + } + + // Set aunt. + toNode.aunt, err = parentNode.getSibling() + if err != nil { + return err + } + // If there's no sibling, it means that toNode is a children of + // the root. + if toNode.aunt == nil { + toNode.aunt = parentNode + } + + // Hash until we get to the root. + for parentNode != nil { + // Grab children of this parent. + leftChild, rightChild, err := parentNode.getChildren() + if err != nil { + return err + } + + parentNode.data = parentHash(leftChild.data, rightChild.data) + + // Grab the next parent that needs the hash updated. + parentNode, err = parentNode.getParent() + if err != nil { + return err + } + } + + return nil +} + +func (p *Pollard) removeSwapless(dels []uint64) error { + sortUint64s(dels) + + p.numDels += uint64(len(dels)) + + totalRows := treeRows(p.numLeaves) + deTwin(&dels, totalRows) + + //fmt.Println("dels", dels) + + for _, del := range dels { + // If a root is being deleted, then we mark it and all the leaves below + // it to be deleted. + if isRootPosition(del, p.numLeaves, totalRows) { + tree, _, _ := detectOffset(del, p.numLeaves) + if tree >= uint8(len(p.roots)) { + return ErrorStrings[ErrorNotEnoughTrees] + } + + // Delete from map. + if p.NodeMap != nil { + delete(p.NodeMap, p.roots[tree].data.Mini()) + } + + if p.roots[tree].leftNiece != nil { + p.roots[tree].leftNiece.aunt = nil + } + if p.roots[tree].rightNiece != nil { + p.roots[tree].rightNiece.aunt = nil + } + p.roots[tree].chop() + p.roots[tree].aunt = nil + p.roots[tree].data = empty + } else { + err := p.deleteNode(del) + if err != nil { + return err + } + } + } + + return nil +} + +func (p *Pollard) ModifySwapless(adds []Leaf, delsOrig []uint64) error { + dels := make([]uint64, len(delsOrig)) + copy(dels, delsOrig) + sortUint64s(dels) + + err := p.removeSwapless(delsOrig) + if err != nil { + return err + } + + p.addSwapless(adds) + + return nil +} + +func (p *Pollard) MakeFull() { + p.NodeMap = make(map[MiniHash]*polNode) + p.full = true +} + +type hashAndPos struct { + hash Hash + pos uint64 +} + +type nodeAndPos struct { + node *polNode + pos uint64 +} + +func sortHashAndPos(s []hashAndPos) { + sort.Slice(s, func(a, b int) bool { return s[a].pos < s[b].pos }) +} + +//func calcRemoveIdx(delHashes []Hash, proof BatchProof, treeRows uint8) { +// for h := 0; h < int(treeRows); h++ { +// detectRow() +// } +//} + +func (p *Pollard) Verify2(delHashes []Hash, proof BatchProof) error { + toProve := make([]nodeAndPos, len(delHashes)) + + for i, delHash := range delHashes { + node, found := p.NodeMap[delHash.Mini()] + if found { + toProve[i] = nodeAndPos{node, proof.Targets[i]} + } else { + n := &polNode{data: delHash, remember: true} + toProve[i] = nodeAndPos{n, proof.Targets[i]} + } + } + + return nil +} + +func (p *Pollard) Verify(delHashes []Hash, proof BatchProof) error { + if len(delHashes) == 0 { + return nil + } + //fmt.Println("targets", proof.Targets) + toProve := make([]hashAndPos, len(delHashes)) + + for i := range toProve { + toProve[i].hash = delHashes[i] + toProve[i].pos = proof.Targets[i] + } + + sortHashAndPos(toProve) + + rootHashes := p.calculateRoots(toProve, proof.Proof) + if len(rootHashes) == 0 { + return fmt.Errorf("No roots calculated but has %d deletions", len(delHashes)) + } + + for i, rootHash := range rootHashes { + fmt.Printf("root %d, hash %s\n", i, hex.EncodeToString(rootHash[:])) + } + + rootMatches := 0 + for i := range p.roots { + if len(rootHashes) > rootMatches && + p.roots[len(p.roots)-(i+1)].data == rootHashes[rootMatches] { + rootMatches++ + } + } + if len(rootHashes) != rootMatches { + // the proof is invalid because some root candidates were not + // included in `roots`. + err := fmt.Errorf("Pollard.Verify: generated %d roots but only"+ + "matched %d roots", len(rootHashes), rootMatches) + return err + } + + return nil +} + +func (p *Pollard) calculateRoots(toProve []hashAndPos, proofHashes []Hash) []Hash { + calculatedRootHashes := make([]Hash, 0, len(p.roots)) + totalRows := treeRows(p.numLeaves) + + var nextProves []hashAndPos + for row := 0; row <= int(totalRows); row++ { + extractedProves := extractRowHash(toProve, totalRows, uint8(row)) + + proves := mergeSortedHashAndPos(nextProves, extractedProves) + nextProves = nextProves[:0] + + for i := 0; i < len(proves); i++ { + prove := proves[i] + + // This means we hashed all the way to the top of this subtree. + if isRootPosition(prove.pos, p.numLeaves, totalRows) { + calculatedRootHashes = append(calculatedRootHashes, prove.hash) + continue + } + + // Check if the next prove is the sibling of this prove. + if i+1 < len(proves) && rightSib(prove.pos) == proves[i+1].pos { + nextProve := hashAndPos{ + hash: parentHash(prove.hash, proves[i+1].hash), + pos: parent(prove.pos, totalRows), + } + nextProves = append(nextProves, nextProve) + + i++ // Increment one more since we procesed another prove. + } else { + hash := proofHashes[0] + proofHashes = proofHashes[1:] + + nextProve := hashAndPos{pos: parent(prove.pos, totalRows)} + if isLeftChild(prove.pos) { + nextProve.hash = parentHash(prove.hash, hash) + } else { + nextProve.hash = parentHash(hash, prove.hash) + } + + nextProves = append(nextProves, nextProve) + } + } + } + + return calculatedRootHashes +} + +func (p *Pollard) VerifyCached(delHashes []Hash, proof BatchProof) error { + if len(delHashes) == 0 { + return nil + } + //fmt.Println("targets", proof.Targets) + toProve := make([]hashAndPos, len(delHashes)) + + for i := range toProve { + _, found := p.NodeMap[delHashes[i].Mini()] + if found { + //if node.data != delHashes[i] { + // // Should never happen as this would mean that the + // // nodeMap is wrong. + // return fmt.Errorf("Passed hash of ") + //} + + toProve[i].hash = empty + toProve[i].pos = proof.Targets[i] + } else { + toProve[i].hash = delHashes[i] + toProve[i].pos = proof.Targets[i] + } + } + + sortHashAndPos(toProve) + + rootHashes, err := p.calculateRootsCached(toProve, proof.Proof) + if err != nil { + return err + } + if len(rootHashes) == 0 { + return fmt.Errorf("No roots calculated but has %d deletions", len(delHashes)) + } + + //for i, rootHash := range rootHashes { + // fmt.Printf("root %d, hash %s\n", i, hex.EncodeToString(rootHash[:])) + //} + + rootMatches := 0 + for i := range p.roots { + if len(rootHashes) > rootMatches && + p.roots[len(p.roots)-(i+1)].data == rootHashes[rootMatches] { + rootMatches++ + } + } + if len(rootHashes) != rootMatches { + // the proof is invalid because some root candidates were not + // included in `roots`. + err := fmt.Errorf("Pollard.Verify: generated %d roots but only"+ + "matched %d roots", len(rootHashes), rootMatches) + return err + } + + return nil +} + +func (p *Pollard) calculateRootsCached(toProve []hashAndPos, proofHashes []Hash) ([]Hash, error) { + calculatedRootHashes := make([]Hash, 0, len(p.roots)) + totalRows := treeRows(p.numLeaves) + + var nextProves []hashAndPos + for row := 0; row <= int(totalRows); row++ { + extractedProves := extractRowHash(toProve, totalRows, uint8(row)) + + proves := mergeSortedHashAndPos(nextProves, extractedProves) + nextProves = nextProves[:0] + + for i := 0; i < len(proves); i++ { + prove := proves[i] + + // This means we hashed all the way to the top of this subtree. + if isRootPosition(prove.pos, p.numLeaves, totalRows) { + if prove.hash == empty { + tree, _, _ := detectOffset(prove.pos, p.numLeaves) + calculatedRootHashes = append(calculatedRootHashes, p.roots[tree].data) + //fmt.Println("cached root") + } else { + calculatedRootHashes = append(calculatedRootHashes, prove.hash) + //fmt.Println("non-cached root") + } + continue + } + + // Check if the next prove is the sibling of this prove. + if i+1 < len(proves) && rightSib(prove.pos) == proves[i+1].pos { + var pHash Hash + if prove.hash != empty && proves[i+1].hash != empty { + pHash = parentHash(prove.hash, proves[i+1].hash) + } else { + if prove.hash != empty { + n, _, _, err := p.readPosition(prove.pos) + if err != nil { + return nil, err + } + + if n.data != prove.hash { + err := fmt.Errorf("Position %d, has cached hash %s "+ + "but got calculated hash of %s ", prove.pos, + hex.EncodeToString(n.data[:]), + hex.EncodeToString(prove.hash[:])) + return nil, err + } + } else if proves[i+1].hash != empty { + n, _, _, err := p.readPosition(proves[i+1].pos) + if err != nil { + return nil, err + } + + if n.data != proves[i+1].hash { + err := fmt.Errorf("Position %d, has cached hash %s "+ + "but got calculated hash of %s ", prove.pos, + hex.EncodeToString(n.data[:]), + hex.EncodeToString(proves[i+1].hash[:])) + return nil, err + } + } else { + // If both are empty than the hashes are cached and + // verified. + } + + pHash = empty + } + nextProve := hashAndPos{ + //hash: parentHash(prove.hash, proves[i+1].hash), + hash: pHash, + pos: parent(prove.pos, totalRows), + } + nextProves = append(nextProves, nextProve) + //fmt.Printf("skipping sib %s\n", hex.EncodeToString(proves[i+1].hash[:])) + + i++ // Increment one more since we procesed another prove. + } else { + hash := proofHashes[0] + proofHashes = proofHashes[1:] + + nextProve := hashAndPos{pos: parent(prove.pos, totalRows)} + if prove.hash == empty { + nextProve.hash = empty + } else { + if isLeftChild(prove.pos) { + nextProve.hash = parentHash(prove.hash, hash) + } else { + nextProve.hash = parentHash(hash, prove.hash) + } + } + + nextProves = append(nextProves, nextProve) + } + //fmt.Printf("Proving %s, got parent %s\n", + // hex.EncodeToString(prove.hash[:]), + // hex.EncodeToString(nextProves[len(nextProves)-1].hash[:])) + } + } + + return calculatedRootHashes, nil +} + +func mergeSortedHashAndPos(a []hashAndPos, b []hashAndPos) (c []hashAndPos) { + maxa := len(a) + maxb := len(b) + + // shortcuts: + if maxa == 0 { + return b + } + if maxb == 0 { + return a + } + + // make it (potentially) too long and truncate later + c = make([]hashAndPos, maxa+maxb) + + idxa, idxb := 0, 0 + for j := 0; j < len(c); j++ { + // if we're out of a or b, just use the remainder of the other one + if idxa >= maxa { + // a is done, copy remainder of b + j += copy(c[j:], b[idxb:]) + c = c[:j] // truncate empty section of c + break + } + if idxb >= maxb { + // b is done, copy remainder of a + j += copy(c[j:], a[idxa:]) + c = c[:j] // truncate empty section of c + break + } + + vala, valb := a[idxa], b[idxb] + if vala.pos < valb.pos { // a is less so append that + c[j] = vala + idxa++ + } else if vala.pos > valb.pos { // b is less so append that + c[j] = valb + idxb++ + } else { // they're equal + c[j] = vala + idxa++ + idxb++ + } + } + return +} + +func extractRowHash(toProve []hashAndPos, forestRows, rowToExtract uint8) []hashAndPos { + if len(toProve) < 0 { + return []hashAndPos{} + } + + start := -1 + end := 0 + + for i := 0; i < len(toProve); i++ { + if detectRow(toProve[i].pos, forestRows) == rowToExtract { + if start == -1 { + start = i + } + + end = i + } else { + // If we're not at the desired row and start has already been set + // once, that means we've extracted everything we can. This is + // possible because the assumption is that the toProve are sorted. + if start != -1 { + break + } + } + } + + if start == -1 { + return []hashAndPos{} + } + + count := (end + 1) - start + row := make([]hashAndPos, count) + + copy(row, toProve[start:end+1]) + + return row +} + +func (p *Pollard) readPosition(pos uint64) (n, sibling, parent *polNode, err error) { + // Tree is the root the position is located under. + // branchLen denotes how far down the root the position is. + // bits tell us if we should go down to the left child or the right child. + tree, branchLen, bits := detectOffset(pos, p.numLeaves) + + // Roots point to their children so we actually need to bitflip this. + // TODO fix detectOffset. + bits ^= 1 + + // Initialize. + n, sibling, parent = p.roots[tree], p.roots[tree], nil + + // Go down the tree to find the node we're looking for. + for h := int(branchLen) - 1; h >= 0; h-- { + // Parent is the sibling of the current node as each of the + // nodes point to their nieces. + parent = sibling + + // Figure out which node we need to follow. + niecePos := uint8(bits>>h) & 1 + if isLeftChild(uint64(niecePos)) { + n, sibling = n.leftNiece, n.rightNiece + } else { + n, sibling = n.rightNiece, n.leftNiece + } + + // Return early if the path to the node we're looking for + // doesn't exist. + if n == nil { + return nil, nil, nil, nil + } + } + + return +} diff --git a/accumulator/transform.go b/accumulator/transform.go index a3a0ebb7..c2a445e2 100644 --- a/accumulator/transform.go +++ b/accumulator/transform.go @@ -1,5 +1,301 @@ package accumulator +import ( + "fmt" + "sort" + "strconv" +) + +// Transform outlines how the accumulator state should be modified. This function itself does +// not modify the accumulator state. +func Transform2(origDels []uint64, numLeaves uint64, forestRows uint8) [][]arrow { + dels := make([]uint64, len(origDels)) + copy(dels, origDels) + + deTwin(&dels, forestRows) + + //fmt.Println("detwined del", dels) + + // Moves indicate where a leaf should move to next. + moves := make([][]arrow, forestRows+1) + + currentRow := uint8(0) + for _, del := range dels { + // If the next del is not in this row, move to the next row until + // we're at the correct row. + for detectRow(del, forestRows) != currentRow { + currentRow++ + } + + // If a root is being deleted, then we mark it and all the leaves below + // it to be deleted. + if isRootPosition(del, numLeaves, forestRows) { + move := arrow{from: del, to: del} + moves[currentRow] = append(moves[currentRow], move) + continue + } + + //fmt.Printf("currentRow: %d, rootPresent: %v, rootPos: %d\n", + // currentRow, rootPresent, rootPos) + + sib := sibling(del) + + move := arrow{from: sib, to: parent(del, forestRows)} + moves[currentRow] = append(moves[currentRow], move) + } + + return moves +} + +// Transform outlines how the accumulator state should be modified. This function itself does +// not modify the accumulator state. +func Transform(origDels []uint64, numLeaves uint64, forestRows uint8) [][]arrow { + dels := make([]uint64, len(origDels)) + copy(dels, origDels) + + deTwin(&dels, forestRows) + + //fmt.Println("detwined del", dels) + + // Moves indicate where a leaf should move to next. + moves := make([][]arrow, forestRows+1) + + currentRow := uint8(0) + for _, del := range dels { + // If the next del is not in this row, move to the next row until + // we're at the correct row. + for detectRow(del, forestRows) != currentRow { + currentRow++ + } + + // If a root is being deleted, then we mark it and all the leaves below + // it to be deleted. + if isRootPosition(del, numLeaves, forestRows) { + move := arrow{from: del, to: del} + moves[currentRow] = append(moves[currentRow], move) + continue + } + + //fmt.Printf("currentRow: %d, rootPresent: %v, rootPos: %d\n", + // currentRow, rootPresent, rootPos) + + sib := sibling(del) + + move := arrow{from: sib, to: parent(del, forestRows)} + moves[currentRow] = append(moves[currentRow], move) + + //fmt.Printf("from %d, to %d\n", sib, parent(del, forestRows)) + + // If 00 -> 16 and 16 -> 24, then what you're really doing is 00 -> 24. + // The loop below tries to find any arrows that can be shortened with + // the newly created arrow by looking at the row below. + if currentRow != 0 { + for i, arw := range moves[currentRow-1] { + if arw.to == sib { + // Change the arrow.from to the arrow.from value from + // the row below. + moves[currentRow][len(moves[currentRow])-1].from = arw.from + + // Delete the previous arrow from the row below. + moves[currentRow-1] = append( + moves[currentRow-1][:i], + moves[currentRow-1][i+1:]...) + break + } + } + } + } + + return moves +} + +// deTwin goes through the list of sorted deletions and finds the parent deletions. +// The caller MUST sort the dels before passing it into the function. +// +// Ex: If we're deleting 00 and 01 in this tree: +// +// 02 +// |--\ +// 00 01 +// +// Then we're really deleting 02. The dels of [00, 01] would be [02]. +func deTwin(dels *[]uint64, forestRows uint8) { + for i := 0; i < len(*dels); i++ { + // 1: Check that there's at least 2 elements in the slice left. + // 2: Check if the right sibling of the current element matches + // up with the next element in the slice. + if i+1 < len(*dels) && rightSib((*dels)[i]) == (*dels)[i+1] { + // Grab the position of the del. + pos := (*dels)[i] + + // Delete both of the child nodes from the slice. + *dels = append((*dels)[:i], (*dels)[i+2:]...) + + // Calculate and Insert the parent in order. + insertSort(dels, parent(pos, forestRows)) + + // Decrement one since the next element we should + // look at is at the same index because the slice decreased + // in length by one. + i-- + } + } +} + +func insertSort(dels *[]uint64, el uint64) { + index := sort.Search(len(*dels), func(i int) bool { return (*dels)[i] > el }) + *dels = append(*dels, 0) + copy((*dels)[index+1:], (*dels)[index:]) + (*dels)[index] = el +} + +func calcDirtyNodes2(moves [][]arrow, numLeaves uint64, forestRows uint8) [][]uint64 { + dirtyNodes := make([][]uint64, len(moves)) + + for currentRow := int(forestRows); currentRow >= 0; currentRow-- { + moveRow := moves[currentRow] + + for _, move := range moveRow { + // If to and from are the same, it means that the whole + // subtree is gonna be deleted, resulting in no dirty nodes. + if move.to == move.from { + continue + } + + // Calculate the dirty position. + dirtyPos := parent(move.to, forestRows) + + // No dirty positions if the node is moving to a root position. + if isRootPosition(move.to, numLeaves, forestRows) { + continue + } + + for i := currentRow; i < len(moves); i++ { + compMoveRow := moves[i] + + for _, compMove := range compMoveRow { + if isAncestor(compMove.from, dirtyPos, forestRows) { + + fromRow := detectRow(compMove.from, forestRows) + toRow := detectRow(compMove.to, forestRows) + + for currentRow := fromRow; currentRow < toRow; currentRow++ { + dirtyPos = calcNextPosition(dirtyPos, numLeaves, currentRow, forestRows) + + } + } else { + if dirtyPos == compMove.from { + dirtyPos = compMove.to + } + } + } + } + + // Grab the row of where the dirty position should be and + // append to that row. + row := detectRow(dirtyPos, forestRows) + dirtyNodes[row] = append(dirtyNodes[row], dirtyPos) + dirtyNodes[row] = removeDuplicateInt(dirtyNodes[row]) + } + } + + return dirtyNodes +} + +func positionsToMove(position, numLeaves uint64, forestRow uint8) { +} + +func calcNextPosition2(position, numLeaves uint64, delRow, forestRows uint8) { + positionRow := detectRow(position, forestRows) + subTreeRows := detectRow(getRootPosition(position, numLeaves, forestRows), forestRows) + + mask := (1 << uint64(subTreeRows-positionRow)) - uint64(1) + lsb := position & mask + fmt.Println(strconv.FormatUint(lsb, 2)) + + rise := delRow - positionRow + fmt.Println(rise) + + hi := uint64(0) + startRow := int(subTreeRows) - int(positionRow) + for i := int(startRow) - 1; i >= 0; i-- { + //for i := 0; i < startRow; i++ { + if i == int(delRow) { + continue + } + + mask := uint64(1 << i) + if (lsb & mask) == mask { + hi = hi & 1 + hi <<= 1 + } else { + hi <<= 1 + } + } + + fmt.Printf("lsb %s, after %s\n", strconv.FormatUint(lsb, 2), strconv.FormatUint(hi, 2)) + + returnPos := parentMany(position, rise, forestRows) + fmt.Println(returnPos) +} + +func calcNextPosition(position, numLeaves uint64, delRow, forestRows uint8) uint64 { + returnPos := getRootPosition(position, numLeaves, forestRows) + + subTreeRows := detectRow(getRootPosition(position, numLeaves, forestRows), forestRows) + positionRow := detectRow(position, forestRows) + startRow := int(subTreeRows) - int(positionRow) + + delRow = delRow - positionRow + + if positionRow > 0 { + mask := (1 << uint64(subTreeRows-positionRow)) - uint64(1) + position = position & mask + } + + for i := int(startRow) - 1; i >= 0; i-- { + // Skip the bit field operation for this row. + if i == int(delRow) { + //fmt.Println("skipping row", i) + continue + } + mask := uint64(1 << i) + //fmt.Println("mask", mask) + + // 1 means right + if (position & mask) == mask { + //fmt.Println("right") + returnPos = rightChild(returnPos, forestRows) + } else { + //fmt.Println("left") + returnPos = child(returnPos, forestRows) + } + } + + return returnPos +} + +func isRootPosition(position, numLeaves uint64, forestRows uint8) bool { + row := detectRow(position, forestRows) + + rootPresent := numLeaves&(1< 0 && len(targets) > 0 && - targets[len(targets)-1] == rootPosition(numLeaves, row, forestRows) { - // remove roots from targets - targets = targets[:len(targets)-1] - } + rowTargs := extractRow(targets, forestRows, row) - // reset nextTargets + rowTargs = append(rowTargs, nextTargets.list...) + sortUint64s(rowTargs) + + // Reset nextTargets nextTargets.list = nextTargets.list[:0] - for len(targets) > 0 { + + computedPositions += int64(len(rowTargs)) + if numLeaves&(1< 0 && len(rowTargs) > 0 && + rowTargs[len(rowTargs)-1] == rootPosition(numLeaves, row, forestRows) { + // remove roots from rowTargs + rowTargs = rowTargs[:len(rowTargs)-1] + } + + for len(rowTargs) > 0 { switch { // look at the first 4 targets - case len(targets) > 3: - if (targets[0]|1)^2 == targets[3]|1 { + case len(rowTargs) > 3: + if (rowTargs[0]|1)^2 == rowTargs[3]|1 { // the first and fourth target are cousins - // => target 2 and 3 are also targets, both parents are - // targets of next row + // => target 2 and 3 are also rowTargs, both parents are + // rowTargs of next row nextTargets.list = append(nextTargets.list, - parent(targets[0], forestRows), parent(targets[3], forestRows)) - targets = targets[4:] + parent(rowTargs[0], forestRows), parent(rowTargs[3], forestRows)) + rowTargs = rowTargs[4:] break } - // handle first three targets + // handle first three rowTargs fallthrough - // look at the first 3 targets - case len(targets) > 2: - if (targets[0]|1)^2 == targets[2]|1 { + // look at the first 3 rowTargs + case len(rowTargs) > 2: + if (rowTargs[0]|1)^2 == rowTargs[2]|1 { // the first and third target are cousins // => the second target is either the sibling of the first // OR the sibiling of the third // => only the sibling that is not a target is appended // to the proof positions - if targets[1]|1 == targets[0]|1 { - *proofPositions = append(*proofPositions, targets[2]^1) + if rowTargs[1]|1 == rowTargs[0]|1 { + *proofPositions = append(*proofPositions, rowTargs[2]^1) } else { - *proofPositions = append(*proofPositions, targets[0]^1) + *proofPositions = append(*proofPositions, rowTargs[0]^1) } - // both parents are targets of next row + // both parents are rowTargs of next row nextTargets.list = append(nextTargets.list, - parent(targets[0], forestRows), parent(targets[2], forestRows)) - targets = targets[3:] + parent(rowTargs[0], forestRows), parent(rowTargs[2], forestRows)) + rowTargs = rowTargs[3:] break } - // handle first two targets + // handle first two rowTargs fallthrough - // look at the first 2 targets - case len(targets) > 1: - if targets[0]|1 == targets[1] { - nextTargets.list = append(nextTargets.list, parent(targets[0], forestRows)) - targets = targets[2:] + // look at the first 2 rowTargs + case len(rowTargs) > 1: + if rowTargs[0]|1 == rowTargs[1] { + nextTargets.list = append(nextTargets.list, parent(rowTargs[0], forestRows)) + rowTargs = rowTargs[2:] break } - if (targets[0]|1)^2 == targets[1]|1 { - *proofPositions = append(*proofPositions, targets[0]^1, targets[1]^1) + if (rowTargs[0]|1)^2 == rowTargs[1]|1 { + *proofPositions = append(*proofPositions, rowTargs[0]^1, rowTargs[1]^1) nextTargets.list = append(nextTargets.list, - parent(targets[0], forestRows), parent(targets[1], forestRows)) - targets = targets[2:] + parent(rowTargs[0], forestRows), parent(rowTargs[1], forestRows)) + rowTargs = rowTargs[2:] break } // not related, handle first target @@ -117,18 +120,79 @@ func ProofPositions( // look at the first target default: - *proofPositions = append(*proofPositions, targets[0]^1) - nextTargets.list = append(nextTargets.list, parent(targets[0], forestRows)) - targets = targets[1:] + *proofPositions = append(*proofPositions, rowTargs[0]^1) + nextTargets.list = append(nextTargets.list, parent(rowTargs[0], forestRows)) + rowTargs = rowTargs[1:] } } - - targets = nextTargets.list } return computedPositions } +func popSlice(a *[]*polNode) { + length := len(*(a)) + i := length - 1 + + copy((*a)[i:], (*a)[i+1:]) + (*a)[length-1] = nil + (*a) = (*a)[:length-1] +} + +//func extractRow(targets []uint64, forestRows uint8) []uint64 { +// if len(targets) < 0 { +// return nil +// } +// +// currentRow := detectRow(targets[0], forestRows) +// end := 0 +// for i := 1; i < len(targets); i++ { +// if detectRow(targets[i], forestRows) != currentRow { +// end = i +// break +// } +// } +// +// return targets[:end] +//} + +func extractRow(targets []uint64, forestRows, rowToExtract uint8) []uint64 { + if len(targets) < 0 { + return []uint64{} + } + + start := -1 + end := 0 + + for i := 0; i < len(targets); i++ { + if detectRow(targets[i], forestRows) == rowToExtract { + if start == -1 { + start = i + } + + end = i + } else { + // If we're not at the desired row and start has already been set + // once, that means we've extracted everything we can. This is + // possible because the assumption is that the targets are sorted. + if start != -1 { + break + } + } + } + + if start == -1 { + return []uint64{} + } + + count := (end + 1) - start + row := make([]uint64, count) + + copy(row, targets[start:end+1]) + + return row +} + // takes a slice of dels, removes the twins (in place) and returns a slice // of parents of twins // @@ -196,9 +260,9 @@ func detectOffset(position uint64, numLeaves uint64) (uint8, uint8, uint64) { // similarities to detectSubTreeRows() with more features // maybe replace detectSubTreeRows with this - // th = tree rows + // tr = tree rows tr := treeRows(numLeaves) - // nh = target node row + // nr = target node row nr := detectRow(position, tr) // there's probably a fancier way with bits... var biggerTrees uint8 @@ -246,6 +310,11 @@ func child(position uint64, forestRows uint8) uint64 { return (position << 1) & mask } +func rightChild(position uint64, forestRows uint8) uint64 { + mask := uint64(2<>rise | (mask << uint64(forestRows-(rise-1)))) & mask } +func isLeftChild(position uint64) bool { + return position&1 == 0 +} + +//func isDescendent( uint64, forestRows uint8) bool { +// for row:=0; row github.com/mit-dci/utcd v0.21.0-beta.0.20210716180138-e7464b93a1b7 diff --git a/go.sum b/go.sum index 85b0e40b..5b25956b 100644 --- a/go.sum +++ b/go.sum @@ -68,20 +68,33 @@ github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFd github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca h1:Ld/zXl5t4+D69SiV4JoN7kkfvJdOWlPpfxrzxpLMoUk= github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 h1:7I4JAnoQBe7ZtJcBaYHi5UtiO8tQHbUSXxL+pnGRANg= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/exp v0.0.0-20220317015231-48e79f11773a h1:DAzrdbxsb5tXNOhMCSwF7ZdfMbW46hE9fSVO6BsmUZM= +golang.org/x/exp v0.0.0-20220317015231-48e79f11773a/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -92,10 +105,21 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200602225109-6fdc65e7d980/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=