diff --git a/CHANGELOG.md b/CHANGELOG.md index 75bbf9f..0683f8f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +- Added an option to disable leaf sorting. + ## 1.0.5 - Make `processMultiProof` more robust by validating invariants. diff --git a/README.md b/README.md index 5583505..0e52b6e 100644 --- a/README.md +++ b/README.md @@ -130,6 +130,18 @@ bytes32 leaf = keccak256(bytes.concat(keccak256(abi.encode(addr, amount)))); This is an opinionated design that we believe will offer the best out of the box experience for most users. We may introduce options for customization in the future based on user requests. +### Leaf ordering + +Each leaf of a merkle tree can be proven individually. The relative ordering of leaves is mostly irrelevant when the only objective is to prove the inclusion of individual leaves in the tree. Proving multiple leaves at once is however a little bit more difficult. + +This library proposes a mechanism to prove (and verify) that sets of leaves are included in the tree. These "multiproofs" can also be verified onchain using the implementation available in `@openzeppelin/contracts`. This mechanism requires the leaves to be ordered respective to their position in the tree. For example, if the tree leaves are (in hex form) `[ 0xAA...AA, 0xBB...BB, 0xCC...CC, 0xDD...DD]`, then you'd be able to prove `[0xBB...BB, 0xDD...DD]` as a subset of the leaves, but not `[0xDD...DD, 0xBB...BB]`. + +Since this library knows the entire tree, you can generate a multiproof with the requested leaves in any order. The library will re-order them so that they appear inside the proof in the correct order. The `MultiProof` object returned by `tree.getMultiProof(...)` will have the leaves ordered according to their position in the tree, and not in the order in which you provided them. + +By default, the library orders the leaves according to their hash when building the tree. This is so that a smart contract can build the hashes of a set of leaves and order them correctly without any knowledge of the tree itself. Said differently, it is simpler for a smart contract to process a multiproof for leaves that it rebuilt itself if the corresponding tree is ordered. + +However, some trees are constructed iteratively from unsorted data, causing the leaves to be unsorted as well. For this library to be able to represent such trees, the call to `StandardMerkleTree.of` includes an option to disable sorting. Using that option, the leaves are kept in the order in which they were provided. Note that this option has no effect on your ability to generate and verify proofs and multiproofs in JavaScript, but that it may introduce challenges when verifying multiproofs onchain. We recommend only using it for building a representation of trees that are built (onchain) using an iterative process. + ## API & Examples ### `StandardMerkleTree` @@ -141,7 +153,7 @@ import { StandardMerkleTree } from "@openzeppelin/merkle-tree"; ### `StandardMerkleTree.of` ```typescript -const tree = StandardMerkleTree.of([[alice, '100'], [bob, '200']], ['address', 'uint']); +const tree = StandardMerkleTree.of([[alice, '100'], [bob, '200']], ['address', 'uint'], options); ``` Creates a standard merkle tree out of an array of the elements in the tree, along with their types for ABI encoding. For documentation on the syntax of the types, including how to encode structs, refer to the documentation for Ethers.js's [`AbiCoder`](https://docs.ethers.org/v5/api/utils/abi/coder/#AbiCoder-encode). @@ -149,6 +161,15 @@ Creates a standard merkle tree out of an array of the elements in the tree, alon > **Note** > Consider reading the array of elements from a CSV file for easy interoperability with spreadsheets or other data processing pipelines. +> **Note** +> By default, leaves are sorted according to their hash. This is done so that multiproof generated by the library can more easily be verified onchain. This can be disabled using the optional third argument. See the [Leaf ordering](#leaf-ordering) section for more details. + +#### Options + +| Option | Description | Default | +| ------------ | ----------------------------------------------------------------------------------- | ------- | +| `sortLeaves` | Enable or disable sorted leaves. Sorting is strongly recommended for multiproofs. | `true` | + ### `StandardMerkleTree.verify` ```typescript diff --git a/src/options.ts b/src/options.ts new file mode 100644 index 0000000..3516460 --- /dev/null +++ b/src/options.ts @@ -0,0 +1,11 @@ +// MerkleTree building options +export type MerkleTreeOptions = Partial<{ + /** Enable or disable sorted leaves. Sorting is strongly recommended for multiproofs. */ + sortLeaves: boolean; +}>; + +// Recommended (default) options. +// - leaves are sorted by default to facilitate onchain verification of multiproofs. +export const defaultOptions: Required = { + sortLeaves: true, +}; diff --git a/src/standard.test.ts b/src/standard.test.ts index 590d836..cd1217d 100644 --- a/src/standard.test.ts +++ b/src/standard.test.ts @@ -1,131 +1,149 @@ import assert from 'assert/strict'; import { keccak256 } from 'ethereum-cryptography/keccak'; import { hex } from './bytes'; +import { MerkleTreeOptions } from './options'; import { StandardMerkleTree } from './standard'; const zeroBytes = new Uint8Array(32); const zero = hex(zeroBytes); -const characters = (s: string) => { +const makeTree = (s: string, opts: MerkleTreeOptions = {}) => { const l = s.split('').map(c => [c]); - const t = StandardMerkleTree.of(l, ['string']); + const t = StandardMerkleTree.of(l, ['string'], opts); return { l, t }; } describe('standard merkle tree', () => { - it('generates valid single proofs for all leaves', () => { - const { t } = characters('abcdef'); - t.validate(); - }); - - it('generates valid single proofs for all leaves', () => { - const { t } = characters('abcdef'); - - for (const [id, leaf] of t.entries()) { - const proof1 = t.getProof(id); - const proof2 = t.getProof(leaf); - - assert.deepEqual(proof1, proof2); - - assert(t.verify(id, proof1)); - assert(t.verify(leaf, proof1)); - assert(StandardMerkleTree.verify(t.root, ['string'], leaf, proof1)); - } - }); - - it('rejects invalid proofs', () => { - const { t } = characters('abcdef'); - const { t: otherTree } = characters('abc'); - - const leaf = ['a']; - const invalidProof = otherTree.getProof(leaf); - - assert(!t.verify(leaf, invalidProof)); - assert(!StandardMerkleTree.verify(t.root, ['string'], leaf, invalidProof)); - }); - - it('generates valid multiproofs', () => { - const { t, l } = characters('abcdef'); - - for (const ids of [[], [0, 1], [0, 1, 5], [1, 3, 4, 5], [0, 2, 4, 5], [0, 1, 2, 3, 4, 5]]) { - const proof1 = t.getMultiProof(ids); - const proof2 = t.getMultiProof(ids.map(i => l[i]!)); - - assert.deepEqual(proof1, proof2); - - assert(t.verifyMultiProof(proof1)); - assert(StandardMerkleTree.verifyMultiProof(t.root, ['string'], proof1)); - } - }); - - it('rejects invalid multiproofs', () => { - const { t } = characters('abcdef'); - const { t: otherTree } = characters('abc'); - - const leaves = [['a'], ['b'], ['c']]; - const multiProof = otherTree.getMultiProof(leaves); - - assert(!t.verifyMultiProof(multiProof)); - assert(!StandardMerkleTree.verifyMultiProof(t.root, ['string'], multiProof)); - }); - - it('renders tree representation', () => { - const { t } = characters('abc'); - - const expected = `\ -0) f2129b5a697531ef818f644564a6552b35c549722385bc52aa7fe46c0b5f46b1 -├─ 1) fa914d99a18dc32d9725b3ef1c50426deb40ec8d0885dac8edcc5bfd6d030016 -│ ├─ 3) 9c15a6a0eaeed500fd9eed4cbeab71f797cefcc67bfd46683e4d2e6ff7f06d1c -│ └─ 4) 19ba6c6333e0e9a15bf67523e0676e2f23eb8e574092552d5e888c64a4bb3681 -└─ 2) 9cf5a63718145ba968a01c1d557020181c5b252f665cf7386d370eddb176517b`; - - assert.equal(t.render(), expected); - }); - - it('dump and load', () => { - const { t } = characters('abcdef'); - const t2 = StandardMerkleTree.load(t.dump()); - - t2.validate(); - assert.deepEqual(t2, t); - }); - - it('reject out of bounds value index', () => { - const { t } = characters('a'); - assert.throws( - () => t.getProof(1), - /^Error: Index out of bounds$/, - ); - }); - - it('reject unrecognized tree dump', () => { - assert.throws( - () => StandardMerkleTree.load({ format: 'nonstandard' } as any), - /^Error: Unknown format 'nonstandard'$/, - ); - }); - - it('reject malformed tree dump', () => { - const t1 = StandardMerkleTree.load({ - format: 'standard-v1', - tree: [zero], - values: [{ value: ['0'], treeIndex: 0 }], - leafEncoding: ['uint256'], + for (const opts of [ + {}, + { sortLeaves: true }, + { sortLeaves: false }, + ]) { + describe(`with options '${JSON.stringify(opts)}'`, () => { + const { l: leaves, t: tree } = makeTree('abcdef', opts); + const { l: otherLeaves, t: otherTree } = makeTree('abc', opts); + + it('generates valid single proofs for all leaves', () => { + tree.validate(); + }); + + it('generates valid single proofs for all leaves', () => { + for (const [id, leaf] of tree.entries()) { + const proof1 = tree.getProof(id); + const proof2 = tree.getProof(leaf); + + assert.deepEqual(proof1, proof2); + + assert(tree.verify(id, proof1)); + assert(tree.verify(leaf, proof1)); + assert(StandardMerkleTree.verify(tree.root, ['string'], leaf, proof1)); + } + }); + + it('rejects invalid proofs', () => { + const leaf = ['a']; + const invalidProof = otherTree.getProof(leaf); + + assert(!tree.verify(leaf, invalidProof)); + assert(!StandardMerkleTree.verify(tree.root, ['string'], leaf, invalidProof)); + }); + + it('generates valid multiproofs', () => { + for (const ids of [[], [0, 1], [0, 1, 5], [1, 3, 4, 5], [0, 2, 4, 5], [0, 1, 2, 3, 4, 5], [4, 1, 5, 0, 2]]) { + const proof1 = tree.getMultiProof(ids); + const proof2 = tree.getMultiProof(ids.map(i => leaves[i]!)); + + assert.deepEqual(proof1, proof2); + + assert(tree.verifyMultiProof(proof1)); + assert(StandardMerkleTree.verifyMultiProof(tree.root, ['string'], proof1)); + } + }); + + it('rejects invalid multiproofs', () => { + const multiProof = otherTree.getMultiProof([['a'], ['b'], ['c']]); + + assert(!tree.verifyMultiProof(multiProof)); + assert(!StandardMerkleTree.verifyMultiProof(tree.root, ['string'], multiProof)); + }); + + it('renders tree representation', () => { + assert.equal( + tree.render(), + opts.sortLeaves == false + ? [ + "0) 23be0977360f08bb0bd7f709a7d543d2cd779c79c66d74e0441919871647de2b", + "├─ 1) 8f7234e8cfe39c08ca84a3a3e3274f574af26fd15165fe29e09cbab742daccd9", + "│ ├─ 3) 03707d7802a71ca56a8ad8028da98c4f1dbec55b31b4a25d536b5309cc20eda9", + "│ │ ├─ 7) eba909cf4bb90c6922771d7f126ad0fd11dfde93f3937a196274e1ac20fd2f5b", + "│ │ └─ 8) 9cf5a63718145ba968a01c1d557020181c5b252f665cf7386d370eddb176517b", + "│ └─ 4) fa914d99a18dc32d9725b3ef1c50426deb40ec8d0885dac8edcc5bfd6d030016", + "│ ├─ 9) 19ba6c6333e0e9a15bf67523e0676e2f23eb8e574092552d5e888c64a4bb3681", + "│ └─ 10) 9c15a6a0eaeed500fd9eed4cbeab71f797cefcc67bfd46683e4d2e6ff7f06d1c", + "└─ 2) 7b0c6cd04b82bfc0e250030a5d2690c52585e0cc6a4f3bc7909d7723b0236ece", + " ├─ 5) c62a8cfa41edc0ef6f6ae27a2985b7d39c7fea770787d7e104696c6e81f64848", + " └─ 6) 9a4f64e953595df82d1b4f570d34c4f4f0cfaf729a61e9d60e83e579e1aa283e", + ].join("\n") + : [ + "0) 6deb52b5da8fd108f79fab00341f38d2587896634c646ee52e49f845680a70c8", + "├─ 1) 52426e0f1f65ff7e209a13b8c29cffe82e3acaf3dad0a9b9088f3b9a61a929c3", + "│ ├─ 3) 8076923e76cf01a7c048400a2304c9a9c23bbbdac3a98ea3946340fdafbba34f", + "│ │ ├─ 7) 9cf5a63718145ba968a01c1d557020181c5b252f665cf7386d370eddb176517b", + "│ │ └─ 8) 9c15a6a0eaeed500fd9eed4cbeab71f797cefcc67bfd46683e4d2e6ff7f06d1c", + "│ └─ 4) 965b92c6cf08303cc4feb7f3e0819c436c2cec17c6f0688a6af139c9a368707c", + "│ ├─ 9) 9a4f64e953595df82d1b4f570d34c4f4f0cfaf729a61e9d60e83e579e1aa283e", + "│ └─ 10) 19ba6c6333e0e9a15bf67523e0676e2f23eb8e574092552d5e888c64a4bb3681", + "└─ 2) fd3cf45654e88d1cc5d663578c82c76f4b5e3826bacaa1216441443504538f51", + " ├─ 5) eba909cf4bb90c6922771d7f126ad0fd11dfde93f3937a196274e1ac20fd2f5b", + " └─ 6) c62a8cfa41edc0ef6f6ae27a2985b7d39c7fea770787d7e104696c6e81f64848", + ].join("\n"), + ); + }); + + it('dump and load', () => { + const recoveredTree = StandardMerkleTree.load(tree.dump()); + + recoveredTree.validate(); + assert.deepEqual(tree, recoveredTree); + }); + + it('reject out of bounds value index', () => { + assert.throws( + () => tree.getProof(leaves.length), + /^Error: Index out of bounds$/, + ); + }); + + it('reject unrecognized tree dump', () => { + assert.throws( + () => StandardMerkleTree.load({ format: 'nonstandard' } as any), + /^Error: Unknown format 'nonstandard'$/, + ); + }); + + it('reject malformed tree dump', () => { + const loadedTree1 = StandardMerkleTree.load({ + format: 'standard-v1', + tree: [zero], + values: [{ value: ['0'], treeIndex: 0 }], + leafEncoding: ['uint256'], + }); + assert.throws( + () => loadedTree1.getProof(0), + /^Error: Merkle tree does not contain the expected value$/, + ); + + const loadedTree2 = StandardMerkleTree.load({ + format: 'standard-v1', + tree: [zero, zero, hex(keccak256(keccak256(zeroBytes)))], + values: [{ value: ['0'], treeIndex: 2 }], + leafEncoding: ['uint256'], + }); + assert.throws( + () => loadedTree2.getProof(0), + /^Error: Unable to prove value$/, + ); + }); }); - assert.throws( - () => t1.getProof(0), - /^Error: Merkle tree does not contain the expected value$/, - ); - - const t2 = StandardMerkleTree.load({ - format: 'standard-v1', - tree: [zero, zero, hex(keccak256(keccak256(zeroBytes)))], - values: [{ value: ['0'], treeIndex: 2 }], - leafEncoding: ['uint256'], - }); - assert.throws( - () => t2.getProof(0), - /^Error: Unable to prove value$/, - ); - }); + } }); diff --git a/src/standard.ts b/src/standard.ts index 110ad36..20b40e1 100644 --- a/src/standard.ts +++ b/src/standard.ts @@ -1,14 +1,10 @@ -import { keccak256 } from 'ethereum-cryptography/keccak'; import { equalsBytes, hexToBytes } from 'ethereum-cryptography/utils'; -import { defaultAbiCoder } from '@ethersproject/abi'; import { Bytes, compareBytes, hex } from './bytes'; import { getProof, isValidMerkleTree, makeMerkleTree, processProof, renderMerkleTree, MultiProof, getMultiProof, processMultiProof } from './core'; +import { MerkleTreeOptions, defaultOptions } from './options'; import { checkBounds } from './utils/check-bounds'; import { throwError } from './utils/throw-error'; - -function standardLeafHash(value: T, types: string[]): Bytes { - return keccak256(keccak256(hexToBytes(defaultAbiCoder.encode(types, value)))); -} +import { standardLeafHash } from './utils/standard-leaf-hash'; interface StandardMerkleTreeData { format: 'standard-v1'; @@ -35,10 +31,14 @@ export class StandardMerkleTree { ])); } - static of(values: T[], leafEncoding: string[]) { - const hashedValues = values - .map((value, valueIndex) => ({ value, valueIndex, hash: standardLeafHash(value, leafEncoding) })) - .sort((a, b) => compareBytes(a.hash, b.hash)); + static of(values: T[], leafEncoding: string[], options: MerkleTreeOptions = {}) { + const sortLeaves = options.sortLeaves ?? defaultOptions.sortLeaves; + + const hashedValues = values.map((value, valueIndex) => ({ value, valueIndex, hash: standardLeafHash(value, leafEncoding) })); + + if (sortLeaves) { + hashedValues.sort((a, b) => compareBytes(a.hash, b.hash)); + } const tree = makeMerkleTree(hashedValues.map(v => v.hash)); diff --git a/src/utils/standard-leaf-hash.ts b/src/utils/standard-leaf-hash.ts new file mode 100644 index 0000000..df65671 --- /dev/null +++ b/src/utils/standard-leaf-hash.ts @@ -0,0 +1,8 @@ +import { keccak256 } from 'ethereum-cryptography/keccak'; +import { hexToBytes } from 'ethereum-cryptography/utils'; +import { defaultAbiCoder } from '@ethersproject/abi'; +import { Bytes } from '../bytes'; + +export function standardLeafHash(value: T, types: string[]): Bytes { + return keccak256(keccak256(hexToBytes(defaultAbiCoder.encode(types, value)))); +}