From 952030cfa6f11be6aef938e5359064c4cf6b30a9 Mon Sep 17 00:00:00 2001 From: Serban Iorga Date: Fri, 30 Sep 2022 13:46:48 +0300 Subject: [PATCH] pallet-mmr: generate historical proofs (#12324) * BEEFY: generate historical proofs Signed-off-by: Serban Iorga * Update frame/merkle-mountain-range/rpc/src/lib.rs Co-authored-by: Adrian Catangiu * Update primitives/merkle-mountain-range/src/lib.rs Co-authored-by: Adrian Catangiu * Update frame/merkle-mountain-range/src/lib.rs Co-authored-by: Adrian Catangiu * cargo fmt * fix off-by-one in leaves powerset generation * test all possible mmr sizes for historical proofs * remove now redundant simple_historical_proof * cargo fmt Signed-off-by: Serban Iorga Co-authored-by: Adrian Catangiu Co-authored-by: Robert Hambrock --- bin/node/runtime/src/lib.rs | 39 ++- client/beefy/src/tests.rs | 7 + frame/merkle-mountain-range/rpc/src/lib.rs | 49 ++++ frame/merkle-mountain-range/src/lib.rs | 22 +- frame/merkle-mountain-range/src/tests.rs | 276 +++++++++++++++++++- primitives/merkle-mountain-range/src/lib.rs | 11 +- 6 files changed, 379 insertions(+), 25 deletions(-) diff --git a/bin/node/runtime/src/lib.rs b/bin/node/runtime/src/lib.rs index 5e4fdb4748d15..4fa4049e22682 100644 --- a/bin/node/runtime/src/lib.rs +++ b/bin/node/runtime/src/lib.rs @@ -2011,10 +2011,7 @@ impl_runtime_apis! { } } - impl pallet_mmr::primitives::MmrApi< - Block, - mmr::Hash, - > for Runtime { + impl pallet_mmr::primitives::MmrApi for Runtime { fn generate_proof(leaf_index: pallet_mmr::primitives::LeafIndex) -> Result<(mmr::EncodableOpaqueLeaf, mmr::Proof), mmr::Error> { @@ -2049,11 +2046,35 @@ impl_runtime_apis! { Ok(Mmr::mmr_root()) } - fn generate_batch_proof(leaf_indices: Vec) - -> Result<(Vec, mmr::BatchProof), mmr::Error> - { - Mmr::generate_batch_proof(leaf_indices) - .map(|(leaves, proof)| (leaves.into_iter().map(|leaf| mmr::EncodableOpaqueLeaf::from_leaf(&leaf)).collect(), proof)) + fn generate_batch_proof( + leaf_indices: Vec, + ) -> Result<(Vec, mmr::BatchProof), mmr::Error> { + Mmr::generate_batch_proof(leaf_indices).map(|(leaves, proof)| { + ( + leaves + .into_iter() + .map(|leaf| mmr::EncodableOpaqueLeaf::from_leaf(&leaf)) + .collect(), + proof, + ) + }) + } + + fn generate_historical_batch_proof( + leaf_indices: Vec, + leaves_count: pallet_mmr::primitives::LeafIndex, + ) -> Result<(Vec, mmr::BatchProof), mmr::Error> { + Mmr::generate_historical_batch_proof(leaf_indices, leaves_count).map( + |(leaves, proof)| { + ( + leaves + .into_iter() + .map(|leaf| mmr::EncodableOpaqueLeaf::from_leaf(&leaf)) + .collect(), + proof, + ) + }, + ) } fn verify_batch_proof(leaves: Vec, proof: mmr::BatchProof) diff --git a/client/beefy/src/tests.rs b/client/beefy/src/tests.rs index 26c85592ecb85..3e49f4e05cc91 100644 --- a/client/beefy/src/tests.rs +++ b/client/beefy/src/tests.rs @@ -277,6 +277,13 @@ macro_rules! create_test_api { unimplemented!() } + fn generate_historical_batch_proof( + _leaf_indices: Vec, + _leaves_count: LeafIndex + ) -> Result<(Vec, BatchProof), MmrError> { + unimplemented!() + } + fn verify_batch_proof(_leaves: Vec, _proof: BatchProof) -> Result<(), MmrError> { unimplemented!() } diff --git a/frame/merkle-mountain-range/rpc/src/lib.rs b/frame/merkle-mountain-range/rpc/src/lib.rs index 75032d40f492a..e939ff8ae7cd0 100644 --- a/frame/merkle-mountain-range/rpc/src/lib.rs +++ b/frame/merkle-mountain-range/rpc/src/lib.rs @@ -128,6 +128,31 @@ pub trait MmrApi { leaf_indices: Vec, at: Option, ) -> RpcResult>; + + /// Generate a MMR proof for the given `leaf_indices` of the MMR that had `leaves_count` leaves. + /// + /// This method calls into a runtime with MMR pallet included and attempts to generate + /// a MMR proof for the set of leaves at the given `leaf_indices` with MMR fixed to the state + /// with exactly `leaves_count` leaves. `leaves_count` must be larger than all `leaf_indices` + /// for the function to succeed. + /// + /// Optionally, a block hash at which the runtime should be queried can be specified. + /// Note that specifying the block hash isn't super-useful here, unless you're generating + /// proof using non-finalized blocks where there are several competing forks. That's because + /// MMR state will be fixed to the state with `leaves_count`, which already points to some + /// historical block. + /// + /// Returns the leaves and a proof for these leaves (compact encoding, i.e. hash of + /// the leaves). Both parameters are SCALE-encoded. + /// The order of entries in the `leaves` field of the returned struct + /// is the same as the order of the entries in `leaf_indices` supplied + #[method(name = "mmr_generateHistoricalBatchProof")] + fn generate_historical_batch_proof( + &self, + leaf_indices: Vec, + leaves_count: LeafIndex, + at: Option, + ) -> RpcResult>; } /// MMR RPC methods. @@ -192,6 +217,30 @@ where Ok(LeafBatchProof::new(block_hash, leaves, proof)) } + + fn generate_historical_batch_proof( + &self, + leaf_indices: Vec, + leaves_count: LeafIndex, + at: Option<::Hash>, + ) -> RpcResult::Hash>> { + let api = self.client.runtime_api(); + let block_hash = at.unwrap_or_else(|| + // If the block hash is not supplied assume the best block. + self.client.info().best_hash); + + let (leaves, proof) = api + .generate_historical_batch_proof_with_context( + &BlockId::hash(block_hash), + sp_core::ExecutionContext::OffchainCall(None), + leaf_indices, + leaves_count, + ) + .map_err(runtime_error_into_rpc_error)? + .map_err(mmr_error_into_rpc_error)?; + + Ok(LeafBatchProof::new(block_hash, leaves, proof)) + } } /// Converts a mmr-specific error into a [`CallError`]. diff --git a/frame/merkle-mountain-range/src/lib.rs b/frame/merkle-mountain-range/src/lib.rs index 9f989847af0f9..8b4f2b60bc198 100644 --- a/frame/merkle-mountain-range/src/lib.rs +++ b/frame/merkle-mountain-range/src/lib.rs @@ -330,7 +330,27 @@ impl, I: 'static> Pallet { (Vec>, primitives::BatchProof<>::Hash>), primitives::Error, > { - let mmr: ModuleMmr = mmr::Mmr::new(Self::mmr_leaves()); + Self::generate_historical_batch_proof(leaf_indices, Self::mmr_leaves()) + } + + /// Generate a MMR proof for the given `leaf_indices` for the MMR of `leaves_count` size. + /// + /// Note this method can only be used from an off-chain context + /// (Offchain Worker or Runtime API call), since it requires + /// all the leaves to be present. + /// It may return an error or panic if used incorrectly. + pub fn generate_historical_batch_proof( + leaf_indices: Vec, + leaves_count: LeafIndex, + ) -> Result< + (Vec>, primitives::BatchProof<>::Hash>), + primitives::Error, + > { + if leaves_count > Self::mmr_leaves() { + return Err(Error::InvalidLeavesCount) + } + + let mmr: ModuleMmr = mmr::Mmr::new(leaves_count); mmr.generate_batch_proof(leaf_indices) } diff --git a/frame/merkle-mountain-range/src/tests.rs b/frame/merkle-mountain-range/src/tests.rs index d6886f90a5da7..bcb775ba02819 100644 --- a/frame/merkle-mountain-range/src/tests.rs +++ b/frame/merkle-mountain-range/src/tests.rs @@ -227,7 +227,8 @@ fn should_generate_proofs_correctly() { let _ = env_logger::try_init(); let mut ext = new_test_ext(); // given - ext.execute_with(|| add_blocks(7)); + let num_blocks: u64 = 7; + ext.execute_with(|| add_blocks(num_blocks as usize)); ext.persist_offchain_overlay(); // Try to generate proofs now. This requires the offchain extensions to be present @@ -241,6 +242,23 @@ fn should_generate_proofs_correctly() { crate::Pallet::::generate_batch_proof(vec![leaf_index]).unwrap() }) .collect::>(); + // when generate historical proofs for all leaves + let historical_proofs = (0_u64..crate::NumberOfLeaves::::get()) + .into_iter() + .map(|leaf_index| { + let mut proofs = vec![]; + for leaves_count in leaf_index + 1..=num_blocks { + proofs.push( + crate::Pallet::::generate_historical_batch_proof( + vec![leaf_index], + leaves_count, + ) + .unwrap(), + ) + } + proofs + }) + .collect::>(); // then assert_eq!( @@ -258,6 +276,79 @@ fn should_generate_proofs_correctly() { } ) ); + assert_eq!( + historical_proofs[0][0], + ( + vec![Compact::new(((0, H256::repeat_byte(1)).into(), LeafData::new(1).into(),))], + BatchProof { leaf_indices: vec![0], leaf_count: 1, items: vec![] } + ) + ); + + // D + // / \ + // / \ + // A B C + // / \ / \ / \ + // 1 2 3 4 5 6 7 + // + // we're proving 3 => we need { 4, A, C++7 } + assert_eq!( + proofs[2], + ( + vec![Compact::new(((2, H256::repeat_byte(3)).into(), LeafData::new(3).into(),))], + BatchProof { + leaf_indices: vec![2], + leaf_count: 7, + items: vec![ + hex("1b14c1dc7d3e4def11acdf31be0584f4b85c3673f1ff72a3af467b69a3b0d9d0"), + hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854"), + hex("dca421199bdcc55bb773c6b6967e8d16675de69062b52285ca63685241fdf626"), + ], + } + ) + ); + // A + // / \ + // 1 2 3 + // + // we're proving 3 => we need { A } + assert_eq!( + historical_proofs[2][0], + ( + vec![Compact::new(((2, H256::repeat_byte(3)).into(), LeafData::new(3).into(),))], + BatchProof { + leaf_indices: vec![2], + leaf_count: 3, + items: vec![hex( + "672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854" + ),], + } + ) + ); + // D + // / \ + // / \ + // A B + // / \ / \ + // 1 2 3 4 5 + // we're proving 3 => we need { 4, A, 5 } + assert_eq!( + historical_proofs[2][2], + ( + vec![Compact::new(((2, H256::repeat_byte(3)).into(), LeafData::new(3).into(),))], + BatchProof { + leaf_indices: vec![2], + leaf_count: 5, + items: vec![ + hex("1b14c1dc7d3e4def11acdf31be0584f4b85c3673f1ff72a3af467b69a3b0d9d0"), + hex("672c04a9cd05a644789d769daa552d35d8de7c33129f8a7cbf49e595234c4854"), + hex("3b031d22e24f1126c8f7d2f394b663f9b960ed7abbedb7152e17ce16112656d0") + ], + } + ) + ); + assert_eq!(historical_proofs[2][4], proofs[2]); + assert_eq!( proofs[4], ( @@ -273,6 +364,21 @@ fn should_generate_proofs_correctly() { } ) ); + assert_eq!( + historical_proofs[4][0], + ( + vec![Compact::new(((4, H256::repeat_byte(5)).into(), LeafData::new(5).into(),))], + BatchProof { + leaf_indices: vec![4], + leaf_count: 5, + items: vec![hex( + "ae88a0825da50e953e7a359c55fe13c8015e48d03d301b8bdfc9193874da9252" + ),], + } + ) + ); + assert_eq!(historical_proofs[4][2], proofs[4]); + assert_eq!( proofs[6], ( @@ -287,6 +393,7 @@ fn should_generate_proofs_correctly() { } ) ); + assert_eq!(historical_proofs[6][0], proofs[6]); }); } @@ -302,9 +409,8 @@ fn should_generate_batch_proof_correctly() { // to retrieve full leaf data. register_offchain_ext(&mut ext); ext.execute_with(|| { - // when generate proofs for all leaves + // when generate proofs for a batch of leaves let (.., proof) = crate::Pallet::::generate_batch_proof(vec![0, 4, 5]).unwrap(); - // then assert_eq!( proof, @@ -318,6 +424,28 @@ fn should_generate_batch_proof_correctly() { ], } ); + + // when generate historical proofs for a batch of leaves + let (.., historical_proof) = + crate::Pallet::::generate_historical_batch_proof(vec![0, 4, 5], 6).unwrap(); + // then + assert_eq!( + historical_proof, + BatchProof { + leaf_indices: vec![0, 4, 5], + leaf_count: 6, + items: vec![ + hex("ad4cbc033833612ccd4626d5f023b9dfc50a35e838514dd1f3c86f8506728705"), + hex("cb24f4614ad5b2a5430344c99545b421d9af83c46fd632d70a332200884b4d46"), + ], + } + ); + + // when generate historical proofs for a batch of leaves + let (.., historical_proof) = + crate::Pallet::::generate_historical_batch_proof(vec![0, 4, 5], 7).unwrap(); + // then + assert_eq!(historical_proof, proof); }); } @@ -338,11 +466,33 @@ fn should_verify() { // when crate::Pallet::::generate_batch_proof(vec![5]).unwrap() }); + let (simple_historical_leaves, simple_historical_proof5) = ext.execute_with(|| { + // when + crate::Pallet::::generate_historical_batch_proof(vec![5], 6).unwrap() + }); + let (advanced_historical_leaves, advanced_historical_proof5) = ext.execute_with(|| { + // when + crate::Pallet::::generate_historical_batch_proof(vec![5], 7).unwrap() + }); ext.execute_with(|| { add_blocks(7); // then assert_eq!(crate::Pallet::::verify_leaves(leaves, proof5), Ok(())); + assert_eq!( + crate::Pallet::::verify_leaves( + simple_historical_leaves, + simple_historical_proof5 + ), + Ok(()) + ); + assert_eq!( + crate::Pallet::::verify_leaves( + advanced_historical_leaves, + advanced_historical_proof5 + ), + Ok(()) + ); }); } @@ -350,16 +500,40 @@ fn should_verify() { fn should_verify_batch_proofs() { fn generate_and_verify_batch_proof( ext: &mut sp_io::TestExternalities, - leaves: &Vec, + leaf_indices: &Vec, blocks_to_add: usize, ) { - let (leaves, proof) = ext - .execute_with(|| crate::Pallet::::generate_batch_proof(leaves.to_vec()).unwrap()); + let (leaves, proof) = ext.execute_with(|| { + crate::Pallet::::generate_batch_proof(leaf_indices.to_vec()).unwrap() + }); + + let mmr_size = ext.execute_with(|| crate::Pallet::::mmr_leaves()); + let min_mmr_size = leaf_indices.iter().max().unwrap() + 1; + + // generate historical proofs for all possible mmr sizes, + // lower bound being index of highest leaf to be proven + let historical_proofs = (min_mmr_size..=mmr_size) + .map(|mmr_size| { + ext.execute_with(|| { + crate::Pallet::::generate_historical_batch_proof( + leaf_indices.to_vec(), + mmr_size, + ) + .unwrap() + }) + }) + .collect::>(); ext.execute_with(|| { add_blocks(blocks_to_add); // then assert_eq!(crate::Pallet::::verify_leaves(leaves, proof), Ok(())); + historical_proofs.iter().for_each(|(leaves, proof)| { + assert_eq!( + crate::Pallet::::verify_leaves(leaves.clone(), proof.clone()), + Ok(()) + ); + }); }) } @@ -378,7 +552,7 @@ fn should_verify_batch_proofs() { ext.persist_offchain_overlay(); // generate powerset (skipping empty set) of all possible leaf combinations for mmr size n - let leaves_set: Vec> = (0..n).into_iter().powerset().skip(1).collect(); + let leaves_set: Vec> = (0..=n).into_iter().powerset().skip(1).collect(); leaves_set.iter().for_each(|leaves_subset| { generate_and_verify_batch_proof(&mut ext, leaves_subset, 0); @@ -393,7 +567,7 @@ fn should_verify_batch_proofs() { ext.persist_offchain_overlay(); // generate all possible 2-leaf combinations for mmr size n - let leaves_set: Vec> = (0..n).into_iter().combinations(2).collect(); + let leaves_set: Vec> = (0..=n).into_iter().combinations(2).collect(); leaves_set.iter().for_each(|leaves_subset| { generate_and_verify_batch_proof(&mut ext, leaves_subset, 0); @@ -414,7 +588,13 @@ fn verification_should_be_stateless() { // Start off with chain initialisation and storing indexing data off-chain // (MMR Leafs) let mut ext = new_test_ext(); - ext.execute_with(|| add_blocks(7)); + let (root_6, root_7) = ext.execute_with(|| { + add_blocks(6); + let root_6 = crate::Pallet::::mmr_root_hash(); + add_blocks(1); + let root_7 = crate::Pallet::::mmr_root_hash(); + (root_6, root_7) + }); ext.persist_offchain_overlay(); // Try to generate proof now. This requires the offchain extensions to be present @@ -424,12 +604,27 @@ fn verification_should_be_stateless() { // when crate::Pallet::::generate_batch_proof(vec![5]).unwrap() }); - let root = ext.execute_with(|| crate::Pallet::::mmr_root_hash()); + let (_, historical_proof5) = ext.execute_with(|| { + // when + crate::Pallet::::generate_historical_batch_proof(vec![5], 6).unwrap() + }); // Verify proof without relying on any on-chain data. let leaf = crate::primitives::DataOrHash::Data(leaves[0].clone()); assert_eq!( - crate::verify_leaves_proof::<::Hashing, _>(root, vec![leaf], proof5), + crate::verify_leaves_proof::<::Hashing, _>( + root_7, + vec![leaf.clone()], + proof5 + ), + Ok(()) + ); + assert_eq!( + crate::verify_leaves_proof::<::Hashing, _>( + root_6, + vec![leaf], + historical_proof5 + ), Ok(()) ); } @@ -441,7 +636,13 @@ fn should_verify_batch_proof_statelessly() { // Start off with chain initialisation and storing indexing data off-chain // (MMR Leafs) let mut ext = new_test_ext(); - ext.execute_with(|| add_blocks(7)); + let (root_6, root_7) = ext.execute_with(|| { + add_blocks(6); + let root_6 = crate::Pallet::::mmr_root_hash(); + add_blocks(1); + let root_7 = crate::Pallet::::mmr_root_hash(); + (root_6, root_7) + }); ext.persist_offchain_overlay(); // Try to generate proof now. This requires the offchain extensions to be present @@ -451,12 +652,15 @@ fn should_verify_batch_proof_statelessly() { // when crate::Pallet::::generate_batch_proof(vec![0, 4, 5]).unwrap() }); - let root = ext.execute_with(|| crate::Pallet::::mmr_root_hash()); + let (historical_leaves, historical_proof) = ext.execute_with(|| { + // when + crate::Pallet::::generate_historical_batch_proof(vec![0, 4, 5], 6).unwrap() + }); // Verify proof without relying on any on-chain data. assert_eq!( crate::verify_leaves_proof::<::Hashing, _>( - root, + root_7, leaves .into_iter() .map(|leaf| crate::primitives::DataOrHash::Data(leaf)) @@ -465,6 +669,17 @@ fn should_verify_batch_proof_statelessly() { ), Ok(()) ); + assert_eq!( + crate::verify_leaves_proof::<::Hashing, _>( + root_6, + historical_leaves + .into_iter() + .map(|leaf| crate::primitives::DataOrHash::Data(leaf)) + .collect(), + historical_proof + ), + Ok(()) + ); } #[test] @@ -721,3 +936,36 @@ fn should_verify_canonicalized() { assert_eq!(crate::Pallet::::verify_leaves(leaves, proofs), Ok(())); }); } + +#[test] +fn does_not_panic_when_generating_historical_proofs() { + let _ = env_logger::try_init(); + let mut ext = new_test_ext(); + + // given 7 blocks (7 MMR leaves) + ext.execute_with(|| add_blocks(7)); + ext.persist_offchain_overlay(); + + // Try to generate historical proof with invalid arguments. This requires the offchain + // extensions to be present to retrieve full leaf data. + register_offchain_ext(&mut ext); + ext.execute_with(|| { + // when leaf index is invalid + assert_eq!( + crate::Pallet::::generate_historical_batch_proof(vec![10], 7), + Err(Error::LeafNotFound), + ); + + // when leaves count is invalid + assert_eq!( + crate::Pallet::::generate_historical_batch_proof(vec![3], 100), + Err(Error::InvalidLeavesCount), + ); + + // when both leaf index and leaves count are invalid + assert_eq!( + crate::Pallet::::generate_historical_batch_proof(vec![10], 100), + Err(Error::InvalidLeavesCount), + ); + }); +} diff --git a/primitives/merkle-mountain-range/src/lib.rs b/primitives/merkle-mountain-range/src/lib.rs index 29a7e3d1a6fb6..c40a594739ec1 100644 --- a/primitives/merkle-mountain-range/src/lib.rs +++ b/primitives/merkle-mountain-range/src/lib.rs @@ -402,6 +402,8 @@ pub enum Error { PalletNotIncluded, /// Cannot find the requested leaf index InvalidLeafIndex, + /// The provided leaves count is larger than the actual leaves count. + InvalidLeavesCount, } impl Error { @@ -455,7 +457,14 @@ sp_api::decl_runtime_apis! { fn mmr_root() -> Result; /// Generate MMR proof for a series of leaves under given indices. - fn generate_batch_proof(leaf_indices: Vec) -> Result<(Vec, BatchProof), Error>; + fn generate_batch_proof(leaf_indices: Vec) + -> Result<(Vec, BatchProof), Error>; + + /// Generate MMR proof for a series of leaves under given indices, using MMR at given `leaves_count` size. + fn generate_historical_batch_proof( + leaf_indices: Vec, + leaves_count: LeafIndex + ) -> Result<(Vec, BatchProof), Error>; /// Verify MMR proof against on-chain MMR for a batch of leaves. ///