From 5891a2bc105744df6cde68139480d610c5ae4d3f Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 28 May 2024 10:20:08 +0200 Subject: [PATCH 1/3] perf: make prover able to certify more than 5 transactions Some SQL query was too complex because depended on the number of block ranges to retrieve in the prover. The query is now made indiviudally on each block range. --- .../src/dependency_injection/builder.rs | 2 +- mithril-aggregator/src/services/prover.rs | 92 +++++++++++-------- 2 files changed, 56 insertions(+), 38 deletions(-) diff --git a/mithril-aggregator/src/dependency_injection/builder.rs b/mithril-aggregator/src/dependency_injection/builder.rs index 18b041e1235..eec877a0a8f 100644 --- a/mithril-aggregator/src/dependency_injection/builder.rs +++ b/mithril-aggregator/src/dependency_injection/builder.rs @@ -1377,7 +1377,7 @@ impl DependenciesBuilder { Ok(self.message_service.as_ref().cloned().unwrap()) } - /// build Prover service + /// Build Prover service pub async fn build_prover_service(&mut self) -> Result> { let mk_map_pool_size = self .configuration diff --git a/mithril-aggregator/src/services/prover.rs b/mithril-aggregator/src/services/prover.rs index 7992fbeb27e..7356f965b4a 100644 --- a/mithril-aggregator/src/services/prover.rs +++ b/mithril-aggregator/src/services/prover.rs @@ -99,10 +99,14 @@ impl MithrilProverService { block_ranges: &[BlockRange], ) -> StdResult>> { let mut block_ranges_map = HashMap::new(); - let transactions = self - .transaction_retriever - .get_by_block_ranges(block_ranges.to_vec()) - .await?; + let mut transactions = vec![]; + for block_range in block_ranges { + let block_range_transactions = self + .transaction_retriever + .get_by_block_ranges(vec![block_range.clone()]) + .await?; + transactions.extend(block_range_transactions); + } for transaction in transactions { let block_range = BlockRange::from_block_number(transaction.block_number); let block_range_transactions: &mut Vec<_> = @@ -374,21 +378,24 @@ mod tests { test_data::filter_transactions_for_indices(&[1, 2, 4], &transactions); let test_data = test_data::build_test_data(&transactions_to_prove, &transactions); let prover = build_prover( - |retriever_mock| { + |transaction_retriever_mock| { let transaction_hashes_to_prove = test_data.transaction_hashes_to_prove.clone(); let transactions_to_prove = transactions_to_prove.clone(); - retriever_mock + transaction_retriever_mock .expect_get_by_hashes() .with(eq(transaction_hashes_to_prove)) .return_once(move |_| Ok(transactions_to_prove)); let block_ranges_to_prove = test_data.block_ranges_to_prove.clone(); - let all_transactions_in_block_ranges_to_prove = - test_data.all_transactions_in_block_ranges_to_prove.clone(); - retriever_mock - .expect_get_by_block_ranges() - .with(eq(block_ranges_to_prove)) - .return_once(move |_| Ok(all_transactions_in_block_ranges_to_prove)); + for block_range_to_prove in block_ranges_to_prove { + let block_ranges = vec![block_range_to_prove.clone()]; + let transactions_in_block_range_to_prove = + test_data.block_ranges_map[&block_range_to_prove].clone(); + transaction_retriever_mock + .expect_get_by_block_ranges() + .with(eq(block_ranges)) + .return_once(move |_| Ok(transactions_in_block_range_to_prove)); + } }, |block_range_root_retriever_mock| { let block_ranges_map = test_data.block_ranges_map.clone(); @@ -428,21 +435,24 @@ mod tests { let mut test_data = test_data::build_test_data(&transactions_to_prove, &transactions); test_data.transaction_hashes_to_prove = vec!["tx-unknown-123".to_string()]; let prover = build_prover( - |retriever_mock| { + |transaction_retriever_mock| { let transaction_hashes_to_prove = test_data.transaction_hashes_to_prove.clone(); let transactions_to_prove = transactions_to_prove.clone(); - retriever_mock + transaction_retriever_mock .expect_get_by_hashes() .with(eq(transaction_hashes_to_prove)) .return_once(move |_| Ok(transactions_to_prove)); let block_ranges_to_prove = test_data.block_ranges_to_prove.clone(); - let all_transactions_in_block_ranges_to_prove = - test_data.all_transactions_in_block_ranges_to_prove.clone(); - retriever_mock - .expect_get_by_block_ranges() - .with(eq(block_ranges_to_prove)) - .return_once(move |_| Ok(all_transactions_in_block_ranges_to_prove)); + for block_range_to_prove in block_ranges_to_prove { + let block_ranges = vec![block_range_to_prove.clone()]; + let transactions_in_block_range_to_prove = + test_data.block_ranges_map[&block_range_to_prove].clone(); + transaction_retriever_mock + .expect_get_by_block_ranges() + .with(eq(block_ranges)) + .return_once(move |_| Ok(transactions_in_block_range_to_prove)); + } }, |block_range_root_retriever_mock| { let block_ranges_map = test_data.block_ranges_map.clone(); @@ -485,21 +495,24 @@ mod tests { ] .concat(); let prover = build_prover( - |retriever_mock| { + |transaction_retriever_mock| { let transaction_hashes_to_prove = test_data.transaction_hashes_to_prove.clone(); let transactions_to_prove = transactions_to_prove.clone(); - retriever_mock + transaction_retriever_mock .expect_get_by_hashes() .with(eq(transaction_hashes_to_prove)) .return_once(move |_| Ok(transactions_to_prove)); let block_ranges_to_prove = test_data.block_ranges_to_prove.clone(); - let all_transactions_in_block_ranges_to_prove = - test_data.all_transactions_in_block_ranges_to_prove.clone(); - retriever_mock - .expect_get_by_block_ranges() - .with(eq(block_ranges_to_prove)) - .return_once(move |_| Ok(all_transactions_in_block_ranges_to_prove)); + for block_range_to_prove in block_ranges_to_prove { + let block_ranges = vec![block_range_to_prove.clone()]; + let transactions_in_block_range_to_prove = + test_data.block_ranges_map[&block_range_to_prove].clone(); + transaction_retriever_mock + .expect_get_by_block_ranges() + .with(eq(block_ranges)) + .return_once(move |_| Ok(transactions_in_block_range_to_prove)); + } }, |block_range_root_retriever_mock| { let block_ranges_map = test_data.block_ranges_map.clone(); @@ -539,8 +552,8 @@ mod tests { test_data::filter_transactions_for_indices(&[1, 2, 4], &transactions); let test_data = test_data::build_test_data(&transactions_to_prove, &transactions); let prover = build_prover( - |retriever_mock| { - retriever_mock + |transaction_retriever_mock| { + transaction_retriever_mock .expect_get_by_hashes() .returning(|_| Err(anyhow!("Error"))); }, @@ -570,17 +583,22 @@ mod tests { test_data::filter_transactions_for_indices(&[1, 2, 4], &transactions); let test_data = test_data::build_test_data(&transactions_to_prove, &transactions); let prover = build_prover( - |retriever_mock| { + |transaction_retriever_mock| { let transactions_to_prove = transactions_to_prove.clone(); - retriever_mock + transaction_retriever_mock .expect_get_by_hashes() .return_once(move |_| Ok(transactions_to_prove)); - let all_transactions_in_block_ranges_to_prove = - test_data.all_transactions_in_block_ranges_to_prove.clone(); - retriever_mock - .expect_get_by_block_ranges() - .return_once(move |_| Ok(all_transactions_in_block_ranges_to_prove)); + let block_ranges_to_prove = test_data.block_ranges_to_prove.clone(); + for block_range_to_prove in block_ranges_to_prove { + let block_ranges = vec![block_range_to_prove.clone()]; + let transactions_in_block_range_to_prove = + test_data.block_ranges_map[&block_range_to_prove].clone(); + transaction_retriever_mock + .expect_get_by_block_ranges() + .with(eq(block_ranges)) + .return_once(move |_| Ok(transactions_in_block_range_to_prove)); + } }, |block_range_root_retriever_mock| { block_range_root_retriever_mock From b604e9f1a6648608e999495389f91204ca0ce291 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Mon, 3 Jun 2024 16:08:19 +0200 Subject: [PATCH 2/3] refactor: move 'get_transaction_by_block_ranges' optimization to repository --- .../cardano_transaction_repository.rs | 13 +++- mithril-aggregator/src/services/prover.rs | 72 +++++++------------ 2 files changed, 38 insertions(+), 47 deletions(-) diff --git a/internal/mithril-persistence/src/database/repository/cardano_transaction_repository.rs b/internal/mithril-persistence/src/database/repository/cardano_transaction_repository.rs index cd72cb82058..0c1b8b4c188 100644 --- a/internal/mithril-persistence/src/database/repository/cardano_transaction_repository.rs +++ b/internal/mithril-persistence/src/database/repository/cardano_transaction_repository.rs @@ -255,8 +255,17 @@ impl CardanoTransactionRepository { &self, block_ranges: Vec, ) -> StdResult> { - self.connection - .fetch_collect(GetCardanoTransactionQuery::by_block_ranges(block_ranges)) + let mut transactions = vec![]; + for block_range in block_ranges { + let block_range_transactions: Vec = self + .connection + .fetch_collect(GetCardanoTransactionQuery::by_block_ranges(vec![ + block_range, + ]))?; + transactions.extend(block_range_transactions); + } + + Ok(transactions) } /// Prune the transactions older than the given number of blocks (based on the block range root diff --git a/mithril-aggregator/src/services/prover.rs b/mithril-aggregator/src/services/prover.rs index 7356f965b4a..c11fc32427f 100644 --- a/mithril-aggregator/src/services/prover.rs +++ b/mithril-aggregator/src/services/prover.rs @@ -99,14 +99,10 @@ impl MithrilProverService { block_ranges: &[BlockRange], ) -> StdResult>> { let mut block_ranges_map = HashMap::new(); - let mut transactions = vec![]; - for block_range in block_ranges { - let block_range_transactions = self - .transaction_retriever - .get_by_block_ranges(vec![block_range.clone()]) - .await?; - transactions.extend(block_range_transactions); - } + let transactions = self + .transaction_retriever + .get_by_block_ranges(block_ranges.to_vec()) + .await?; for transaction in transactions { let block_range = BlockRange::from_block_number(transaction.block_number); let block_range_transactions: &mut Vec<_> = @@ -387,15 +383,12 @@ mod tests { .return_once(move |_| Ok(transactions_to_prove)); let block_ranges_to_prove = test_data.block_ranges_to_prove.clone(); - for block_range_to_prove in block_ranges_to_prove { - let block_ranges = vec![block_range_to_prove.clone()]; - let transactions_in_block_range_to_prove = - test_data.block_ranges_map[&block_range_to_prove].clone(); - transaction_retriever_mock - .expect_get_by_block_ranges() - .with(eq(block_ranges)) - .return_once(move |_| Ok(transactions_in_block_range_to_prove)); - } + let all_transactions_in_block_ranges_to_prove = + test_data.all_transactions_in_block_ranges_to_prove.clone(); + transaction_retriever_mock + .expect_get_by_block_ranges() + .with(eq(block_ranges_to_prove)) + .return_once(move |_| Ok(all_transactions_in_block_ranges_to_prove)); }, |block_range_root_retriever_mock| { let block_ranges_map = test_data.block_ranges_map.clone(); @@ -444,15 +437,12 @@ mod tests { .return_once(move |_| Ok(transactions_to_prove)); let block_ranges_to_prove = test_data.block_ranges_to_prove.clone(); - for block_range_to_prove in block_ranges_to_prove { - let block_ranges = vec![block_range_to_prove.clone()]; - let transactions_in_block_range_to_prove = - test_data.block_ranges_map[&block_range_to_prove].clone(); - transaction_retriever_mock - .expect_get_by_block_ranges() - .with(eq(block_ranges)) - .return_once(move |_| Ok(transactions_in_block_range_to_prove)); - } + let all_transactions_in_block_ranges_to_prove = + test_data.all_transactions_in_block_ranges_to_prove.clone(); + transaction_retriever_mock + .expect_get_by_block_ranges() + .with(eq(block_ranges_to_prove)) + .return_once(move |_| Ok(all_transactions_in_block_ranges_to_prove)); }, |block_range_root_retriever_mock| { let block_ranges_map = test_data.block_ranges_map.clone(); @@ -504,15 +494,12 @@ mod tests { .return_once(move |_| Ok(transactions_to_prove)); let block_ranges_to_prove = test_data.block_ranges_to_prove.clone(); - for block_range_to_prove in block_ranges_to_prove { - let block_ranges = vec![block_range_to_prove.clone()]; - let transactions_in_block_range_to_prove = - test_data.block_ranges_map[&block_range_to_prove].clone(); - transaction_retriever_mock - .expect_get_by_block_ranges() - .with(eq(block_ranges)) - .return_once(move |_| Ok(transactions_in_block_range_to_prove)); - } + let all_transactions_in_block_ranges_to_prove = + test_data.all_transactions_in_block_ranges_to_prove.clone(); + transaction_retriever_mock + .expect_get_by_block_ranges() + .with(eq(block_ranges_to_prove)) + .return_once(move |_| Ok(all_transactions_in_block_ranges_to_prove)); }, |block_range_root_retriever_mock| { let block_ranges_map = test_data.block_ranges_map.clone(); @@ -589,16 +576,11 @@ mod tests { .expect_get_by_hashes() .return_once(move |_| Ok(transactions_to_prove)); - let block_ranges_to_prove = test_data.block_ranges_to_prove.clone(); - for block_range_to_prove in block_ranges_to_prove { - let block_ranges = vec![block_range_to_prove.clone()]; - let transactions_in_block_range_to_prove = - test_data.block_ranges_map[&block_range_to_prove].clone(); - transaction_retriever_mock - .expect_get_by_block_ranges() - .with(eq(block_ranges)) - .return_once(move |_| Ok(transactions_in_block_range_to_prove)); - } + let all_transactions_in_block_ranges_to_prove = + test_data.all_transactions_in_block_ranges_to_prove.clone(); + transaction_retriever_mock + .expect_get_by_block_ranges() + .return_once(move |_| Ok(all_transactions_in_block_ranges_to_prove)); }, |block_range_root_retriever_mock| { block_range_root_retriever_mock From c188939d983fe357cbe4552deb55c8afff206620 Mon Sep 17 00:00:00 2001 From: Jean-Philippe Raynaud Date: Tue, 4 Jun 2024 09:56:54 +0200 Subject: [PATCH 3/3] chore: bump crates versions - 'mithril-aggregator' from '0.5.13' to '0.5.14' - 'mithril-persistence' from '0.2.0' to '0.2.1'. --- Cargo.lock | 4 ++-- internal/mithril-persistence/Cargo.toml | 2 +- mithril-aggregator/Cargo.toml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e126f16035f..95f915c1606 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3452,7 +3452,7 @@ dependencies = [ [[package]] name = "mithril-aggregator" -version = "0.5.13" +version = "0.5.14" dependencies = [ "anyhow", "async-trait", @@ -3706,7 +3706,7 @@ dependencies = [ [[package]] name = "mithril-persistence" -version = "0.2.0" +version = "0.2.1" dependencies = [ "anyhow", "async-trait", diff --git a/internal/mithril-persistence/Cargo.toml b/internal/mithril-persistence/Cargo.toml index a83e2cde432..caf2f3395e2 100644 --- a/internal/mithril-persistence/Cargo.toml +++ b/internal/mithril-persistence/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-persistence" -version = "0.2.0" +version = "0.2.1" description = "Common types, interfaces, and utilities to persist data for Mithril nodes." authors = { workspace = true } edition = { workspace = true } diff --git a/mithril-aggregator/Cargo.toml b/mithril-aggregator/Cargo.toml index 982f101a79f..dd87c322a64 100644 --- a/mithril-aggregator/Cargo.toml +++ b/mithril-aggregator/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "mithril-aggregator" -version = "0.5.13" +version = "0.5.14" description = "A Mithril Aggregator server" authors = { workspace = true } edition = { workspace = true }