-
Notifications
You must be signed in to change notification settings - Fork 7
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge branch 'master' into aszepieniec/relativize-genesis-block
- Loading branch information
Showing
90 changed files
with
11,228 additions
and
2,923 deletions.
There are no files selected for viewing
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,246 @@ | ||
| use divan::Bencher; | ||
| use leveldb::options::Options; | ||
| use leveldb::options::ReadOptions; | ||
| use leveldb::options::WriteOptions; | ||
| use leveldb_sys::Compression; | ||
| use neptune_core::database::storage::storage_schema::traits::*; | ||
| use neptune_core::database::storage::storage_schema::DbtVec; | ||
| use neptune_core::database::storage::storage_schema::SimpleRustyStorage; | ||
| use neptune_core::database::storage::storage_vec::traits::StorageVecBase; | ||
| use neptune_core::database::NeptuneLevelDb; | ||
| use neptune_core::util_types::mutator_set::archival_mmr::ArchivalMmr; | ||
| use rand::random; | ||
| use tasm_lib::twenty_first::shared_math::tip5::Tip5; | ||
| use tasm_lib::twenty_first::util_types::mmr::shared_advanced::leaf_count_to_node_count; | ||
| use tasm_lib::Digest; | ||
|
|
||
| fn main() { | ||
| divan::main(); | ||
| } | ||
|
|
||
| /// These settings affect DB performance and correctness. | ||
| /// | ||
| /// Adjust and re-run the benchmarks to see effects. | ||
| /// | ||
| /// Rust docs: (basic) | ||
| /// https://docs.rs/rs-leveldb/0.1.5/leveldb/database/options/struct.Options.html | ||
| /// | ||
| /// C++ docs: (complete) | ||
| /// https://github.com/google/leveldb/blob/068d5ee1a3ac40dabd00d211d5013af44be55bea/include/leveldb/options.h | ||
| fn db_options() -> Option<Options> { | ||
| Some(Options { | ||
| // default: false | ||
| create_if_missing: true, | ||
|
|
||
| // default: false | ||
| error_if_exists: true, | ||
|
|
||
| // default: false | ||
| paranoid_checks: false, | ||
|
|
||
| // default: None --> (4 * 1024 * 1024) | ||
| write_buffer_size: None, | ||
|
|
||
| // default: None --> 1000 | ||
| max_open_files: None, | ||
|
|
||
| // default: None --> 4 * 1024 | ||
| block_size: None, | ||
|
|
||
| // default: None --> 16 | ||
| block_restart_interval: None, | ||
|
|
||
| // default: Compression::No | ||
| // or: Compression::Snappy | ||
| compression: Compression::No, | ||
|
|
||
| // default: None --> 8MB | ||
| cache: None, | ||
| // cache: Some(Cache::new(1024)), | ||
| // note: tests put 128 bytes in each entry. | ||
| // 100 entries = 12,800 bytes. | ||
| // So Cache of 1024 bytes is 8% of total data set. | ||
| // that seems reasonably realistic to get some | ||
| // hits/misses. | ||
| }) | ||
| } | ||
|
|
||
| async fn new_ammr(leaf_count: u64) -> (SimpleRustyStorage, ArchivalMmr<Tip5, DbtVec<Digest>>) { | ||
| let db = NeptuneLevelDb::open_new_test_database( | ||
| false, | ||
| db_options(), | ||
| Some(ReadOptions { | ||
| verify_checksums: false, | ||
| fill_cache: false, | ||
| }), | ||
| Some(WriteOptions { sync: true }), | ||
| ) | ||
| .await | ||
| .unwrap(); | ||
| let mut rusty_storage = SimpleRustyStorage::new(db); | ||
| let mut nv = rusty_storage | ||
| .schema | ||
| .new_vec::<Digest>("test-archival-mmr") | ||
| .await; | ||
|
|
||
| // Add the dummy node since nodes are 1-indexed in AMMRs. | ||
| nv.push(Digest::default()).await; | ||
|
|
||
| let num_nodes = leaf_count_to_node_count(leaf_count); | ||
| for _ in 0..num_nodes { | ||
| nv.push(random()).await; | ||
| } | ||
|
|
||
| (rusty_storage, ArchivalMmr::new(nv).await) | ||
| } | ||
|
|
||
| mod append { | ||
| use super::*; | ||
|
|
||
| mod append_5000 { | ||
| const NUM_WRITE_ITEMS: usize = 5000; | ||
| const INIT_AMMR_LEAF_COUNT: u64 = 0; | ||
| use tasm_lib::twenty_first::shared_math::other::random_elements; | ||
|
|
||
| use super::*; | ||
|
|
||
| fn append_impl(bencher: Bencher, persist: bool) { | ||
| let rt = tokio::runtime::Runtime::new().unwrap(); | ||
| let (mut storage, mut ammr) = rt.block_on(new_ammr(INIT_AMMR_LEAF_COUNT)); | ||
| let digests = random_elements(NUM_WRITE_ITEMS); | ||
|
|
||
| bencher.bench_local(|| { | ||
| rt.block_on(async { | ||
| for new_leaf in digests.iter() { | ||
| ammr.append(*new_leaf).await; | ||
| } | ||
| if persist { | ||
| storage.persist().await; | ||
| } | ||
| }); | ||
| }); | ||
| } | ||
|
|
||
| #[divan::bench] | ||
| fn append(bencher: Bencher) { | ||
| append_impl(bencher, false); | ||
| } | ||
|
|
||
| #[divan::bench] | ||
| fn append_and_persist(bencher: Bencher) { | ||
| append_impl(bencher, true); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| mod mutate { | ||
| use super::*; | ||
|
|
||
| mod mutate_100_of_10000 { | ||
| const NUM_MUTATIONS: usize = 100; | ||
| const AMMR_LEAF_COUNT: u64 = 10000; | ||
| use itertools::Itertools; | ||
| use rand::{thread_rng, Rng}; | ||
| use tasm_lib::twenty_first::shared_math::other::random_elements; | ||
|
|
||
| use super::*; | ||
|
|
||
| fn leaf_mutation_impl(bencher: Bencher, persist: bool) { | ||
| let rt = tokio::runtime::Runtime::new().unwrap(); | ||
| let (mut storage, mut ammr) = rt.block_on(new_ammr(AMMR_LEAF_COUNT)); | ||
| let mut rng = thread_rng(); | ||
| let digests = random_elements(NUM_MUTATIONS); | ||
| let leaf_index_of_mutated_leafs = (0..NUM_MUTATIONS as u64) | ||
| .map(|_| rng.gen_range(0..AMMR_LEAF_COUNT)) | ||
| .collect_vec(); | ||
|
|
||
| bencher.bench_local(|| { | ||
| rt.block_on(async { | ||
| for (new_leaf, leaf_index) in | ||
| digests.iter().zip(leaf_index_of_mutated_leafs.iter()) | ||
| { | ||
| ammr.mutate_leaf(*leaf_index, *new_leaf).await; | ||
| } | ||
| if persist { | ||
| storage.persist().await; | ||
| } | ||
| }); | ||
| }); | ||
| } | ||
|
|
||
| #[divan::bench] | ||
| fn leaf_mutation(bencher: Bencher) { | ||
| leaf_mutation_impl(bencher, false); | ||
| } | ||
|
|
||
| #[divan::bench] | ||
| fn leaf_mutation_and_persist(bencher: Bencher) { | ||
| leaf_mutation_impl(bencher, true); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| mod batch_mutate_leaf_and_update_mps { | ||
| use super::*; | ||
|
|
||
| mod mutate_100_of_10000 { | ||
| const NUM_MUTATIONS_IN_BATCH: usize = 100; | ||
| const AMMR_LEAF_COUNT: u64 = 10000; | ||
| use itertools::Itertools; | ||
| use rand::{thread_rng, Rng}; | ||
| use tasm_lib::twenty_first::shared_math::other::random_elements; | ||
|
|
||
| use super::*; | ||
|
|
||
| fn batch_leaf_mutation_impl(bencher: Bencher, persist: bool) { | ||
| let rt = tokio::runtime::Runtime::new().unwrap(); | ||
| let (mut storage, mut ammr) = rt.block_on(new_ammr(AMMR_LEAF_COUNT)); | ||
| let mut rng = thread_rng(); | ||
| let new_digests = random_elements(NUM_MUTATIONS_IN_BATCH); | ||
| let mut leaf_index_of_mutated_leafs = (0..NUM_MUTATIONS_IN_BATCH as u64) | ||
| .map(|_| rng.gen_range(0..AMMR_LEAF_COUNT)) | ||
| .collect_vec(); | ||
| leaf_index_of_mutated_leafs.sort(); | ||
| leaf_index_of_mutated_leafs.dedup(); | ||
|
|
||
| let mutation_data = leaf_index_of_mutated_leafs | ||
| .into_iter() | ||
| .zip(new_digests) | ||
| .collect_vec(); | ||
|
|
||
| let mut leaf_indices_for_mps_to_preserve = (0..NUM_MUTATIONS_IN_BATCH as u64) | ||
| .map(|_| rng.gen_range(0..AMMR_LEAF_COUNT)) | ||
| .collect_vec(); | ||
| leaf_indices_for_mps_to_preserve.sort(); | ||
| leaf_indices_for_mps_to_preserve.dedup(); | ||
|
|
||
| let mut mps = leaf_indices_for_mps_to_preserve | ||
| .iter() | ||
| .map(|i| rt.block_on(async { ammr.prove_membership_async(*i).await })) | ||
| .collect_vec(); | ||
|
|
||
| bencher.bench_local(|| { | ||
| rt.block_on(async { | ||
| ammr.batch_mutate_leaf_and_update_mps( | ||
| &mut mps.iter_mut().collect_vec(), | ||
| mutation_data.clone(), | ||
| ) | ||
| .await; | ||
| if persist { | ||
| storage.persist().await; | ||
| } | ||
| }); | ||
| }); | ||
| } | ||
|
|
||
| #[divan::bench] | ||
| fn leaf_mutation(bencher: Bencher) { | ||
| batch_leaf_mutation_impl(bencher, false); | ||
| } | ||
|
|
||
| #[divan::bench] | ||
| fn leaf_mutation_and_persist(bencher: Bencher) { | ||
| batch_leaf_mutation_impl(bencher, true); | ||
| } | ||
| } | ||
| } |
Oops, something went wrong.
7de6c67There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I prefer rebasing development branches on top of
master. This makes the commit history confusing, I think.7de6c67There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
So do I and that is what I always do for my branches, and I also set the repo default to rebase instead of merge for PRs.
However in this case, I first needed to get changes from master into @aszepieniec branch. I could have rebased his branch on top of master, but felt that is more intrusive on his work as if he later did a git pull on his branch it would break. So I merged from master to his branch instead.
Later, when I went to merge his branch back into master, github informs that it cannot be rebased, but can be merged instead. So that's why there are two merge commits.
I don't plan to do this again, but anyway that's what happened.