Skip to content

Commit

Permalink
Merge branch 'master' into aszepieniec/relativize-genesis-block
Browse files Browse the repository at this point in the history
  • Loading branch information
dan-da committed Apr 9, 2024
2 parents 990cf77 + a40aee0 commit 7de6c67
Show file tree
Hide file tree
Showing 90 changed files with 11,228 additions and 2,923 deletions.
671 changes: 276 additions & 395 deletions Cargo.lock

Large diffs are not rendered by default.

109 changes: 70 additions & 39 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -7,60 +7,71 @@ default-run = "neptune-core"
publish = false

[dependencies]
aead = "0"
aes-gcm = "0"
anyhow = "1"
arbitrary = { version = "1", features = ["derive"] }
bech32 = "0"
bincode = "1"
bytes = "1"
bytesize = "1"
chrono = "^0.4.31"
clap = { version = "4", features = ["derive"] }
clap_complete = "4.4.6"
console-subscriber = "0.2.0"
crossterm = "0"
directories = "5"
field_count = "0.1.1"
futures = "0"
get-size = { version = "0", features = ["derive"] }
aead = "0.5"
aes-gcm = "0.10"
anyhow = "1.0"
arbitrary = { version = "1.3", features = ["derive"] }
bech32 = "0.9"
bincode = "1.3"
bytes = "1.6"
bytesize = "1.3"
chrono = "=0.4.34"
clap = { version = "4.5", features = ["derive"] }
clap_complete = "4.4"
console-subscriber = "0.2"
crossterm = "0.27"
directories = "5.0"
field_count = "0.1"
futures = "0.3"
get-size = { version = "0.1", features = ["derive"] }
itertools = "0.11"
memmap2 = "0.9"
num-bigint = { version = "0", features = ["serde"] }
num-rational = "0"
num-traits = "0"
priority-queue = "1"
num-bigint = { version = "0.4", features = ["serde"] }
num-rational = "0.4"
num-traits = "0.2"
priority-queue = "1.4"
proptest = "1.4"
proptest-arbitrary-interop = "0.1"
rand = "0.8"
ratatui = "0.23"
regex = "1.10.3"
semver = "^1.0.21"
serde = { version = "1", features = ["derive"] }
serde_derive = "1"
serde_json = "1"
serde = { version = "1.0", features = ["derive"] }
serde_derive = "1.0"
serde_json = "1.0"
strum = { version = "0.25", features = ["derive"] }
tarpc = { version = "^0.34", features = ["tokio1", "serde-transport", "serde-transport-json", "tcp"] }
tasm-lib = "0.2.1"
tiny-bip39 = "1.0.0"
tokio = { version = "1", features = ["full", "tracing"] }
tokio-serde = { version = "0", features = ["bincode", "json"] }
tokio-util = { version = "0", features = ["codec"] }
tracing = "0"
tracing-subscriber = { version = "0", features = [
tarpc = { version = "^0.34", features = [
"tokio1",
"serde-transport",
"serde-transport-json",
"tcp",
] }
tasm-lib = "0.2"
tiny-bip39 = "1.0"
tokio = { version = "1.37", features = ["full", "tracing"] }
tokio-serde = { version = "0.8", features = ["bincode", "json"] }
tokio-util = { version = "0.7", features = ["codec"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3", features = [
"std",
"env-filter",
"time",
"fmt",
] }
tracing-test = "0"
unicode-width = "0"
tracing-test = "0.2"
unicode-width = "0.1"
zeroize = "1.7.0"
rs-leveldb = "0.1.5"
leveldb-sys = "2.0.9"
async-trait = "0.1.77"
async-stream = "0.3.5"

[dev-dependencies]
test-strategy = "0.3"
pin-project-lite = "0.2.13"
tokio-test = "0"
tokio-test = "0.4"
blake3 = "1.5.1"
divan = "0.1.14"

[dev-dependencies.cargo-husky]
default-features = false
Expand Down Expand Up @@ -90,8 +101,28 @@ opt-level = 0
# codegen-units = 256
# rpath = false

## We use harness = false on these so that the divan reports are output on stdout.

[[bench]]
name = "sync_atomic"
harness = false

[[bench]]
name = "db_leveldb"
harness = false

[[bench]]
name = "db_dbtvec"
harness = false

[[bench]]
name = "archival_mmr"
harness = false


[patch.crates-io]
# # rev = "f711ae27d1402d733989624bbadd59b7e82a1972" is tip of tasm-lib master as of 2024-01-25
# tasm-lib = { git = "https://github.com/TritonVM/tasm-lib.git", rev = "f711ae27d1402d733989624bbadd59b7e82a1972" }
# rev = "928b1fa7522aacdb055c4c04d144b44af241f2e6" is tip of tasm-lib master as of 2024-02-27
tasm-lib = { git = "https://github.com/TritonVM/tasm-lib.git", rev = "928b1fa7522aacdb055c4c04d144b44af241f2e6" }
# 15a708cf6fcbd9ed3e65e5e2067be6e622176328 is tip of branch: make_storage_async as of 2024-03-18
tasm-lib = { git = "https://github.com/dan-da/tasm-lib.git", rev = "15a708cf6fcbd9ed3e65e5e2067be6e622176328" }

# 81573735e9df4836ca16655fd31884271984bac7 = tip of branch: make_storage_async on 2024-03-18
twenty-first = { git = "https://github.com/Neptune-Crypto/twenty-first.git", rev = "81573735e9df4836ca16655fd31884271984bac7" }
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ run:
test: export RUST_BACKTRACE = 1
test:
$(info RUSTFLAGS is $(RUSTFLAGS))
cargo test -- --test-threads=1
cargo nextest r

bench:
$(info RUSTFLAGS is $(RUSTFLAGS))
Expand Down
246 changes: 246 additions & 0 deletions benches/archival_mmr.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,246 @@
use divan::Bencher;
use leveldb::options::Options;
use leveldb::options::ReadOptions;
use leveldb::options::WriteOptions;
use leveldb_sys::Compression;
use neptune_core::database::storage::storage_schema::traits::*;
use neptune_core::database::storage::storage_schema::DbtVec;
use neptune_core::database::storage::storage_schema::SimpleRustyStorage;
use neptune_core::database::storage::storage_vec::traits::StorageVecBase;
use neptune_core::database::NeptuneLevelDb;
use neptune_core::util_types::mutator_set::archival_mmr::ArchivalMmr;
use rand::random;
use tasm_lib::twenty_first::shared_math::tip5::Tip5;
use tasm_lib::twenty_first::util_types::mmr::shared_advanced::leaf_count_to_node_count;
use tasm_lib::Digest;

fn main() {
divan::main();
}

/// These settings affect DB performance and correctness.
///
/// Adjust and re-run the benchmarks to see effects.
///
/// Rust docs: (basic)
/// https://docs.rs/rs-leveldb/0.1.5/leveldb/database/options/struct.Options.html
///
/// C++ docs: (complete)
/// https://github.com/google/leveldb/blob/068d5ee1a3ac40dabd00d211d5013af44be55bea/include/leveldb/options.h
fn db_options() -> Option<Options> {
Some(Options {
// default: false
create_if_missing: true,

// default: false
error_if_exists: true,

// default: false
paranoid_checks: false,

// default: None --> (4 * 1024 * 1024)
write_buffer_size: None,

// default: None --> 1000
max_open_files: None,

// default: None --> 4 * 1024
block_size: None,

// default: None --> 16
block_restart_interval: None,

// default: Compression::No
// or: Compression::Snappy
compression: Compression::No,

// default: None --> 8MB
cache: None,
// cache: Some(Cache::new(1024)),
// note: tests put 128 bytes in each entry.
// 100 entries = 12,800 bytes.
// So Cache of 1024 bytes is 8% of total data set.
// that seems reasonably realistic to get some
// hits/misses.
})
}

async fn new_ammr(leaf_count: u64) -> (SimpleRustyStorage, ArchivalMmr<Tip5, DbtVec<Digest>>) {
let db = NeptuneLevelDb::open_new_test_database(
false,
db_options(),
Some(ReadOptions {
verify_checksums: false,
fill_cache: false,
}),
Some(WriteOptions { sync: true }),
)
.await
.unwrap();
let mut rusty_storage = SimpleRustyStorage::new(db);
let mut nv = rusty_storage
.schema
.new_vec::<Digest>("test-archival-mmr")
.await;

// Add the dummy node since nodes are 1-indexed in AMMRs.
nv.push(Digest::default()).await;

let num_nodes = leaf_count_to_node_count(leaf_count);
for _ in 0..num_nodes {
nv.push(random()).await;
}

(rusty_storage, ArchivalMmr::new(nv).await)
}

mod append {
use super::*;

mod append_5000 {
const NUM_WRITE_ITEMS: usize = 5000;
const INIT_AMMR_LEAF_COUNT: u64 = 0;
use tasm_lib::twenty_first::shared_math::other::random_elements;

use super::*;

fn append_impl(bencher: Bencher, persist: bool) {
let rt = tokio::runtime::Runtime::new().unwrap();
let (mut storage, mut ammr) = rt.block_on(new_ammr(INIT_AMMR_LEAF_COUNT));
let digests = random_elements(NUM_WRITE_ITEMS);

bencher.bench_local(|| {
rt.block_on(async {
for new_leaf in digests.iter() {
ammr.append(*new_leaf).await;
}
if persist {
storage.persist().await;
}
});
});
}

#[divan::bench]
fn append(bencher: Bencher) {
append_impl(bencher, false);
}

#[divan::bench]
fn append_and_persist(bencher: Bencher) {
append_impl(bencher, true);
}
}
}

mod mutate {
use super::*;

mod mutate_100_of_10000 {
const NUM_MUTATIONS: usize = 100;
const AMMR_LEAF_COUNT: u64 = 10000;
use itertools::Itertools;
use rand::{thread_rng, Rng};
use tasm_lib::twenty_first::shared_math::other::random_elements;

use super::*;

fn leaf_mutation_impl(bencher: Bencher, persist: bool) {
let rt = tokio::runtime::Runtime::new().unwrap();
let (mut storage, mut ammr) = rt.block_on(new_ammr(AMMR_LEAF_COUNT));
let mut rng = thread_rng();
let digests = random_elements(NUM_MUTATIONS);
let leaf_index_of_mutated_leafs = (0..NUM_MUTATIONS as u64)
.map(|_| rng.gen_range(0..AMMR_LEAF_COUNT))
.collect_vec();

bencher.bench_local(|| {
rt.block_on(async {
for (new_leaf, leaf_index) in
digests.iter().zip(leaf_index_of_mutated_leafs.iter())
{
ammr.mutate_leaf(*leaf_index, *new_leaf).await;
}
if persist {
storage.persist().await;
}
});
});
}

#[divan::bench]
fn leaf_mutation(bencher: Bencher) {
leaf_mutation_impl(bencher, false);
}

#[divan::bench]
fn leaf_mutation_and_persist(bencher: Bencher) {
leaf_mutation_impl(bencher, true);
}
}
}

mod batch_mutate_leaf_and_update_mps {
use super::*;

mod mutate_100_of_10000 {
const NUM_MUTATIONS_IN_BATCH: usize = 100;
const AMMR_LEAF_COUNT: u64 = 10000;
use itertools::Itertools;
use rand::{thread_rng, Rng};
use tasm_lib::twenty_first::shared_math::other::random_elements;

use super::*;

fn batch_leaf_mutation_impl(bencher: Bencher, persist: bool) {
let rt = tokio::runtime::Runtime::new().unwrap();
let (mut storage, mut ammr) = rt.block_on(new_ammr(AMMR_LEAF_COUNT));
let mut rng = thread_rng();
let new_digests = random_elements(NUM_MUTATIONS_IN_BATCH);
let mut leaf_index_of_mutated_leafs = (0..NUM_MUTATIONS_IN_BATCH as u64)
.map(|_| rng.gen_range(0..AMMR_LEAF_COUNT))
.collect_vec();
leaf_index_of_mutated_leafs.sort();
leaf_index_of_mutated_leafs.dedup();

let mutation_data = leaf_index_of_mutated_leafs
.into_iter()
.zip(new_digests)
.collect_vec();

let mut leaf_indices_for_mps_to_preserve = (0..NUM_MUTATIONS_IN_BATCH as u64)
.map(|_| rng.gen_range(0..AMMR_LEAF_COUNT))
.collect_vec();
leaf_indices_for_mps_to_preserve.sort();
leaf_indices_for_mps_to_preserve.dedup();

let mut mps = leaf_indices_for_mps_to_preserve
.iter()
.map(|i| rt.block_on(async { ammr.prove_membership_async(*i).await }))
.collect_vec();

bencher.bench_local(|| {
rt.block_on(async {
ammr.batch_mutate_leaf_and_update_mps(
&mut mps.iter_mut().collect_vec(),
mutation_data.clone(),
)
.await;
if persist {
storage.persist().await;
}
});
});
}

#[divan::bench]
fn leaf_mutation(bencher: Bencher) {
batch_leaf_mutation_impl(bencher, false);
}

#[divan::bench]
fn leaf_mutation_and_persist(bencher: Bencher) {
batch_leaf_mutation_impl(bencher, true);
}
}
}
Loading

2 comments on commit 7de6c67

@Sword-Smith
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I prefer rebasing development branches on top of master. This makes the commit history confusing, I think.

@dan-da
Copy link
Collaborator Author

@dan-da dan-da commented on 7de6c67 Apr 9, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So do I and that is what I always do for my branches, and I also set the repo default to rebase instead of merge for PRs.

However in this case, I first needed to get changes from master into @aszepieniec branch. I could have rebased his branch on top of master, but felt that is more intrusive on his work as if he later did a git pull on his branch it would break. So I merged from master to his branch instead.

Later, when I went to merge his branch back into master, github informs that it cannot be rebased, but can be merged instead. So that's why there are two merge commits.

I don't plan to do this again, but anyway that's what happened.

Please sign in to comment.