Skip to content

Commit

Permalink
migrate to criterion for benching
Browse files Browse the repository at this point in the history
  • Loading branch information
RCasatta committed Aug 21, 2024
1 parent 94ae478 commit 035fd87
Show file tree
Hide file tree
Showing 8 changed files with 271 additions and 290 deletions.
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,5 +50,5 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: dtolnay/rust-toolchain@nightly
- run: RUSTFLAGS='--cfg=bench' cargo +nightly bench --all-features
- uses: dtolnay/rust-toolchain@stable
- run: cargo bench --all-features
6 changes: 6 additions & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,13 @@ hex_lit = { version = "0.1", features = ["rust_v_1_46"] }
bitcoin = { version = "0.32.0", features = ["rand"] }
bitcoin-test-data = "0.2.0"
tempfile = "3.4.0"
criterion = "0.3"

[package.metadata.docs.rs]
all-features = true
rustdoc-args = ["--cfg", "docsrs"]

[[bench]]
name = "benches"
harness = false
required-features = ["bitcoin_hashes", "bitcoin", "sha2"]
52 changes: 33 additions & 19 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -120,31 +120,45 @@ cargo test

## Bench

[Criterion](https://bheisler.github.io/criterion.rs/book/) is used for benching mainly because we are not required to use nightly, and it is also nicer.

```sh
cargo bench --all-features
```

To have compact results, similar to standard libtests launch with:

```sh
RUSTFLAGS='--cfg=bench' cargo +nightly bench --all-features
cargo bench --bench benches --all-features -- --output-format bencher
```

```sh
test bsl::block::bench::block_deserialize ... bench: 289,421 ns/iter (+/- 46,179)
test bsl::block::bench::block_deserialize_bitcoin ... bench: 2,719,666 ns/iter (+/- 459,186)
test bsl::block::bench::block_sum_outputs ... bench: 288,248 ns/iter (+/- 39,013)
test bsl::block::bench::block_sum_outputs_bitcoin ... bench: 2,607,791 ns/iter (+/- 321,212)
test bsl::block::bench::find_tx ... bench: 1,012,297 ns/iter (+/- 6,278)
test bsl::block::bench::find_tx_bitcoin ... bench: 8,632,416 ns/iter (+/- 89,751)
test bsl::block::bench::hash_block_txs ... bench: 8,406,341 ns/iter (+/- 938,119)
test bsl::block::bench::hash_block_txs_bitcoin ... bench: 11,843,590 ns/iter (+/- 1,052,109)
test bsl::block::bench::hash_block_txs_sha2 ... bench: 7,891,956 ns/iter (+/- 1,047,439)
test bsl::block_header::bench::block_hash ... bench: 1,399 ns/iter (+/- 205)
test bsl::block_header::bench::block_hash_bitcoin ... bench: 1,510 ns/iter (+/- 222)
test bsl::transaction::bench::tx_deserialize ... bench: 38 ns/iter (+/- 8)
test bsl::transaction::bench::tx_deserialize_bitcoin ... bench: 219 ns/iter (+/- 30)
test bsl::transaction::bench::txid ... bench: 2,185 ns/iter (+/- 166)
test bsl::transaction::bench::txid_bitcoin ... bench: 2,416 ns/iter (+/- 213)
test bsl::transaction::bench::txid_sha2 ... bench: 2,085 ns/iter (+/- 216)
test tx_deserialize/slices ... bench: 29 ns/iter (+/- 0)
test tx_deserialize/bitcoin ... bench: 211 ns/iter (+/- 0)

test tx_id/slices_bitcoin_hashes ... bench: 183 ns/iter (+/- 0)
test tx_id/slices_sha2 ... bench: 158 ns/iter (+/- 0)
test tx_id/bitcoin ... bench: 234 ns/iter (+/- 1)

test block_deserialize/slices ... bench: 230872 ns/iter (+/- 1686)
test block_deserialize/bitcoin ... bench: 1462784 ns/iter (+/- 115792)

test block_sum_outputs/slices ... bench: 235757 ns/iter (+/- 1318)
test block_sum_outputs/bitcoin ... bench: 1459730 ns/iter (+/- 95817)

test hash_block_txs/slices ... bench: 881940 ns/iter (+/- 4961)
test hash_block_txs/slices_sha2 ... bench: 789365 ns/iter (+/- 932)
test hash_block_txs/bitcoin ... bench: 2301561 ns/iter (+/- 15406)

test find_tx/slices ... bench: 406519 ns/iter (+/- 1423)
test find_tx/bitcoin ... bench: 1826147 ns/iter (+/- 122216)

test block_hash/slices ... bench: 112 ns/iter (+/- 0)
test block_hash/bitcoin ... bench: 146 ns/iter (+/- 2)
```

* benches ending with `_bitcoin` use `rust-bitcoin`
* benches ending with `_sha2` use `sha2` lib instead of `bitcoin_hashes`
* benches variants with `/bitcoin` use `rust-bitcoin`
* benches ending with `/slices_sha2` use this lib and `sha2` lib instead of `bitcoin_hashes`

### Comparison against rust-bitcoin

Expand Down
229 changes: 229 additions & 0 deletions benches/benches.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,229 @@
use bitcoin::consensus::deserialize;
use bitcoin_hashes::sha256d;
use bitcoin_slices::bsl::{Block, BlockHeader, FindTransaction, Transaction, TxOut};
use bitcoin_slices::{Parse, Visit, Visitor};
use bitcoin_test_data::blocks::mainnet_702861;
use core::ops::ControlFlow;
use criterion::{black_box, criterion_group, criterion_main, Criterion};
use hex_lit::hex;
use std::str::FromStr;

const BENCH_TX: [u8; 193] = hex!("0100000001a15d57094aa7a21a28cb20b59aab8fc7d1149a3bdbcddba9c622e4f5f6a99ece010000006c493046022100f93bb0e7d8db7bd46e40132d1f8242026e045f03a0efe71bbb8e3f475e970d790221009337cd7f1f929f00cc6ff01f03729b069a7c21b59b1736ddfee5db5946c5da8c0121033b9b137ee87d5a812d6f506efdd37f0affa7ffc310711c06c7f3e097c9447c52ffffffff0100e1f505000000001976a9140389035a9225b3839e2bbf32d826a1e222031fd888ac00000000");
const GENESIS_BLOCK_HEADER: [u8; 80] = hex!("0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c");

criterion_group!(
benches,
tx_deserialize,
tx_id,
block_deserialize,
block_sum_outputs,
hash_block_txs,
find_tx,
block_hash
);
criterion_main!(benches);

pub fn tx_deserialize(c: &mut Criterion) {
c.benchmark_group("tx_deserialize")
.throughput(criterion::Throughput::Bytes(BENCH_TX.len() as u64))
.bench_function("slices", |b| {
b.iter(|| {
let tx = Transaction::parse(&BENCH_TX[..]).unwrap().parsed_owned();
black_box(&tx);
})
})
.bench_function("bitcoin", |b| {
b.iter(|| {
let tx: bitcoin::Transaction = deserialize(&BENCH_TX).unwrap();
black_box(&tx);
})
});
}

pub fn tx_id(c: &mut Criterion) {
c.benchmark_group("tx_id")
.throughput(criterion::Throughput::Bytes(BENCH_TX.len() as u64))
.bench_function("slices_bitcoin_hashes", |b| {
let tx = Transaction::parse(&BENCH_TX[..]).unwrap().parsed_owned();
b.iter(|| {
black_box(tx.txid());
})
})
.bench_function("slices_sha2", |b| {
let tx = Transaction::parse(&BENCH_TX[..]).unwrap().parsed_owned();
b.iter(|| {
black_box(tx.txid_sha2());
})
})
.bench_function("bitcoin", |b| {
let tx: bitcoin::Transaction = deserialize(&BENCH_TX[..]).unwrap();
b.iter(|| {
black_box(tx.compute_txid());
})
});
}

pub fn block_deserialize(c: &mut Criterion) {
c.benchmark_group("block_deserialize")
.throughput(criterion::Throughput::Bytes(mainnet_702861().len() as u64))
.bench_function("slices", |b| {
b.iter(|| {
let block = Block::parse(mainnet_702861()).unwrap();
black_box(&block);
})
})
.bench_function("bitcoin", |b| {
b.iter(|| {
let block: bitcoin::Block = deserialize(mainnet_702861()).unwrap();
black_box(&block);
})
});
}

pub fn block_sum_outputs(c: &mut Criterion) {
c.benchmark_group("block_sum_outputs")
.throughput(criterion::Throughput::Bytes(mainnet_702861().len() as u64))
.bench_function("slices", |b| {
b.iter(|| {
struct Sum(u64);
impl Visitor for Sum {
fn visit_tx_out(&mut self, _vout: usize, tx_out: &TxOut) -> ControlFlow<()> {
self.0 += tx_out.value();
ControlFlow::Continue(())
}
}
let mut sum = Sum(0);
let block = Block::visit(mainnet_702861(), &mut sum).unwrap();
assert_eq!(sum.0, 2883682728990);
black_box(&block);
})
})
.bench_function("bitcoin", |b| {
b.iter(|| {
let block: bitcoin::Block = deserialize(mainnet_702861()).unwrap();
let sum: u64 = block
.txdata
.iter()
.flat_map(|t| t.output.iter())
.fold(0, |acc, e| acc + e.value.to_sat());
assert_eq!(sum, 2883682728990);

black_box(&block);
})
});
}

pub fn hash_block_txs(c: &mut Criterion) {
c.benchmark_group("hash_block_txs")
.throughput(criterion::Throughput::Bytes(mainnet_702861().len() as u64))
.bench_function("slices", |b| {
b.iter(|| {
struct VisitTx(Vec<sha256d::Hash>);
let mut v = VisitTx(vec![]);
impl Visitor for VisitTx {
fn visit_block_begin(&mut self, total_transactions: usize) {
self.0.reserve(total_transactions);
}
fn visit_transaction(&mut self, tx: &Transaction) -> ControlFlow<()> {
self.0.push(tx.txid());
ControlFlow::Continue(())
}
}

let block = Block::visit(mainnet_702861(), &mut v).unwrap();

assert_eq!(v.0.len(), 2500);

black_box((&block, v));
})
})
.bench_function("slices_sha2", |b| {
b.iter(|| {
struct VisitTx(
Vec<sha2::digest::generic_array::GenericArray<u8, sha2::digest::typenum::U32>>,
);
let mut v = VisitTx(vec![]);
impl Visitor for VisitTx {
fn visit_block_begin(&mut self, total_transactions: usize) {
self.0.reserve(total_transactions);
}
fn visit_transaction(&mut self, tx: &Transaction) -> ControlFlow<()> {
self.0.push(tx.txid_sha2());
ControlFlow::Continue(())
}
}

let block = Block::visit(mainnet_702861(), &mut v).unwrap();

assert_eq!(v.0.len(), 2500);

black_box((&block, v));
})
})
.bench_function("bitcoin", |b| {
b.iter(|| {
let block: bitcoin::Block = deserialize(mainnet_702861()).unwrap();
let mut tx_hashes = Vec::with_capacity(block.txdata.len());

for tx in block.txdata.iter() {
tx_hashes.push(tx.compute_txid())
}
assert_eq!(tx_hashes.len(), 2500);
black_box((&block, tx_hashes));
})
});
}

const TXID: &str = "416a5f96cb63e7649f6f272e7f82a43a97bcf6cfc46184c733344de96ff1e433";
pub fn find_tx(c: &mut Criterion) {
c.benchmark_group("find_tx")
.throughput(criterion::Throughput::Bytes(mainnet_702861().len() as u64))
.bench_function("slices", |b| {
let txid = bitcoin::Txid::from_str(TXID).unwrap();
b.iter(|| {
let mut visitor = FindTransaction::new(txid.clone());
let _ = Block::visit(&mainnet_702861(), &mut visitor);
let tx = visitor.tx_found().unwrap();
assert_eq!(tx.compute_txid(), txid);
core::hint::black_box(tx);
})
})
.bench_function("bitcoin", |b| {
let txid = bitcoin::Txid::from_str(TXID).unwrap();

b.iter(|| {
let block: bitcoin::Block = deserialize(mainnet_702861()).unwrap();
let mut tx = None;
for current in block.txdata {
if current.compute_txid() == txid {
tx = Some(current);
break;
}
}
let tx = tx.unwrap();
assert_eq!(tx.compute_txid(), txid);
core::hint::black_box(&tx);
})
});
}

pub fn block_hash(c: &mut Criterion) {
c.benchmark_group("block_hash")
.bench_function("slices", |b| {
let block_header = BlockHeader::parse(&GENESIS_BLOCK_HEADER)
.unwrap()
.parsed_owned();
b.iter(|| {
let hash = block_header.block_hash();
black_box(&hash);
})
})
.bench_function("bitcoin", |b| {
let block_header: bitcoin::blockdata::block::Header =
deserialize(&GENESIS_BLOCK_HEADER).unwrap();
b.iter(|| {
let hash = block_header.block_hash();
black_box(&hash);
})
});
}
Loading

0 comments on commit 035fd87

Please sign in to comment.