Use CLI instead of binaries for configuring benchmarks (#357)

refactor: use CLI for benchmarks

README.md

@@ -49,7 +49,7 @@ $ cargo test
 *bench*
 ```bash
 $ cd bench
-$ cargo run --bin confirmation_slot ./bench-config.toml
+$ RUST_LOG=info cargo run -- --help
 ```
 
 Find a new file named `metrics.csv` in the project root.

bench/Cargo.toml

@@ -3,24 +3,8 @@ name = "bench"
 version = "0.2.4"
 edition = "2021"
 
-[lib]
-name = "bench_lib"
-path = "src/lib.rs"
-
-[[bin]]
-name = "api_load"
-path = "src/benches/api_load.rs"
-
-[[bin]]
-name = "confirmation_slot"
-path = "src/benches/confirmation_slot.rs"
-
-[[bin]]
-name = "confirmation_rate"
-path = "src/benches/confirmation_rate.rs"
-
-
 [dependencies]
+clap = { workspace = true }
 solana-sdk = { workspace = true }
 solana-rpc-client = { workspace = true }
 solana-transaction-status = { workspace = true }
@@ -39,7 +23,6 @@ dashmap = { workspace = true }
 bincode = { workspace = true }
 itertools = "0.10.5"
 spl-memo = "4.0.0"
-toml = "0.8.10"
 
 [dev-dependencies]
 bincode = { workspace = true }

bench/src/benches/api_load.rs

@@ -1,34 +1,32 @@
-use bench_lib::{config::BenchConfig, create_memo_tx_small};
 use log::{info, warn};
-use std::sync::{
-    atomic::{AtomicUsize, Ordering},
-    Arc,
+use std::{
+    path::Path,
+    sync::{
+        atomic::{AtomicUsize, Ordering},
+        Arc,
+    },
 };
 
 use solana_rpc_client::nonblocking::rpc_client::RpcClient;
 use solana_sdk::signature::{read_keypair_file, Keypair, Signer};
 
-// TC3 measure how much load the API endpoint can take
-#[tokio::main]
-async fn main() -> anyhow::Result<()> {
-    tracing_subscriber::fmt::init();
+use crate::create_memo_tx_small;
 
+// TC3 measure how much load the API endpoint can take
+pub async fn api_load(payer_path: &Path, rpc_url: String, time_ms: u64) -> anyhow::Result<()> {
     warn!("THIS IS WORK IN PROGRESS");
 
-    let config = BenchConfig::load().unwrap();
-
-    let rpc = Arc::new(RpcClient::new(config.lite_rpc_url.clone()));
+    let rpc = Arc::new(RpcClient::new(rpc_url));
     info!("RPC: {}", rpc.as_ref().url());
 
-    let payer: Arc<Keypair> = Arc::new(read_keypair_file(&config.payer_path).unwrap());
+    let payer: Arc<Keypair> = Arc::new(read_keypair_file(payer_path).unwrap());
     info!("Payer: {}", payer.pubkey().to_string());
 
     let mut txs = 0;
     let failed = Arc::new(AtomicUsize::new(0));
     let success = Arc::new(AtomicUsize::new(0));
 
     let hash = rpc.get_latest_blockhash().await?;
-    let time_ms = config.api_load.time_ms;
     let time = tokio::time::Instant::now();
 
     while time.elapsed().as_millis() < time_ms.into() {

bench/src/benches/confirmation_rate.rs

@@ -1,13 +1,13 @@
+use crate::tx_size::TxSize;
+use crate::{create_rng, generate_txs};
 use anyhow::{bail, Error};
-use bench_lib::config::{BenchConfig, ConfirmationRateConfig};
-use bench_lib::tx_size::TxSize;
-use bench_lib::{create_rng, generate_txs};
 use futures::future::join_all;
 use futures::TryFutureExt;
 use itertools::Itertools;
 use log::{debug, info, trace, warn};
 use std::collections::{HashMap, HashSet};
 use std::iter::zip;
+use std::path::Path;
 use std::sync::Arc;
 use std::time::Duration;
 
@@ -31,29 +31,25 @@ pub struct RpcStat {
 }
 
 /// TC2 send multiple runs of num_txns, measure the confirmation rate
-#[tokio::main]
-async fn main() -> anyhow::Result<()> {
-    tracing_subscriber::fmt::init();
-
+pub async fn confirmation_rate(
+    payer_path: &Path,
+    rpc_url: String,
+    tx_size: TxSize,
+    txns_per_round: usize,
+    num_rounds: usize,
+) -> anyhow::Result<()> {
     warn!("THIS IS WORK IN PROGRESS");
 
-    let config = BenchConfig::load().unwrap();
-    let ConfirmationRateConfig {
-        tx_size,
-        num_txns,
-        num_runs,
-    } = config.confirmation_rate;
-
-    let rpc = Arc::new(RpcClient::new(config.lite_rpc_url.clone()));
+    let rpc = Arc::new(RpcClient::new(rpc_url));
     info!("RPC: {}", rpc.as_ref().url());
 
-    let payer: Arc<Keypair> = Arc::new(read_keypair_file(&config.payer_path).unwrap());
+    let payer: Arc<Keypair> = Arc::new(read_keypair_file(payer_path).unwrap());
     info!("Payer: {}", payer.pubkey().to_string());
 
-    let mut rpc_results = Vec::with_capacity(num_runs);
+    let mut rpc_results = Vec::with_capacity(num_rounds);
 
-    for _ in 0..num_runs {
-        let stat: RpcStat = send_bulk_txs_and_wait(&rpc, &payer, num_txns, tx_size).await?;
+    for _ in 0..num_rounds {
+        let stat: RpcStat = send_bulk_txs_and_wait(&rpc, &payer, txns_per_round, tx_size).await?;
         rpc_results.push(stat);
     }
 

bench/src/benches/confirmation_slot.rs

@@ -1,39 +1,39 @@
+use std::path::Path;
+
+use crate::tx_size::TxSize;
+use crate::{create_memo_tx, create_rng, send_and_confirm_transactions, Rng8};
 use anyhow::Context;
-use bench_lib::config::BenchConfig;
-use bench_lib::tx_size::TxSize;
-use bench_lib::{create_memo_tx, create_rng, send_and_confirm_transactions, Rng8};
 use log::{info, warn};
 use solana_rpc_client::nonblocking::rpc_client::RpcClient;
 use solana_sdk::signature::{read_keypair_file, Signer};
 use solana_sdk::transaction::Transaction;
 use solana_sdk::{commitment_config::CommitmentConfig, signature::Keypair};
 
 /// TC1 send 2 txs (one via LiteRPC, one via Solana RPC) and compare confirmation slot (=slot distance)
-#[tokio::main]
-async fn main() -> anyhow::Result<()> {
-    tracing_subscriber::fmt::init();
-
+pub async fn confirmation_slot(
+    payer_path: &Path,
+    rpc_a_url: String,
+    rpc_b_url: String,
+    tx_size: TxSize,
+) -> anyhow::Result<()> {
     warn!("THIS IS WORK IN PROGRESS");
 
-    let config = BenchConfig::load().unwrap();
-    let tx_size = config.confirmation_slot.tx_size;
-
-    let lite_rpc = RpcClient::new(config.lite_rpc_url.clone());
-    info!("Lite RPC: {}", lite_rpc.url());
+    let rpc_a = RpcClient::new(rpc_a_url);
+    info!("RPC A: {}", rpc_a.url());
 
-    let rpc = RpcClient::new(config.rpc_url.clone());
-    info!("RPC: {}", rpc.url());
+    let rpc_b = RpcClient::new(rpc_b_url);
+    info!("RPC B: {}", rpc_b.url());
 
     let mut rng = create_rng(None);
-    let payer = read_keypair_file(&config.payer_path).expect("payer file");
+    let payer = read_keypair_file(payer_path).expect("payer file");
     info!("Payer: {}", payer.pubkey().to_string());
 
-    let rpc_tx = create_tx(&rpc, &payer, &mut rng, tx_size).await?;
-    let lite_rpc_tx = create_tx(&lite_rpc, &payer, &mut rng, tx_size).await?;
+    let rpc_a_tx = create_tx(&rpc_a, &payer, &mut rng, tx_size).await?;
+    let rpc_b_tx = create_tx(&rpc_b, &payer, &mut rng, tx_size).await?;
 
     let (rpc_slot, lite_rpc_slot) = tokio::join!(
-        send_transaction_and_get_slot(&rpc, rpc_tx),
-        send_transaction_and_get_slot(&lite_rpc, lite_rpc_tx)
+        send_transaction_and_get_slot(&rpc_a, rpc_a_tx),
+        send_transaction_and_get_slot(&rpc_b, rpc_b_tx)
     );
 
     info!("rpc_slot: {}", rpc_slot?);

bench/src/mod.rs → bench/src/benches/mod.rs

@@ -1,7 +1,3 @@
-pub mod confirmation_slot;
-pub mod confirmation_rate;
 pub mod api_load;
-// pub mod tx_broadcast;
-
-
-
+pub mod confirmation_rate;
+pub mod confirmation_slot;

bench/src/lib.rs

@@ -20,7 +20,8 @@ use solana_transaction_status::TransactionStatus;
 use std::{str::FromStr, time::Duration};
 use tokio::time::Instant;
 
-pub mod config;
+pub mod benches;
+pub mod metrics;
 pub mod tx_size;
 
 //TODO: use CLAP