From 935cc9823d82efefcb5d2b1e318eb7fee1e6e760 Mon Sep 17 00:00:00 2001 From: Jan Ciolek <149345204+jancionear@users.noreply.github.com> Date: Thu, 30 Nov 2023 18:07:00 +0100 Subject: [PATCH] feat: add database analyse-gas-usage command (#10240) Add a command which allows to analyse gas usage on some part of the blockchain. The command goes over the specified blocks and gathers information about how much gas was used by each block, shard and account. Then it displays an analysis of all the data. The analysis contains: * Total gas used * Gas used on each shard * Top 10 accounts by gas usage * The optimal account by which a shard could be split into two halves with similar gas usage Help page: ```bash $ ./neard database analyse-gas-usage --help Analyse gas usage in a chosen sequnce of blocks Usage: neard database analyse-gas-usage [OPTIONS] Options: --last-blocks Analyse the last N blocks in the blockchain --from-block-height Analyse blocks from the given block height, inclusive --to-block-height Analyse blocks up to the given block height, inclusive -h, --help Print help ``` Example usage: ```bash # Analyse gas usage in the last 1000 blocks cargo run --bin neard -- database analyse-gas-usage --last-blocks 1000 # Analyse gas usage in blocks with heights between 200 and 300 (inclusive) ./neard database analyse-gas-usage --from-block-height 200 --to-block-height 300 ``` --- Cargo.lock | 1 + tools/database/Cargo.toml | 3 + tools/database/src/analyse_gas_usage.rs | 589 ++++++++++++++++++ .../src/block_iterators/height_range.rs | 86 +++ .../src/block_iterators/last_blocks.rs | 43 ++ tools/database/src/block_iterators/mod.rs | 61 ++ tools/database/src/commands.rs | 5 + tools/database/src/lib.rs | 2 + 8 files changed, 790 insertions(+) create mode 100644 tools/database/src/analyse_gas_usage.rs create mode 100644 tools/database/src/block_iterators/height_range.rs create mode 100644 tools/database/src/block_iterators/last_blocks.rs create mode 100644 tools/database/src/block_iterators/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 98930fbd499..9111f02e41a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3722,6 +3722,7 @@ dependencies = [ "borsh 1.0.0", "clap", "indicatif", + "near-chain", "near-chain-configs", "near-epoch-manager", "near-primitives", diff --git a/tools/database/Cargo.toml b/tools/database/Cargo.toml index 90dcd4b537d..d4526f18c6d 100644 --- a/tools/database/Cargo.toml +++ b/tools/database/Cargo.toml @@ -24,6 +24,7 @@ tempfile.workspace = true nearcore.workspace = true near-epoch-manager.workspace = true +near-chain.workspace = true near-chain-configs.workspace = true near-store.workspace = true near-primitives.workspace = true @@ -32,6 +33,7 @@ near-primitives.workspace = true nightly = [ "nightly_protocol", "near-chain-configs/nightly", + "near-chain/nightly", "near-epoch-manager/nightly", "near-primitives/nightly", "near-store/nightly", @@ -39,6 +41,7 @@ nightly = [ ] nightly_protocol = [ "near-chain-configs/nightly_protocol", + "near-chain/nightly_protocol", "near-epoch-manager/nightly_protocol", "near-primitives/nightly_protocol", "near-store/nightly_protocol", diff --git a/tools/database/src/analyse_gas_usage.rs b/tools/database/src/analyse_gas_usage.rs new file mode 100644 index 00000000000..34c0a6b87c2 --- /dev/null +++ b/tools/database/src/analyse_gas_usage.rs @@ -0,0 +1,589 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::path::PathBuf; +use std::rc::Rc; + +use clap::Parser; +use near_chain::{Block, ChainStore}; +use near_chain_configs::GenesisValidationMode; +use near_epoch_manager::EpochManager; +use nearcore::config::load_config; + +use near_primitives::hash::CryptoHash; +use near_primitives::shard_layout::{account_id_to_shard_id, ShardUId}; +use near_primitives::types::{AccountId, BlockHeight}; + +use nearcore::open_storage; + +use crate::block_iterators::{ + make_block_iterator_from_command_args, CommandArgs, LastNBlocksIterator, +}; + +/// `Gas` is an u64, but it stil might overflow when analysing a large amount of blocks. +/// 1ms of compute is about 1TGas = 10^12 gas. One epoch takes 43200 seconds (43200000ms). +/// This means that the amount of gas consumed during a single epoch can reach 43200000 * 10^12 = 4.32 * 10^19 +/// 10^19 doesn't fit in u64, so we need to use u128 +/// To avoid overflows, let's use `BigGas` for storing gas amounts in the code. +type BigGas = u128; + +/// Display gas amount in a human-friendly way +fn display_gas(gas: BigGas) -> String { + let tera_gas = gas as f64 / 1e12; + format!("{:.2} TGas", tera_gas) +} + +#[derive(Parser)] +pub(crate) struct AnalyseGasUsageCommand { + /// Analyse the last N blocks in the blockchain + #[arg(long)] + last_blocks: Option, + + /// Analyse blocks from the given block height, inclusive + #[arg(long)] + from_block_height: Option, + + /// Analyse blocks up to the given block height, inclusive + #[arg(long)] + to_block_height: Option, +} + +impl AnalyseGasUsageCommand { + pub(crate) fn run(&self, home: &PathBuf) -> anyhow::Result<()> { + // Create a ChainStore and EpochManager that will be used to read blockchain data. + let mut near_config = load_config(home, GenesisValidationMode::Full).unwrap(); + let node_storage = open_storage(&home, &mut near_config).unwrap(); + let store = node_storage.get_split_store().unwrap_or_else(|| node_storage.get_hot_store()); + let chain_store = Rc::new(ChainStore::new( + store.clone(), + near_config.genesis.config.genesis_height, + false, + )); + let epoch_manager = + EpochManager::new_from_genesis_config(store, &near_config.genesis.config).unwrap(); + + // Create an iterator over the blocks that should be analysed + let blocks_iter_opt = make_block_iterator_from_command_args( + CommandArgs { + last_blocks: self.last_blocks, + from_block_height: self.from_block_height, + to_block_height: self.to_block_height, + }, + chain_store.clone(), + ); + + let blocks_iter = match blocks_iter_opt { + Some(iter) => iter, + None => { + println!("No arguments, defaulting to last 100 blocks"); + Box::new(LastNBlocksIterator::new(100, chain_store.clone())) + } + }; + + // Analyse + analyse_gas_usage(blocks_iter, &chain_store, &epoch_manager); + + Ok(()) + } +} + +#[derive(Clone, Debug, Default)] +struct GasUsageInShard { + pub used_gas_per_account: BTreeMap, +} + +/// A shard can be split into two halves. +/// This struct represents the result of splitting a shard at `boundary_account`. +#[derive(Debug, Clone, PartialEq, Eq)] +struct ShardSplit { + /// Account on which the shard would be split + pub boundary_account: AccountId, + /// Gas used by accounts < boundary_account + pub gas_left: BigGas, + /// Gas used by accounts => boundary_account + pub gas_right: BigGas, +} + +impl GasUsageInShard { + pub fn new() -> GasUsageInShard { + GasUsageInShard { used_gas_per_account: BTreeMap::new() } + } + + pub fn add_used_gas(&mut self, account: AccountId, used_gas: BigGas) { + let account_gas = self.used_gas_per_account.entry(account).or_insert(0); + *account_gas = account_gas.checked_add(used_gas).unwrap(); + } + + pub fn used_gas_total(&self) -> BigGas { + let mut result: BigGas = 0; + for used_gas in self.used_gas_per_account.values() { + result = result.checked_add(*used_gas).unwrap(); + } + result + } + + pub fn merge(&mut self, other: &GasUsageInShard) { + for (account_id, used_gas) in &other.used_gas_per_account { + self.add_used_gas(account_id.clone(), *used_gas); + } + } + + /// Calculate the optimal point at which this shard could be split into two halves with similar gas usage + pub fn calculate_split(&self) -> Option { + let total_gas = self.used_gas_total(); + if total_gas == 0 || self.used_gas_per_account.len() < 2 { + return None; + } + + // Find a split with the smallest difference between the two halves + let mut best_split: Option = None; + let mut best_difference: BigGas = total_gas; + + let mut gas_left: BigGas = 0; + let mut gas_right: BigGas = total_gas; + + for (account, used_gas) in &self.used_gas_per_account { + // We are now considering a split of (left < account) and (right >= account) + + let difference: BigGas = gas_left.abs_diff(gas_right); + if difference < best_difference { + best_difference = difference; + best_split = + Some(ShardSplit { boundary_account: account.clone(), gas_left, gas_right }); + } + + gas_left = gas_left.checked_add(*used_gas).unwrap(); + gas_right = gas_right.checked_sub(*used_gas).unwrap(); + } + best_split + } + + pub fn biggest_account(&self) -> Option<(AccountId, BigGas)> { + let mut result: Option<(AccountId, BigGas)> = None; + + for (account, used_gas) in &self.used_gas_per_account { + match &mut result { + None => result = Some((account.clone(), *used_gas)), + Some((best_account, best_gas)) => { + if *used_gas > *best_gas { + *best_account = account.clone(); + *best_gas = *used_gas + } + } + } + } + + result + } +} + +#[derive(Clone, Debug)] +struct GasUsageStats { + pub shards: BTreeMap, +} + +impl GasUsageStats { + pub fn new() -> GasUsageStats { + GasUsageStats { shards: BTreeMap::new() } + } + + pub fn add_gas_usage_in_shard(&mut self, shard_uid: ShardUId, shard_usage: GasUsageInShard) { + match self.shards.get_mut(&shard_uid) { + Some(existing_shard_usage) => existing_shard_usage.merge(&shard_usage), + None => { + let _ = self.shards.insert(shard_uid, shard_usage); + } + } + } + + pub fn used_gas_total(&self) -> BigGas { + let mut result: BigGas = 0; + for shard_usage in self.shards.values() { + result = result.checked_add(shard_usage.used_gas_total()).unwrap(); + } + result + } + + pub fn merge(&mut self, other: GasUsageStats) { + for (shard_uid, shard_usage) in other.shards { + self.add_gas_usage_in_shard(shard_uid, shard_usage); + } + } +} + +fn get_gas_usage_in_block( + block: &Block, + chain_store: &ChainStore, + epoch_manager: &EpochManager, +) -> GasUsageStats { + let block_info = epoch_manager.get_block_info(block.hash()).unwrap(); + let epoch_id = block_info.epoch_id(); + let shard_layout = epoch_manager.get_shard_layout(epoch_id).unwrap(); + + let mut result = GasUsageStats::new(); + + // Go over every chunk in this block and gather data + for chunk_header in block.chunks().iter() { + let shard_id = chunk_header.shard_id(); + let shard_uid = ShardUId::from_shard_id_and_layout(shard_id, &shard_layout); + + let mut gas_usage_in_shard = GasUsageInShard::new(); + + // The outcome of each transaction and receipt executed in this chunk is saved in the database as an ExecutionOutcome. + // Go through all ExecutionOutcomes from this chunk and record the gas usage. + let outcome_ids = + chain_store.get_outcomes_by_block_hash_and_shard_id(block.hash(), shard_id).unwrap(); + for outcome_id in outcome_ids { + let outcome = chain_store + .get_outcome_by_id_and_block_hash(&outcome_id, block.hash()) + .unwrap() + .unwrap() + .outcome; + + // Sanity check - make sure that the executor of this outcome belongs to this shard + let account_shard_id = account_id_to_shard_id(&outcome.executor_id, &shard_layout); + assert_eq!(account_shard_id, shard_id); + + gas_usage_in_shard.add_used_gas(outcome.executor_id, outcome.gas_burnt.into()); + } + + result.add_gas_usage_in_shard(shard_uid, gas_usage_in_shard); + } + + result +} + +/// A struct that can be used to find N biggest accounts by gas usage in an efficient manner. +struct BiggestAccountsFinder { + accounts: BTreeSet<(BigGas, AccountId)>, + accounts_num: usize, +} + +impl BiggestAccountsFinder { + pub fn new(accounts_num: usize) -> BiggestAccountsFinder { + BiggestAccountsFinder { accounts: BTreeSet::new(), accounts_num } + } + + pub fn add_account_stats(&mut self, account: AccountId, used_gas: BigGas) { + self.accounts.insert((used_gas, account)); + + // If there are more accounts than desired, remove the one with the smallest gas usage + while self.accounts.len() > self.accounts_num { + self.accounts.pop_first(); + } + } + + pub fn get_biggest_accounts(&self) -> impl Iterator + '_ { + self.accounts.iter().rev().map(|(gas, account)| (account.clone(), *gas)) + } +} + +// Calculates how much percent of `big` is `small` and returns it as a string. +// Example: as_percentage_of(10, 100) == "10.0%" +fn as_percentage_of(small: BigGas, big: BigGas) -> String { + if big > 0 { + format!("{:.1}%", small as f64 / big as f64 * 100.0) + } else { + format!("-") + } +} + +fn display_shard_split_stats<'a>( + accounts: impl Iterator, + total_shard_gas: BigGas, +) { + let mut accounts_num: u64 = 0; + let mut total_split_half_gas: BigGas = 0; + let mut top_3_finder = BiggestAccountsFinder::new(3); + + for (account, used_gas) in accounts { + accounts_num += 1; + total_split_half_gas = total_split_half_gas.checked_add(*used_gas).unwrap(); + top_3_finder.add_account_stats(account.clone(), *used_gas); + } + + let indent = " "; + println!( + "{}Gas: {} ({} of shard)", + indent, + display_gas(total_split_half_gas), + as_percentage_of(total_split_half_gas, total_shard_gas) + ); + println!("{}Accounts: {}", indent, accounts_num); + println!("{}Top 3 accounts:", indent); + for (i, (account, used_gas)) in top_3_finder.get_biggest_accounts().enumerate() { + println!("{} #{}: {}", indent, i + 1, account); + println!( + "{} Used gas: {} ({} of shard)", + indent, + display_gas(used_gas), + as_percentage_of(used_gas, total_shard_gas) + ) + } +} + +fn analyse_gas_usage( + blocks_iter: impl Iterator, + chain_store: &ChainStore, + epoch_manager: &EpochManager, +) { + // Gather statistics about gas usage in all of the blocks + let mut blocks_count: usize = 0; + let mut first_analysed_block: Option<(BlockHeight, CryptoHash)> = None; + let mut last_analysed_block: Option<(BlockHeight, CryptoHash)> = None; + + let mut gas_usage_stats = GasUsageStats::new(); + + for block in blocks_iter { + blocks_count += 1; + if first_analysed_block.is_none() { + first_analysed_block = Some((block.header().height(), *block.hash())); + } + last_analysed_block = Some((block.header().height(), *block.hash())); + + let gas_usage_in_block = get_gas_usage_in_block(&block, chain_store, epoch_manager); + gas_usage_stats.merge(gas_usage_in_block); + } + + // Print out the analysis + if blocks_count == 0 { + println!("No blocks to analyse!"); + return; + } + println!(""); + println!("Analysed {} blocks between:", blocks_count); + if let Some((block_height, block_hash)) = first_analysed_block { + println!("Block: height = {block_height}, hash = {block_hash}"); + } + if let Some((block_height, block_hash)) = last_analysed_block { + println!("Block: height = {block_height}, hash = {block_hash}"); + } + let total_gas = gas_usage_stats.used_gas_total(); + println!(""); + println!("Total gas used: {}", display_gas(total_gas)); + println!(""); + for (shard_uid, shard_usage) in &gas_usage_stats.shards { + println!("Shard: {}", shard_uid); + let shard_total_gas = shard_usage.used_gas_total(); + println!( + " Gas usage: {} ({} of total)", + display_gas(shard_usage.used_gas_total()), + as_percentage_of(shard_total_gas, total_gas) + ); + println!(" Number of accounts: {}", shard_usage.used_gas_per_account.len()); + if let Some((biggest_account, biggest_account_gas)) = shard_usage.biggest_account() { + println!(" Biggest account: {}", biggest_account); + println!( + " Biggest account gas: {} ({} of shard)", + display_gas(biggest_account_gas), + as_percentage_of(biggest_account_gas, shard_total_gas) + ); + } + match shard_usage.calculate_split() { + Some(shard_split) => { + println!(" Optimal split:"); + println!(" boundary_account: {}", shard_split.boundary_account); + let boundary_account_gas = + *shard_usage.used_gas_per_account.get(&shard_split.boundary_account).unwrap(); + println!(" gas(boundary_account): {}", display_gas(boundary_account_gas)); + println!( + " Gas distribution (left, boundary_acc, right): ({}, {}, {})", + as_percentage_of(shard_split.gas_left, shard_total_gas), + as_percentage_of(boundary_account_gas, shard_total_gas), + as_percentage_of( + shard_split.gas_right.saturating_sub(boundary_account_gas), + shard_total_gas + ) + ); + println!(" Left (account < boundary_account):"); + let left_accounts = + shard_usage.used_gas_per_account.range(..shard_split.boundary_account.clone()); + display_shard_split_stats(left_accounts, shard_total_gas); + println!(" Right (account >= boundary_account):"); + let right_accounts = + shard_usage.used_gas_per_account.range(shard_split.boundary_account..); + display_shard_split_stats(right_accounts, shard_total_gas); + } + None => println!(" No optimal split for this shard"), + } + println!(""); + } + + // Find 10 biggest accounts by gas usage + let mut biggest_accounts_finder = BiggestAccountsFinder::new(10); + for shard in gas_usage_stats.shards.values() { + for (account, used_gas) in &shard.used_gas_per_account { + biggest_accounts_finder.add_account_stats(account.clone(), *used_gas); + } + } + println!("10 biggest accounts by gas usage:"); + for (i, (account, gas_usage)) in biggest_accounts_finder.get_biggest_accounts().enumerate() { + println!("#{}: {}", i + 1, account); + println!( + " Used gas: {} ({} of total)", + display_gas(gas_usage), + as_percentage_of(gas_usage, total_gas) + ) + } +} + +#[cfg(test)] +mod tests { + use std::str::FromStr; + + use near_primitives::types::AccountId; + + use super::{GasUsageInShard, ShardSplit}; + + fn account(name: &str) -> AccountId { + AccountId::from_str(&format!("{name}.near")).unwrap() + } + + // There is no optimal split for a shard with no accounts + #[test] + fn empty_shard_no_split() { + let empty_shard = GasUsageInShard::new(); + assert_eq!(empty_shard.calculate_split(), None); + } + + // There is no optimal split for a shard with a single account + #[test] + fn one_account_no_split() { + let mut shard_usage = GasUsageInShard::new(); + + shard_usage.add_used_gas(account("a"), 12345); + + assert_eq!(shard_usage.calculate_split(), None); + } + + // A shard with two equally sized accounts should be split in half + #[test] + fn two_accounts_equal_split() { + let mut shard_usage = GasUsageInShard::new(); + + shard_usage.add_used_gas(account("a"), 12345); + shard_usage.add_used_gas(account("b"), 12345); + + let optimal_split = + ShardSplit { boundary_account: account("b"), gas_left: 12345, gas_right: 12345 }; + assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); + } + + // A shard with two accounts where the first is slightly smaller should be split in half + #[test] + fn two_accounts_first_smaller_split() { + let mut shard_usage = GasUsageInShard::new(); + + shard_usage.add_used_gas(account("a"), 123); + shard_usage.add_used_gas(account("b"), 12345); + + let optimal_split = + ShardSplit { boundary_account: account("b"), gas_left: 123, gas_right: 12345 }; + assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); + } + + // A shard with two accounts where the second one is slightly smaller should be split in half + #[test] + fn two_accounts_second_smaller_split() { + let mut shard_usage = GasUsageInShard::new(); + + shard_usage.add_used_gas(account("a"), 12345); + shard_usage.add_used_gas(account("b"), 123); + + let optimal_split = + ShardSplit { boundary_account: account("b"), gas_left: 12345, gas_right: 123 }; + assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); + } + + // A shard with multiple accounts where all of them use 0 gas has optimal split + #[test] + fn many_accounts_zero_no_split() { + let mut shard_usage = GasUsageInShard::new(); + + shard_usage.add_used_gas(account("a"), 0); + shard_usage.add_used_gas(account("b"), 0); + shard_usage.add_used_gas(account("c"), 0); + shard_usage.add_used_gas(account("d"), 0); + shard_usage.add_used_gas(account("e"), 0); + shard_usage.add_used_gas(account("f"), 0); + shard_usage.add_used_gas(account("g"), 0); + shard_usage.add_used_gas(account("h"), 0); + + assert_eq!(shard_usage.calculate_split(), None); + } + + // A shard with multiple accounts where only one is nonzero has no optimal split + #[test] + fn many_accounts_one_nonzero_no_split() { + let mut shard_usage = GasUsageInShard::new(); + + shard_usage.add_used_gas(account("a"), 0); + shard_usage.add_used_gas(account("b"), 0); + shard_usage.add_used_gas(account("c"), 0); + shard_usage.add_used_gas(account("d"), 0); + shard_usage.add_used_gas(account("e"), 12345); + shard_usage.add_used_gas(account("f"), 0); + shard_usage.add_used_gas(account("g"), 0); + shard_usage.add_used_gas(account("h"), 0); + + assert_eq!(shard_usage.calculate_split(), None); + } + + // An example set of accounts is split correctly + #[test] + fn many_accounts_split() { + let mut shard_usage = GasUsageInShard::new(); + + shard_usage.add_used_gas(account("a"), 1); + shard_usage.add_used_gas(account("b"), 3); + shard_usage.add_used_gas(account("c"), 5); + shard_usage.add_used_gas(account("d"), 2); + shard_usage.add_used_gas(account("e"), 8); + shard_usage.add_used_gas(account("f"), 1); + shard_usage.add_used_gas(account("g"), 2); + shard_usage.add_used_gas(account("h"), 8); + + // Optimal split: + // 1 + 3 + 5 + 2 = 11 + // 8 + 1 + 2 + 8 = 19 + let optimal_split = + ShardSplit { boundary_account: account("e"), gas_left: 11, gas_right: 19 }; + assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); + } + + // The first account uses the most gas, it should be the only one in its half of the split + #[test] + fn first_heavy_split() { + let mut shard_usage = GasUsageInShard::new(); + + shard_usage.add_used_gas(account("a"), 10000); + shard_usage.add_used_gas(account("b"), 1); + shard_usage.add_used_gas(account("c"), 1); + shard_usage.add_used_gas(account("d"), 1); + shard_usage.add_used_gas(account("e"), 1); + shard_usage.add_used_gas(account("f"), 1); + shard_usage.add_used_gas(account("g"), 1); + shard_usage.add_used_gas(account("h"), 1); + + let optimal_split = + ShardSplit { boundary_account: account("b"), gas_left: 10000, gas_right: 7 }; + assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); + } + + // The last account uses the most gas, it should be the only one in its half of the split + #[test] + fn last_heavy_split() { + let mut shard_usage = GasUsageInShard::new(); + + shard_usage.add_used_gas(account("a"), 1); + shard_usage.add_used_gas(account("b"), 1); + shard_usage.add_used_gas(account("c"), 1); + shard_usage.add_used_gas(account("d"), 1); + shard_usage.add_used_gas(account("e"), 1); + shard_usage.add_used_gas(account("f"), 1); + shard_usage.add_used_gas(account("g"), 1); + shard_usage.add_used_gas(account("h"), 10000); + + let optimal_split = + ShardSplit { boundary_account: account("h"), gas_left: 7, gas_right: 10000 }; + assert_eq!(shard_usage.calculate_split(), Some(optimal_split)); + } +} diff --git a/tools/database/src/block_iterators/height_range.rs b/tools/database/src/block_iterators/height_range.rs new file mode 100644 index 00000000000..9b79675e4b1 --- /dev/null +++ b/tools/database/src/block_iterators/height_range.rs @@ -0,0 +1,86 @@ +use std::rc::Rc; + +use near_chain::{Block, ChainStore, ChainStoreAccess, Error}; +use near_primitives::{hash::CryptoHash, types::BlockHeight}; + +/// Iterate over blocks between two block heights. +/// `from_height` and `to_height` are inclusive +pub struct BlockHeightRangeIterator { + chain_store: Rc, + from_block_height: BlockHeight, + /// Hash of the block that will be returned when next() is called + current_block_hash: Option, +} + +impl BlockHeightRangeIterator { + /// Create an iterator which iterates over blocks between from_height and to_height. + /// `from_height` and `to_height` are inclusive. + /// Both arguments are optional, passing `None`` means that the limit is +- infinity. + pub fn new( + from_height_opt: Option, + to_height_opt: Option, + chain_store: Rc, + ) -> BlockHeightRangeIterator { + if let (Some(from), Some(to)) = (&from_height_opt, &to_height_opt) { + if *from > *to { + // Empty iterator + return BlockHeightRangeIterator { + chain_store, + from_block_height: 0, + current_block_hash: None, + }; + } + } + + let min_height = chain_store.get_genesis_height(); + let max_height = chain_store.head().unwrap().height; + + let from_height = from_height_opt.unwrap_or(min_height); + let mut to_height = to_height_opt.unwrap_or(max_height); + + // There is no point in going over nonexisting blocks past the highest height + if to_height > max_height { + to_height = max_height; + } + + // A block with height `to_height` might not exist. + // Go over the range in reverse and find the highest block that exists. + let mut current_block_hash: Option = None; + for height in (from_height..=to_height).rev() { + match chain_store.get_block_hash_by_height(height) { + Ok(hash) => { + current_block_hash = Some(hash); + break; + } + Err(Error::DBNotFoundErr(_)) => continue, + err => err.unwrap(), + }; + } + + BlockHeightRangeIterator { chain_store, from_block_height: from_height, current_block_hash } + } +} + +impl Iterator for BlockHeightRangeIterator { + type Item = Block; + + fn next(&mut self) -> Option { + let current_block_hash = match self.current_block_hash.take() { + Some(hash) => hash, + None => return None, + }; + let current_block = self.chain_store.get_block(¤t_block_hash).unwrap(); + + // Make sure that the block is within the from..=to range, stop iterating if it isn't + if current_block.header().height() >= self.from_block_height { + // Set the previous block as "current" one, as long as the current one isn't the genesis block + if current_block.header().height() != self.chain_store.get_genesis_height() { + self.current_block_hash = Some(*current_block.header().prev_hash()); + } + + return Some(current_block); + } + + None + } +} diff --git a/tools/database/src/block_iterators/last_blocks.rs b/tools/database/src/block_iterators/last_blocks.rs new file mode 100644 index 00000000000..8704391be56 --- /dev/null +++ b/tools/database/src/block_iterators/last_blocks.rs @@ -0,0 +1,43 @@ +use std::rc::Rc; + +use near_chain::{Block, ChainStore, ChainStoreAccess}; +use near_primitives::hash::CryptoHash; + +/// Iterate over the last N blocks in the blockchain +pub struct LastNBlocksIterator { + chain_store: Rc, + blocks_left: u64, + /// Hash of the block that will be returned when next() is called + current_block_hash: Option, +} + +impl LastNBlocksIterator { + pub fn new(blocks_num: u64, chain_store: Rc) -> LastNBlocksIterator { + let current_block_hash = Some(chain_store.head().unwrap().last_block_hash); + LastNBlocksIterator { chain_store, blocks_left: blocks_num, current_block_hash } + } +} + +impl Iterator for LastNBlocksIterator { + type Item = Block; + + fn next(&mut self) -> Option { + // Decrease the amount of blocks left to produce + match self.blocks_left.checked_sub(1) { + Some(new_blocks_left) => self.blocks_left = new_blocks_left, + None => return None, + }; + + if let Some(current_block_hash) = self.current_block_hash.take() { + let current_block = self.chain_store.get_block(¤t_block_hash).unwrap(); + + // Set the previous block as "current" one, as long as the current one isn't the genesis block + if current_block.header().height() != self.chain_store.get_genesis_height() { + self.current_block_hash = Some(*current_block.header().prev_hash()); + } + return Some(current_block); + } + + None + } +} diff --git a/tools/database/src/block_iterators/mod.rs b/tools/database/src/block_iterators/mod.rs new file mode 100644 index 00000000000..f8c49c0995e --- /dev/null +++ b/tools/database/src/block_iterators/mod.rs @@ -0,0 +1,61 @@ +//! This module contains iterators that can be used to iterate over blocks in the database + +mod height_range; +mod last_blocks; + +use std::rc::Rc; + +use near_chain::{Block, ChainStore}; +use near_primitives::types::BlockHeight; + +/// Iterate over blocks between two block heights. +/// `from_height` and `to_height` are inclusive +pub use height_range::BlockHeightRangeIterator; + +/// Iterate over the last N blocks in the blockchain +pub use last_blocks::LastNBlocksIterator; + +/// Arguments that user can pass to a command to choose some subset of blocks +pub struct CommandArgs { + /// Analyse the last N blocks + pub last_blocks: Option, + + /// Analyse blocks from the given block height, inclusive + pub from_block_height: Option, + + /// Analyse blocks up to the given block height, inclusive + pub to_block_height: Option, +} + +/// Produce to right iterator for a given set of command line arguments +pub fn make_block_iterator_from_command_args( + command_args: CommandArgs, + chain_store: Rc, +) -> Option>> { + // Make sure that only one type of argument is used (there is no mixing of last_blocks and from_block_height) + let mut arg_types_used: u64 = 0; + if command_args.last_blocks.is_some() { + arg_types_used += 1; + } + if command_args.from_block_height.is_some() || command_args.from_block_height.is_some() { + arg_types_used += 1; + } + + if arg_types_used > 1 { + panic!("It is illegal to mix multiple types of arguments specifying a subset of blocks"); + } + + if let Some(last_blocks) = command_args.last_blocks { + return Some(Box::new(LastNBlocksIterator::new(last_blocks, chain_store))); + } + + if command_args.from_block_height.is_some() || command_args.to_block_height.is_some() { + return Some(Box::new(BlockHeightRangeIterator::new( + command_args.from_block_height, + command_args.to_block_height, + chain_store, + ))); + } + + None +} diff --git a/tools/database/src/commands.rs b/tools/database/src/commands.rs index f3e656e68e7..ef9060b7118 100644 --- a/tools/database/src/commands.rs +++ b/tools/database/src/commands.rs @@ -1,5 +1,6 @@ use crate::adjust_database::ChangeDbKindCommand; use crate::analyse_data_size_distribution::AnalyseDataSizeDistributionCommand; +use crate::analyse_gas_usage::AnalyseGasUsageCommand; use crate::compact::RunCompactionCommand; use crate::corrupt::CorruptStateSnapshotCommand; use crate::make_snapshot::MakeSnapshotCommand; @@ -21,6 +22,9 @@ enum SubCommand { /// Analyse data size distribution in RocksDB AnalyseDataSizeDistribution(AnalyseDataSizeDistributionCommand), + /// Analyse gas usage in a chosen sequnce of blocks + AnalyseGasUsage(AnalyseGasUsageCommand), + /// Change DbKind of hot or cold db. ChangeDbKind(ChangeDbKindCommand), @@ -48,6 +52,7 @@ impl DatabaseCommand { pub fn run(&self, home: &PathBuf) -> anyhow::Result<()> { match &self.subcmd { SubCommand::AnalyseDataSizeDistribution(cmd) => cmd.run(home), + SubCommand::AnalyseGasUsage(cmd) => cmd.run(home), SubCommand::ChangeDbKind(cmd) => cmd.run(home), SubCommand::CompactDatabase(cmd) => cmd.run(home), SubCommand::CorruptStateSnapshot(cmd) => cmd.run(home), diff --git a/tools/database/src/lib.rs b/tools/database/src/lib.rs index 6d7e70f1691..827b5da4a63 100644 --- a/tools/database/src/lib.rs +++ b/tools/database/src/lib.rs @@ -1,5 +1,7 @@ mod adjust_database; mod analyse_data_size_distribution; +mod analyse_gas_usage; +mod block_iterators; pub mod commands; mod compact; mod corrupt;