From 3add40fc07bd5a922ed443023264d14fa0a9d41f Mon Sep 17 00:00:00 2001 From: steviez Date: Fri, 26 Jan 2024 00:55:05 -0600 Subject: [PATCH] ledger-tool: Refactor accounts subcommand output code (#34915) The accounts command currently dumps every single account in the AccountsDb. This is obviously a lot of output, so a previous change streamed the accounts instead of collecting and dumping at the end. The streaming approach is much more performant, but the implementation is non-trivial. This change - Moves the accounts output code to output.rs - Refactor the logic to several objects that implment the functionality - Adjust the json output to also include the summary This change lays the groundwork for cleanly adding several more flags that will allow for querying different subsets of accounts. --- ledger-tool/src/main.rs | 148 ++++++++----------------------- ledger-tool/src/output.rs | 181 +++++++++++++++++++++++++++++++++++++- runtime/src/bank.rs | 3 +- 3 files changed, 218 insertions(+), 114 deletions(-) diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs index d4fd8a3b2588a9..50bc3a40263743 100644 --- a/ledger-tool/src/main.rs +++ b/ledger-tool/src/main.rs @@ -1,19 +1,24 @@ #![allow(clippy::arithmetic_side_effects)] use { - crate::{args::*, bigtable::*, blockstore::*, ledger_path::*, ledger_utils::*, program::*}, + crate::{ + args::*, + bigtable::*, + blockstore::*, + ledger_path::*, + ledger_utils::*, + output::{output_account, AccountsOutputConfig, AccountsOutputStreamer}, + program::*, + }, clap::{ crate_description, crate_name, value_t, value_t_or_exit, values_t_or_exit, App, AppSettings, Arg, ArgMatches, SubCommand, }, dashmap::DashMap, log::*, - serde::{ - ser::{SerializeSeq, Serializer}, - Serialize, - }, - solana_account_decoder::{UiAccount, UiAccountData, UiAccountEncoding}, + serde::Serialize, + solana_account_decoder::UiAccountEncoding, solana_accounts_db::{ - accounts::Accounts, accounts_db::CalcAccountsHashDataSource, accounts_index::ScanConfig, + accounts_db::CalcAccountsHashDataSource, accounts_index::ScanConfig, hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE, }, solana_clap_utils::{ @@ -25,7 +30,7 @@ use { validate_maximum_incremental_snapshot_archives_to_retain, }, }, - solana_cli_output::{CliAccount, CliAccountNewConfig, OutputFormat}, + solana_cli_output::OutputFormat, solana_core::{ system_monitor_service::{SystemMonitorService, SystemMonitorStatsReportConfig}, validator::BlockVerificationMethod, @@ -38,7 +43,7 @@ use { }, solana_measure::{measure, measure::Measure}, solana_runtime::{ - bank::{bank_hash_details, Bank, RewardCalculationEvent, TotalAccountsStats}, + bank::{bank_hash_details, Bank, RewardCalculationEvent}, bank_forks::BankForks, snapshot_archive_info::SnapshotArchiveInfoGetter, snapshot_bank_utils, @@ -73,7 +78,7 @@ use { collections::{HashMap, HashSet}, ffi::OsStr, fs::File, - io::{self, stdout, Write}, + io::{self, Write}, num::NonZeroUsize, path::{Path, PathBuf}, process::{exit, Command, Stdio}, @@ -102,44 +107,6 @@ fn parse_encoding_format(matches: &ArgMatches<'_>) -> UiAccountEncoding { } } -fn output_account( - pubkey: &Pubkey, - account: &AccountSharedData, - modified_slot: Option, - print_account_data: bool, - encoding: UiAccountEncoding, -) { - println!("{pubkey}:"); - println!(" balance: {} SOL", lamports_to_sol(account.lamports())); - println!(" owner: '{}'", account.owner()); - println!(" executable: {}", account.executable()); - if let Some(slot) = modified_slot { - println!(" slot: {slot}"); - } - println!(" rent_epoch: {}", account.rent_epoch()); - println!(" data_len: {}", account.data().len()); - if print_account_data { - let account_data = UiAccount::encode(pubkey, account, encoding, None, None).data; - match account_data { - UiAccountData::Binary(data, data_encoding) => { - println!(" data: '{data}'"); - println!( - " encoding: {}", - serde_json::to_string(&data_encoding).unwrap() - ); - } - UiAccountData::Json(account_data) => { - println!( - " data: '{}'", - serde_json::to_string(&account_data).unwrap() - ); - println!(" encoding: \"jsonParsed\""); - } - UiAccountData::LegacyBinary(_) => {} - }; - } -} - fn render_dot(dot: String, output_file: &str, output_format: &str) -> io::Result<()> { let mut child = Command::new("dot") .arg(format!("-T{output_format}")) @@ -2192,7 +2159,6 @@ fn main() { ("accounts", Some(arg_matches)) => { let process_options = parse_process_options(&ledger_path, arg_matches); let genesis_config = open_genesis_config_by(&ledger_path, arg_matches); - let include_sysvars = arg_matches.is_present("include_sysvars"); let blockstore = open_blockstore( &ledger_path, arg_matches, @@ -2206,70 +2172,30 @@ fn main() { snapshot_archive_path, incremental_snapshot_archive_path, ); - let bank = bank_forks.read().unwrap().working_bank(); - let mut serializer = serde_json::Serializer::new(stdout()); - let (summarize, mut json_serializer) = - match OutputFormat::from_matches(arg_matches, "output_format", false) { - OutputFormat::Json | OutputFormat::JsonCompact => { - (false, Some(serializer.serialize_seq(None).unwrap())) - } - _ => (true, None), - }; - let mut total_accounts_stats = TotalAccountsStats::default(); - let rent_collector = bank.rent_collector(); - let print_account_contents = !arg_matches.is_present("no_account_contents"); - let print_account_data = !arg_matches.is_present("no_account_data"); - let data_encoding = parse_encoding_format(arg_matches); - let cli_account_new_config = CliAccountNewConfig { - data_encoding, - ..CliAccountNewConfig::default() - }; - let scan_func = - |some_account_tuple: Option<(&Pubkey, AccountSharedData, Slot)>| { - if let Some((pubkey, account, slot)) = some_account_tuple - .filter(|(_, account, _)| Accounts::is_loadable(account.lamports())) - { - if !include_sysvars && solana_sdk::sysvar::is_sysvar_id(pubkey) { - return; - } - total_accounts_stats.accumulate_account( - pubkey, - &account, - rent_collector, - ); - - if print_account_contents { - if let Some(json_serializer) = json_serializer.as_mut() { - let cli_account = CliAccount::new_with_config( - pubkey, - &account, - &cli_account_new_config, - ); - json_serializer.serialize_element(&cli_account).unwrap(); - } else { - output_account( - pubkey, - &account, - Some(slot), - print_account_data, - data_encoding, - ); - } - } - } - }; - let mut measure = Measure::start("scanning accounts"); - bank.scan_all_accounts(scan_func).unwrap(); - measure.stop(); - info!("{}", measure); - if let Some(json_serializer) = json_serializer { - json_serializer.end().unwrap(); - } - if summarize { - println!("\n{total_accounts_stats:#?}"); - } + let include_sysvars = arg_matches.is_present("include_sysvars"); + let include_account_contents = !arg_matches.is_present("no_account_contents"); + let include_account_data = !arg_matches.is_present("no_account_data"); + let account_data_encoding = parse_encoding_format(arg_matches); + let config = AccountsOutputConfig { + include_sysvars, + include_account_contents, + include_account_data, + account_data_encoding, + }; + let output_format = + OutputFormat::from_matches(arg_matches, "output_format", false); + + let accounts_streamer = + AccountsOutputStreamer::new(bank, output_format, config); + let (_, scan_time) = measure!( + accounts_streamer + .output() + .map_err(|err| error!("Error while outputting accounts: {err}")), + "accounts scan" + ); + info!("{scan_time}"); } ("capitalization", Some(arg_matches)) => { let process_options = parse_process_options(&ledger_path, arg_matches); diff --git a/ledger-tool/src/output.rs b/ledger-tool/src/output.rs index 4c953b37baa0f2..e21676771d598f 100644 --- a/ledger-tool/src/output.rs +++ b/ledger-tool/src/output.rs @@ -1,11 +1,20 @@ use { crate::ledger_utils::get_program_ids, chrono::{Local, TimeZone}, - serde::{Deserialize, Serialize}, - solana_cli_output::{display::writeln_transaction, OutputFormat, QuietDisplay, VerboseDisplay}, + serde::{ + ser::{Impossible, SerializeSeq, SerializeStruct, Serializer}, + Deserialize, Serialize, + }, + solana_account_decoder::{UiAccount, UiAccountData, UiAccountEncoding}, + solana_cli_output::{ + display::writeln_transaction, CliAccount, CliAccountNewConfig, OutputFormat, QuietDisplay, + VerboseDisplay, + }, solana_entry::entry::Entry, solana_ledger::blockstore::Blockstore, + solana_runtime::bank::{Bank, TotalAccountsStats}, solana_sdk::{ + account::{AccountSharedData, ReadableAccount}, clock::{Slot, UnixTimestamp}, hash::Hash, native_token::lamports_to_sol, @@ -15,10 +24,13 @@ use { EncodedConfirmedBlock, EncodedTransactionWithStatusMeta, EntrySummary, Rewards, }, std::{ + cell::RefCell, collections::HashMap, fmt::{self, Display, Formatter}, io::{stdout, Write}, + rc::Rc, result::Result, + sync::Arc, }, }; @@ -548,3 +560,168 @@ pub fn output_sorted_program_ids(program_ids: HashMap) { println!("{:<44}: {}", program_id.to_string(), count); } } + +/// A type to facilitate streaming account information to an output destination +/// +/// This type scans every account, so streaming is preferred over the simpler +/// approach of accumulating all the accounts into a Vec and printing or +/// serializing the Vec directly. +pub struct AccountsOutputStreamer { + account_scanner: AccountsScanner, + total_accounts_stats: Rc>, + output_format: OutputFormat, +} + +pub struct AccountsOutputConfig { + pub include_sysvars: bool, + pub include_account_contents: bool, + pub include_account_data: bool, + pub account_data_encoding: UiAccountEncoding, +} + +impl AccountsOutputStreamer { + pub fn new(bank: Arc, output_format: OutputFormat, config: AccountsOutputConfig) -> Self { + let total_accounts_stats = Rc::new(RefCell::new(TotalAccountsStats::default())); + let account_scanner = AccountsScanner { + bank, + total_accounts_stats: total_accounts_stats.clone(), + config, + }; + Self { + account_scanner, + total_accounts_stats, + output_format, + } + } + + pub fn output(&self) -> Result<(), String> { + match self.output_format { + OutputFormat::Json | OutputFormat::JsonCompact => { + let mut serializer = serde_json::Serializer::new(stdout()); + let mut struct_serializer = serializer + .serialize_struct("accountInfo", 2) + .map_err(|err| format!("unable to start serialization: {err}"))?; + struct_serializer + .serialize_field("accounts", &self.account_scanner) + .map_err(|err| format!("unable to serialize accounts scanner: {err}"))?; + struct_serializer + .serialize_field("summary", &*self.total_accounts_stats.borrow()) + .map_err(|err| format!("unable to serialize accounts summary: {err}"))?; + SerializeStruct::end(struct_serializer) + .map_err(|err| format!("unable to end serialization: {err}")) + } + _ => { + // The compiler needs a placeholder type to satisfy the generic + // SerializeSeq trait on AccountScanner::output(). The type + // doesn't really matter since we're passing None, so just use + // serde::ser::Impossible as it already implements SerializeSeq + self.account_scanner + .output::>(&mut None); + println!("\n{:#?}", self.total_accounts_stats.borrow()); + Ok(()) + } + } + } +} + +struct AccountsScanner { + bank: Arc, + total_accounts_stats: Rc>, + config: AccountsOutputConfig, +} + +impl AccountsScanner { + /// Returns true if this account should be included in the output + fn should_process_account(&self, account: &AccountSharedData, pubkey: &Pubkey) -> bool { + solana_accounts_db::accounts::Accounts::is_loadable(account.lamports()) + && (self.config.include_sysvars || !solana_sdk::sysvar::is_sysvar_id(pubkey)) + } + + pub fn output(&self, seq_serializer: &mut Option) + where + S: SerializeSeq, + { + let mut total_accounts_stats = self.total_accounts_stats.borrow_mut(); + let rent_collector = self.bank.rent_collector(); + + let cli_account_new_config = CliAccountNewConfig { + data_encoding: self.config.account_data_encoding, + ..CliAccountNewConfig::default() + }; + + let scan_func = |account_tuple: Option<(&Pubkey, AccountSharedData, Slot)>| { + if let Some((pubkey, account, slot)) = account_tuple + .filter(|(pubkey, account, _)| self.should_process_account(account, pubkey)) + { + total_accounts_stats.accumulate_account(pubkey, &account, rent_collector); + + if self.config.include_account_contents { + if let Some(serializer) = seq_serializer { + let cli_account = + CliAccount::new_with_config(pubkey, &account, &cli_account_new_config); + serializer.serialize_element(&cli_account).unwrap(); + } else { + output_account( + pubkey, + &account, + Some(slot), + self.config.include_account_data, + self.config.account_data_encoding, + ); + } + } + } + }; + + self.bank.scan_all_accounts(scan_func).unwrap(); + } +} + +impl Serialize for AccountsScanner { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let mut seq_serializer = Some(serializer.serialize_seq(None)?); + self.output(&mut seq_serializer); + seq_serializer.unwrap().end() + } +} + +pub fn output_account( + pubkey: &Pubkey, + account: &AccountSharedData, + modified_slot: Option, + print_account_data: bool, + encoding: UiAccountEncoding, +) { + println!("{pubkey}:"); + println!(" balance: {} SOL", lamports_to_sol(account.lamports())); + println!(" owner: '{}'", account.owner()); + println!(" executable: {}", account.executable()); + if let Some(slot) = modified_slot { + println!(" slot: {slot}"); + } + println!(" rent_epoch: {}", account.rent_epoch()); + println!(" data_len: {}", account.data().len()); + if print_account_data { + let account_data = UiAccount::encode(pubkey, account, encoding, None, None).data; + match account_data { + UiAccountData::Binary(data, data_encoding) => { + println!(" data: '{data}'"); + println!( + " encoding: {}", + serde_json::to_string(&data_encoding).unwrap() + ); + } + UiAccountData::Json(account_data) => { + println!( + " data: '{}'", + serde_json::to_string(&account_data).unwrap() + ); + println!(" encoding: \"jsonParsed\""); + } + UiAccountData::LegacyBinary(_) => {} + }; + } +} diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs index e00b04354697cf..03971724438dc9 100644 --- a/runtime/src/bank.rs +++ b/runtime/src/bank.rs @@ -72,6 +72,7 @@ use { slice::ParallelSlice, ThreadPool, ThreadPoolBuilder, }, + serde::Serialize, solana_accounts_db::{ account_overrides::AccountOverrides, accounts::{ @@ -8485,7 +8486,7 @@ impl CollectRentInPartitionInfo { } /// Struct to collect stats when scanning all accounts in `get_total_accounts_stats()` -#[derive(Debug, Default, Copy, Clone)] +#[derive(Debug, Default, Copy, Clone, Serialize)] pub struct TotalAccountsStats { /// Total number of accounts pub num_accounts: usize,