Skip to content
This repository has been archived by the owner on Jan 22, 2025. It is now read-only.

Commit

Permalink
ledger-tool: Refactor accounts subcommand output code (#34915)
Browse files Browse the repository at this point in the history 
The accounts command currently dumps every single account in the
AccountsDb. This is obviously a lot of output, so a previous change
streamed the accounts instead of collecting and dumping at the end.

The streaming approach is much more performant, but the implementation
is non-trivial. This change
- Moves the accounts output code to output.rs
- Refactor the logic to several objects that implment the functionality
- Adjust the json output to also include the summary

This change lays the groundwork for cleanly adding several more flags
that will allow for querying different subsets of accounts.
  • Loading branch information
steviez authored Jan 26, 2024
1 parent 89fd6ac commit 3add40f
Show file tree
Hide file tree
Showing 3 changed files with 218 additions and 114 deletions.
148 changes: 37 additions & 111 deletions ledger-tool/src/main.rs
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,24 @@
#![allow(clippy::arithmetic_side_effects)]
use {
crate::{args::*, bigtable::*, blockstore::*, ledger_path::*, ledger_utils::*, program::*},
crate::{
args::*,
bigtable::*,
blockstore::*,
ledger_path::*,
ledger_utils::*,
output::{output_account, AccountsOutputConfig, AccountsOutputStreamer},
program::*,
},
clap::{
crate_description, crate_name, value_t, value_t_or_exit, values_t_or_exit, App,
AppSettings, Arg, ArgMatches, SubCommand,
},
dashmap::DashMap,
log::*,
serde::{
ser::{SerializeSeq, Serializer},
Serialize,
},
solana_account_decoder::{UiAccount, UiAccountData, UiAccountEncoding},
serde::Serialize,
solana_account_decoder::UiAccountEncoding,
solana_accounts_db::{
accounts::Accounts, accounts_db::CalcAccountsHashDataSource, accounts_index::ScanConfig,
accounts_db::CalcAccountsHashDataSource, accounts_index::ScanConfig,
hardened_unpack::MAX_GENESIS_ARCHIVE_UNPACKED_SIZE,
},
solana_clap_utils::{
Expand All @@ -25,7 +30,7 @@ use {
validate_maximum_incremental_snapshot_archives_to_retain,
},
},
solana_cli_output::{CliAccount, CliAccountNewConfig, OutputFormat},
solana_cli_output::OutputFormat,
solana_core::{
system_monitor_service::{SystemMonitorService, SystemMonitorStatsReportConfig},
validator::BlockVerificationMethod,
Expand All @@ -38,7 +43,7 @@ use {
},
solana_measure::{measure, measure::Measure},
solana_runtime::{
bank::{bank_hash_details, Bank, RewardCalculationEvent, TotalAccountsStats},
bank::{bank_hash_details, Bank, RewardCalculationEvent},
bank_forks::BankForks,
snapshot_archive_info::SnapshotArchiveInfoGetter,
snapshot_bank_utils,
Expand Down Expand Up @@ -73,7 +78,7 @@ use {
collections::{HashMap, HashSet},
ffi::OsStr,
fs::File,
io::{self, stdout, Write},
io::{self, Write},
num::NonZeroUsize,
path::{Path, PathBuf},
process::{exit, Command, Stdio},
Expand Down Expand Up @@ -102,44 +107,6 @@ fn parse_encoding_format(matches: &ArgMatches<'_>) -> UiAccountEncoding {
}
}

fn output_account(
pubkey: &Pubkey,
account: &AccountSharedData,
modified_slot: Option<Slot>,
print_account_data: bool,
encoding: UiAccountEncoding,
) {
println!("{pubkey}:");
println!(" balance: {} SOL", lamports_to_sol(account.lamports()));
println!(" owner: '{}'", account.owner());
println!(" executable: {}", account.executable());
if let Some(slot) = modified_slot {
println!(" slot: {slot}");
}
println!(" rent_epoch: {}", account.rent_epoch());
println!(" data_len: {}", account.data().len());
if print_account_data {
let account_data = UiAccount::encode(pubkey, account, encoding, None, None).data;
match account_data {
UiAccountData::Binary(data, data_encoding) => {
println!(" data: '{data}'");
println!(
" encoding: {}",
serde_json::to_string(&data_encoding).unwrap()
);
}
UiAccountData::Json(account_data) => {
println!(
" data: '{}'",
serde_json::to_string(&account_data).unwrap()
);
println!(" encoding: \"jsonParsed\"");
}
UiAccountData::LegacyBinary(_) => {}
};
}
}

fn render_dot(dot: String, output_file: &str, output_format: &str) -> io::Result<()> {
let mut child = Command::new("dot")
.arg(format!("-T{output_format}"))
Expand Down Expand Up @@ -2192,7 +2159,6 @@ fn main() {
("accounts", Some(arg_matches)) => {
let process_options = parse_process_options(&ledger_path, arg_matches);
let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
let include_sysvars = arg_matches.is_present("include_sysvars");
let blockstore = open_blockstore(
&ledger_path,
arg_matches,
Expand All @@ -2206,70 +2172,30 @@ fn main() {
snapshot_archive_path,
incremental_snapshot_archive_path,
);

let bank = bank_forks.read().unwrap().working_bank();
let mut serializer = serde_json::Serializer::new(stdout());
let (summarize, mut json_serializer) =
match OutputFormat::from_matches(arg_matches, "output_format", false) {
OutputFormat::Json | OutputFormat::JsonCompact => {
(false, Some(serializer.serialize_seq(None).unwrap()))
}
_ => (true, None),
};
let mut total_accounts_stats = TotalAccountsStats::default();
let rent_collector = bank.rent_collector();
let print_account_contents = !arg_matches.is_present("no_account_contents");
let print_account_data = !arg_matches.is_present("no_account_data");
let data_encoding = parse_encoding_format(arg_matches);
let cli_account_new_config = CliAccountNewConfig {
data_encoding,
..CliAccountNewConfig::default()
};
let scan_func =
|some_account_tuple: Option<(&Pubkey, AccountSharedData, Slot)>| {
if let Some((pubkey, account, slot)) = some_account_tuple
.filter(|(_, account, _)| Accounts::is_loadable(account.lamports()))
{
if !include_sysvars && solana_sdk::sysvar::is_sysvar_id(pubkey) {
return;
}

total_accounts_stats.accumulate_account(
pubkey,
&account,
rent_collector,
);

if print_account_contents {
if let Some(json_serializer) = json_serializer.as_mut() {
let cli_account = CliAccount::new_with_config(
pubkey,
&account,
&cli_account_new_config,
);
json_serializer.serialize_element(&cli_account).unwrap();
} else {
output_account(
pubkey,
&account,
Some(slot),
print_account_data,
data_encoding,
);
}
}
}
};
let mut measure = Measure::start("scanning accounts");
bank.scan_all_accounts(scan_func).unwrap();
measure.stop();
info!("{}", measure);
if let Some(json_serializer) = json_serializer {
json_serializer.end().unwrap();
}
if summarize {
println!("\n{total_accounts_stats:#?}");
}
let include_sysvars = arg_matches.is_present("include_sysvars");
let include_account_contents = !arg_matches.is_present("no_account_contents");
let include_account_data = !arg_matches.is_present("no_account_data");
let account_data_encoding = parse_encoding_format(arg_matches);
let config = AccountsOutputConfig {
include_sysvars,
include_account_contents,
include_account_data,
account_data_encoding,
};
let output_format =
OutputFormat::from_matches(arg_matches, "output_format", false);

let accounts_streamer =
AccountsOutputStreamer::new(bank, output_format, config);
let (_, scan_time) = measure!(
accounts_streamer
.output()
.map_err(|err| error!("Error while outputting accounts: {err}")),
"accounts scan"
);
info!("{scan_time}");
}
("capitalization", Some(arg_matches)) => {
let process_options = parse_process_options(&ledger_path, arg_matches);
Expand Down
Loading

0 comments on commit 3add40f

Please sign in to comment.