apply-range: add ability to operate with memtries & a new benchmark m…

…ode (#12522)

With this PR the original `benchmarking` functionality has been subsumed
by the `sequential` mode which has been extended to operate on flat and
memtrie storage modes.

`benchmark` is then changed to not commit any results to the storage,
effectively leading to the same starting block being applied over and
over again, thus ideally demonstrating the time spent inside
`Runtime::apply`.

I've seen this to sometimes handle a huge number of blocks per second
which would make the output somewhat difficult keep track of, so I went
ahead and integrated indicatif-based progress bar (which then also comes
with a built-in mechanism to estimate duration and rate of blocks per
second.)

Cargo.toml

@@ -199,7 +199,7 @@ hyper = { version = "0.14", features = ["full"] }
 hyper-tls = "0.5.0"
 im = "15"
 indexmap = "2"
-indicatif = { version = "0.15.0", features = ["with_rayon"] }
+indicatif = { version = "0.17.0", features = ["rayon"] }
 insta = { version = "1.41.0", features = ["json", "yaml", "redactions"] }
 integration-tests = { path = "integration-tests" }
 inventory = "0.3.15"

deny.toml

@@ -136,4 +136,7 @@ skip = [
     { name = "thiserror", version = "<2.0" },
     { name = "thiserror-impl", version = "<2.0" },
     { name = "derive_more", version = "<1" },
+
+    # indicatif brings in a newer version
+    { name = "unicode-width", version = "<0.2" },
 ]

genesis-tools/genesis-populate/src/lib.rs

@@ -164,7 +164,7 @@ impl GenesisBuilder {
         let bar = ProgressBar::new(total_accounts_num as _);
         bar.set_style(ProgressStyle::default_bar().template(
             "[elapsed {elapsed_precise} remaining {eta_precise}] Writing into storage {bar} {pos:>7}/{len:7}",
-        ));
+        ).unwrap());
         // Add records in chunks of 3000 per shard for memory efficiency reasons.
         for i in 0..total_accounts_num {
             let account_id = get_account_id(i);

nearcore/src/download_file.rs

@@ -56,14 +56,15 @@ async fn download_file_impl(
         bar.set_style(
             ProgressStyle::default_bar().template(
                 "{spinner:.green} [{elapsed_precise}] [{wide_bar:.cyan/blue}] {bytes}/{total_bytes} [{bytes_per_sec}] ({eta})"
-            ).progress_chars("#>-")
+            ).unwrap().progress_chars("#>-")
         );
         bar
     } else {
         let bar = ProgressBar::new_spinner();
         bar.set_style(
             ProgressStyle::default_bar()
-                .template("{spinner:.green} [{elapsed_precise}] {bytes} [{bytes_per_sec}]"),
+                .template("{spinner:.green} [{elapsed_precise}] {bytes} [{bytes_per_sec}]")
+                .unwrap(),
         );
         bar
     };

tools/flat-storage/src/commands.rs

@@ -46,7 +46,7 @@ enum SubCommand {
     /// The trie is constructed for the block height equal to flat_head
     ConstructTrieFromFlat(ConstructTriedFromFlatCmd),
 
-    /// Move flat head forward.
+    /// Move flat head forward or backward.
     MoveFlatHead(MoveFlatHeadCmd),
 }
 
@@ -106,8 +106,9 @@ pub enum MoveFlatHeadMode {
         new_flat_head_height: BlockHeight,
     },
     /// Moves head back by specific number of blocks.
-    /// Note: it doesn't record deltas on the way and should be used
-    /// only for replaying chain forward.
+    ///
+    /// Note: it doesn't record deltas on the way and should be used only for replaying chain
+    /// forward.
     Back {
         #[clap(long)]
         blocks: usize,

tools/replay-archive/Cargo.toml

@@ -16,6 +16,7 @@ anyhow.workspace = true
 borsh.workspace = true
 clap.workspace = true
 itertools.workspace = true
+indicatif.workspace = true
 
 near-chain.workspace = true
 near-chain-primitives.workspace = true

tools/replay-archive/src/cli.rs

@@ -27,7 +27,7 @@ use near_primitives::sharding::{ReceiptProof, ShardChunk, ShardChunkHeader, Shar
 use near_primitives::types::chunk_extra::ChunkExtra;
 use near_primitives::types::{BlockHeight, Gas, ProtocolVersion, ShardId};
 use near_primitives::version::ProtocolFeature;
-use near_state_viewer::progress_reporter::{timestamp_ms, ProgressReporter};
+use near_state_viewer::progress_reporter::ProgressReporter;
 use near_store::{get_genesis_state_roots, ShardUId, Store};
 use nearcore::{load_config, NearConfig, NightshadeRuntime, NightshadeRuntimeExt};
 use std::collections::HashMap;
@@ -132,12 +132,13 @@ impl ReplayController {
 
         let progress_reporter = ProgressReporter {
             cnt: AtomicU64::new(0),
-            ts: AtomicU64::new(timestamp_ms()),
-            all: (end_height + 1).saturating_sub(start_height),
             skipped: AtomicU64::new(0),
             empty_blocks: AtomicU64::new(0),
             non_empty_blocks: AtomicU64::new(0),
             tgas_burned: AtomicU64::new(0),
+            indicatif: near_state_viewer::progress_reporter::default_indicatif(
+                (end_height + 1).checked_sub(start_height),
+            ),
         };
 
         Ok(Self {
@@ -192,7 +193,8 @@ impl ReplayController {
                 total_gas_burnt = Some(gas_burnt);
             }
         }
-        self.progress_reporter.inc_and_report_progress(total_gas_burnt.unwrap_or(0));
+        self.progress_reporter
+            .inc_and_report_progress(self.next_height, total_gas_burnt.unwrap_or(0));
         self.next_height += 1;
         Ok(self.next_height <= self.end_height)
     }

tools/state-viewer/Cargo.toml

@@ -33,6 +33,7 @@ tempfile.workspace = true
 thiserror.workspace = true
 tracing.workspace = true
 yansi.workspace = true
+indicatif.workspace = true
 
 near-time.workspace = true
 near-chain-configs.workspace = true