From 09bc50f7568a3f7995713ea0c9031c66b366f99d Mon Sep 17 00:00:00 2001 From: cchudant Date: Tue, 15 Oct 2024 11:36:03 +0000 Subject: [PATCH 1/2] fix(db): max rocksdb LOG files count and size and add more memory metrics --- CHANGELOG.md | 1 + crates/client/db/src/db_metrics.rs | 54 ++++++++++++++++++++++++++---- crates/client/db/src/lib.rs | 33 ++++++++++-------- 3 files changed, 67 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6aecddaa1..43208a624 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Next release +- fix(db): max rocksdb LOG files count and size and add more memory metrics - fix(rpc): handle batched requests in middleware - chore: padded devnet address display with 64 chars - feat(script): added more capabilities to the launcher script diff --git a/crates/client/db/src/db_metrics.rs b/crates/client/db/src/db_metrics.rs index 2f6a42943..027a5e8a4 100644 --- a/crates/client/db/src/db_metrics.rs +++ b/crates/client/db/src/db_metrics.rs @@ -1,23 +1,42 @@ use crate::{Column, DatabaseExt, DB}; -use mc_metrics::{Gauge, IntGaugeVec, MetricsRegistry, Opts, PrometheusError, F64}; +use anyhow::Context; +use mc_metrics::{prometheus::IntGauge, IntGaugeVec, MetricsRegistry, Opts, PrometheusError}; +use rocksdb::perf::MemoryUsageBuilder; #[derive(Clone, Debug)] pub struct DbMetrics { - pub db_size: Gauge, + pub size: IntGauge, pub column_sizes: IntGaugeVec, + pub mem_table_total: IntGauge, + pub mem_table_unflushed: IntGauge, + pub mem_table_readers_total: IntGauge, + pub cache_total: IntGauge, } impl DbMetrics { pub fn register(registry: &MetricsRegistry) -> Result { Ok(Self { - db_size: registry.register(Gauge::new("db_size", "Node storage usage in GB")?)?, + size: registry.register(IntGauge::new("db_size", "Node storage usage in bytes")?)?, column_sizes: registry - .register(IntGaugeVec::new(Opts::new("column_sizes", "Sizes of RocksDB columns"), &["column"])?)?, + .register(IntGaugeVec::new(Opts::new("db_column_sizes", "Sizes of RocksDB columns"), &["column"])?)?, + mem_table_total: registry.register(IntGauge::new( + "db_mem_table_total", + "Approximate memory usage of all the mem-tables in bytes", + )?)?, + mem_table_unflushed: registry.register(IntGauge::new( + "db_mem_table_unflushed", + "Approximate memory usage of un-flushed mem-tables in bytes", + )?)?, + mem_table_readers_total: registry.register(IntGauge::new( + "db_mem_table_readers_total", + "Approximate memory usage of all the table readers in bytes", + )?)?, + cache_total: registry + .register(IntGauge::new("db_cache_total", "Approximate memory usage by cache in bytes")?)?, }) } - /// Returns the total storage size - pub fn update(&self, db: &DB) -> u64 { + pub fn try_update(&self, db: &DB) -> anyhow::Result { let mut storage_size = 0; for &column in Column::ALL.iter() { @@ -29,6 +48,27 @@ impl DbMetrics { self.column_sizes.with_label_values(&[column.rocksdb_name()]).set(column_size as i64); } - storage_size + self.size.set(storage_size as _); + + let mut builder = MemoryUsageBuilder::new().context("Creating memory usage builder")?; + builder.add_db(db); + let mem_usage = builder.build().context("Getting memory usage")?; + self.mem_table_total.set(mem_usage.approximate_mem_table_total() as _); + self.mem_table_unflushed.set(mem_usage.approximate_mem_table_unflushed() as _); + self.mem_table_readers_total.set(mem_usage.approximate_mem_table_readers_total() as _); + self.cache_total.set(mem_usage.approximate_cache_total() as _); + + Ok(storage_size) + } + + /// Returns the total storage size + pub fn update(&self, db: &DB) -> u64 { + match self.try_update(db) { + Ok(res) => res, + Err(err) => { + log::warn!("Error updating db metrics: {err:#}"); + 0 + } + } } } diff --git a/crates/client/db/src/lib.rs b/crates/client/db/src/lib.rs index c76f0a7df..78912b234 100644 --- a/crates/client/db/src/lib.rs +++ b/crates/client/db/src/lib.rs @@ -1,10 +1,5 @@ //! Madara database -use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex}; -use std::time::{Duration, Instant}; -use std::{fmt, fs}; - use anyhow::{Context, Result}; use bonsai_db::{BonsaiDb, DatabaseKeyMapping}; use bonsai_trie::id::BasicId; @@ -14,43 +9,50 @@ use mc_metrics::MetricsRegistry; use mp_chain_config::ChainConfig; use mp_utils::service::Service; use rocksdb::backup::{BackupEngine, BackupEngineOptions}; - -pub mod block_db; -mod error; use rocksdb::{ BoundColumnFamily, ColumnFamilyDescriptor, DBCompressionType, DBWithThreadMode, Env, FlushOptions, MultiThreaded, Options, SliceTransform, }; +use starknet_types_core::hash::{Pedersen, Poseidon, StarkHash}; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant}; +use std::{fmt, fs}; +use tokio::sync::{mpsc, oneshot}; + +pub mod block_db; pub mod bonsai_db; pub mod class_db; pub mod contract_db; pub mod db_block_id; pub mod db_metrics; pub mod devnet_db; +mod error; pub mod l1_db; pub mod storage_updates; pub mod tests; pub use error::{MadaraStorageError, TrieType}; -use starknet_types_core::hash::{Pedersen, Poseidon, StarkHash}; -use tokio::sync::{mpsc, oneshot}; - pub type DB = DBWithThreadMode; - pub use rocksdb; pub type WriteBatchWithTransaction = rocksdb::WriteBatchWithTransaction; const DB_UPDATES_BATCH_SIZE: usize = 1024; +#[allow(clippy::identity_op)] // allow 1 * MiB +#[allow(non_upper_case_globals)] // allow KiB/MiB/GiB names pub fn open_rocksdb(path: &Path, create: bool) -> Result> { + const KiB: usize = 1024; + const MiB: usize = 1024 * KiB; + const GiB: usize = 1024 * MiB; + let mut opts = Options::default(); opts.set_report_bg_io_stats(true); opts.set_use_fsync(false); opts.create_if_missing(create); opts.create_missing_column_families(true); - opts.set_bytes_per_sync(1024 * 1024); opts.set_keep_log_file_num(1); - opts.optimize_level_style_compaction(4096 * 1024 * 1024); + opts.optimize_level_style_compaction(4 * GiB); opts.set_compression_type(DBCompressionType::Zstd); let cores = std::thread::available_parallelism().map(|e| e.get() as i32).unwrap_or(1); opts.increase_parallelism(cores); @@ -59,6 +61,9 @@ pub fn open_rocksdb(path: &Path, create: bool) -> Result> { opts.set_manual_wal_flush(true); opts.set_max_subcompactions(cores as _); + opts.set_max_log_file_size(1 * MiB); + opts.set_keep_log_file_num(3); + let mut env = Env::new().context("Creating rocksdb env")?; // env.set_high_priority_background_threads(cores); // flushes env.set_low_priority_background_threads(cores); // compaction From fe340e7646dabd8083022820d9f1a228267b9d7d Mon Sep 17 00:00:00 2001 From: cchudant Date: Tue, 15 Oct 2024 11:45:46 +0000 Subject: [PATCH 2/2] fix(db): set log level to warn --- crates/client/db/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/crates/client/db/src/lib.rs b/crates/client/db/src/lib.rs index 78912b234..7dba6e4a4 100644 --- a/crates/client/db/src/lib.rs +++ b/crates/client/db/src/lib.rs @@ -63,6 +63,7 @@ pub fn open_rocksdb(path: &Path, create: bool) -> Result> { opts.set_max_log_file_size(1 * MiB); opts.set_keep_log_file_num(3); + opts.set_log_level(rocksdb::LogLevel::Warn); let mut env = Env::new().context("Creating rocksdb env")?; // env.set_high_priority_background_threads(cores); // flushes