From 3cdf809afc8492b2728eadadb9c9ca615c0b8d95 Mon Sep 17 00:00:00 2001 From: canepat <16927169+canepat@users.noreply.github.com> Date: Sun, 15 Oct 2023 11:41:17 +0200 Subject: [PATCH] node: read page size from existing MDBX database (#1580) --- cmd/dev/toolbox.cpp | 2 +- silkworm/node/db/mdbx.cpp | 30 +++++----- silkworm/node/db/mdbx.hpp | 27 ++++----- silkworm/node/db/mdbx_test.cpp | 56 +++++++++++++------ .../node/db/memory_mutation_cursor_test.cpp | 2 +- 5 files changed, 68 insertions(+), 49 deletions(-) diff --git a/cmd/dev/toolbox.cpp b/cmd/dev/toolbox.cpp index 1e511c85fd..0c3e8c93de 100644 --- a/cmd/dev/toolbox.cpp +++ b/cmd/dev/toolbox.cpp @@ -330,7 +330,7 @@ dbTablesInfo get_tablesInfo(::mdbx::txn& txn) { return ret; } -void do_scan(const db::EnvConfig& config) { +void do_scan(db::EnvConfig& config) { static std::string fmt_hdr{" %3s %-24s %=50s %13s %13s %13s"}; auto env{silkworm::db::open_env(config)}; diff --git a/silkworm/node/db/mdbx.cpp b/silkworm/node/db/mdbx.cpp index e4a1066f76..223951512b 100644 --- a/silkworm/node/db/mdbx.cpp +++ b/silkworm/node/db/mdbx.cpp @@ -66,7 +66,7 @@ static inline mdbx::cursor::move_operation move_operation(CursorMoveDirection di : mdbx::cursor::move_operation::previous; } -::mdbx::env_managed open_env(const EnvConfig& config) { +::mdbx::env_managed open_env(EnvConfig& config) { namespace fs = std::filesystem; if (config.path.empty()) { @@ -148,36 +148,32 @@ ::mdbx::env_managed open_env(const EnvConfig& config) { op.max_maps = config.max_tables; op.max_readers = config.max_readers; - ::mdbx::env_managed ret{db_path.native(), cp, op, config.shared}; - if (size_t db_page_size{ret.get_pagesize()}; db_page_size != config.page_size) { - throw std::runtime_error( - "Incompatible page size. " - "Requested " + - human_size(config.page_size) + - " db has " + human_size(db_page_size)); - } + ::mdbx::env_managed env{db_path.native(), cp, op, config.shared}; + + // MDBX will not change the page size if db already exists, so we need to read value + config.page_size = env.get_pagesize(); if (!config.shared) { // C++ bindings don't have setoptions - ::mdbx::error::success_or_throw(::mdbx_env_set_option(ret, MDBX_opt_rp_augment_limit, 32_Mebi)); + ::mdbx::error::success_or_throw(::mdbx_env_set_option(env, MDBX_opt_rp_augment_limit, 32_Mebi)); if (!config.readonly) { - ::mdbx::error::success_or_throw(::mdbx_env_set_option(ret, MDBX_opt_txn_dp_initial, 16_Kibi)); - ::mdbx::error::success_or_throw(::mdbx_env_set_option(ret, MDBX_opt_dp_reserve_limit, 16_Kibi)); + ::mdbx::error::success_or_throw(::mdbx_env_set_option(env, MDBX_opt_txn_dp_initial, 16_Kibi)); + ::mdbx::error::success_or_throw(::mdbx_env_set_option(env, MDBX_opt_dp_reserve_limit, 16_Kibi)); uint64_t dirty_pages_limit{0}; - ::mdbx::error::success_or_throw(::mdbx_env_get_option(ret, MDBX_opt_txn_dp_limit, &dirty_pages_limit)); - ::mdbx::error::success_or_throw(::mdbx_env_set_option(ret, MDBX_opt_txn_dp_limit, dirty_pages_limit * 2)); + ::mdbx::error::success_or_throw(::mdbx_env_get_option(env, MDBX_opt_txn_dp_limit, &dirty_pages_limit)); + ::mdbx::error::success_or_throw(::mdbx_env_set_option(env, MDBX_opt_txn_dp_limit, dirty_pages_limit * 2)); // must be in the range from 12.5% (almost empty) to 50% (half empty) // which corresponds to the range from 8192 and to 32768 in units respectively ::mdbx::error::success_or_throw( - ::mdbx_env_set_option(ret, MDBX_opt_merge_threshold_16dot16_percent, 32_Kibi)); + ::mdbx_env_set_option(env, MDBX_opt_merge_threshold_16dot16_percent, 32_Kibi)); } } if (!config.in_memory) { - ret.check_readers(); + env.check_readers(); } - return ret; + return env; } ::mdbx::map_handle open_map(::mdbx::txn& tx, const MapConfig& config) { diff --git a/silkworm/node/db/mdbx.hpp b/silkworm/node/db/mdbx.hpp index 79eb02830f..69163a1c9a 100644 --- a/silkworm/node/db/mdbx.hpp +++ b/silkworm/node/db/mdbx.hpp @@ -35,6 +35,7 @@ #include #include #include +#include namespace silkworm::db { @@ -358,25 +359,25 @@ using WalkFuncRef = absl::FunctionRef; //! \brief Essential environment settings struct EnvConfig { std::string path{}; - bool create{false}; // Whether db file must be created - bool readonly{false}; // Whether db should be opened in RO mode - bool exclusive{false}; // Whether this process has exclusive access - bool in_memory{false}; // Whether this db is in memory - bool shared{false}; // Whether this process opens a db already opened by another process - bool read_ahead{false}; // Whether to enable mdbx read ahead - bool write_map{false}; // Whether to enable mdbx write map - size_t page_size{4_Kibi}; // Mdbx page size - size_t max_size{3_Tebi}; // Mdbx max map size - size_t growth_size{2_Gibi}; // Increment size for each extension - uint32_t max_tables{128}; // Default max number of named tables - uint32_t max_readers{100}; // Default max number of readers + bool create{false}; // Whether db file must be created + bool readonly{false}; // Whether db should be opened in RO mode + bool exclusive{false}; // Whether this process has exclusive access + bool in_memory{false}; // Whether this db is in memory + bool shared{false}; // Whether this process opens a db already opened by another process + bool read_ahead{false}; // Whether to enable mdbx read ahead + bool write_map{false}; // Whether to enable mdbx write map + size_t page_size{os::page_size()}; // Mdbx page size + size_t max_size{3_Tebi}; // Mdbx max map size + size_t growth_size{2_Gibi}; // Increment size for each extension + uint32_t max_tables{128}; // Default max number of named tables + uint32_t max_readers{100}; // Default max number of readers }; //! \brief Opens an mdbx environment using the provided environment config //! \param [in] config : A structure containing essential environment settings //! \return A handler to mdbx::env_managed class //! \remarks May throw exceptions -::mdbx::env_managed open_env(const EnvConfig& config); +::mdbx::env_managed open_env(EnvConfig& config); //! \brief Opens an mdbx "map" (aka table) //! \param [in] tx : a reference to a valid mdbx transaction diff --git a/silkworm/node/db/mdbx_test.cpp b/silkworm/node/db/mdbx_test.cpp index 6f0b8f43e3..a6cf1d45ca 100644 --- a/silkworm/node/db/mdbx_test.cpp +++ b/silkworm/node/db/mdbx_test.cpp @@ -92,31 +92,53 @@ static const std::map kGeneticCode{ namespace silkworm::db { -TEST_CASE("Env opening") { - SECTION("Non default page size") { +TEST_CASE("Environment opening") { + SECTION("Default page size on creation") { const TemporaryDirectory tmp_dir; - db::EnvConfig db_config{tmp_dir.path().string(), /*create*/ true}; - db_config.in_memory = true; - db_config.page_size = 8_Kibi; - auto env{db::open_env(db_config)}; - REQUIRE(env.get_pagesize() == db_config.page_size); + db::EnvConfig db_config{ + .path = tmp_dir.path().string(), + .create = true, + .in_memory = true, + }; + REQUIRE(db_config.page_size == os::page_size()); + const auto env{db::open_env(db_config)}; + CHECK(env.get_pagesize() == db_config.page_size); } - SECTION("Incompatible page size") { + SECTION("Non default page size on creation") { + const TemporaryDirectory tmp_dir; + db::EnvConfig db_config{ + .path = tmp_dir.path().string(), + .create = true, + .in_memory = true, + .page_size = os::page_size() / 2, + }; + const auto env{db::open_env(db_config)}; + CHECK(env.get_pagesize() == db_config.page_size); + } + + SECTION("Read page size on opening") { const TemporaryDirectory tmp_dir; { - db::EnvConfig db_config{tmp_dir.path().string(), /*create*/ true}; - db_config.in_memory = true; - db_config.page_size = 4_Kibi; - REQUIRE_NOTHROW((void)db::open_env(db_config)); + db::EnvConfig db_config{ + .path = tmp_dir.path().string(), + .create = true, + .in_memory = true, + .page_size = os::page_size() / 2, + }; + (void)db::open_env(db_config); } { - // Try to reopen same db with 16KB page size - db::EnvConfig db_config{tmp_dir.path().string(), /*create*/ false}; - db_config.in_memory = true; - db_config.page_size = 16_Kibi; - REQUIRE_THROWS((void)db::open_env(db_config)); + // Try to reopen same db with another page size + db::EnvConfig db_config{ + .path = tmp_dir.path().string(), + .create = false, + .in_memory = true, + .page_size = os::page_size() * 2, + }; + const auto env{db::open_env(db_config)}; + CHECK(env.get_pagesize() == os::page_size() / 2); } } } diff --git a/silkworm/node/db/memory_mutation_cursor_test.cpp b/silkworm/node/db/memory_mutation_cursor_test.cpp index 3b099d0d24..093addceb6 100644 --- a/silkworm/node/db/memory_mutation_cursor_test.cpp +++ b/silkworm/node/db/memory_mutation_cursor_test.cpp @@ -23,7 +23,7 @@ namespace silkworm::db { -static mdbx::env_managed create_main_env(const db::EnvConfig& main_db_config) { +static mdbx::env_managed create_main_env(db::EnvConfig& main_db_config) { auto main_env = db::open_env(main_db_config); RWTxnManaged main_txn{main_env}; table::check_or_create_chaindata_tables(main_txn);