node: read page size from existing MDBX database (#1580)

erigontech · Oct 15, 2023 · 3cdf809 · 3cdf809
1 parent 953aae4
commit 3cdf809
Show file tree

Hide file tree

Showing 5 changed files with 68 additions and 49 deletions.
diff --git a/cmd/dev/toolbox.cpp b/cmd/dev/toolbox.cpp
@@ -330,7 +330,7 @@ dbTablesInfo get_tablesInfo(::mdbx::txn& txn) {
     return ret;
 }
 
-void do_scan(const db::EnvConfig& config) {
+void do_scan(db::EnvConfig& config) {
     static std::string fmt_hdr{" %3s %-24s %=50s %13s %13s %13s"};
 
     auto env{silkworm::db::open_env(config)};

diff --git a/silkworm/node/db/mdbx.cpp b/silkworm/node/db/mdbx.cpp
@@ -66,7 +66,7 @@ static inline mdbx::cursor::move_operation move_operation(CursorMoveDirection di
                : mdbx::cursor::move_operation::previous;
 }
 
-::mdbx::env_managed open_env(const EnvConfig& config) {
+::mdbx::env_managed open_env(EnvConfig& config) {
     namespace fs = std::filesystem;
 
     if (config.path.empty()) {
@@ -148,36 +148,32 @@ ::mdbx::env_managed open_env(const EnvConfig& config) {
     op.max_maps = config.max_tables;
     op.max_readers = config.max_readers;
 
-    ::mdbx::env_managed ret{db_path.native(), cp, op, config.shared};
-    if (size_t db_page_size{ret.get_pagesize()}; db_page_size != config.page_size) {
-        throw std::runtime_error(
-            "Incompatible page size. "
-            "Requested " +
-            human_size(config.page_size) +
-            " db has " + human_size(db_page_size));
-    }
+    ::mdbx::env_managed env{db_path.native(), cp, op, config.shared};
+
+    // MDBX will not change the page size if db already exists, so we need to read value
+    config.page_size = env.get_pagesize();
 
     if (!config.shared) {
         // C++ bindings don't have setoptions
-        ::mdbx::error::success_or_throw(::mdbx_env_set_option(ret, MDBX_opt_rp_augment_limit, 32_Mebi));
+        ::mdbx::error::success_or_throw(::mdbx_env_set_option(env, MDBX_opt_rp_augment_limit, 32_Mebi));
         if (!config.readonly) {
-            ::mdbx::error::success_or_throw(::mdbx_env_set_option(ret, MDBX_opt_txn_dp_initial, 16_Kibi));
-            ::mdbx::error::success_or_throw(::mdbx_env_set_option(ret, MDBX_opt_dp_reserve_limit, 16_Kibi));
+            ::mdbx::error::success_or_throw(::mdbx_env_set_option(env, MDBX_opt_txn_dp_initial, 16_Kibi));
+            ::mdbx::error::success_or_throw(::mdbx_env_set_option(env, MDBX_opt_dp_reserve_limit, 16_Kibi));
 
             uint64_t dirty_pages_limit{0};
-            ::mdbx::error::success_or_throw(::mdbx_env_get_option(ret, MDBX_opt_txn_dp_limit, &dirty_pages_limit));
-            ::mdbx::error::success_or_throw(::mdbx_env_set_option(ret, MDBX_opt_txn_dp_limit, dirty_pages_limit * 2));
+            ::mdbx::error::success_or_throw(::mdbx_env_get_option(env, MDBX_opt_txn_dp_limit, &dirty_pages_limit));
+            ::mdbx::error::success_or_throw(::mdbx_env_set_option(env, MDBX_opt_txn_dp_limit, dirty_pages_limit * 2));
 
             // must be in the range from 12.5% (almost empty) to 50% (half empty)
             // which corresponds to the range from 8192 and to 32768 in units respectively
             ::mdbx::error::success_or_throw(
-                ::mdbx_env_set_option(ret, MDBX_opt_merge_threshold_16dot16_percent, 32_Kibi));
+                ::mdbx_env_set_option(env, MDBX_opt_merge_threshold_16dot16_percent, 32_Kibi));
         }
     }
     if (!config.in_memory) {
-        ret.check_readers();
+        env.check_readers();
     }
-    return ret;
+    return env;
 }
 
 ::mdbx::map_handle open_map(::mdbx::txn& tx, const MapConfig& config) {

diff --git a/silkworm/node/db/mdbx.hpp b/silkworm/node/db/mdbx.hpp
@@ -35,6 +35,7 @@
 #include <silkworm/core/common/bytes.hpp>
 #include <silkworm/core/common/object_pool.hpp>
 #include <silkworm/core/common/util.hpp>
+#include <silkworm/infra/common/os.hpp>
 
 namespace silkworm::db {
 
@@ -358,25 +359,25 @@ using WalkFuncRef = absl::FunctionRef<void(ByteView key, ByteView value)>;
 //! \brief Essential environment settings
 struct EnvConfig {
     std::string path{};
-    bool create{false};          // Whether db file must be created
-    bool readonly{false};        // Whether db should be opened in RO mode
-    bool exclusive{false};       // Whether this process has exclusive access
-    bool in_memory{false};       // Whether this db is in memory
-    bool shared{false};          // Whether this process opens a db already opened by another process
-    bool read_ahead{false};      // Whether to enable mdbx read ahead
-    bool write_map{false};       // Whether to enable mdbx write map
-    size_t page_size{4_Kibi};    // Mdbx page size
-    size_t max_size{3_Tebi};     // Mdbx max map size
-    size_t growth_size{2_Gibi};  // Increment size for each extension
-    uint32_t max_tables{128};    // Default max number of named tables
-    uint32_t max_readers{100};   // Default max number of readers
+    bool create{false};                 // Whether db file must be created
+    bool readonly{false};               // Whether db should be opened in RO mode
+    bool exclusive{false};              // Whether this process has exclusive access
+    bool in_memory{false};              // Whether this db is in memory
+    bool shared{false};                 // Whether this process opens a db already opened by another process
+    bool read_ahead{false};             // Whether to enable mdbx read ahead
+    bool write_map{false};              // Whether to enable mdbx write map
+    size_t page_size{os::page_size()};  // Mdbx page size
+    size_t max_size{3_Tebi};            // Mdbx max map size
+    size_t growth_size{2_Gibi};         // Increment size for each extension
+    uint32_t max_tables{128};           // Default max number of named tables
+    uint32_t max_readers{100};          // Default max number of readers
 };
 
 //! \brief Opens an mdbx environment using the provided environment config
 //! \param [in] config : A structure containing essential environment settings
 //! \return A handler to mdbx::env_managed class
 //! \remarks May throw exceptions
-::mdbx::env_managed open_env(const EnvConfig& config);
+::mdbx::env_managed open_env(EnvConfig& config);
 
 //! \brief Opens an mdbx "map" (aka table)
 //! \param [in] tx : a reference to a valid mdbx transaction

diff --git a/silkworm/node/db/mdbx_test.cpp b/silkworm/node/db/mdbx_test.cpp
@@ -92,31 +92,53 @@ static const std::map<std::string, std::string> kGeneticCode{
 
 namespace silkworm::db {
 
-TEST_CASE("Env opening") {
-    SECTION("Non default page size") {
+TEST_CASE("Environment opening") {
+    SECTION("Default page size on creation") {
         const TemporaryDirectory tmp_dir;
-        db::EnvConfig db_config{tmp_dir.path().string(), /*create*/ true};
-        db_config.in_memory = true;
-        db_config.page_size = 8_Kibi;
-        auto env{db::open_env(db_config)};
-        REQUIRE(env.get_pagesize() == db_config.page_size);
+        db::EnvConfig db_config{
+            .path = tmp_dir.path().string(),
+            .create = true,
+            .in_memory = true,
+        };
+        REQUIRE(db_config.page_size == os::page_size());
+        const auto env{db::open_env(db_config)};
+        CHECK(env.get_pagesize() == db_config.page_size);
     }
 
-    SECTION("Incompatible page size") {
+    SECTION("Non default page size on creation") {
+        const TemporaryDirectory tmp_dir;
+        db::EnvConfig db_config{
+            .path = tmp_dir.path().string(),
+            .create = true,
+            .in_memory = true,
+            .page_size = os::page_size() / 2,
+        };
+        const auto env{db::open_env(db_config)};
+        CHECK(env.get_pagesize() == db_config.page_size);
+    }
+
+    SECTION("Read page size on opening") {
         const TemporaryDirectory tmp_dir;
         {
-            db::EnvConfig db_config{tmp_dir.path().string(), /*create*/ true};
-            db_config.in_memory = true;
-            db_config.page_size = 4_Kibi;
-            REQUIRE_NOTHROW((void)db::open_env(db_config));
+            db::EnvConfig db_config{
+                .path = tmp_dir.path().string(),
+                .create = true,
+                .in_memory = true,
+                .page_size = os::page_size() / 2,
+            };
+            (void)db::open_env(db_config);
         }
 
         {
-            // Try to reopen same db with 16KB page size
-            db::EnvConfig db_config{tmp_dir.path().string(), /*create*/ false};
-            db_config.in_memory = true;
-            db_config.page_size = 16_Kibi;
-            REQUIRE_THROWS((void)db::open_env(db_config));
+            // Try to reopen same db with another page size
+            db::EnvConfig db_config{
+                .path = tmp_dir.path().string(),
+                .create = false,
+                .in_memory = true,
+                .page_size = os::page_size() * 2,
+            };
+            const auto env{db::open_env(db_config)};
+            CHECK(env.get_pagesize() == os::page_size() / 2);
         }
     }
 }

diff --git a/silkworm/node/db/memory_mutation_cursor_test.cpp b/silkworm/node/db/memory_mutation_cursor_test.cpp
@@ -23,7 +23,7 @@
 
 namespace silkworm::db {
 
-static mdbx::env_managed create_main_env(const db::EnvConfig& main_db_config) {
+static mdbx::env_managed create_main_env(db::EnvConfig& main_db_config) {
     auto main_env = db::open_env(main_db_config);
     RWTxnManaged main_txn{main_env};
     table::check_or_create_chaindata_tables(main_txn);