Merge branch 'master' into ci/bit_cast

cmd/dev/backend_kv_server.cpp

@@ -243,11 +243,11 @@ int main(int argc, char* argv[]) {
                 auto stop = [&state_changes_timer]() {
                     state_changes_timer.cancel();
                 };
-                co_await concurrency::async_thread(std::move(run), std::move(stop));
+                co_await concurrency::async_thread(std::move(run), std::move(stop), "state-c-sim");
             };
-            tasks = state_changes_simulator() && server.async_run();
+            tasks = state_changes_simulator() && server.async_run("bekv-server");
         } else {
-            tasks = server.async_run();
+            tasks = server.async_run("bekv-server");
         }
 
         ShutdownSignal shutdown_signal{context_pool.next_io_context()};

docs/JSON-RPC-API.md

@@ -140,7 +140,7 @@ The following table shows the current [JSON RPC API](https://eth.wiki/json-rpc/A
 | erigon_cumulativeChainTraffic              |     Yes      |                                           |
 | erigon_getHeaderByHash                     |     Yes      |                                           |
 | erigon_getHeaderByNumber                   |     Yes      |                                           |
-| erigon_getBalanceChangesInBlock            |      -       |                       not yet implemented |
+| erigon_getBalanceChangesInBlock            |     Yes      |                                           |
 | erigon_getBlockByTimestamp                 |     Yes      |                                           |
 | erigon_getBlockReceiptsByBlockHash         |      -       |                       not yet implemented |
 | erigon_getLogsByHash                       |     Yes      |                                           |

silkworm/api/silkworm_api.cpp

@@ -29,6 +29,10 @@
 
 using namespace silkworm;
 
+static MemoryMappedRegion make_region(const SilkwormMemoryMappedFile& mmf) {
+    return {mmf.memory_address, mmf.memory_length};
+}
+
 SILKWORM_EXPORT int silkworm_init(SilkwormHandle** handle) SILKWORM_NOEXCEPT {
     if (!handle) {
         return SILKWORM_INVALID_HANDLE;
@@ -48,46 +52,50 @@ SILKWORM_EXPORT int silkworm_add_snapshot(SilkwormHandle* handle, SilkwormChainS
     }
     const auto snapshot_repository = reinterpret_cast<snapshot::SnapshotRepository*>(handle);
 
-    const SilkwormHeadersSnapshot& headers_snapshot = snapshot->headers;
-    const auto headers_segment_path = snapshot::SnapshotPath::parse(headers_snapshot.segment.file_path);
+    const SilkwormHeadersSnapshot& hs = snapshot->headers;
+    const auto headers_segment_path = snapshot::SnapshotPath::parse(hs.segment.file_path);
     if (!headers_segment_path) {
         return SILKWORM_INVALID_PATH;
     }
-    // TODO(canepat) HeaderSnapshot must be created w/ segment_address+segment_length because mmap already done by Erigon
-    // TODO(canepat) The same holds for its index
-    auto headers_segment = std::make_unique<snapshot::HeaderSnapshot>(*headers_segment_path);
-    headers_segment->reopen_segment();  // TODO(canepat) must not be called hence throw exception if called when snapshot already mapped
-    headers_segment->reopen_index();    // TODO(canepat) must not be called hence throw exception if called when snapshot already mapped
-
-    const SilkwormBodiesSnapshot& bodies_snapshot = snapshot->bodies;
-    const auto bodies_segment_path = snapshot::SnapshotPath::parse(bodies_snapshot.segment.file_path);
+    snapshot::MappedHeadersSnapshot mapped_h_snapshot{
+        .segment = make_region(hs.segment),
+        .header_hash_index = make_region(hs.header_hash_index)};
+    auto headers_snapshot = std::make_unique<snapshot::HeaderSnapshot>(*headers_segment_path, mapped_h_snapshot);
+    headers_snapshot->reopen_segment();
+    headers_snapshot->reopen_index();
+
+    const SilkwormBodiesSnapshot& bs = snapshot->bodies;
+    const auto bodies_segment_path = snapshot::SnapshotPath::parse(bs.segment.file_path);
     if (!bodies_segment_path) {
         return SILKWORM_INVALID_PATH;
     }
-    // TODO(canepat) BodySnapshot must be created w/ segment_address+segment_length because mmap already done by Erigon
-    // TODO(canepat) The same holds for its index
-    auto bodies_segment = std::make_unique<snapshot::BodySnapshot>(*bodies_segment_path);
-    bodies_segment->reopen_segment();  // TODO(canepat) must not be called hence throw exception if called when snapshot already mapped
-    bodies_segment->reopen_index();    // TODO(canepat) must not be called hence throw exception if called when snapshot already mapped
-
-    const SilkwormTransactionsSnapshot& transactions_snapshot = snapshot->transactions;
-    const auto transactions_segment_path = snapshot::SnapshotPath::parse(transactions_snapshot.segment.file_path);
+    snapshot::MappedBodiesSnapshot mapped_b_snapshot{
+        .segment = make_region(bs.segment),
+        .block_num_index = make_region(bs.block_num_index)};
+    auto bodies_snapshot = std::make_unique<snapshot::BodySnapshot>(*bodies_segment_path, mapped_b_snapshot);
+    bodies_snapshot->reopen_segment();
+    bodies_snapshot->reopen_index();
+
+    const SilkwormTransactionsSnapshot& ts = snapshot->transactions;
+    const auto transactions_segment_path = snapshot::SnapshotPath::parse(ts.segment.file_path);
     if (!transactions_segment_path) {
         return SILKWORM_INVALID_PATH;
     }
-    // TODO(canepat) TransactionSnapshot must be created w/ segment_address+segment_length because mmap already done by Erigon
-    // TODO(canepat) The same holds for its index
-    auto transactions_segment = std::make_unique<snapshot::TransactionSnapshot>(*transactions_segment_path);
-    transactions_segment->reopen_segment();  // TODO(canepat) must not be called hence throw exception if called when snapshot already mapped
-    transactions_segment->reopen_index();    // TODO(canepat) must not be called hence throw exception if called when snapshot already mapped
+    snapshot::MappedTransactionsSnapshot mapped_t_snapshot{
+        .segment = make_region(ts.segment),
+        .tx_hash_index = make_region(ts.tx_hash_index),
+        .tx_hash_2_block_index = make_region(ts.tx_hash_2_block_index)};
+    auto transactions_snapshot = std::make_unique<snapshot::TransactionSnapshot>(*transactions_segment_path, mapped_t_snapshot);
+    transactions_snapshot->reopen_segment();
+    transactions_snapshot->reopen_index();
 
     snapshot::SnapshotBundle bundle{
         .headers_snapshot_path = *headers_segment_path,
-        .headers_snapshot = std::move(headers_segment),
+        .headers_snapshot = std::move(headers_snapshot),
         .bodies_snapshot_path = *bodies_segment_path,
-        .bodies_snapshot = std::move(bodies_segment),
+        .bodies_snapshot = std::move(bodies_snapshot),
         .tx_snapshot_path = *transactions_segment_path,
-        .tx_snapshot = std::move(transactions_segment)};
+        .tx_snapshot = std::move(transactions_snapshot)};
     snapshot_repository->add_snapshot_bundle(std::move(bundle));
     return SILKWORM_OK;
 }

silkworm/api/silkworm_api.h

@@ -63,7 +63,7 @@ SILKWORM_EXPORT int silkworm_init(SilkwormHandle** handle) SILKWORM_NOEXCEPT;
 
 struct SilkwormMemoryMappedFile {
     const char* file_path;
-    const void* memory_address;
+    uint8_t* memory_address;
     size_t memory_length;
 };
 

silkworm/infra/common/memory_mapped_file.cpp

@@ -36,26 +36,44 @@
 
 namespace silkworm {
 
-MemoryMappedFile::MemoryMappedFile(std::filesystem::path path, bool read_only) : path_(std::move(path)), managed_{true} {
-    map_existing(read_only);
-}
-
-MemoryMappedFile::MemoryMappedFile(std::filesystem::path path, uint8_t* address, std::size_t length)
-    : path_{std::move(path)}, address_{address}, length_{length}, managed_{false} {
+MemoryMappedFile::MemoryMappedFile(std::filesystem::path path, std::optional<MemoryMappedRegion> region, bool read_only)
+    : path_(std::move(path)), managed_{not region.has_value()} {
     ensure(std::filesystem::exists(path_), "MemoryMappedFile: " + path_.string() + " does not exist");
     ensure(std::filesystem::is_regular_file(path_), "MemoryMappedFile: " + path_.string() + " is not regular file");
-    ensure(address != nullptr, "MemoryMappedFile: address is null");
-    ensure(length > 0, "MemoryMappedFile: length is zero");
+
+    if (region) {
+        ensure(region->address != nullptr, "MemoryMappedFile: address is null");
+        ensure(region->length > 0, "MemoryMappedFile: length is zero");
+        address_ = region->address;
+        length_ = region->length;
+    } else {
+        map_existing(read_only);
+    }
 }
 
 MemoryMappedFile::~MemoryMappedFile() {
+    if (not managed_) {
+        return;
+    }
+
     unmap();
 
 #ifdef _WIN32
     cleanup();
 #endif
 }
 
+MemoryMappedFile::MemoryMappedFile(MemoryMappedFile&& source) noexcept
+    : path_(std::move(source.path_)), address_(source.address_), length_(source.length_), managed_(source.managed_) {}
+
+MemoryMappedFile& MemoryMappedFile::operator=(MemoryMappedFile&& other) noexcept {
+    path_ = std::move(other.path_);
+    address_ = other.address_;
+    length_ = other.length_;
+    managed_ = other.managed_;
+    return *this;
+}
+
 #ifdef _WIN32
 void MemoryMappedFile::map_existing(bool read_only) {
     DWORD desired_access = read_only ? GENERIC_READ : (GENERIC_READ | GENERIC_WRITE);
@@ -163,7 +181,7 @@ void* MemoryMappedFile::mmap(FileDescriptor fd, bool read_only) {
 }
 
 void MemoryMappedFile::unmap() {
-    if (managed_ and address_ != nullptr) {
+    if (address_ != nullptr) {
         const int result = ::munmap(address_, length_);
         if (result == -1) {
             throw std::runtime_error{"munmap failed for: " + path_.string() + " error: " + strerror(errno)};

silkworm/infra/common/memory_mapped_file.hpp

@@ -46,6 +46,7 @@
 #include <cstdio>
 #include <filesystem>
 #include <istream>
+#include <optional>
 #include <streambuf>
 #include <tuple>
 
@@ -68,12 +69,24 @@ using FileDescriptor = HANDLE;
 using FileDescriptor = int;
 #endif
 
+struct MemoryMappedRegion {
+    uint8_t* address{nullptr};
+    std::size_t length{0};
+};
+
 class MemoryMappedFile {
   public:
-    explicit MemoryMappedFile(std::filesystem::path path, bool read_only = true);
-    MemoryMappedFile(std::filesystem::path path, uint8_t* address, std::size_t length);
+    explicit MemoryMappedFile(std::filesystem::path path, std::optional<MemoryMappedRegion> region = {}, bool read_only = true);
     ~MemoryMappedFile();
 
+    // Not copyable
+    MemoryMappedFile(const MemoryMappedFile&) = delete;
+    MemoryMappedFile& operator=(const MemoryMappedFile&) = delete;
+
+    // Only movable
+    MemoryMappedFile(MemoryMappedFile&& source) noexcept;
+    MemoryMappedFile& operator=(MemoryMappedFile&& other) noexcept;
+
     [[nodiscard]] std::filesystem::path path() const {
         return path_;
     }

silkworm/infra/common/memory_mapped_file_test.cpp

@@ -30,7 +30,7 @@ using namespace std::chrono_literals;
 
 TEST_CASE("MemoryMappedFile from file", "[silkworm][infra][common][memory_mapped_file]") {
     SECTION("constructor fails for nonexistent file") {
-        CHECK_THROWS_AS(MemoryMappedFile{"nonexistent.txt"}, std::runtime_error);
+        CHECK_THROWS_AS(MemoryMappedFile{"nonexistent.txt"}, std::logic_error);
     }
 
     SECTION("constructor fails for existent empty file") {
@@ -46,9 +46,9 @@ TEST_CASE("MemoryMappedFile from file", "[silkworm][infra][common][memory_mapped
         tmp_stream.write("\x01", 1);
         tmp_stream.close();
         CHECK_NOTHROW(MemoryMappedFile{tmp_file});
-        CHECK_NOTHROW(MemoryMappedFile{tmp_file, false});
+        CHECK_NOTHROW(MemoryMappedFile{tmp_file, {}, false});
         CHECK_NOTHROW(MemoryMappedFile{tmp_file.string()});
-        CHECK_NOTHROW(MemoryMappedFile{tmp_file.string(), false});
+        CHECK_NOTHROW(MemoryMappedFile{tmp_file.string(), {}, false});
     }
 
     const std::string kFileContent{"\x01\x02\x03"};
@@ -100,12 +100,12 @@ TEST_CASE("MemoryMappedFile from file", "[silkworm][infra][common][memory_mapped
 
 TEST_CASE("MemoryMappedFile from memory", "[silkworm][infra][common][memory_mapped_file]") {
     SECTION("constructor fails for null address") {
-        CHECK_THROWS_AS(MemoryMappedFile("", nullptr, 100), std::logic_error);
+        CHECK_THROWS_AS(MemoryMappedFile("", MemoryMappedRegion{nullptr, 100}), std::logic_error);
     }
 
     SECTION("constructor fails for zero length") {
         uint8_t u;
-        CHECK_THROWS_AS(MemoryMappedFile("", &u, 0), std::logic_error);
+        CHECK_THROWS_AS(MemoryMappedFile("", MemoryMappedRegion{&u, 0}), std::logic_error);
     }
 
     SECTION("constructor succeeds for existent nonempty file") {
@@ -114,7 +114,7 @@ TEST_CASE("MemoryMappedFile from memory", "[silkworm][infra][common][memory_mapp
         tmp_stream.write("\x01", 1);
         tmp_stream.close();
         MemoryMappedFile mmf_from_file{tmp_file};
-        CHECK_NOTHROW(MemoryMappedFile(tmp_file, mmf_from_file.address(), mmf_from_file.length()));
+        CHECK_NOTHROW(MemoryMappedFile(tmp_file, MemoryMappedRegion{mmf_from_file.address(), mmf_from_file.length()}));
     }
 
     const std::string kFileContent{"\x01\x02\x03"};
@@ -125,7 +125,7 @@ TEST_CASE("MemoryMappedFile from memory", "[silkworm][infra][common][memory_mapp
     MemoryMappedFile mmf_from_file{tmp_file};
     const auto address{mmf_from_file.address()};
     const auto length{mmf_from_file.length()};
-    MemoryMappedFile mmf{tmp_file, mmf_from_file.address(), mmf_from_file.length()};
+    MemoryMappedFile mmf{tmp_file, MemoryMappedRegion{mmf_from_file.address(), mmf_from_file.length()}};
 
     SECTION("has expected memory address and size") {
         CHECK(mmf.address() == address);
@@ -158,7 +158,7 @@ TEST_CASE("MemoryMappedFile from memory", "[silkworm][infra][common][memory_mapp
         const auto tmp_path = std::filesystem::temp_directory_path() / "example.bin";
         std::ofstream{tmp_path.c_str()}.put('a');
         MemoryMappedFile mmf_from_path{tmp_path};
-        MemoryMappedFile mmf_from_memory{tmp_path, mmf_from_path.address(), mmf_from_path.length()};
+        MemoryMappedFile mmf_from_memory{tmp_path, MemoryMappedRegion{mmf_from_path.address(), mmf_from_path.length()}};
         const auto ftime = mmf_from_memory.last_write_time();
         // Move file write time 1 hour to the future
         std::filesystem::last_write_time(tmp_path, ftime + 1h);

silkworm/infra/common/os.cpp

@@ -72,13 +72,16 @@ bool set_max_file_descriptors(uint64_t max_descriptors) {
 }
 
 std::size_t page_size() noexcept {
+    static auto system_page_size = []() -> std::size_t {
 #ifdef _WIN32
-    SYSTEM_INFO system_info;
-    ::GetSystemInfo(&system_info);
-    return static_cast<std::size_t>(system_info.dwPageSize);
+        SYSTEM_INFO system_info;
+        ::GetSystemInfo(&system_info);
+        return static_cast<std::size_t>(system_info.dwPageSize);
 #else
-    return static_cast<std::size_t>(::getpagesize());
+        return static_cast<std::size_t>(::getpagesize());
 #endif  // _WIN32
+    }();
+    return system_page_size;
 }
 
 }  // namespace silkworm::os

silkworm/infra/concurrency/active_component.hpp

@@ -29,21 +29,17 @@
 
 namespace silkworm {
 
-/*
- * Abstract interface for active components
- * i.e. component that have an infinite loop and need a dedicated thread to run the loop (if the application
- * has also other things to do).
- * Here we prefer not to provide a thread facility and let the user provide one more suitable to the context,
- * so perhaps a better name is LongRunningComponent.
- */
+//! Abstract interface for active components i.e. components that have an infinite loop and need a dedicated thread
+//! to run the loop (if the application has also other things to do).
 class ActiveComponent : public Stoppable {
   public:
     virtual void execution_loop() = 0;
 
-    boost::asio::awaitable<void> async_run(std::optional<std::size_t> stack_size = {}) {
+    //! This adapter method makes ActiveComponent suitable to be used as asynchronous task
+    boost::asio::awaitable<void> async_run(const char* thread_name, std::optional<std::size_t> stack_size = {}) {
         auto run = [this] { this->execution_loop(); };
         auto stop = [this] { this->stop(); };
-        co_await concurrency::async_thread(std::move(run), std::move(stop), stack_size);
+        co_await concurrency::async_thread(std::move(run), std::move(stop), thread_name, stack_size);
     }
 };
 

silkworm/infra/concurrency/async_thread.cpp

@@ -29,7 +29,8 @@
 
 namespace silkworm::concurrency {
 
-boost::asio::awaitable<void> async_thread(std::function<void()> run, std::function<void()> stop, std::optional<std::size_t> stack_size) {
+boost::asio::awaitable<void> async_thread(std::function<void()> run, std::function<void()> stop,
+                                          const char* name, std::optional<std::size_t> stack_size) {
     std::exception_ptr run_exception;
 
     auto executor = co_await boost::asio::this_coro::executor;
@@ -39,9 +40,13 @@ boost::asio::awaitable<void> async_thread(std::function<void()> run, std::functi
     if (stack_size) {
         attributes.set_stack_size(*stack_size);
     }
-    boost::thread thread{attributes, [run = std::move(run), &run_exception, &thread_finished_notifier] {
+
+    boost::thread thread{attributes, [run = std::move(run), name = name, &run_exception, &thread_finished_notifier] {
+                             log::set_thread_name(name);
                              try {
+                                 log::Info() << "Async thread [" << name << "] run started";
                                  run();
+                                 log::Info() << "Async thread [" << name << "] run completed";
                              } catch (...) {
                                  run_exception = std::current_exception();
                              }

silkworm/infra/concurrency/async_thread.hpp

@@ -34,10 +34,11 @@ namespace silkworm::concurrency {
  *
  * @param run thread procedure
  * @param stop a callback to signal the thread procedure to exit
+ * @param name the name appearing in log traces for the created thread
  * @param stack_size optional custom stack size for the created thread
  * @return an awaitable that is pending until the thread finishes
  */
 boost::asio::awaitable<void> async_thread(std::function<void()> run, std::function<void()> stop,
-                                          std::optional<std::size_t> stack_size = {});
+                                          const char* name, std::optional<std::size_t> stack_size = {});
 
 }  // namespace silkworm::concurrency