rpcdaemon: erigon_getBalanceChangesInBlock API (#1382)

docs/JSON-RPC-API.md

@@ -140,7 +140,7 @@ The following table shows the current [JSON RPC API](https://eth.wiki/json-rpc/A
 | erigon_cumulativeChainTraffic              |     Yes      |                                           |
 | erigon_getHeaderByHash                     |     Yes      |                                           |
 | erigon_getHeaderByNumber                   |     Yes      |                                           |
-| erigon_getBalanceChangesInBlock            |      -       |                       not yet implemented |
+| erigon_getBalanceChangesInBlock            |     Yes      |                                           |
 | erigon_getBlockByTimestamp                 |     Yes      |                                           |
 | erigon_getBlockReceiptsByBlockHash         |      -       |                       not yet implemented |
 | erigon_getLogsByHash                       |     Yes      |                                           |

silkworm/silkrpc/commands/erigon_api.cpp

@@ -26,6 +26,7 @@
 #include <silkworm/infra/common/log.hpp>
 #include <silkworm/silkrpc/common/binary_search.hpp>
 #include <silkworm/silkrpc/common/util.hpp>
+#include <silkworm/silkrpc/core/block_reader.hpp>
 #include <silkworm/silkrpc/core/blocks.hpp>
 #include <silkworm/silkrpc/core/cached_chain.hpp>
 #include <silkworm/silkrpc/core/rawdb/chain.hpp>
@@ -57,12 +58,38 @@ awaitable<void> ErigonRpcApi::handle_erigon_cache_check(const nlohmann::json& re
 
 // https://eth.wiki/json-rpc/API#erigon_getbalancechangesinblock
 awaitable<void> ErigonRpcApi::handle_erigon_get_balance_changes_in_block(const nlohmann::json& request, nlohmann::json& reply) {
+    const auto& params = request["params"];
+    if (params.size() < 1) {
+        auto error_msg = "invalid erigon_getBalanceChangesInBlock params: " + params.dump();
+        SILK_ERROR << error_msg;
+        reply = make_json_error(request["id"], 100, error_msg);
+
+        co_return;
+    }
+    const auto block_number_or_hash = params[0].get<BlockNumberOrHash>();
+
+    SILK_DEBUG << "block_number_or_hash: " << block_number_or_hash;
+
     auto tx = co_await database_->begin();
 
     try {
         ethdb::TransactionDatabase tx_database{*tx};
+        const auto chain_storage = tx->create_storage(tx_database, backend_);
 
-        reply = make_json_content(request["id"], to_quantity(0));
+        auto start = std::chrono::system_clock::now();
+
+        rpc::BlockReader block_reader{tx_database, *chain_storage, *tx};
+        rpc::BalanceChanges balance_changes;
+        co_await block_reader.read_balance_changes(*block_cache_, block_number_or_hash, balance_changes);
+
+        auto end = std::chrono::system_clock::now();
+        std::chrono::duration<double> elapsed_seconds = end - start;
+        SILK_DEBUG << "balance_changes: elapsed " << elapsed_seconds.count() << " sec";
+
+        nlohmann::json json;
+        to_json(json, balance_changes);
+
+        reply = make_json_content(request["id"], json);
     } catch (const std::exception& e) {
         SILK_ERROR << "exception: " << e.what() << " processing request: " << request.dump();
         reply = make_json_error(request["id"], 100, e.what());

silkworm/silkrpc/commands/erigon_api.hpp

@@ -55,8 +55,8 @@ class ErigonRpcApi {
     awaitable<void> handle_erigon_cache_check(const nlohmann::json& request, nlohmann::json& reply);
     awaitable<void> handle_erigon_get_balance_changes_in_block(const nlohmann::json& request, nlohmann::json& reply);
     awaitable<void> handle_erigon_get_block_by_timestamp(const nlohmann::json& request, nlohmann::json& reply);
-    awaitable<void> handle_erigon_get_header_by_hash(const nlohmann::json& request, nlohmann::json& reply);
     awaitable<void> handle_erigon_get_block_receipts_by_block_hash(const nlohmann::json& request, nlohmann::json& reply);
+    awaitable<void> handle_erigon_get_header_by_hash(const nlohmann::json& request, nlohmann::json& reply);
     awaitable<void> handle_erigon_get_header_by_number(const nlohmann::json& request, nlohmann::json& reply);
     awaitable<void> handle_erigon_get_latest_logs(const nlohmann::json& request, nlohmann::json& reply);
     awaitable<void> handle_erigon_get_logs_by_hash(const nlohmann::json& request, nlohmann::json& reply);

silkworm/silkrpc/commands/rpc_api_table.cpp

@@ -183,14 +183,18 @@ void RpcApiTable::add_parity_handlers() {
 }
 
 void RpcApiTable::add_erigon_handlers() {
-    method_handlers_[http::method::k_erigon_getBlockByTimestamp] = &commands::RpcApi::handle_erigon_get_block_by_timestamp;
+    method_handlers_[http::method::k_erigon_blockNumber] = &commands::RpcApi::handle_erigon_block_number;
+    method_handlers_[http::method::k_erigon_cacheCheck] = &commands::RpcApi::handle_erigon_cache_check;
+    method_handlers_[http::method::k_erigon_getBalanceChangesInBlock] = &commands::RpcApi::handle_erigon_get_balance_changes_in_block;
+    method_handlers_[http::method::k_erigon_getBlockByTimestamp] = &commands::RpcApi::handle_erigon_cache_check;
+    method_handlers_[http::method::k_erigon_getBlockReceiptsByBlockHash] = &commands::RpcApi::handle_erigon_get_block_receipts_by_block_hash;
     method_handlers_[http::method::k_erigon_getHeaderByHash] = &commands::RpcApi::handle_erigon_get_header_by_hash;
     method_handlers_[http::method::k_erigon_getHeaderByNumber] = &commands::RpcApi::handle_erigon_get_header_by_number;
+    method_handlers_[http::method::k_erigon_getLatestLogs] = &commands::RpcApi::handle_erigon_get_latest_logs;
     method_handlers_[http::method::k_erigon_getLogsByHash] = &commands::RpcApi::handle_erigon_get_logs_by_hash;
     method_handlers_[http::method::k_erigon_forks] = &commands::RpcApi::handle_erigon_forks;
     method_handlers_[http::method::k_erigon_watchTheBurn] = &commands::RpcApi::handle_erigon_watch_the_burn;
     method_handlers_[http::method::k_erigon_cumulative_chain_traffic] = &commands::RpcApi::handle_erigon_cumulative_chain_traffic;
-    method_handlers_[http::method::k_erigon_blockNumber] = &commands::RpcApi::handle_erigon_block_number;
     method_handlers_[http::method::k_erigon_nodeInfo] = &commands::RpcApi::handle_erigon_node_info;
 }
 

silkworm/silkrpc/core/block_reader.cpp

@@ -0,0 +1,101 @@
+/*
+   Copyright 2023 The Silkworm Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+#include "block_reader.hpp"
+
+#include <set>
+
+#include <silkworm/core/common/endian.hpp>
+#include <silkworm/core/common/util.hpp>
+#include <silkworm/core/types/account.hpp>
+#include <silkworm/infra/common/decoding_exception.hpp>
+#include <silkworm/infra/common/log.hpp>
+#include <silkworm/node/db/bitmap.hpp>
+#include <silkworm/node/db/tables.hpp>
+#include <silkworm/node/db/util.hpp>
+#include <silkworm/silkrpc/common/util.hpp>
+#include <silkworm/silkrpc/core/cached_chain.hpp>
+#include <silkworm/silkrpc/core/rawdb/util.hpp>
+#include <silkworm/silkrpc/core/state_reader.hpp>
+#include <silkworm/silkrpc/ethdb/cursor.hpp>
+#include <silkworm/silkrpc/json/types.hpp>
+
+namespace silkworm::rpc {
+
+void to_json(nlohmann::json& json, const BalanceChanges& balance_changes) {
+    for (const auto& entry : balance_changes) {
+        json["0x" + silkworm::to_hex(entry.first)] = to_quantity(entry.second);
+    }
+}
+
+awaitable<void> BlockReader::read_balance_changes(BlockCache& cache, const BlockNumberOrHash& bnoh, BalanceChanges& balance_changes) const {
+    ethdb::TransactionDatabase tx_database{transaction_};
+
+    const auto block_with_hash = co_await core::read_block_by_number_or_hash(cache, chain_storage_, tx_database, bnoh);
+    const auto block_number = block_with_hash->block.header.number;
+
+    SILK_INFO << "read_balance_changes: block_number: " << block_number;
+
+    StateReader state_reader(database_reader_);
+
+    co_await load_addresses(block_number, balance_changes);
+    BalanceChanges::iterator it;
+    for (it = balance_changes.begin(); it != balance_changes.end();) {
+        auto account = co_await state_reader.read_account(it->first, block_number + 1);
+        if (account.has_value()) {
+            auto balance = account.value().balance;
+            if (it->second == balance) {
+                it = balance_changes.erase(it);
+            } else {
+                SILK_DEBUG << "Address "
+                           << "0x" + silkworm::to_hex(it->first) << ": balance changed from " << to_quantity(it->second) << " to " << to_quantity(balance);
+                it->second = balance;
+                it++;
+            }
+        }
+    }
+
+    SILK_DEBUG << "Changed balances " << balance_changes.size();
+
+    co_return;
+}
+
+awaitable<void> BlockReader::load_addresses(uint64_t block_number, BalanceChanges& balance_changes) const {
+    auto acs_cursor = co_await transaction_.cursor(db::table::kAccountChangeSetName);
+    const auto block_number_key = silkworm::db::block_key(block_number);
+
+    auto decode = [](silkworm::ByteView value) {
+        auto address = silkworm::to_evmc_address(value.substr(0, silkworm::kAddressLength));
+        auto remain = value.substr(silkworm::kAddressLength);
+        auto account{silkworm::Account::from_encoded_storage(remain)};
+
+        return std::pair<evmc::address, intx::uint256>{address, account.value().balance};
+    };
+
+    auto kv = co_await acs_cursor->seek(block_number_key);
+    auto pair = decode(kv.value);
+    balance_changes.emplace(pair.first, pair.second);
+
+    auto number = block_number;
+    while (number == block_number) {
+        kv = co_await acs_cursor->next();
+        pair = decode(kv.value);
+        balance_changes.emplace(pair.first, pair.second);
+        number = static_cast<uint64_t>(std::stol(silkworm::to_hex(kv.key), nullptr, 16));
+    }
+}
+
+}  // namespace silkworm::rpc

silkworm/silkrpc/core/block_reader.hpp

@@ -0,0 +1,63 @@
+/*
+   Copyright 2023 The Silkworm Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+#pragma once
+
+#include <optional>
+
+#include <silkworm/infra/concurrency/coroutine.hpp>
+
+#include <boost/asio/awaitable.hpp>
+#include <boost/asio/compose.hpp>
+#include <boost/asio/post.hpp>
+#include <boost/asio/this_coro.hpp>
+#include <evmc/evmc.hpp>
+
+#include <silkworm/core/common/block_cache.hpp>
+#include <silkworm/core/common/util.hpp>
+#include <silkworm/core/types/account.hpp>
+#include <silkworm/silkrpc/common/util.hpp>
+#include <silkworm/silkrpc/core/rawdb/accessors.hpp>
+#include <silkworm/silkrpc/ethdb/transaction_database.hpp>
+#include <silkworm/silkrpc/types/block.hpp>
+
+namespace silkworm::rpc {
+
+using boost::asio::awaitable;
+
+using BalanceChanges = std::map<evmc::address, intx::uint256>;
+
+void to_json(nlohmann::json& json, const BalanceChanges& bc);
+
+class BlockReader {
+  public:
+    explicit BlockReader(const core::rawdb::DatabaseReader& database_reader, const ChainStorage& chain_storage, ethdb::Transaction& transaction)
+        : database_reader_(database_reader), chain_storage_(chain_storage), transaction_(transaction) {}
+
+    BlockReader(const BlockReader&) = delete;
+    BlockReader& operator=(const BlockReader&) = delete;
+
+    [[nodiscard]] awaitable<void> read_balance_changes(BlockCache& cache, const BlockNumberOrHash& bnoh, BalanceChanges& balance_changes) const;
+
+  private:
+    [[nodiscard]] awaitable<void> load_addresses(uint64_t block_number, BalanceChanges& balance_changes) const;
+
+    const core::rawdb::DatabaseReader& database_reader_;
+    const ChainStorage& chain_storage_;
+    ethdb::Transaction& transaction_;
+};
+
+}  // namespace silkworm::rpc

silkworm/silkrpc/http/methods.hpp

@@ -113,14 +113,18 @@ constexpr const char* k_trace_filter{"trace_filter"};
 constexpr const char* k_trace_get{"trace_get"};
 constexpr const char* k_trace_transaction{"trace_transaction"};
 
+constexpr const char* k_erigon_blockNumber{"erigon_blockNumber"};
+constexpr const char* k_erigon_cacheCheck{"erigon_cacheCheck"};
+constexpr const char* k_erigon_getBalanceChangesInBlock{"erigon_getBalanceChangesInBlock"};
 constexpr const char* k_erigon_getBlockByTimestamp{"erigon_getBlockByTimestamp"};
+constexpr const char* k_erigon_getBlockReceiptsByBlockHash{"erigon_getBlockReceiptsByBlockHash"};
 constexpr const char* k_erigon_getHeaderByHash{"erigon_getHeaderByHash"};
 constexpr const char* k_erigon_getHeaderByNumber{"erigon_getHeaderByNumber"};
+constexpr const char* k_erigon_getLatestLogs{"erigon_getLatestLogs"};
 constexpr const char* k_erigon_getLogsByHash{"erigon_getLogsByHash"};
 constexpr const char* k_erigon_forks{"erigon_forks"};
 constexpr const char* k_erigon_watchTheBurn{"erigon_watchTheBurn"};
 constexpr const char* k_erigon_cumulative_chain_traffic{"erigon_cumulativeChainTraffic"};
-constexpr const char* k_erigon_blockNumber{"erigon_blockNumber"};
 constexpr const char* k_erigon_nodeInfo{"erigon_nodeInfo"};
 
 constexpr const char* k_parity_getBlockReceipts{"parity_getBlockReceipts"};

tests/integration/goerly/erigon_getBalanceChangesInBlock/test_01.json

@@ -0,0 +1,33 @@
+[
+  {
+    "request": {
+      "id": 240,
+      "jsonrpc": "2.0",
+      "method": "erigon_getBalanceChangesInBlock",
+      "params": [
+        "0xe64899e6fe64ebb72b8f65565e9dd765776da064aff9af4601c1efa445dbb0a1"
+      ]
+    },
+    "response": {
+      "id": 240,
+      "jsonrpc": "2.0",
+      "result": {
+        "0x23b790f50dacb056c5e1ef6bc33fde744a739633": "0x84ab022dcba41",
+        "0x3dfbfdf2fdb29d1976d70483eff7552de991be5c": "0xddd6237c3fefb84",
+        "0x4ba4880d287d504e503bc5883848cbcce839e495": "0x23227aea72fca9",
+        "0x56768b032fc12d2e911ef654b0054e26a58cef74": "0x6e461b6c7994564",
+        "0x784798960e52dde47705f1aa1c21243ea8222dda": "0xddf02fef0b85854",
+        "0x79a4d418f7887dd4d5123a41b6c8c186686ae8cb": "0x157fe18268af2da8",
+        "0x861ca2f5ff2e03f90d2c3eafda88752fbffc6a69": "0x470de4df820000",
+        "0x8bb2dc06b366a48fbf98824e2d30387b1d8c7488": "0x6991613e8c3f0",
+        "0xb1b19eff752019cd5108dbef2ff56eb1dd0bb063": "0x7e1ace4440b93",
+        "0xca3cd40edc45d29b28442e87892a32b020076d59": "0x424eb1265123692",
+        "0xcb9ec8584681f4ffc23029eb5d303370e2112b64": "0xddef0e1d1a5b66c",
+        "0xd978cc9c7a93935fecd66c96e2df5f363dc63bc8": "0x2c296662b0c7378",
+        "0xd9a5179f091d85051d3c982785efd1455cec8699": "0x432ce766ab116d2952b2",
+        "0xf14cd6286564e44223ad6aee242623bf4398f99d": "0x7b332e5a5ca6f",
+        "0xf3a3956d084e3f2a24add02c35c8afd09e3e9bf5": "0xc9e3b55575c08d7"
+      }
+    }
+  }
+]

tests/integration/goerly/erigon_getBalanceChangesInBlock/test_03.json

@@ -0,0 +1,68 @@
+[
+  {
+    "request": {
+      "id": 240,
+      "jsonrpc": "2.0",
+      "method": "erigon_getBalanceChangesInBlock",
+      "params": [
+        "0x78865b467633789a830da9aa53f3bbb4c2b1beb1fc35055e775fa858db285f11"
+      ]
+    },
+    "response": {
+      "id": 240,
+      "jsonrpc": "2.0",
+      "result": {
+        "0x115d325dea8796d611429ce0e5d9fdb5c5a49790": "0xddd60f979387832",
+        "0x1331e5d70d53bf7a90c245cdf14a10ee635108cc": "0x7baaa51eb004b",
+        "0x17318ec6498099d287ba470093b8b5deef23293c": "0x42887c361f9ed6f",
+        "0x1a99fb83141a5129a79ed062f6b643b0d4f4770e": "0x8ae7b7158a7f2000",
+        "0x1a9e72a7b783abfdd6b55420922ccb571ae8bd31": "0x0",
+        "0x1d80996d872274e7aba3af413dafad3996a432c1": "0x42887bb003571bf",
+        "0x206aaad240cc085903a00bea05ad41c5ad276d46": "0x42887c2b1e7fbcf",
+        "0x28ece1aa90827453962b9e7815c54bf2c47f8cc9": "0x8f9ce02be4d10",
+        "0x2da234c7f29ebbf2f6e557bcc6451cce5420d142": "0x0",
+        "0x3dfbfdf2fdb29d1976d70483eff7552de991be5c": "0xddcabdaa90712b2",
+        "0x49929b28ac71bc5c9ed0cf07268e44d94b7a976e": "0x0",
+        "0x4ab32a38645ae7a2e05202b11d754ea35e830efc": "0x4251e4aa720b362",
+        "0x550bbb7b44be2022cf0fcdd451efb3f933ed4944": "0x0",
+        "0x56768b032fc12d2e911ef654b0054e26a58cef74": "0x6e401a321e828b5",
+        "0x5c3f649ffdbc91a247ac45fc2c4c63f9319e5135": "0xe9ef0e7bf94223e1",
+        "0x620e55708292748d2c752e19a206fbf502a0dde7": "0x0",
+        "0x6743d755309c9e81669f1309db1ded1e3e78a624": "0x10c53a210d5f7538",
+        "0x6f493f8a07b9be6a6a5781240dcb7c05c040064b": "0x0",
+        "0x6fbbf0c75c2c0e494a0eee8d21b59049df044fda": "0xddf02fef0b85854",
+        "0x7afe758564b8b53c44e2d8af569e42876a26e788": "0x0",
+        "0x7ef1ce4e30c4bfb767d5d46190715c52e1c47624": "0x8e1bc9bf04000",
+        "0x7f91a74f6440274cac74138471fc22bafd4425f0": "0xcf967b511c56e425",
+        "0x8be621962f89ab3841962496696e65064c89bc69": "0x23227aea72fca9",
+        "0x8c52dcb703c92d257f79852b4f93e82296ce405b": "0x0",
+        "0x8e32062936bdc84fc1f8a4e611ec2d48fd5a7798": "0x0",
+        "0x8e4c5e550a25495758c89cae1b761aa8876a3ddc": "0xa18e67c80e6e",
+        "0x90df7882bb9098da50553fa6bc11383936bcbd22": "0x0",
+        "0x9202584ac2a5081c6d1f27d637d1dd1fb2aec6b7": "0x1e8ee8db15d80c9f79",
+        "0x9219742142cd54ea1323f559997c3e41d4d30d7d": "0x7894023fab7bb",
+        "0x922149fd6791e643464081c8ff2f0e6f6e5f21c7": "0x7b58ca5d816c9",
+        "0x940a71ff5a0da095be90f34a396d0c4b52ccc8be": "0x0",
+        "0x96c1584a81f822fab0299a122014c02bc534314f": "0x0",
+        "0x9e8056cd8e96a5d132f3e8133af9511bdf649e37": "0x0",
+        "0xa4365a8033a75581dbe8317a3ba10dba9136b412": "0x0",
+        "0xa4b7fba0f43065685fbfc06c3044f0ba2fd40254": "0x0",
+        "0xa6dd2974b96e959f2c8930024451a30afec24203": "0x189c05f7e02e98365da4",
+        "0xaebe6d2e6ff6f41c0f6d803c3519295385045e00": "0x8570e207e6c00",
+        "0xafe1168b2ede9614151f29a0ca1bd28cc667627f": "0xfefbf724663579b",
+        "0xb344147ea92cf102cd92ec996b8986ddca4a918e": "0x3e4a19daff75045ce",
+        "0xb43291d2ac3728affe731511c5aa84278487a856": "0x0",
+        "0xb62dc6c1fb071ba73bbafa32887760bd72b0512b": "0x0",
+        "0xc9de9e8c51627823fcd5ef6c729bbef15bf8f815": "0x7b0e4016ed84c",
+        "0xcffeae6ec98c1d90f86f9601009a5a0f59fd8371": "0x5441828457a71180",
+        "0xd052d7b19f874bdb731f6ab24725ef5acad046a6": "0x4220efad69612ff",
+        "0xd13aaffd3fc710ebe5faf47425d2b6e4a59673e8": "0x0",
+        "0xdaf3a3abd10ee39ca7b8426236fde0b6d3ff87bc": "0x7b3371fd0d72f",
+        "0xf03c622de476e71f546c3b688eb8f4b3157f61fd": "0x0",
+        "0xf3a3956d084e3f2a24add02c35c8afd09e3e9bf5": "0xc9cdd1123471e3b",
+        "0xf48ba2394d263824394f04a12d81aff4b390b656": "0x0",
+        "0xfb2cc747e949476720cc1fd2921c04c23653aeb6": "0x7808a3aebd207"
+      }
+    }
+  }
+]