integration: use shell scripts & enable workflow for E3

.clang-tidy

@@ -97,6 +97,12 @@ CheckOptions:
   readability-identifier-naming.GlobalConstantCase: CamelCase
   readability-identifier-naming.GlobalConstantPrefix: k
   readability-identifier-naming.MacroDefinitionCase: UPPER_CASE
+  readability-identifier-naming.PrivateMemberCase: lower_case
+  readability-identifier-naming.PrivateMemberSuffix: _
+  readability-identifier-naming.ProtectedMemberCase: lower_case
+  readability-identifier-naming.ProtectedMemberSuffix: _
+  readability-identifier-naming.PublicMemberCase: lower_case
+  readability-identifier-naming.PublicMemberSuffix: ''
   readability-identifier-naming.StaticConstantCase: CamelCase
   readability-identifier-naming.StaticConstantPrefix: k
   readability-simplify-boolean-expr.SimplifyDeMorgan: false

.github/workflows/kill_integration_daemon.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+if [ "$#" -ne 1 ]; then
+  echo "Usage: $0 <daemon-pid>"
+  exit -1
+fi
+
+# Clean up rpcdaemon process if it's still running
+if kill -0 $RPC_DAEMON_PID 2> /dev/null; then
+   echo "Silkworm RpcDaemon stopping..."
+   kill $RPC_DAEMON_PID
+   echo "Silkworm RpcDaemon stopped"
+else
+   echo "Silkworm RpcDaemon has already terminated"
+fi
+

.github/workflows/rpc-integration-tests.yml

@@ -12,9 +12,12 @@ on:
 jobs:
   integration-test-suite:
     strategy:
+      fail-fast: false
       matrix:
-        backend: [ Erigon2 ]
+        backend: [ Erigon2, Erigon3 ]
     runs-on: [ self-hosted, "${{ matrix.backend }}" ]
+    concurrency:
+      group: ${{ github.workflow }}-${{ matrix.backend }}
     env:
       ERIGON_DATA_DIR: /opt/erigon-versions/reference-version/datadir
       RPC_PAST_TEST_DIR: /opt/rpc-past-tests
@@ -30,7 +33,11 @@ jobs:
       - name: Checkout RPC Tests Repository & Install Requirements
         run: |
           rm -rf ${{runner.workspace}}/rpc-tests
-          git -c advice.detachedHead=false clone --depth 1 --branch v0.52.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests
+          if [ ${{ matrix.backend }} == 'Erigon2' ]; then
+             git -c advice.detachedHead=false clone --depth 1 --branch v0.52.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests
+          else
+             git -c advice.detachedHead=false clone --depth 1 --branch v0.52.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests
+          fi
           cd ${{runner.workspace}}/rpc-tests
           pip3 install -r requirements.txt
 
@@ -50,58 +57,40 @@ jobs:
         run: cmake --build . --config Release --target rpcdaemon -j 8
 
       - name: Pause the Erigon instance dedicated to db maintenance
+        if: matrix.backend == 'Erigon2'
         run: |
           python3 $ERIGON_QA_PATH/test_system/db-producer/pause_production.py || true
 
       - name: Start Silkworm RpcDaemon
         working-directory: ${{runner.workspace}}/silkworm/build/cmd
         run: |
-          echo "Silkworm RpcDaemon starting..."
-          ./rpcdaemon --datadir $ERIGON_DATA_DIR --api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net --log.verbosity 1 --erigon_compatibility --jwt ./jwt.hex --skip_protocol_check --ws &
+          ${{runner.workspace}}/silkworm/.github/workflows/run_integration_daemon.sh ${{ matrix.backend }} $ERIGON_DATA_DIR ./jwt.hex &
           RPC_DAEMON_PID=$!
           echo "RPC_DAEMON_PID=$RPC_DAEMON_PID" >> $GITHUB_ENV
-          echo "Silkworm RpcDaemon started"
 
       - name: Run RPC Integration Tests
         id: test_step
         run: |
-          set +e # Disable exit on error
-
-          cd ${{runner.workspace}}/rpc-tests/integration
-          rm -rf ./mainnet/results/
-
           # Run RPC integration test runner via http
-          python3 ./run_tests.py --continue --blockchain mainnet --jwt ${{runner.workspace}}/silkworm/build/cmd/jwt.hex --display-only-fail --port 8545 -x admin_,eth_mining,eth_getWork,eth_coinbase,eth_createAccessList/test_16.json,engine_,net_,web3_,txpool_,eth_submitWork,eth_submitHashrate,eth_protocolVersion,erigon_nodeInfo,trace_rawTransaction/test_02.json --transport_type http,websocket
-         
+          ${{runner.workspace}}/silkworm/.github/workflows/run_integration_tests.sh ${{ matrix.backend }} ${{runner.workspace}}/rpc-tests/integration ${{runner.workspace}}/silkworm/build/cmd/jwt.hex $RPC_PAST_TEST_DIR/mainnet_$(date +%Y%m%d_%H%M%S)_integration_$(git -C ${{runner.workspace}}/silkworm rev-parse --short HEAD)_http/ 
+
           # Capture test runner script exit status
           test_exit_status=$?
 
           # Check test runner exit status
           if [ $test_exit_status -eq 0 ]; then
-            echo "tests completed successfully"
-            echo
             echo "TEST_RESULT=success" >> "$GITHUB_OUTPUT"
           else
-            echo "error detected during tests"
             echo "TEST_RESULT=failure" >> "$GITHUB_OUTPUT"
-
-            # Save failed results to a directory with timestamp and commit hash
-            cp -r ${{runner.workspace}}/rpc-tests/integration/mainnet/results/ $RPC_PAST_TEST_DIR/mainnet_$(date +%Y%m%d_%H%M%S)_integration_$(git -C ${{runner.workspace}}/silkworm rev-parse --short HEAD)_http/
           fi
 
       - name: Stop Silkworm RpcDaemon
         working-directory: ${{runner.workspace}}/silkworm/build/cmd
         run: |
-          # Clean up rpcdaemon process if it's still running
-          if kill -0 $RPC_DAEMON_PID 2> /dev/null; then
-            echo "Silkworm RpcDaemon stopping..."
-            kill $RPC_DAEMON_PID
-            echo "Silkworm RpcDaemon stopped"
-          else
-            echo "Silkworm RpcDaemon has already terminated"
-          fi
+          ${{runner.workspace}}/silkworm/.github/workflows/kill_integration_daemon.sh $RPC_DAEMON_PID
 
       - name: Resume the Erigon instance dedicated to db maintenance
+        if: matrix.backend == 'Erigon2'
         run: |
           python3 $ERIGON_QA_PATH/test_system/db-producer/resume_production.py || true
 

.github/workflows/run_integration_daemon.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+trap : SIGTERM SIGINT
+
+if [ "$#" -ne 3 ]; then
+  echo "Usage: $0 Erigon2/Erigon3 <datadir> <jwtfile>"
+  exit -1
+fi
+
+echo "Silkworm RpcDaemon starting..."
+if [ $1 == 'Erigon2' ]; then
+./rpcdaemon --datadir $2 --api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net --log.verbosity 1 --erigon_compatibility --jwt $3 --skip_protocol_check --ws &
+else
+./rpcdaemon --erigon3 --eth.addr 127.0.0.1:51515 --engine.addr 127.0.0.1:51516 --api admin,debug,eth,parity,erigon,trace,web3,txpool,ots,net --log.verbosity 1 --erigon_compatibility --jwt $3 --skip_protocol_check --ws &
+fi
+
+PID=$!
+
+wait $PID
+
+if [[ $? -gt 128 ]]
+then
+    kill $PID
+fi
+
+exit 0 

.github/workflows/run_integration_tests.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+if [ "$#" -ne 4 ]; then
+  echo "Usage: $0 Erigon2/Erigon3 <integration-dir> <jwt-file> <saved failed tests>"
+  exit 1
+fi
+
+set +e # Disable exit on error
+cd $2
+rm -rf ./mainnet/results/
+
+if [ $1 == 'Erigon2' ]; then
+   python3 ./run_tests.py --continue --blockchain mainnet --jwt $3 --display-only-fail --port 8545 -x admin_,eth_mining,eth_getWork,eth_coinbase,eth_createAccessList/test_16.json,engine_,net_,web3_,txpool_,eth_submitWork,eth_submitHashrate,eth_protocolVersion,erigon_nodeInfo --transport_type http,websocket
+else
+   python3 ./run_tests.py --continue --blockchain mainnet --jwt $3 --display-only-fail --port 51515 -x engine_,\
+debug_accountRange,\
+debug_getModifiedAccounts,\
+debug_storageRangeAt,\
+erigon_getBalanceChangesInBlock,\
+ots_getTransactionBySenderAndNonce,\
+parity_listStorageKeys,\
+ots_getContractCreator,\
+erigon_getLatestLogs,\
+eth_getLogs,\
+erigon_getBlockReceiptsByBlockHash/test_05.json,\
+eth_getBlockReceipts/test_04.json,\
+ots_getBlockDetails/test_05.json,\
+ots_getBlockDetailsByHash/test_03.json,\
+ots_getBlockTransactions/test_04.json,\
+ots_getBlockTransactions/test_07.json,\
+ots_searchTransactionsAfter,\
+txpool_content,\
+ots_searchTransactionsBefore --transport_type http,websocket
+fi
+
+failed_test=$?
+
+# Check test runner exit status
+if [ $failed_test -eq 0 ]; then
+    echo "tests completed successfully"
+else
+    echo "error detected during tests"
+
+    # Save failed results to a directory with timestamp and commit hash
+    cp -r $2/mainnet/results/ $4
+fi
+exit $failed_test
+

cmd/common/common.cpp

@@ -107,25 +107,8 @@ void add_option_num_contexts(CLI::App& cli, uint32_t& num_contexts) {
         ->default_val(std::thread::hardware_concurrency() / 2);
 }
 
-//! \brief Set up parsing of the wait mode (e.g. block, sleep, spin...) in RPC execution contexts
-void add_option_wait_mode(CLI::App& cli, concurrency::WaitMode& wait_mode) {
-    std::map<std::string, concurrency::WaitMode> wait_mode_mapping{
-        {"backoff", concurrency::WaitMode::kBackoff},
-        {"blocking", concurrency::WaitMode::kBlocking},
-        {"busy_spin", concurrency::WaitMode::kBusySpin},
-        {"sleeping", concurrency::WaitMode::kSleeping},
-        {"yielding", concurrency::WaitMode::kYielding},
-    };
-    cli.add_option("--wait.mode", wait_mode, "The waiting mode for execution loops during idle cycles")
-        ->capture_default_str()
-        ->check(CLI::Range(concurrency::WaitMode::kBackoff, concurrency::WaitMode::kBusySpin))
-        ->transform(CLI::Transformer(wait_mode_mapping, CLI::ignore_case))
-        ->default_val(concurrency::WaitMode::kBlocking);
-}
-
 void add_context_pool_options(CLI::App& cli, concurrency::ContextPoolSettings& settings) {
     add_option_num_contexts(cli, settings.num_contexts);
-    add_option_wait_mode(cli, settings.wait_mode);
 }
 
 }  // namespace silkworm::cmd::common

cmd/common/rpcdaemon_options.cpp

@@ -136,13 +136,17 @@ void add_rpcdaemon_options(CLI::App& cli, silkworm::rpc::DaemonSettings& setting
         ->description("Enable WebSocket protocol for Execution Layer and Engine JSON RPC API, same port as HTTP(S)")
         ->capture_default_str();
 
-    cli.add_flag("--ws-compression", settings.ws_compression)
+    cli.add_flag("--ws.compression", settings.ws_compression)
         ->description("Enable compression on WebSocket protocol for Execution Layer and Engine JSON RPC API")
         ->capture_default_str();
 
-    cli.add_flag("--http-compression", settings.http_compression)
+    cli.add_flag("--http.compression", settings.http_compression)
         ->description("Enable compression on HTTP protocol for Execution Layer and Engine JSON RPC API")
         ->capture_default_str();
+
+    cli.add_flag("--erigon3", settings.use_erigon3_data_format)
+        ->description("Enable usage of Erigon3 data storage model")
+        ->capture_default_str();
 }
 
 }  // namespace silkworm::cmd::common

cmd/dev/CMakeLists.txt

@@ -71,7 +71,7 @@ add_executable(scan_txs scan_txs.cpp)
 target_link_libraries(scan_txs PRIVATE silkworm_node CLI11::CLI11 absl::time)
 
 add_executable(snapshots snapshots.cpp)
-target_link_libraries(snapshots PRIVATE silkworm_node cmd_common magic_enum::magic_enum)
+target_link_libraries(snapshots PRIVATE absl::strings cmd_common magic_enum::magic_enum silkworm_node)
 
 add_executable(db_toolbox db_toolbox.cpp)
 target_link_libraries(db_toolbox PRIVATE silkworm_node cmd_common CLI11::CLI11 magic_enum::magic_enum)

cmd/dev/backend_kv_server.cpp

@@ -14,7 +14,6 @@
    limitations under the License.
 */
 
-#include <cassert>
 #include <filesystem>
 #include <stdexcept>
 #include <string>
@@ -29,6 +28,7 @@
 #include <grpcpp/grpcpp.h>
 
 #include <silkworm/buildinfo.h>
+#include <silkworm/core/common/assert.hpp>
 #include <silkworm/core/types/address.hpp>
 #include <silkworm/core/types/evmc_bytes32.hpp>
 #include <silkworm/db/access_layer.hpp>
@@ -120,7 +120,7 @@ std::shared_ptr<silkworm::sentry::api::SentryClient> make_sentry_client(
     silkworm::sentry::eth::StatusDataProvider eth_status_data_provider{std::move(chain_head_provider), node_settings.chain_config.value()};
 
     if (node_settings.remote_sentry_addresses.empty()) {
-        assert(false);
+        SILKWORM_ASSERT(false);
     } else if (node_settings.remote_sentry_addresses.size() == 1) {
         // remote client
         auto remote_sentry_client = std::make_shared<silkworm::sentry::grpc::client::SentryClient>(

cmd/dev/check_log_indices.cpp

@@ -133,7 +133,7 @@ void trace(const Log& log) {
     int i{0};
     for (const auto& t : log.topics) {
         log::Trace() << "topic[" << i << "]: " << to_hex(t);
-        i++;
+        ++i;
     }
 }
 
@@ -219,7 +219,7 @@ int main(int argc, char* argv[]) {
             }
             if (reached_block_number != block_number) {
                 reached_block_number = block_number;
-                processed_block_numbers++;
+                ++processed_block_numbers;
             }
 
             std::vector<Log> transaction_logs;
@@ -238,7 +238,7 @@ int main(int argc, char* argv[]) {
                         log::Info() << "block " << block_number << " tx " << tx_id << " generated log for " << log.address;
                     }
                     check_address_index(block_number, log.address, log_address_cursor.get());
-                    processed_addresses_count++;
+                    ++processed_addresses_count;
                 }
 
                 if (settings.index != TargetIndex::kLogAddress) {
@@ -253,7 +253,7 @@ int main(int argc, char* argv[]) {
             }
             processed_logs_count += transaction_logs.size();
 
-            processed_transaction_count++;
+            ++processed_transaction_count;
             if (processed_transaction_count % 100'000 == 0) {
                 log::Info() << "Scanned transactions " << processed_transaction_count << " processed logs " << processed_logs_count;
             }

cmd/dev/check_pow.cpp

@@ -105,7 +105,7 @@ int main(int argc, char* argv[]) {
         StopWatch sw;
         sw.start();
         for (uint32_t block_num{options.block_from}; block_num <= options.block_to && !SignalHandler::signalled();
-             block_num++) {
+             ++block_num) {
             if (epoch_context->epoch_number != static_cast<int>(block_num / ethash::epoch_length)) {
                 epoch_num = (block_num / ethash::epoch_length);
                 log::Info() << "Initializing Light Cache for DAG epoch " << epoch_num;

cmd/dev/check_senders.cpp

@@ -140,7 +140,7 @@ int main(int argc, char* argv[]) {
                 std::vector<Transaction> transactions;
                 uint64_t i{0};
                 auto tx_data{tx_cursor->find(db::to_slice(tx_key), false)};
-                for (; i < body.txn_count && tx_data.done; i++, tx_data = tx_cursor->to_next(false)) {
+                for (; i < body.txn_count && tx_data.done; ++i, tx_data = tx_cursor->to_next(false)) {
                     if (!tx_data) {
                         log::Error() << "Block " << block_number << " tx " << i << " not found in " << db::table::kBlockTransactions.name << " table";
                         continue;
@@ -158,7 +158,7 @@ int main(int argc, char* argv[]) {
                         log::Error() << "Block " << block_number << " tx " << i << " recovered sender " << senders[i]
                                      << " does not match computed sender " << *tx.sender();
                     }
-                    processed_senders_count++;
+                    ++processed_senders_count;
 
                     const auto transaction_hash{keccak256(transaction_rlp)};
                     log::Debug() << "Tx hash: " << to_hex(transaction_hash.bytes) << " has sender: " << to_hex(senders[i].bytes);
@@ -183,7 +183,7 @@ int main(int argc, char* argv[]) {
             }
 
             // Move to next block body
-            expected_block_number++;
+            ++expected_block_number;
             bodies_data = bodies_cursor->to_next(false);
         }
 

cmd/dev/check_tx_lookup.cpp

@@ -74,7 +74,7 @@ int main(int argc, char* argv[]) {
                 uint64_t i{0};
                 auto transaction_data{transactions_table.find(db::to_slice(transaction_key), false)};
                 for (; i < body.txn_count && transaction_data.done;
-                     i++, transaction_data = transactions_table.to_next(false)) {
+                     ++i, transaction_data = transactions_table.to_next(false)) {
                     if (!transaction_data) {
                         log::Error() << "Block " << block_number << " transaction " << i << " not found in "
                                      << db::table::kBlockTransactions.name << " table";
@@ -117,7 +117,7 @@ int main(int argc, char* argv[]) {
                 break;
             }
 
-            expected_block_number++;
+            ++expected_block_number;
             bodies_data = bodies_table.to_next(false);
         }
 

cmd/dev/db_toolbox.cpp

@@ -85,7 +85,7 @@ class Progress {
         return static_cast<uint32_t>(current_counter_ * 100 / max_counter_);
     }
 
-    void step() { current_counter_++; }
+    void step() { ++current_counter_; }
     void set_current(size_t count) { current_counter_ = std::max(count, current_counter_); }
     [[nodiscard]] size_t get_current() const noexcept { return current_counter_; }
     [[nodiscard]] size_t get_increment_count() const noexcept { return bar_width_ ? (max_counter_ / bar_width_) : 0u; }

cmd/dev/scan_txs.cpp

@@ -104,7 +104,7 @@ int main(int argc, char* argv[]) {
             processor.flush_state();
 
             // There is one receipt per transaction
-            assert(block.transactions.size() == receipts.size());
+            SILKWORM_ASSERT(block.transactions.size() == receipts.size());
 
             // Erigon returns success in the receipt even for pre-Byzantium txs.
             for (const auto& receipt : receipts) {