From 4901ee8feaf534791bd3e16eac19f62cc24b9232 Mon Sep 17 00:00:00 2001 From: Sixtysixter <20945591+Sixtysixter@users.noreply.github.com> Date: Tue, 5 Nov 2024 00:27:12 +0100 Subject: [PATCH 1/9] Step 1 --- silkworm/db/tables.hpp | 6 ++ silkworm/rpc/commands/ots_api.cpp | 124 +++++++++++++++++++++--------- 2 files changed, 92 insertions(+), 38 deletions(-) diff --git a/silkworm/db/tables.hpp b/silkworm/db/tables.hpp index cd7c98ccd3..e21ead29f7 100644 --- a/silkworm/db/tables.hpp +++ b/silkworm/db/tables.hpp @@ -436,4 +436,10 @@ inline constexpr const char* kAccountsHistory{"AccountsHistory"}; //! \details Inverted Index storing the account common information inline constexpr const char* kAccountsHistoryIdx{"AccountsHistoryIdx"}; +//! \details TODO +inline constexpr const char* kTracesFromIdx{"TracesFromIdx"}; + +//! \details TODO +inline constexpr const char* kTracesToIdx{"TracesToIdx"}; + } // namespace silkworm::db::table diff --git a/silkworm/rpc/commands/ots_api.cpp b/silkworm/rpc/commands/ots_api.cpp index 835c7c5872..43e48492ff 100644 --- a/silkworm/rpc/commands/ots_api.cpp +++ b/silkworm/rpc/commands/ots_api.cpp @@ -694,7 +694,7 @@ Task OtsRpcApi::handle_ots_search_transactions_before(const nlohmann::json auto block_number = params[1].get(); const auto page_size = params[2].get(); - SILK_DEBUG << "address: " << address << " block_number: " << block_number << " page_size: " << page_size; + SILK_LOG << "address: " << address << " block_number: " << block_number << " page_size: " << page_size; if (page_size > kMaxPageSize) { auto error_msg = "max allowed page size: " + std::to_string(kMaxPageSize); @@ -703,52 +703,100 @@ Task OtsRpcApi::handle_ots_search_transactions_before(const nlohmann::json co_return; } + if (block_number > 0) { + --block_number; + } auto tx = co_await database_->begin(); - try { - auto call_from_cursor = co_await tx->cursor(table::kCallFromIndexName); - auto call_to_cursor = co_await tx->cursor(table::kCallToIndexName); - - bool is_first_page = false; - - if (block_number == 0) { - is_first_page = true; - } else { - // Internal search code considers blockNum [including], so adjust the value - --block_number; - } - - BackwardBlockProvider from_provider{call_from_cursor.get(), address, block_number}; - BackwardBlockProvider to_provider{call_to_cursor.get(), address, block_number}; - FromToBlockProvider from_to_provider{false, &from_provider, &to_provider}; - - uint64_t result_count = 0; - bool has_more = true; - - TransactionsWithReceipts results{ - .first_page = is_first_page}; - - while (result_count < page_size && has_more) { - std::vector transactions_with_receipts_vec; - - has_more = co_await trace_blocks(from_to_provider, *tx, address, page_size, result_count, transactions_with_receipts_vec); - - for (const auto& item : transactions_with_receipts_vec) { - results.receipts.insert(results.receipts.end(), item.receipts.rbegin(), item.receipts.rend()); - results.transactions.insert(results.transactions.end(), item.transactions.rbegin(), item.transactions.rend()); - results.blocks.insert(results.blocks.end(), item.blocks.rbegin(), item.blocks.rend()); + auto provider = ethdb::kv::canonical_body_for_storage_provider(backend_); + const auto key = db::code_domain_key(address); - result_count += item.transactions.size(); + const auto max_tx_num = co_await db::txn::max_tx_num(*tx, block_number, provider); + SILK_LOG << "block_number: " << block_number << " max_tx_num: " << max_tx_num; - if (result_count >= page_size) { - break; - } + db::kv::api::IndexRangeQuery query_to{ + .table = db::table::kTracesToIdx, + .key = key, + .from_timestamp = static_cast(max_tx_num), + .to_timestamp = -1, + .ascending_order = false}; + auto paginated_result_to = co_await tx->index_range(std::move(query_to)); + auto it_to = co_await paginated_result_to.begin(); + + db::kv::api::IndexRangeQuery query_from{ + .table = db::table::kTracesFromIdx, + .key = key, + .from_timestamp = static_cast(max_tx_num), + .to_timestamp = -1, + .ascending_order = false}; + auto paginated_result_from = co_await tx->index_range(std::move(query_from)); + auto it_from = co_await paginated_result_from.begin(); + +// auto it = db::kv::api::UnionIterator{it_from, it_to, false, 0}; + auto it = db::kv::api::UnionIterator{it_to, it_from, false, 5}; +// std::vector keys; +// std::uint64_t count = 0; +// TxnId prev_txn_id = 0; +// TxnId next_txn_id = 0; + while (const auto value = co_await it.next()) { + const auto txn_id = static_cast(*value); + SILK_LOG << "txn_id: " << txn_id; + const auto block_number_opt = co_await db::txn::block_num_from_tx_num(*tx, txn_id, provider); + if (block_number_opt) { + const auto bn = block_number_opt.value(); + SILK_LOG << "txn_id: " << txn_id << " block_number: " << bn; } } - results.last_page = !has_more; + TransactionsWithReceipts results{ + .first_page = false, + .last_page = false}; reply = make_json_content(request, results); +// try { +// auto call_from_cursor = co_await tx->cursor(table::kCallFromIndexName); +// auto call_to_cursor = co_await tx->cursor(table::kCallToIndexName); +// +// bool is_first_page = false; +// +// if (block_number == 0) { +// is_first_page = true; +// } else { +// // Internal search code considers blockNum [including], so adjust the value +// --block_number; +// } +// +// BackwardBlockProvider from_provider{call_from_cursor.get(), address, block_number}; +// BackwardBlockProvider to_provider{call_to_cursor.get(), address, block_number}; +// FromToBlockProvider from_to_provider{false, &from_provider, &to_provider}; +// +// uint64_t result_count = 0; +// bool has_more = true; +// +// TransactionsWithReceipts results{ +// .first_page = is_first_page}; +// +// while (result_count < page_size && has_more) { +// std::vector transactions_with_receipts_vec; +// +// has_more = co_await trace_blocks(from_to_provider, *tx, address, page_size, result_count, transactions_with_receipts_vec); +// +// for (const auto& item : transactions_with_receipts_vec) { +// results.receipts.insert(results.receipts.end(), item.receipts.rbegin(), item.receipts.rend()); +// results.transactions.insert(results.transactions.end(), item.transactions.rbegin(), item.transactions.rend()); +// results.blocks.insert(results.blocks.end(), item.blocks.rbegin(), item.blocks.rend()); +// +// result_count += item.transactions.size(); +// +// if (result_count >= page_size) { +// break; +// } +// } +// } +// +// results.last_page = !has_more; +// reply = make_json_content(request, results); + } catch (const std::invalid_argument& iv) { SILK_WARN << "invalid_argument: " << iv.what() << " processing request: " << request.dump(); reply = make_json_content(request, nlohmann::detail::value_t::null); From d418e469f0614b2f1c0ffe94269851c2d2e14b93 Mon Sep 17 00:00:00 2001 From: Sixtysixter <20945591+Sixtysixter@users.noreply.github.com> Date: Fri, 8 Nov 2024 10:21:17 +0100 Subject: [PATCH 2/9] step 2 --- silkworm/rpc/commands/ots_api.cpp | 50 ++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 8 deletions(-) diff --git a/silkworm/rpc/commands/ots_api.cpp b/silkworm/rpc/commands/ots_api.cpp index 43e48492ff..ce486b1924 100644 --- a/silkworm/rpc/commands/ots_api.cpp +++ b/silkworm/rpc/commands/ots_api.cpp @@ -712,7 +712,8 @@ Task OtsRpcApi::handle_ots_search_transactions_before(const nlohmann::json const auto key = db::code_domain_key(address); const auto max_tx_num = co_await db::txn::max_tx_num(*tx, block_number, provider); - SILK_LOG << "block_number: " << block_number << " max_tx_num: " << max_tx_num; + const auto min_txn_num = co_await db::txn::min_tx_num(*tx, block_number, provider); + SILK_LOG << "block_number: " << block_number << " max_tx_num: " << max_tx_num << " min_txn_num: " << min_txn_num; db::kv::api::IndexRangeQuery query_to{ .table = db::table::kTracesToIdx, @@ -723,29 +724,62 @@ Task OtsRpcApi::handle_ots_search_transactions_before(const nlohmann::json auto paginated_result_to = co_await tx->index_range(std::move(query_to)); auto it_to = co_await paginated_result_to.begin(); + size_t count = 20; +// auto it_to1 = co_await paginated_result_to.begin(); +// SILK_LOG << "TRACES TO IDX"; +// while (const auto value = co_await it_to1.next()) { +// const auto txn_id = static_cast(*value); +// SILK_LOG << "txn_id: " << txn_id; +// +// if (--count == 0) { +// break; +// } +// } + db::kv::api::IndexRangeQuery query_from{ .table = db::table::kTracesFromIdx, .key = key, - .from_timestamp = static_cast(max_tx_num), + .from_timestamp = -1,//static_cast(max_tx_num), .to_timestamp = -1, .ascending_order = false}; auto paginated_result_from = co_await tx->index_range(std::move(query_from)); auto it_from = co_await paginated_result_from.begin(); -// auto it = db::kv::api::UnionIterator{it_from, it_to, false, 0}; - auto it = db::kv::api::UnionIterator{it_to, it_from, false, 5}; + auto it_from1 = co_await paginated_result_from.begin(); + count = 20; + SILK_LOG << "TRACES FROM IDX"; + while (const auto value = co_await it_from1.next()) { + const auto txn_id = static_cast(*value); + SILK_LOG << "txn_id: " << txn_id; + + if (--count == 0) { + break; + } + } + + auto it = db::kv::api::set_union(it_to, it_from, false, 20); // std::vector keys; // std::uint64_t count = 0; // TxnId prev_txn_id = 0; // TxnId next_txn_id = 0; while (const auto value = co_await it.next()) { const auto txn_id = static_cast(*value); - SILK_LOG << "txn_id: " << txn_id; const auto block_number_opt = co_await db::txn::block_num_from_tx_num(*tx, txn_id, provider); - if (block_number_opt) { - const auto bn = block_number_opt.value(); - SILK_LOG << "txn_id: " << txn_id << " block_number: " << bn; + if (!block_number_opt) { + break; // TODO } + const auto bn = block_number_opt.value(); + const auto max_txn_id = co_await db::txn::max_tx_num(*tx, bn, provider); + const auto min_txn_id = co_await db::txn::min_tx_num(*tx, bn, provider); + const auto txn_index = txn_id - min_txn_id - 1; + SILK_LOG + << "txn_id: " << txn_id + << " block_number: " << bn + << ", txn_index: " << txn_index + << ", max_txn_id: " << max_txn_id + << ", min_txn_id: " << min_txn_id + << ", final txn: " << (txn_id == max_txn_id); + } TransactionsWithReceipts results{ From 6fd0fd1c2ed5573cb549025fcf1e21314cb75ef5 Mon Sep 17 00:00:00 2001 From: Sixtysixter <20945591+Sixtysixter@users.noreply.github.com> Date: Sun, 10 Nov 2024 23:38:34 +0100 Subject: [PATCH 3/9] Step 2 --- silkworm/rpc/commands/ots_api.cpp | 288 +++++++++++++++++++++--------- silkworm/rpc/commands/ots_api.hpp | 12 +- 2 files changed, 218 insertions(+), 82 deletions(-) diff --git a/silkworm/rpc/commands/ots_api.cpp b/silkworm/rpc/commands/ots_api.cpp index ce486b1924..19dc00f8a6 100644 --- a/silkworm/rpc/commands/ots_api.cpp +++ b/silkworm/rpc/commands/ots_api.cpp @@ -681,6 +681,11 @@ Task OtsRpcApi::handle_ots_get_internal_operations(const nlohmann::json& r co_await tx->close(); // RAII not (yet) available with coroutines } +struct BlockInfo { + uint64_t number{0}; + BlockDetails details; +}; + Task OtsRpcApi::handle_ots_search_transactions_before(const nlohmann::json& request, nlohmann::json& reply) { const auto& params = request["params"]; if (params.size() != 3) { @@ -711,61 +716,67 @@ Task OtsRpcApi::handle_ots_search_transactions_before(const nlohmann::json auto provider = ethdb::kv::canonical_body_for_storage_provider(backend_); const auto key = db::code_domain_key(address); - const auto max_tx_num = co_await db::txn::max_tx_num(*tx, block_number, provider); - const auto min_txn_num = co_await db::txn::min_tx_num(*tx, block_number, provider); - SILK_LOG << "block_number: " << block_number << " max_tx_num: " << max_tx_num << " min_txn_num: " << min_txn_num; + db::kv::api::Timestamp from_timestamp{-1}; + if (block_number > 0) { + const auto max_tx_num = co_await db::txn::max_tx_num(*tx, block_number, provider); + from_timestamp = static_cast(max_tx_num); + SILK_LOG << "block_number: " << block_number << " max_tx_num: " << max_tx_num; + } +// const auto min_txn_num = co_await db::txn::min_tx_num(*tx, block_number, provider); +// SILK_LOG << "block_number: " << block_number << " max_tx_num: " << max_tx_num << " min_txn_num: " << min_txn_num; db::kv::api::IndexRangeQuery query_to{ .table = db::table::kTracesToIdx, .key = key, - .from_timestamp = static_cast(max_tx_num), + .from_timestamp = from_timestamp, .to_timestamp = -1, .ascending_order = false}; auto paginated_result_to = co_await tx->index_range(std::move(query_to)); - auto it_to = co_await paginated_result_to.begin(); - size_t count = 20; -// auto it_to1 = co_await paginated_result_to.begin(); -// SILK_LOG << "TRACES TO IDX"; -// while (const auto value = co_await it_to1.next()) { -// const auto txn_id = static_cast(*value); -// SILK_LOG << "txn_id: " << txn_id; -// -// if (--count == 0) { -// break; -// } -// } + SILK_LOG << "TEST *************************************"; + auto it_to1 = co_await paginated_result_to.begin(); + auto cc{0}; + while (const auto value = co_await it_to1.next()) { + const auto txn_id = static_cast(*value); + SILK_LOG << "ITERATE: " << cc++ << ", txn_id: " << txn_id; + if (cc > 10) { + break; + } + } db::kv::api::IndexRangeQuery query_from{ .table = db::table::kTracesFromIdx, .key = key, - .from_timestamp = -1,//static_cast(max_tx_num), + .from_timestamp = from_timestamp, .to_timestamp = -1, .ascending_order = false}; auto paginated_result_from = co_await tx->index_range(std::move(query_from)); - auto it_from = co_await paginated_result_from.begin(); - auto it_from1 = co_await paginated_result_from.begin(); - count = 20; - SILK_LOG << "TRACES FROM IDX"; - while (const auto value = co_await it_from1.next()) { - const auto txn_id = static_cast(*value); - SILK_LOG << "txn_id: " << txn_id; + const auto resultssss = co_await collect_results(*tx, block_number, provider, paginated_result_from, paginated_result_to, false, page_size); + SILK_LOG << "********************************** size: " << resultssss.transactions.size(); - if (--count == 0) { - break; - } - } + auto it_to = co_await paginated_result_to.begin(); + auto it_from = co_await paginated_result_from.begin(); + TransactionsWithReceipts results{ + .first_page = block_number == 0, + .last_page = true}; - auto it = db::kv::api::set_union(it_to, it_from, false, 20); -// std::vector keys; -// std::uint64_t count = 0; -// TxnId prev_txn_id = 0; -// TxnId next_txn_id = 0; + std::map receipts; + std::optional block_info; + auto block_num_changed = false; + auto it = db::kv::api::set_union(it_from, it_to, false); + const auto chain_storage = tx->create_storage(); + auto count{0}; while (const auto value = co_await it.next()) { + if (!value.has_value()) { + SILK_LOG << "NO VALUE FROM ITR"; + break; + } const auto txn_id = static_cast(*value); + SILK_LOG << "ITERATE: " << count++ << ", txn_id: " << txn_id; const auto block_number_opt = co_await db::txn::block_num_from_tx_num(*tx, txn_id, provider); if (!block_number_opt) { + SILK_LOG << "No block found for txn_id " << txn_id; break; // TODO } const auto bn = block_number_opt.value(); @@ -780,57 +791,61 @@ Task OtsRpcApi::handle_ots_search_transactions_before(const nlohmann::json << ", min_txn_id: " << min_txn_id << ", final txn: " << (txn_id == max_txn_id); - } + if (txn_id == max_txn_id) { + continue; + } - TransactionsWithReceipts results{ - .first_page = false, - .last_page = false}; - reply = make_json_content(request, results); + block_num_changed = false; + if (block_info && block_info.value().number != bn) { + block_info.reset(); + } -// try { -// auto call_from_cursor = co_await tx->cursor(table::kCallFromIndexName); -// auto call_to_cursor = co_await tx->cursor(table::kCallToIndexName); -// -// bool is_first_page = false; -// -// if (block_number == 0) { -// is_first_page = true; -// } else { -// // Internal search code considers blockNum [including], so adjust the value -// --block_number; -// } -// -// BackwardBlockProvider from_provider{call_from_cursor.get(), address, block_number}; -// BackwardBlockProvider to_provider{call_to_cursor.get(), address, block_number}; -// FromToBlockProvider from_to_provider{false, &from_provider, &to_provider}; -// -// uint64_t result_count = 0; -// bool has_more = true; -// -// TransactionsWithReceipts results{ -// .first_page = is_first_page}; -// -// while (result_count < page_size && has_more) { -// std::vector transactions_with_receipts_vec; -// -// has_more = co_await trace_blocks(from_to_provider, *tx, address, page_size, result_count, transactions_with_receipts_vec); -// -// for (const auto& item : transactions_with_receipts_vec) { -// results.receipts.insert(results.receipts.end(), item.receipts.rbegin(), item.receipts.rend()); -// results.transactions.insert(results.transactions.end(), item.transactions.rbegin(), item.transactions.rend()); -// results.blocks.insert(results.blocks.end(), item.blocks.rbegin(), item.blocks.rend()); -// -// result_count += item.transactions.size(); -// -// if (result_count >= page_size) { -// break; -// } -// } -// } -// -// results.last_page = !has_more; -// reply = make_json_content(request, results); + if (!block_info) { + const auto block_with_hash = co_await rpc::core::read_block_by_number(*block_cache_, *chain_storage, bn); + if (!block_with_hash) { + SILK_DEBUG << "Not found block no. " << bn; + co_return; + } + + auto rr = co_await core::get_receipts(*tx, *block_with_hash, *chain_storage, workers_); + SILK_LOG << "Read #" << rr.size() << " receipts from block " << bn; + + std::for_each(rr.begin(), rr.end(), [&receipts](const auto& item) { + receipts[silkworm::to_hex(item.tx_hash, false)] = std::move(item); + }); + + const Block extended_block{block_with_hash, false}; + const auto block_size = extended_block.get_block_size(); + const BlockDetails block_details{block_size, block_with_hash->hash, block_with_hash->block.header, + block_with_hash->block.transactions.size(), block_with_hash->block.ommers, + block_with_hash->block.withdrawals}; + block_info = BlockInfo{block_with_hash->block.header.number, block_details}; + block_num_changed = true; + } + + if (results.transactions.size() >= page_size && block_num_changed) { + results.last_page = false; + break; + } + + auto transaction = co_await chain_storage->read_transaction_by_idx_in_block(bn, txn_index); + if (!transaction) { + SILK_LOG << "No transaction found in block " << bn << " for index " << txn_index; + co_return; + } + results.receipts.push_back(std::move(receipts.at(silkworm::to_hex(transaction.value().hash(), false)))); + results.transactions.push_back(std::move(transaction.value())); + results.blocks.push_back(block_info.value().details); + + SILK_LOG << "PageSie " << page_size << ", result size: " << results.transactions.size(); + } + SILK_LOG << "Results" + << " transactions size: " << results.transactions.size() + << " receipts size: " << results.receipts.size() + << " block details size: " << results.blocks.size(); + + reply = make_json_content(request, results); } catch (const std::invalid_argument& iv) { SILK_WARN << "invalid_argument: " << iv.what() << " processing request: " << request.dump(); reply = make_json_content(request, nlohmann::detail::value_t::null); @@ -932,6 +947,117 @@ Task OtsRpcApi::handle_ots_search_transactions_after(const nlohmann::json& co_await tx->close(); // RAII not (yet) available with coroutines } +Task OtsRpcApi::collect_results(kv::api::Transaction& tx, BlockNum block_number, db::chain::CanonicalBodyForStorageProvider& provider, + db::kv::api::PaginatedTimestamps paginated_result_from, db::kv::api::PaginatedTimestamps paginated_result_to, + bool ascending, uint64_t page_size) { + SILK_LOG << "collect_results **************************************"; + auto it_from = co_await paginated_result_from.begin(); + auto it_to = co_await paginated_result_to.begin(); + + SILK_LOG << "TEST *************************************"; + auto it_to1 = co_await paginated_result_to.begin(); + auto cc{0}; + while (const auto value = co_await it_to1.next()) { + const auto txn_id = static_cast(*value); + SILK_LOG << "ITERATE: " << cc++ << ", txn_id: " << txn_id; + if (cc > 10) { + break; + } + } + + TransactionsWithReceipts results{ + .first_page = block_number == 0, + .last_page = true}; + + std::map receipts; + std::optional block_info; + auto block_num_changed = false; + + auto it = db::kv::api::set_union(it_from, it_to, ascending); + const auto chain_storage = tx.create_storage(); + auto count{0}; + while (const auto value = co_await it.next()) { + if (!value.has_value()) { + SILK_LOG << "NO VALUE FROM ITR"; + break; + } + const auto txn_id = static_cast(*value); + SILK_LOG << "ITERATE: " << count++ << ", txn_id: " << txn_id; + const auto block_number_opt = co_await db::txn::block_num_from_tx_num(tx, txn_id, provider); + if (!block_number_opt) { + SILK_LOG << "No block found for txn_id " << txn_id; + break; // TODO + } + const auto bn = block_number_opt.value(); + const auto max_txn_id = co_await db::txn::max_tx_num(tx, bn, provider); + const auto min_txn_id = co_await db::txn::min_tx_num(tx, bn, provider); + const auto txn_index = txn_id - min_txn_id - 1; + SILK_LOG + << "txn_id: " << txn_id + << " block_number: " << bn + << ", txn_index: " << txn_index + << ", max_txn_id: " << max_txn_id + << ", min_txn_id: " << min_txn_id + << ", final txn: " << (txn_id == max_txn_id) + << ", ascending: " << ascending; + + if (txn_id == max_txn_id) { + continue; + } + + block_num_changed = false; + if (block_info && block_info.value().number != bn) { + block_info.reset(); + } + + if (!block_info) { + const auto block_with_hash = co_await rpc::core::read_block_by_number(*block_cache_, *chain_storage, bn); + if (!block_with_hash) { + SILK_DEBUG << "Not found block no. " << bn; + co_return results; + } + + auto rr = co_await core::get_receipts(tx, *block_with_hash, *chain_storage, workers_); + SILK_LOG << "Read #" << rr.size() << " receipts from block " << bn; + + std::for_each(rr.begin(), rr.end(), [&receipts](const auto& item) { + receipts[silkworm::to_hex(item.tx_hash, false)] = std::move(item); + }); + + const Block extended_block{block_with_hash, false}; + const auto block_size = extended_block.get_block_size(); + const BlockDetails block_details{block_size, block_with_hash->hash, block_with_hash->block.header, + block_with_hash->block.transactions.size(), block_with_hash->block.ommers, + block_with_hash->block.withdrawals}; + block_info = BlockInfo{block_with_hash->block.header.number, block_details}; + block_num_changed = true; + } + + if (results.transactions.size() >= page_size && block_num_changed) { + results.last_page = false; + break; + } + + auto transaction = co_await chain_storage->read_transaction_by_idx_in_block(bn, txn_index); + if (!transaction) { + SILK_LOG << "No transaction found in block " << bn << " for index " << txn_index; + co_return results; + } + results.receipts.push_back(std::move(receipts.at(silkworm::to_hex(transaction.value().hash(), false)))); + results.transactions.push_back(std::move(transaction.value())); + results.blocks.push_back(block_info.value().details); + + SILK_LOG << "PageSie " << page_size << ", result size: " << results.transactions.size(); + } + + SILK_LOG << "Results" + << " transactions size: " << results.transactions.size() + << " receipts size: " << results.receipts.size() + << " block details size: " << results.blocks.size(); + + co_return results; +} + Task OtsRpcApi::trace_blocks( FromToBlockProvider& from_to_provider, kv::api::Transaction& tx, diff --git a/silkworm/rpc/commands/ots_api.hpp b/silkworm/rpc/commands/ots_api.hpp index 5696cea187..45c2d76af1 100644 --- a/silkworm/rpc/commands/ots_api.hpp +++ b/silkworm/rpc/commands/ots_api.hpp @@ -23,10 +23,12 @@ #include #include #include +#include #include #include #include #include +#include #include #include #include @@ -228,7 +230,15 @@ class OtsRpcApi { friend class silkworm::rpc::json_rpc::RequestHandler; private: - Task trace_blocks( + Task collect_results( + db::kv::api::Transaction& tx, BlockNum block_number, + db::chain::CanonicalBodyForStorageProvider& provider, + db::kv::api::PaginatedTimestamps paginated_result_from, + db::kv::api::PaginatedTimestamps paginated_result_to, + bool ascending, + uint64_t page_size); + + Task trace_blocks( FromToBlockProvider& from_to_provider, db::kv::api::Transaction& tx, const evmc::address& address, From 93f08cbfa4c427991866470d2eb2180f9d2569a2 Mon Sep 17 00:00:00 2001 From: Sixtysixter <20945591+Sixtysixter@users.noreply.github.com> Date: Fri, 8 Nov 2024 10:21:17 +0100 Subject: [PATCH 4/9] step 2 --- silkworm/rpc/commands/ots_api.cpp | 35 +++++++++++++++++++++++++------ 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/silkworm/rpc/commands/ots_api.cpp b/silkworm/rpc/commands/ots_api.cpp index 19dc00f8a6..7e86fbb3d0 100644 --- a/silkworm/rpc/commands/ots_api.cpp +++ b/silkworm/rpc/commands/ots_api.cpp @@ -744,6 +744,18 @@ Task OtsRpcApi::handle_ots_search_transactions_before(const nlohmann::json } } +// size_t count = 20; +// auto it_to1 = co_await paginated_result_to.begin(); +// SILK_LOG << "TRACES TO IDX"; +// while (const auto value = co_await it_to1.next()) { +// const auto txn_id = static_cast(*value); +// SILK_LOG << "txn_id: " << txn_id; +// +// if (--count == 0) { +// break; +// } +// } + db::kv::api::IndexRangeQuery query_from{ .table = db::table::kTracesFromIdx, .key = key, @@ -767,16 +779,27 @@ Task OtsRpcApi::handle_ots_search_transactions_before(const nlohmann::json auto it = db::kv::api::set_union(it_from, it_to, false); const auto chain_storage = tx->create_storage(); auto count{0}; +// auto it_from1 = co_await paginated_result_from.begin(); +// count = 20; +// SILK_LOG << "TRACES FROM IDX"; +// while (const auto value = co_await it_from1.next()) { +// const auto txn_id = static_cast(*value); +// SILK_LOG << "txn_id: " << txn_id; +// +// if (--count == 0) { +// break; +// } +// } +// +// auto it = db::kv::api::set_union(it_to, it_from, false, 20); +// std::vector keys; +// std::uint64_t count = 0; +// TxnId prev_txn_id = 0; +// TxnId next_txn_id = 0; while (const auto value = co_await it.next()) { - if (!value.has_value()) { - SILK_LOG << "NO VALUE FROM ITR"; - break; - } const auto txn_id = static_cast(*value); - SILK_LOG << "ITERATE: " << count++ << ", txn_id: " << txn_id; const auto block_number_opt = co_await db::txn::block_num_from_tx_num(*tx, txn_id, provider); if (!block_number_opt) { - SILK_LOG << "No block found for txn_id " << txn_id; break; // TODO } const auto bn = block_number_opt.value(); From 1708a37dd51ee4d79baf7ffaf0e7ca2170a1505f Mon Sep 17 00:00:00 2001 From: Sixtysixter <20945591+Sixtysixter@users.noreply.github.com> Date: Sun, 10 Nov 2024 23:38:34 +0100 Subject: [PATCH 5/9] Step 2 --- silkworm/rpc/commands/ots_api.cpp | 35 ++++++------------------------- 1 file changed, 6 insertions(+), 29 deletions(-) diff --git a/silkworm/rpc/commands/ots_api.cpp b/silkworm/rpc/commands/ots_api.cpp index 7e86fbb3d0..19dc00f8a6 100644 --- a/silkworm/rpc/commands/ots_api.cpp +++ b/silkworm/rpc/commands/ots_api.cpp @@ -744,18 +744,6 @@ Task OtsRpcApi::handle_ots_search_transactions_before(const nlohmann::json } } -// size_t count = 20; -// auto it_to1 = co_await paginated_result_to.begin(); -// SILK_LOG << "TRACES TO IDX"; -// while (const auto value = co_await it_to1.next()) { -// const auto txn_id = static_cast(*value); -// SILK_LOG << "txn_id: " << txn_id; -// -// if (--count == 0) { -// break; -// } -// } - db::kv::api::IndexRangeQuery query_from{ .table = db::table::kTracesFromIdx, .key = key, @@ -779,27 +767,16 @@ Task OtsRpcApi::handle_ots_search_transactions_before(const nlohmann::json auto it = db::kv::api::set_union(it_from, it_to, false); const auto chain_storage = tx->create_storage(); auto count{0}; -// auto it_from1 = co_await paginated_result_from.begin(); -// count = 20; -// SILK_LOG << "TRACES FROM IDX"; -// while (const auto value = co_await it_from1.next()) { -// const auto txn_id = static_cast(*value); -// SILK_LOG << "txn_id: " << txn_id; -// -// if (--count == 0) { -// break; -// } -// } -// -// auto it = db::kv::api::set_union(it_to, it_from, false, 20); -// std::vector keys; -// std::uint64_t count = 0; -// TxnId prev_txn_id = 0; -// TxnId next_txn_id = 0; while (const auto value = co_await it.next()) { + if (!value.has_value()) { + SILK_LOG << "NO VALUE FROM ITR"; + break; + } const auto txn_id = static_cast(*value); + SILK_LOG << "ITERATE: " << count++ << ", txn_id: " << txn_id; const auto block_number_opt = co_await db::txn::block_num_from_tx_num(*tx, txn_id, provider); if (!block_number_opt) { + SILK_LOG << "No block found for txn_id " << txn_id; break; // TODO } const auto bn = block_number_opt.value(); From 19e51d7a5bc793fe59ec1dab1b19554bfbadd9b5 Mon Sep 17 00:00:00 2001 From: GitHub Date: Sun, 10 Nov 2024 22:41:27 +0000 Subject: [PATCH 6/9] make fmt --- silkworm/rpc/commands/ots_api.cpp | 46 +++++++++++++++---------------- silkworm/rpc/commands/ots_api.hpp | 16 +++++------ 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/silkworm/rpc/commands/ots_api.cpp b/silkworm/rpc/commands/ots_api.cpp index 19dc00f8a6..cc4aed55d5 100644 --- a/silkworm/rpc/commands/ots_api.cpp +++ b/silkworm/rpc/commands/ots_api.cpp @@ -722,15 +722,15 @@ Task OtsRpcApi::handle_ots_search_transactions_before(const nlohmann::json from_timestamp = static_cast(max_tx_num); SILK_LOG << "block_number: " << block_number << " max_tx_num: " << max_tx_num; } -// const auto min_txn_num = co_await db::txn::min_tx_num(*tx, block_number, provider); -// SILK_LOG << "block_number: " << block_number << " max_tx_num: " << max_tx_num << " min_txn_num: " << min_txn_num; + // const auto min_txn_num = co_await db::txn::min_tx_num(*tx, block_number, provider); + // SILK_LOG << "block_number: " << block_number << " max_tx_num: " << max_tx_num << " min_txn_num: " << min_txn_num; db::kv::api::IndexRangeQuery query_to{ - .table = db::table::kTracesToIdx, - .key = key, - .from_timestamp = from_timestamp, - .to_timestamp = -1, - .ascending_order = false}; + .table = db::table::kTracesToIdx, + .key = key, + .from_timestamp = from_timestamp, + .to_timestamp = -1, + .ascending_order = false}; auto paginated_result_to = co_await tx->index_range(std::move(query_to)); SILK_LOG << "TEST *************************************"; @@ -745,11 +745,11 @@ Task OtsRpcApi::handle_ots_search_transactions_before(const nlohmann::json } db::kv::api::IndexRangeQuery query_from{ - .table = db::table::kTracesFromIdx, - .key = key, - .from_timestamp = from_timestamp, - .to_timestamp = -1, - .ascending_order = false}; + .table = db::table::kTracesFromIdx, + .key = key, + .from_timestamp = from_timestamp, + .to_timestamp = -1, + .ascending_order = false}; auto paginated_result_from = co_await tx->index_range(std::move(query_from)); const auto resultssss = co_await collect_results(*tx, block_number, provider, paginated_result_from, paginated_result_to, false, page_size); @@ -758,8 +758,8 @@ Task OtsRpcApi::handle_ots_search_transactions_before(const nlohmann::json auto it_to = co_await paginated_result_to.begin(); auto it_from = co_await paginated_result_from.begin(); TransactionsWithReceipts results{ - .first_page = block_number == 0, - .last_page = true}; + .first_page = block_number == 0, + .last_page = true}; std::map receipts; std::optional block_info; @@ -777,7 +777,7 @@ Task OtsRpcApi::handle_ots_search_transactions_before(const nlohmann::json const auto block_number_opt = co_await db::txn::block_num_from_tx_num(*tx, txn_id, provider); if (!block_number_opt) { SILK_LOG << "No block found for txn_id " << txn_id; - break; // TODO + break; // TODO } const auto bn = block_number_opt.value(); const auto max_txn_id = co_await db::txn::max_tx_num(*tx, bn, provider); @@ -841,9 +841,9 @@ Task OtsRpcApi::handle_ots_search_transactions_before(const nlohmann::json } SILK_LOG << "Results" - << " transactions size: " << results.transactions.size() - << " receipts size: " << results.receipts.size() - << " block details size: " << results.blocks.size(); + << " transactions size: " << results.transactions.size() + << " receipts size: " << results.receipts.size() + << " block details size: " << results.blocks.size(); reply = make_json_content(request, results); } catch (const std::invalid_argument& iv) { @@ -948,8 +948,8 @@ Task OtsRpcApi::handle_ots_search_transactions_after(const nlohmann::json& } Task OtsRpcApi::collect_results(kv::api::Transaction& tx, BlockNum block_number, db::chain::CanonicalBodyForStorageProvider& provider, - db::kv::api::PaginatedTimestamps paginated_result_from, db::kv::api::PaginatedTimestamps paginated_result_to, - bool ascending, uint64_t page_size) { + db::kv::api::PaginatedTimestamps paginated_result_from, db::kv::api::PaginatedTimestamps paginated_result_to, + bool ascending, uint64_t page_size) { SILK_LOG << "collect_results **************************************"; auto it_from = co_await paginated_result_from.begin(); auto it_to = co_await paginated_result_to.begin(); @@ -966,8 +966,8 @@ Task OtsRpcApi::collect_results(kv::api::Transaction& } TransactionsWithReceipts results{ - .first_page = block_number == 0, - .last_page = true}; + .first_page = block_number == 0, + .last_page = true}; std::map receipts; std::optional block_info; @@ -986,7 +986,7 @@ Task OtsRpcApi::collect_results(kv::api::Transaction& const auto block_number_opt = co_await db::txn::block_num_from_tx_num(tx, txn_id, provider); if (!block_number_opt) { SILK_LOG << "No block found for txn_id " << txn_id; - break; // TODO + break; // TODO } const auto bn = block_number_opt.value(); const auto max_txn_id = co_await db::txn::max_tx_num(tx, bn, provider); diff --git a/silkworm/rpc/commands/ots_api.hpp b/silkworm/rpc/commands/ots_api.hpp index 45c2d76af1..f8644b28aa 100644 --- a/silkworm/rpc/commands/ots_api.hpp +++ b/silkworm/rpc/commands/ots_api.hpp @@ -231,14 +231,14 @@ class OtsRpcApi { private: Task collect_results( - db::kv::api::Transaction& tx, BlockNum block_number, - db::chain::CanonicalBodyForStorageProvider& provider, - db::kv::api::PaginatedTimestamps paginated_result_from, - db::kv::api::PaginatedTimestamps paginated_result_to, - bool ascending, - uint64_t page_size); - - Task trace_blocks( + db::kv::api::Transaction& tx, BlockNum block_number, + db::chain::CanonicalBodyForStorageProvider& provider, + db::kv::api::PaginatedTimestamps paginated_result_from, + db::kv::api::PaginatedTimestamps paginated_result_to, + bool ascending, + uint64_t page_size); + + Task trace_blocks( FromToBlockProvider& from_to_provider, db::kv::api::Transaction& tx, const evmc::address& address, From 58e4334e2a3ea6f1ee68cca92dfa3e1b953a4f3f Mon Sep 17 00:00:00 2001 From: Sixtysixter <20945591+Sixtysixter@users.noreply.github.com> Date: Tue, 12 Nov 2024 21:36:04 +0100 Subject: [PATCH 7/9] Review --- silkworm/rpc/commands/ots_api.cpp | 298 +++++++++++++++++------------- silkworm/rpc/commands/ots_api.hpp | 11 ++ 2 files changed, 179 insertions(+), 130 deletions(-) diff --git a/silkworm/rpc/commands/ots_api.cpp b/silkworm/rpc/commands/ots_api.cpp index cc4aed55d5..826db64d5a 100644 --- a/silkworm/rpc/commands/ots_api.cpp +++ b/silkworm/rpc/commands/ots_api.cpp @@ -714,7 +714,6 @@ Task OtsRpcApi::handle_ots_search_transactions_before(const nlohmann::json auto tx = co_await database_->begin(); try { auto provider = ethdb::kv::canonical_body_for_storage_provider(backend_); - const auto key = db::code_domain_key(address); db::kv::api::Timestamp from_timestamp{-1}; if (block_number > 0) { @@ -722,128 +721,129 @@ Task OtsRpcApi::handle_ots_search_transactions_before(const nlohmann::json from_timestamp = static_cast(max_tx_num); SILK_LOG << "block_number: " << block_number << " max_tx_num: " << max_tx_num; } - // const auto min_txn_num = co_await db::txn::min_tx_num(*tx, block_number, provider); - // SILK_LOG << "block_number: " << block_number << " max_tx_num: " << max_tx_num << " min_txn_num: " << min_txn_num; - - db::kv::api::IndexRangeQuery query_to{ - .table = db::table::kTracesToIdx, - .key = key, - .from_timestamp = from_timestamp, - .to_timestamp = -1, - .ascending_order = false}; - auto paginated_result_to = co_await tx->index_range(std::move(query_to)); - - SILK_LOG << "TEST *************************************"; - auto it_to1 = co_await paginated_result_to.begin(); - auto cc{0}; - while (const auto value = co_await it_to1.next()) { - const auto txn_id = static_cast(*value); - SILK_LOG << "ITERATE: " << cc++ << ", txn_id: " << txn_id; - if (cc > 10) { - break; - } - } - - db::kv::api::IndexRangeQuery query_from{ - .table = db::table::kTracesFromIdx, - .key = key, - .from_timestamp = from_timestamp, - .to_timestamp = -1, - .ascending_order = false}; - auto paginated_result_from = co_await tx->index_range(std::move(query_from)); - - const auto resultssss = co_await collect_results(*tx, block_number, provider, paginated_result_from, paginated_result_to, false, page_size); - SILK_LOG << "********************************** size: " << resultssss.transactions.size(); - - auto it_to = co_await paginated_result_to.begin(); - auto it_from = co_await paginated_result_from.begin(); - TransactionsWithReceipts results{ - .first_page = block_number == 0, - .last_page = true}; - - std::map receipts; - std::optional block_info; - auto block_num_changed = false; - auto it = db::kv::api::set_union(it_from, it_to, false); - const auto chain_storage = tx->create_storage(); - auto count{0}; - while (const auto value = co_await it.next()) { - if (!value.has_value()) { - SILK_LOG << "NO VALUE FROM ITR"; - break; - } - const auto txn_id = static_cast(*value); - SILK_LOG << "ITERATE: " << count++ << ", txn_id: " << txn_id; - const auto block_number_opt = co_await db::txn::block_num_from_tx_num(*tx, txn_id, provider); - if (!block_number_opt) { - SILK_LOG << "No block found for txn_id " << txn_id; - break; // TODO - } - const auto bn = block_number_opt.value(); - const auto max_txn_id = co_await db::txn::max_tx_num(*tx, bn, provider); - const auto min_txn_id = co_await db::txn::min_tx_num(*tx, bn, provider); - const auto txn_index = txn_id - min_txn_id - 1; - SILK_LOG - << "txn_id: " << txn_id - << " block_number: " << bn - << ", txn_index: " << txn_index - << ", max_txn_id: " << max_txn_id - << ", min_txn_id: " << min_txn_id - << ", final txn: " << (txn_id == max_txn_id); - - if (txn_id == max_txn_id) { - continue; - } - - block_num_changed = false; - if (block_info && block_info.value().number != bn) { - block_info.reset(); - } - - if (!block_info) { - const auto block_with_hash = co_await rpc::core::read_block_by_number(*block_cache_, *chain_storage, bn); - if (!block_with_hash) { - SILK_DEBUG << "Not found block no. " << bn; - co_return; - } - - auto rr = co_await core::get_receipts(*tx, *block_with_hash, *chain_storage, workers_); - SILK_LOG << "Read #" << rr.size() << " receipts from block " << bn; - std::for_each(rr.begin(), rr.end(), [&receipts](const auto& item) { - receipts[silkworm::to_hex(item.tx_hash, false)] = std::move(item); - }); + const auto results = co_await collect_transactions_with_receipts(*tx, provider, block_number, address, from_timestamp, false, page_size); - const Block extended_block{block_with_hash, false}; - const auto block_size = extended_block.get_block_size(); - const BlockDetails block_details{block_size, block_with_hash->hash, block_with_hash->block.header, - block_with_hash->block.transactions.size(), block_with_hash->block.ommers, - block_with_hash->block.withdrawals}; - block_info = BlockInfo{block_with_hash->block.header.number, block_details}; - block_num_changed = true; - } - - if (results.transactions.size() >= page_size && block_num_changed) { - results.last_page = false; - break; - } - - auto transaction = co_await chain_storage->read_transaction_by_idx_in_block(bn, txn_index); - if (!transaction) { - SILK_LOG << "No transaction found in block " << bn << " for index " << txn_index; - co_return; - } - results.receipts.push_back(std::move(receipts.at(silkworm::to_hex(transaction.value().hash(), false)))); - results.transactions.push_back(std::move(transaction.value())); - results.blocks.push_back(block_info.value().details); - - SILK_LOG << "PageSie " << page_size << ", result size: " << results.transactions.size(); - } + // const auto min_txn_num = co_await db::txn::min_tx_num(*tx, block_number, provider); + // SILK_LOG << "block_number: " << block_number << " max_tx_num: " << max_tx_num << " min_txn_num: " << min_txn_num; - SILK_LOG << "Results" - << " transactions size: " << results.transactions.size() - << " receipts size: " << results.receipts.size() - << " block details size: " << results.blocks.size(); + // db::kv::api::IndexRangeQuery query_to{ + // .table = db::table::kTracesToIdx, + // .key = key, + // .from_timestamp = from_timestamp, + // .to_timestamp = -1, + // .ascending_order = false}; + // auto paginated_result_to = co_await tx->index_range(std::move(query_to)); + // + // SILK_LOG << "TEST *************************************"; + // auto it_to1 = co_await paginated_result_to.begin(); + // auto cc{0}; + // while (const auto value = co_await it_to1.next()) { + // const auto txn_id = static_cast(*value); + // SILK_LOG << "ITERATE: " << cc++ << ", txn_id: " << txn_id; + // if (cc > 10) { + // break; + // } + // } + // + // db::kv::api::IndexRangeQuery query_from{ + // .table = db::table::kTracesFromIdx, + // .key = key, + // .from_timestamp = from_timestamp, + // .to_timestamp = -1, + // .ascending_order = false}; + // auto paginated_result_from = co_await tx->index_range(std::move(query_from)); + // SILK_LOG << "********************************** size: " << resultssss.transactions.size(); + // + // auto it_to = co_await paginated_result_to.begin(); + // auto it_from = co_await paginated_result_from.begin(); + // TransactionsWithReceipts results{ + // .first_page = block_number == 0, + // .last_page = true}; + // + // std::map receipts; + // std::optional block_info; + // auto block_num_changed = false; + // auto it = db::kv::api::set_union(it_from, it_to, false); + // const auto chain_storage = tx->create_storage(); + // auto count{0}; + // while (const auto value = co_await it.next()) { + // if (!value.has_value()) { + // SILK_LOG << "NO VALUE FROM ITR"; + // break; + // } + // const auto txn_id = static_cast(*value); + // SILK_LOG << "ITERATE: " << count++ << ", txn_id: " << txn_id; + // const auto block_number_opt = co_await db::txn::block_num_from_tx_num(*tx, txn_id, provider); + // if (!block_number_opt) { + // SILK_LOG << "No block found for txn_id " << txn_id; + // break; // TODO + // } + // const auto bn = block_number_opt.value(); + // const auto max_txn_id = co_await db::txn::max_tx_num(*tx, bn, provider); + // const auto min_txn_id = co_await db::txn::min_tx_num(*tx, bn, provider); + // const auto txn_index = txn_id - min_txn_id - 1; + // SILK_LOG + // << "txn_id: " << txn_id + // << " block_number: " << bn + // << ", txn_index: " << txn_index + // << ", max_txn_id: " << max_txn_id + // << ", min_txn_id: " << min_txn_id + // << ", final txn: " << (txn_id == max_txn_id); + // + // if (txn_id == max_txn_id) { + // continue; + // } + // + // block_num_changed = false; + // if (block_info && block_info.value().number != bn) { + // block_info.reset(); + // } + // + // if (!block_info) { + // const auto block_with_hash = co_await rpc::core::read_block_by_number(*block_cache_, *chain_storage, bn); + // if (!block_with_hash) { + // SILK_DEBUG << "Not found block no. " << bn; + // co_return; + // } + // + // auto rr = co_await core::get_receipts(*tx, *block_with_hash, *chain_storage, workers_); + // SILK_LOG << "Read #" << rr.size() << " receipts from block " << bn; + // + // std::for_each(rr.begin(), rr.end(), [&receipts](const auto& item) { + // receipts[silkworm::to_hex(item.tx_hash, false)] = std::move(item); + // }); + // + // const Block extended_block{block_with_hash, false}; + // const auto block_size = extended_block.get_block_size(); + // const BlockDetails block_details{block_size, block_with_hash->hash, block_with_hash->block.header, + // block_with_hash->block.transactions.size(), block_with_hash->block.ommers, + // block_with_hash->block.withdrawals}; + // block_info = BlockInfo{block_with_hash->block.header.number, block_details}; + // block_num_changed = true; + // } + // + // if (results.transactions.size() >= page_size && block_num_changed) { + // results.last_page = false; + // break; + // } + // + // auto transaction = co_await chain_storage->read_transaction_by_idx_in_block(bn, txn_index); + // if (!transaction) { + // SILK_LOG << "No transaction found in block " << bn << " for index " << txn_index; + // co_return; + // } + // results.receipts.push_back(std::move(receipts.at(silkworm::to_hex(transaction.value().hash(), false)))); + // results.transactions.push_back(std::move(transaction.value())); + // results.blocks.push_back(block_info.value().details); + // + // SILK_LOG << "PageSie " << page_size << ", result size: " << results.transactions.size(); + // } + // + // SILK_LOG << "Results" + // << " transactions size: " << results.transactions.size() + // << " receipts size: " << results.receipts.size() + // << " block details size: " << results.blocks.size(); reply = make_json_content(request, results); } catch (const std::invalid_argument& iv) { @@ -947,12 +947,21 @@ Task OtsRpcApi::handle_ots_search_transactions_after(const nlohmann::json& co_await tx->close(); // RAII not (yet) available with coroutines } -Task OtsRpcApi::collect_results(kv::api::Transaction& tx, BlockNum block_number, db::chain::CanonicalBodyForStorageProvider& provider, - db::kv::api::PaginatedTimestamps paginated_result_from, db::kv::api::PaginatedTimestamps paginated_result_to, - bool ascending, uint64_t page_size) { - SILK_LOG << "collect_results **************************************"; - auto it_from = co_await paginated_result_from.begin(); - auto it_to = co_await paginated_result_to.begin(); +Task OtsRpcApi::collect_transactions_with_receipts( + kv::api::Transaction& tx, + db::chain::CanonicalBodyForStorageProvider& provider, + BlockNum block_number, + const evmc::address& address, + db::kv::api::Timestamp from_timestamp, + bool ascending, uint64_t page_size) { + const auto key = db::code_domain_key(address); + db::kv::api::IndexRangeQuery query_to{ + .table = db::table::kTracesToIdx, + .key = key, + .from_timestamp = from_timestamp, + .to_timestamp = -1, + .ascending_order = ascending}; + auto paginated_result_to = co_await tx.index_range(std::move(query_to)); SILK_LOG << "TEST *************************************"; auto it_to1 = co_await paginated_result_to.begin(); @@ -965,6 +974,39 @@ Task OtsRpcApi::collect_results(kv::api::Transaction& } } + db::kv::api::IndexRangeQuery query_from{ + .table = db::table::kTracesFromIdx, + .key = key, + .from_timestamp = from_timestamp, + .to_timestamp = -1, + .ascending_order = ascending}; + auto paginated_result_from = co_await tx.index_range(std::move(query_from)); + + cc = 0; + auto it_from1 = co_await paginated_result_from.begin(); + while (const auto value = co_await it_from1.next()) { + const auto txn_id = static_cast(*value); + SILK_LOG << "ITERATE: " << cc++ << ", txn_id: " << txn_id; + if (cc > 10) { + break; + } + } + + SILK_LOG << "collect_transactions_with_receipts **************************************"; + auto it_from = co_await paginated_result_from.begin(); + auto it_to = co_await paginated_result_to.begin(); + + // SILK_LOG << "TEST *************************************"; + // auto it_to1 = co_await paginated_result_to.begin(); + // auto cc{0}; + // while (const auto value = co_await it_to1.next()) { + // const auto txn_id = static_cast(*value); + // SILK_LOG << "ITERATE: " << cc++ << ", txn_id: " << txn_id; + // if (cc > 10) { + // break; + // } + // } + TransactionsWithReceipts results{ .first_page = block_number == 0, .last_page = true}; @@ -977,10 +1019,6 @@ Task OtsRpcApi::collect_results(kv::api::Transaction& const auto chain_storage = tx.create_storage(); auto count{0}; while (const auto value = co_await it.next()) { - if (!value.has_value()) { - SILK_LOG << "NO VALUE FROM ITR"; - break; - } const auto txn_id = static_cast(*value); SILK_LOG << "ITERATE: " << count++ << ", txn_id: " << txn_id; const auto block_number_opt = co_await db::txn::block_num_from_tx_num(tx, txn_id, provider); diff --git a/silkworm/rpc/commands/ots_api.hpp b/silkworm/rpc/commands/ots_api.hpp index f8644b28aa..8ff0bb1a9a 100644 --- a/silkworm/rpc/commands/ots_api.hpp +++ b/silkworm/rpc/commands/ots_api.hpp @@ -230,6 +230,7 @@ class OtsRpcApi { friend class silkworm::rpc::json_rpc::RequestHandler; private: +<<<<<<< HEAD Task collect_results( db::kv::api::Transaction& tx, BlockNum block_number, db::chain::CanonicalBodyForStorageProvider& provider, @@ -237,6 +238,16 @@ class OtsRpcApi { db::kv::api::PaginatedTimestamps paginated_result_to, bool ascending, uint64_t page_size); +======= + Task collect_transactions_with_receipts( + db::kv::api::Transaction& tx, + db::chain::CanonicalBodyForStorageProvider& provider, + BlockNum block_number, + const evmc::address& address, + db::kv::api::Timestamp from_timestamp, + bool ascending, + uint64_t page_size); +>>>>>>> bed339eb (Review) Task trace_blocks( FromToBlockProvider& from_to_provider, From be3174302929d01b5094171a1742891b41775b04 Mon Sep 17 00:00:00 2001 From: GitHub Date: Tue, 12 Nov 2024 20:36:41 +0000 Subject: [PATCH 8/9] make fmt --- silkworm/rpc/commands/ots_api.hpp | 20 +++++--------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/silkworm/rpc/commands/ots_api.hpp b/silkworm/rpc/commands/ots_api.hpp index 8ff0bb1a9a..94c1f2c322 100644 --- a/silkworm/rpc/commands/ots_api.hpp +++ b/silkworm/rpc/commands/ots_api.hpp @@ -230,24 +230,14 @@ class OtsRpcApi { friend class silkworm::rpc::json_rpc::RequestHandler; private: -<<<<<<< HEAD - Task collect_results( - db::kv::api::Transaction& tx, BlockNum block_number, + Task collect_transactions_with_receipts( + db::kv::api::Transaction& tx, db::chain::CanonicalBodyForStorageProvider& provider, - db::kv::api::PaginatedTimestamps paginated_result_from, - db::kv::api::PaginatedTimestamps paginated_result_to, + BlockNum block_number, + const evmc::address& address, + db::kv::api::Timestamp from_timestamp, bool ascending, uint64_t page_size); -======= - Task collect_transactions_with_receipts( - db::kv::api::Transaction& tx, - db::chain::CanonicalBodyForStorageProvider& provider, - BlockNum block_number, - const evmc::address& address, - db::kv::api::Timestamp from_timestamp, - bool ascending, - uint64_t page_size); ->>>>>>> bed339eb (Review) Task trace_blocks( FromToBlockProvider& from_to_provider, From b58dd89e2334a9467b2c19d31a1accd1facc3b0d Mon Sep 17 00:00:00 2001 From: Sixtysixter <20945591+Sixtysixter@users.noreply.github.com> Date: Wed, 13 Nov 2024 10:23:01 +0100 Subject: [PATCH 9/9] Added unit test --- silkworm/db/kv/api/endpoint/paginated_sequence_test.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/silkworm/db/kv/api/endpoint/paginated_sequence_test.cpp b/silkworm/db/kv/api/endpoint/paginated_sequence_test.cpp index c5a09a8ca1..bdb7fb53e2 100644 --- a/silkworm/db/kv/api/endpoint/paginated_sequence_test.cpp +++ b/silkworm/db/kv/api/endpoint/paginated_sequence_test.cpp @@ -197,7 +197,12 @@ TEST_CASE_METHOD(PaginatedSequenceTest, "set_intersection", "[db][kv][api][pagin TEST_CASE_METHOD(PaginatedSequenceTest, "set_union", "[db][kv][api][paginated_sequence]") { const Fixtures, std::vector> fixtures{ {{/*v1=*/{}, /*v2=*/{}}, /*v1_or_v2=*/{}}, // both empty => empty + {{/*v1=*/{}, /*v2=*/{{1}}}, /*v1_or_v2=*/{1}}, // one empty => other + {{/*v1=*/{{1}}, /*v2=*/{}}, /*v1_or_v2=*/{1}}, // one empty => other {{/*v1=*/{{1, 2, 3}, {4, 5, 6}, {7, 8}}, /*v2=*/{}}, /*v1_or_v2=*/{1, 2, 3, 4, 5, 6, 7, 8}}, // one empty => other + {{/*v1=*/{{1, 3, 5}}, /*v2=*/{{2, 4, 6}}}, /*v1_or_v2=*/{1, 2, 3, 4, 5, 6}}, + {{/*v1=*/{{1, 3, 5, 7}}, /*v2=*/{{1, 4, 6, 7}}}, /*v1_or_v2=*/{1, 3, 4, 5, 6, 7}}, + {{/*v1=*/{{1, 2, 3, 4}}, /*v2=*/{{1, 2, 3, 5}}}, /*v1_or_v2=*/{1, 2, 3, 4, 5}}, {{/*v1=*/{{1, 2, 3}, {4, 5, 6}, {7, 8}}, /*v2=*/{{10, 11, 12}, {13}}}, /*v1_or_v2=*/{1, 2, 3, 4, 5, 6, 7, 8, 10, 11, 12, 13}}, {{/*v1=*/{{1, 2, 3}, {4, 5, 6}, {7, 8}}, /*v2=*/{{7, 8, 9}, {10, 11, 12}, {13}}}, /*v1_or_v2=*/{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}}, {{/*v1=*/{{1, 2, 3}, {4, 5, 6}, {7, 8}}, /*v2=*/{{1, 2, 3}, {4, 5, 6}, {7, 8}}}, /*v1_and_v2=*/{1, 2, 3, 4, 5, 6, 7, 8}},