From 190aa6bd4af9bd53f95474be2cde78df121788fc Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 15 Apr 2024 13:44:12 +0300 Subject: [PATCH 01/28] Use shard client timestamp to determine expiration of temp archives (#965) Co-authored-by: SpyCheese --- validator/manager.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index e186902ad..d88a50154 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2418,9 +2418,9 @@ void ValidatorManagerImpl::state_serializer_update(BlockSeqno seqno) { void ValidatorManagerImpl::alarm() { try_advance_gc_masterchain_block(); alarm_timestamp() = td::Timestamp::in(1.0); - if (last_masterchain_block_handle_ && gc_masterchain_handle_) { - td::actor::send_closure(db_, &Db::run_gc, last_masterchain_block_handle_->unix_time(), - gc_masterchain_handle_->unix_time(), static_cast(opts_->archive_ttl())); + if (shard_client_handle_ && gc_masterchain_handle_) { + td::actor::send_closure(db_, &Db::run_gc, shard_client_handle_->unix_time(), gc_masterchain_handle_->unix_time(), + static_cast(opts_->archive_ttl())); } if (log_status_at_.is_in_past()) { if (last_masterchain_block_handle_) { From 25f61dff161b9c76dce0fc62dc51da911a208b68 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 15 Apr 2024 13:44:30 +0300 Subject: [PATCH 02/28] Tonlib patch (#966) * Bugfix in tonlib * Fix calling actors in RunEmulator * Fix checking proofs in blocks.getTransactions and blocks.getShards * tonlib-cli: Fix printing special cells --------- Co-authored-by: SpyCheese --- tonlib/tonlib/TonlibClient.cpp | 24 +++++++++++++++++------- tonlib/tonlib/tonlib-cli.cpp | 3 ++- 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/tonlib/tonlib/TonlibClient.cpp b/tonlib/tonlib/TonlibClient.cpp index 86acbcc37..39029ca4e 100644 --- a/tonlib/tonlib/TonlibClient.cpp +++ b/tonlib/tonlib/TonlibClient.cpp @@ -1891,7 +1891,9 @@ class RunEmulator : public TonlibQueryActor { if (stopped_) { return; } - get_block_id([self = this](td::Result&& block_id) { self->set_block_id(std::move(block_id)); }); + get_block_id([SelfId = actor_id(this)](td::Result&& block_id) { + td::actor::send_closure(SelfId, &RunEmulator::set_block_id, std::move(block_id)); + }); } void set_block_id(td::Result&& block_id) { @@ -1900,8 +1902,12 @@ class RunEmulator : public TonlibQueryActor { } else { block_id_ = block_id.move_as_ok(); - get_mc_state_root([self = this](td::Result>&& mc_state_root) { self->set_mc_state_root(std::move(mc_state_root)); }); - get_account_state([self = this](td::Result>&& state) { self->set_account_state(std::move(state)); }); + get_mc_state_root([SelfId = actor_id(this)](td::Result>&& mc_state_root) { + td::actor::send_closure(SelfId, &RunEmulator::set_mc_state_root, std::move(mc_state_root)); + }); + get_account_state([SelfId = actor_id(this)](td::Result>&& state) { + td::actor::send_closure(SelfId, &RunEmulator::set_account_state, std::move(state)); + }); check(get_transactions(0)); inc(); @@ -1923,7 +1929,9 @@ class RunEmulator : public TonlibQueryActor { } else { account_state_ = account_state.move_as_ok(); send_query(int_api::ScanAndLoadGlobalLibs{account_state_->get_raw_state()}, - [self = this](td::Result R) { self->set_global_libraries(std::move(R)); }); + [SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &RunEmulator::set_global_libraries, std::move(R)); + }); } } @@ -5521,7 +5529,7 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getShards& request, } block::ShardConfig sh_conf; - if (!sh_conf.unpack(mc_extra.shard_hashes)) { + if (!sh_conf.unpack(data_csr)) { return td::Status::Error("cannot extract shard block list from shard configuration"); } auto ids = sh_conf.get_shard_hash_ids(true); @@ -5544,7 +5552,9 @@ td::Status TonlibClient::do_request(const tonlib_api::blocks_getShards& request, return td::Status::OK(); } -td::Status check_lookup_block_proof(lite_api_ptr& result, int mode, ton::BlockId blkid, ton::BlockIdExt client_mc_blkid, td::uint64 lt, td::uint32 utime); +td::Status check_lookup_block_proof(lite_api_ptr& result, int mode, + ton::BlockId blkid, ton::BlockIdExt client_mc_blkid, td::uint64 lt, + td::uint32 utime); td::Status TonlibClient::do_request(const tonlib_api::blocks_lookupBlock& request, td::Promise>&& promise) { @@ -5730,7 +5740,7 @@ auto to_tonlib_api(const ton::lite_api::liteServer_transactionId& txid) td::Status check_block_transactions_proof(lite_api_ptr& bTxes, int32_t mode, ton::LogicalTime start_lt, td::Bits256 start_addr, td::Bits256 root_hash, int req_count) { - if (mode & ton::lite_api::liteServer_listBlockTransactions::WANT_PROOF_MASK == 0) { + if ((mode & ton::lite_api::liteServer_listBlockTransactions::WANT_PROOF_MASK) == 0) { return td::Status::OK(); } constexpr int max_answer_transactions = 256; diff --git a/tonlib/tonlib/tonlib-cli.cpp b/tonlib/tonlib/tonlib-cli.cpp index 1107a5d04..5a32b50f3 100644 --- a/tonlib/tonlib/tonlib-cli.cpp +++ b/tonlib/tonlib/tonlib-cli.cpp @@ -1423,7 +1423,8 @@ class TonlibCli : public td::actor::Actor { if (r_cell.is_error()) { sb << ""; } - auto cs = vm::load_cell_slice(r_cell.move_as_ok()); + bool spec = true; + auto cs = vm::load_cell_slice_special(r_cell.move_as_ok(), spec); std::stringstream ss; cs.print_rec(ss); sb << ss.str(); From d2b012c883f0406e8d031b202d7c9b53e2de1236 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Thu, 25 Apr 2024 18:45:39 +0300 Subject: [PATCH 03/28] Add write time stats to celldb/db_stats.txt (#972) Co-authored-by: SpyCheese --- tdutils/td/utils/Timer.cpp | 4 +++ tdutils/td/utils/Timer.h | 1 + validator/CMakeLists.txt | 2 ++ validator/db/archive-slice.cpp | 66 +++++++++------------------------- validator/db/celldb.cpp | 22 ++++++++++-- validator/db/celldb.hpp | 25 ++++++++++--- validator/db/db-utils.cpp | 54 ++++++++++++++++++++++++++++ validator/db/db-utils.h | 33 +++++++++++++++++ 8 files changed, 149 insertions(+), 58 deletions(-) create mode 100644 validator/db/db-utils.cpp create mode 100644 validator/db/db-utils.h diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index 0f6a7d6ed..1f72fba96 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -87,4 +87,8 @@ void PerfWarningTimer::reset() { start_at_ = 0; } +double PerfWarningTimer::elapsed() const { + return Time::now() - start_at_; +} + } // namespace td diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index 18ea35d7c..3e0cafbf5 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -53,6 +53,7 @@ class PerfWarningTimer { PerfWarningTimer &operator=(PerfWarningTimer &&) = delete; ~PerfWarningTimer(); void reset(); + double elapsed() const; private: string name_; diff --git a/validator/CMakeLists.txt b/validator/CMakeLists.txt index a25ac7e1f..20462caeb 100644 --- a/validator/CMakeLists.txt +++ b/validator/CMakeLists.txt @@ -25,6 +25,8 @@ set(VALIDATOR_DB_SOURCE db/statedb.cpp db/staticfilesdb.cpp db/staticfilesdb.hpp + db/db-utils.cpp + db/db-utils.h db/package.hpp db/package.cpp diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index 86c2ee932..daee546b0 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -24,6 +24,7 @@ #include "td/utils/port/path.h" #include "common/delay.h" #include "files-async.hpp" +#include "db-utils.h" namespace ton { @@ -41,16 +42,14 @@ class PackageStatistics { void record_read(double time, uint64_t bytes) { read_bytes.fetch_add(bytes, std::memory_order_relaxed); - std::lock_guard guard(read_mutex); + std::lock_guard guard(read_mutex); read_time.insert(time); - read_time_sum += time; } void record_write(double time, uint64_t bytes) { write_bytes.fetch_add(bytes, std::memory_order_relaxed); - std::lock_guard guard(write_mutex); + std::lock_guard guard(write_mutex); write_time.insert(time); - write_time_sum += time; } std::string to_string_and_reset() { @@ -64,68 +63,35 @@ class PackageStatistics { ss << "ton.pack.read.bytes COUNT : " << read_bytes.exchange(0, std::memory_order_relaxed) << "\n"; ss << "ton.pack.write.bytes COUNT : " << write_bytes.exchange(0, std::memory_order_relaxed) << "\n"; - std::multiset temp_read_time; - double temp_read_time_sum; + PercentileStats temp_read_time; { - std::lock_guard guard(read_mutex); + std::lock_guard guard(read_mutex); temp_read_time = std::move(read_time); read_time.clear(); - temp_read_time_sum = read_time_sum; - read_time_sum = 0; } - auto read_stats = calculate_statistics(temp_read_time); - ss << "ton.pack.read.micros P50 : " << read_stats[0] << - " P95 : " << read_stats[1] << - " P99 : " << read_stats[2] << - " P100 : " << read_stats[3] << - " COUNT : " << temp_read_time.size() << - " SUM : " << temp_read_time_sum << "\n"; - - std::multiset temp_write_time; - double temp_write_time_sum; + ss << "ton.pack.read.micros " << temp_read_time.to_string() << "\n"; + + PercentileStats temp_write_time; { - std::lock_guard guard(write_mutex); + std::lock_guard guard(write_mutex); temp_write_time = std::move(write_time); write_time.clear(); - temp_write_time_sum = write_time_sum; - write_time_sum = 0; } - auto write_stats = calculate_statistics(temp_write_time); - ss << "ton.pack.write.micros P50 : " << write_stats[0] << - " P95 : " << write_stats[1] << - " P99 : " << write_stats[2] << - " P100 : " << write_stats[3] << - " COUNT : " << temp_write_time.size() << - " SUM : " << temp_write_time_sum << "\n"; + ss << "ton.pack.write.micros " << temp_write_time.to_string() << "\n"; return ss.str(); } private: - std::atomic_uint64_t open_count; - std::atomic_uint64_t close_count; - std::multiset read_time; - std::atomic_uint64_t read_bytes; - std::multiset write_time; - std::atomic_uint64_t write_bytes; - double read_time_sum; - double write_time_sum; + std::atomic_uint64_t open_count{0}; + std::atomic_uint64_t close_count{0}; + PercentileStats read_time; + std::atomic_uint64_t read_bytes{0}; + PercentileStats write_time; + std::atomic_uint64_t write_bytes{0}; mutable std::mutex read_mutex; mutable std::mutex write_mutex; - - std::vector calculate_statistics(const std::multiset& data) const { - if (data.empty()) return {0, 0, 0, 0}; - - auto size = data.size(); - auto calc_percentile = [&](double p) -> double { - auto it = data.begin(); - std::advance(it, static_cast(std::ceil(p * double(size)) - 1)); - return *it; - }; - - return {calc_percentile(0.5), calc_percentile(0.95), calc_percentile(0.99), *data.rbegin()}; - } }; void DbStatistics::init() { diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index d9803cbe2..294515a31 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -89,7 +89,7 @@ void CellDbIn::start_up() { statistics_flush_at_ = td::Timestamp::in(60.0); } cell_db_ = std::make_shared(td::RocksDb::open(path_, statistics_).move_as_ok()); - + boc_ = vm::DynamicBagOfCellsDb::create(); boc_->set_celldb_compress_depth(opts_->get_celldb_compress_depth()); @@ -155,6 +155,9 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); promise.set_result(boc_->load_cell(cell->get_hash().as_slice())); + if (!opts_->get_disable_rocksdb_stats()) { + cell_db_statistics_.store_cell_time_.insert(timer.elapsed() * 1e6); + } } void CellDbIn::get_cell_db_reader(td::Promise> promise) { @@ -162,8 +165,9 @@ void CellDbIn::get_cell_db_reader(td::Promise> } void CellDbIn::flush_db_stats() { - auto stats = td::RocksDb::statistics_to_string(statistics_); - auto to_file_r = td::FileFd::open(path_ + "/db_stats.txt", td::FileFd::Truncate | td::FileFd::Create | td::FileFd::Write, 0644); + auto stats = td::RocksDb::statistics_to_string(statistics_) + cell_db_statistics_.to_string(); + auto to_file_r = + td::FileFd::open(path_ + "/db_stats.txt", td::FileFd::Truncate | td::FileFd::Create | td::FileFd::Write, 0644); if (to_file_r.is_error()) { LOG(ERROR) << "Failed to open db_stats.txt: " << to_file_r.move_as_error(); return; @@ -176,6 +180,7 @@ void CellDbIn::flush_db_stats() { return; } td::RocksDb::reset_statistics(statistics_); + cell_db_statistics_.clear(); } void CellDbIn::alarm() { @@ -278,6 +283,9 @@ void CellDbIn::gc_cont2(BlockHandle handle) { td::actor::send_closure(parent_, &CellDb::update_snapshot, cell_db_->snapshot()); DCHECK(get_block(key_hash).is_error()); + if (!opts_->get_disable_rocksdb_stats()) { + cell_db_statistics_.gc_cell_time_.insert(timer.elapsed() * 1e6); + } } void CellDbIn::skip_gc() { @@ -441,6 +449,14 @@ td::BufferSlice CellDbIn::DbEntry::release() { return create_serialize_tl_object(create_tl_block_id(block_id), prev, next, root_hash); } +std::string CellDbIn::CellDbStatistics::to_string() { + td::StringBuilder ss; + ss << "ton.celldb.store_cell.micros " << store_cell_time_.to_string() << "\n"; + ss << "ton.celldb.gc_cell.micros " << gc_cell_time_.to_string() << "\n"; + ss << "ton.celldb.total_time.micros : " << (td::Timestamp::now().at() - stats_start_time_.at()) * 1e6 << "\n"; + return ss.as_cslice().str(); +} + } // namespace validator } // namespace ton diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index 573d4b995..7dc1fa781 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -26,6 +26,7 @@ #include "interfaces/block-handle.h" #include "auto/tl/ton_api.h" #include "validator.h" +#include "db-utils.h" namespace rocksdb { class Statistics; @@ -42,9 +43,11 @@ class CellDbAsyncExecutor; class CellDbBase : public td::actor::Actor { public: - virtual void start_up(); + void start_up() override; + protected: std::shared_ptr async_executor; + private: void execute_sync(std::function f); friend CellDbAsyncExecutor; @@ -76,8 +79,7 @@ class CellDbIn : public CellDbBase { RootHash root_hash; DbEntry(tl_object_ptr entry); - DbEntry() { - } + DbEntry() = default; DbEntry(BlockIdExt block_id, KeyHash prev, KeyHash next, RootHash root_hash) : block_id(block_id), prev(prev), next(next), root_hash(root_hash) { } @@ -109,8 +111,6 @@ class CellDbIn : public CellDbBase { std::unique_ptr boc_; std::shared_ptr cell_db_; - std::shared_ptr statistics_; - td::Timestamp statistics_flush_at_ = td::Timestamp::never(); std::function on_load_callback_; std::set cells_to_migrate_; @@ -127,6 +127,21 @@ class CellDbIn : public CellDbBase { }; std::unique_ptr migration_stats_; + struct CellDbStatistics { + PercentileStats store_cell_time_; + PercentileStats gc_cell_time_; + td::Timestamp stats_start_time_ = td::Timestamp::now(); + + std::string to_string(); + void clear() { + *this = CellDbStatistics{}; + } + }; + + std::shared_ptr statistics_; + CellDbStatistics cell_db_statistics_; + td::Timestamp statistics_flush_at_ = td::Timestamp::never(); + public: class MigrationProxy : public td::actor::Actor { public: diff --git a/validator/db/db-utils.cpp b/validator/db/db-utils.cpp new file mode 100644 index 000000000..9375f97e6 --- /dev/null +++ b/validator/db/db-utils.cpp @@ -0,0 +1,54 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#include "db-utils.h" + +#include "td/utils/logging.h" + +#include + +namespace ton::validator { + +void PercentileStats::insert(double value) { + values_.insert(value); +} + +std::string PercentileStats::to_string() const { + double percentiles[4] = {0.0, 0.0, 0.0, 0.0}; + double sum = 0.0; + size_t size = values_.size(); + if (!values_.empty()) { + size_t indices[4] = {(size_t)std::ceil(0.5 * (double)size) - 1, (size_t)std::ceil(0.95 * (double)size) - 1, + (size_t)std::ceil(0.99 * (double)size) - 1, size - 1}; + size_t i = 0; + for (auto it = values_.begin(); it != values_.end(); ++it, ++i) { + for (size_t j = 0; j < 4; ++j) { + if (indices[j] == i) { + percentiles[j] = *it; + } + } + sum += *it; + } + } + return PSTRING() << "P50 : " << percentiles[0] << " P95 : " << percentiles[1] << " P99 : " << percentiles[2] + << " P100 : " << percentiles[3] << " COUNT : " << size << " SUM : " << sum; +} + +void PercentileStats::clear() { + values_.clear(); +} + +} // namespace ton::validator \ No newline at end of file diff --git a/validator/db/db-utils.h b/validator/db/db-utils.h new file mode 100644 index 000000000..8ffbf5c97 --- /dev/null +++ b/validator/db/db-utils.h @@ -0,0 +1,33 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ +#pragma once +#include +#include + +namespace ton::validator { + +class PercentileStats { + public: + void insert(double value); + std::string to_string() const; + void clear(); + + private: + std::multiset values_; +}; + +} // namespace ton::validator \ No newline at end of file From 9a543c6b28b4f7d952f4b0225b5f8ee433a3f392 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Thu, 25 Apr 2024 18:47:15 +0300 Subject: [PATCH 04/28] Add more info to session stats (#973) * Improve validator session stats * Improve validator session stats Add got_submit_at, collation_cached, validation_cached. Fix stats cleanup. * Fix setting timestamp for cached blocks * Add serialize/deserialize time, serialized size to validator session stats, fix setting is_accepted --------- Co-authored-by: SpyCheese --- tl/generate/scheme/ton_api.tl | 9 +- tl/generate/scheme/ton_api.tlo | Bin 87952 -> 88712 bytes validator-session/candidate-serializer.cpp | 1 - validator-session/validator-session-types.h | 51 ++++- validator-session/validator-session.cpp | 223 ++++++++++++++++---- validator-session/validator-session.h | 14 +- validator-session/validator-session.hpp | 22 +- validator/manager.cpp | 8 +- validator/validator-group.cpp | 43 ++-- validator/validator-group.hpp | 6 +- 10 files changed, 309 insertions(+), 68 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index c61bad858..af9df3fd0 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -752,7 +752,14 @@ http.server.config dhs:(vector http.server.dnsEntry) local_hosts:(vector http.se ---types--- -validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int block_timestamp:long comment:string = validatorSession.StatsProducer; +validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int comment:string + block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double + collation_time:double collated_at:double collation_cached:Bool + validation_time:double validated_at:double validation_cached:Bool + gen_utime:double + approved_weight:long approved_33pct_at:double approved_66pct_at:double + signed_weight:long signed_33pct_at:double signed_66pct_at:double + serialize_time:double deserialize_time:double serialized_size:int = validatorSession.StatsProducer; validatorSession.statsRound timestamp:long producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index b8f5d5a4936db19e0d67c2f4bf647954b93cccf4..c97fa2feefc0767775d31b5c072aaf671c979e35 100644 GIT binary patch delta 460 zcmbQRp0#5u>xLbMES~D@>YMi%^4tcqrk~+qRNw@O-8?4bBQd$KR(ZMvFJr*;3A~Il zlQ&FenV!MRC^7lOECEjL%;NaO Mn6=%7kI}~y0408<1poj5 delta 92 zcmeC!$~s{^>xLbMEGHyC=x*L)$a9+&#FCqSP=Ha4H=rm#r8GIUhyf%vxu8d5x&$v{ n!1NAYMv2KYN(858@-j+KZZPHm>z=+rfUyJ2+HNDr=;H|hYdj%F diff --git a/validator-session/candidate-serializer.cpp b/validator-session/candidate-serializer.cpp index c220b4a95..a85c4b72e 100644 --- a/validator-session/candidate-serializer.cpp +++ b/validator-session/candidate-serializer.cpp @@ -14,7 +14,6 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ -#pragma once #include "candidate-serializer.h" #include "tl-utils/tl-utils.hpp" #include "vm/boc.h" diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index bcbaa8f71..db06bbd9c 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -76,8 +76,57 @@ struct ValidatorSessionStats { PublicKeyHash id = PublicKeyHash::zero(); ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero(); int block_status = status_none; - td::uint64 block_timestamp = 0; + double block_timestamp = -1.0; std::string comment; + + bool is_accepted = false; + bool is_ours = false; + double got_submit_at = -1.0; + double collation_time = -1.0; + double validation_time = -1.0; + double collated_at = -1.0; + double validated_at = -1.0; + bool collation_cached = false; + bool validation_cached = false; + double gen_utime = -1.0; + + std::vector approvers, signers; + ValidatorWeight approved_weight = 0; + ValidatorWeight signed_weight = 0; + double approved_33pct_at = -1.0; + double approved_66pct_at = -1.0; + double signed_33pct_at = -1.0; + double signed_66pct_at = -1.0; + + double serialize_time = -1.0; + double deserialize_time = -1.0; + td::int32 serialized_size = -1; + + void set_approved_by(td::uint32 id, ValidatorWeight weight, ValidatorWeight total_weight) { + if (!approvers.at(id)) { + approvers.at(id) = true; + approved_weight += weight; + if (approved_33pct_at <= 0.0 && approved_weight >= total_weight / 3 + 1) { + approved_33pct_at = td::Clocks::system(); + } + if (approved_66pct_at <= 0.0 && approved_weight >= (total_weight * 2) / 3 + 1) { + approved_66pct_at = td::Clocks::system(); + } + } + } + + void set_signed_by(td::uint32 id, ValidatorWeight weight, ValidatorWeight total_weight) { + if (!signers.at(id)) { + signers.at(id) = true; + signed_weight += weight; + if (signed_33pct_at <= 0.0 && signed_weight >= total_weight / 3 + 1) { + signed_33pct_at = td::Clocks::system(); + } + if (signed_66pct_at <= 0.0 && signed_weight >= (total_weight * 2) / 3 + 1) { + signed_66pct_at = td::Clocks::system(); + } + } + } }; struct Round { td::uint64 timestamp = 0; diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 6b328b2bd..215b6b12d 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -20,6 +20,7 @@ #include "td/utils/Random.h" #include "td/utils/crypto.h" #include "candidate-serializer.h" +#include "td/utils/overloaded.h" namespace ton { @@ -86,6 +87,7 @@ void ValidatorSessionImpl::process_blocks(std::vector for (auto &msg : msgs) { VLOG(VALIDATOR_SESSION_INFO) << this << ": applying action: " << msg.get(); + stats_process_action(local_idx(), *msg); real_state_ = ValidatorSessionState::action(description(), real_state_, local_idx(), att, msg.get()); } @@ -167,6 +169,7 @@ void ValidatorSessionImpl::preprocess_block(catchain::CatChainBlock *block) { for (auto &msg : B->actions_) { VLOG(VALIDATOR_SESSION_INFO) << this << "[node " << description().get_source_id(block->source()) << "][block " << block->hash() << "]: applying action " << msg.get(); + stats_process_action(block->source(), *msg); state = ValidatorSessionState::action(description(), state, block->source(), att, msg.get()); } state = ValidatorSessionState::make_all(description(), state, block->source(), att); @@ -222,9 +225,11 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice // Note: src is not necessarily equal to the sender of this message: // If requested using get_broadcast_p2p, src is the creator of the block, sender possibly is some other node. auto src_idx = description().get_source_idx(src); + td::Timer deserialize_timer; auto R = deserialize_candidate(data, compress_block_candidates_, description().opts().max_block_size + description().opts().max_collated_data_size + 1024); + double deserialize_time = deserialize_timer.elapsed(); if (R.is_error()) { VLOG(VALIDATOR_SESSION_WARNING) << this << "[node " << src << "][broadcast " << sha256_bits256(data.as_slice()) << "]: failed to parse: " << R.move_as_error(); @@ -255,6 +260,18 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice return; } + auto stat = stats_get_candidate_stat(block_round, src, block_id); + if (stat) { + if (stat->block_status == ValidatorSessionStats::status_none) { + stat->block_status = ValidatorSessionStats::status_received; + } + if (stat->block_timestamp <= 0.0) { + stat->block_timestamp = td::Clocks::system(); + } + stat->deserialize_time = deserialize_time; + stat->serialized_size = data.size(); + } + if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || block_round >= cur_round_ + MAX_FUTURE_ROUND_BLOCK) { VLOG(VALIDATOR_SESSION_NOTICE) << this << "[node " << src << "][broadcast " << block_id @@ -291,7 +308,6 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice CHECK(!pending_reject_.count(block_id)); CHECK(!rejected_.count(block_id)); - stats_set_candidate_status(cur_round_, src, block_id, ValidatorSessionStats::status_received); auto v = virtual_state_->choose_blocks_to_approve(description(), local_idx()); for (auto &b : v) { if (b && SentBlock::get_block_id(b) == block_id) { @@ -363,9 +379,16 @@ void ValidatorSessionImpl::process_query(PublicKeyHash src, td::BufferSlice data } void ValidatorSessionImpl::candidate_decision_fail(td::uint32 round, ValidatorSessionCandidateId hash, - std::string result, td::uint32 src, td::BufferSlice proof) { - stats_set_candidate_status(round, description().get_source_id(src), hash, ValidatorSessionStats::status_rejected, - result); + std::string result, td::uint32 src, td::BufferSlice proof, + double validation_time, bool validation_cached) { + auto stat = stats_get_candidate_stat(round, description().get_source_id(src), hash); + if (stat) { + stat->block_status = ValidatorSessionStats::status_rejected; + stat->comment = result; + stat->validation_time = validation_time; + stat->validated_at = td::Clocks::system(); + stat->validation_cached = validation_cached; + } if (round != cur_round_) { return; } @@ -379,9 +402,17 @@ void ValidatorSessionImpl::candidate_decision_fail(td::uint32 round, ValidatorSe } void ValidatorSessionImpl::candidate_decision_ok(td::uint32 round, ValidatorSessionCandidateId hash, RootHash root_hash, - FileHash file_hash, td::uint32 src, td::uint32 ok_from) { - stats_set_candidate_status(round, description().get_source_id(src), hash, ValidatorSessionStats::status_approved, - PSTRING() << "ts=" << ok_from); + FileHash file_hash, td::uint32 src, td::uint32 ok_from, + double validation_time, bool validation_cached) { + auto stat = stats_get_candidate_stat(round, description().get_source_id(src), hash); + if (stat) { + stat->block_status = ValidatorSessionStats::status_approved; + stat->comment = PSTRING() << "ts=" << ok_from; + stat->validation_time = validation_time; + stat->gen_utime = (double)ok_from; + stat->validated_at = td::Clocks::system(); + stat->validation_cached = validation_cached; + } if (round != cur_round_) { return; } @@ -418,10 +449,8 @@ void ValidatorSessionImpl::candidate_approved_signed(td::uint32 round, Validator } void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCandidateId root_hash, - td::BufferSlice data, td::BufferSlice collated_data) { - if (round != cur_round_) { - return; - } + td::BufferSlice data, td::BufferSlice collated_data, double collation_time, + bool collation_cached) { if (data.size() > description().opts().max_block_size || collated_data.size() > description().opts().max_collated_data_size) { LOG(ERROR) << this << ": generated candidate is too big. Dropping. size=" << data.size() << " " @@ -430,13 +459,27 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCan } auto file_hash = sha256_bits256(data.as_slice()); auto collated_data_file_hash = sha256_bits256(collated_data.as_slice()); + auto block_id = description().candidate_id(local_idx(), root_hash, file_hash, collated_data_file_hash); + auto stat = stats_get_candidate_stat(round, local_id(), block_id); + if (stat) { + stat->block_status = ValidatorSessionStats::status_received; + stat->collation_time = collation_time; + stat->collated_at = td::Clocks::system(); + stat->block_timestamp = td::Clocks::system(); + stat->collation_cached = collation_cached; + } + if (round != cur_round_) { + return; + } + td::Timer serialize_timer; auto b = create_tl_object(local_id().tl(), round, root_hash, std::move(data), std::move(collated_data)); - auto B = serialize_candidate(b, compress_block_candidates_).move_as_ok(); - - auto block_id = description().candidate_id(local_idx(), root_hash, file_hash, collated_data_file_hash); + if (stat) { + stat->serialize_time = serialize_timer.elapsed(); + stat->serialized_size = B.size(); + } td::actor::send_closure(catchain_, &catchain::CatChain::send_broadcast, std::move(B)); @@ -496,11 +539,11 @@ void ValidatorSessionImpl::check_generate_slot() { td::PerfWarningTimer timer{"too long block generation", 1.0}; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), print_id = print_id(), timer = std::move(timer), - round = cur_round_](td::Result R) { + round = cur_round_](td::Result R) { if (R.is_ok()) { - auto c = R.move_as_ok(); + auto c = std::move(R.ok_ref().candidate); td::actor::send_closure(SelfId, &ValidatorSessionImpl::generated_block, round, c.id.root_hash, - c.data.clone(), c.collated_data.clone()); + c.data.clone(), c.collated_data.clone(), timer.elapsed(), R.ok().is_cached); } else { LOG(WARNING) << print_id << ": failed to generate block candidate: " << R.move_as_error(); } @@ -550,6 +593,16 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { it->second->round_ = std::max(it->second->round_, cur_round_); td::PerfWarningTimer timer{"too long block validation", 1.0}; auto &B = it->second; + auto stat = stats_get_candidate_stat(B->round_, PublicKeyHash{B->src_}); + if (stat) { + // Can happen if block is cached from previous round + if (stat->block_status == ValidatorSessionStats::status_none) { + stat->block_status = ValidatorSessionStats::status_received; + } + if (stat->block_timestamp <= 0.0) { + stat->block_timestamp = td::Clocks::system(); + } + } auto P = td::PromiseCreator::lambda([round = cur_round_, hash = block_id, root_hash = block->get_root_hash(), file_hash = block->get_file_hash(), timer = std::move(timer), @@ -563,10 +616,10 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { auto R = res.move_as_ok(); if (R.is_ok()) { td::actor::send_closure(SelfId, &ValidatorSessionImpl::candidate_decision_ok, round, hash, root_hash, - file_hash, src, R.ok_from()); + file_hash, src, R.ok_from(), timer.elapsed(), R.is_cached()); } else { td::actor::send_closure(SelfId, &ValidatorSessionImpl::candidate_decision_fail, round, hash, R.reason(), - src, R.proof()); + src, R.proof(), timer.elapsed(), R.is_cached()); } }); pending_approve_.insert(block_id); @@ -824,22 +877,28 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { cur_stats_.approve_signatures = (td::uint32)export_approve_sigs.size(); cur_stats_.approve_signatures_weight = approve_signatures_weight; cur_stats_.creator = description().get_source_id(block->get_src_idx()); + auto stat = stats_get_candidate_stat(cur_round_, cur_stats_.creator); + if (stat) { + stat->is_accepted = true; + } + auto stats = cur_stats_; + while (!stats.rounds.empty() && stats.rounds.size() + stats.first_round - 1 > cur_round_) { + stats.rounds.pop_back(); + } if (it == blocks_.end()) { callback_->on_block_committed(cur_round_, description().get_source_public_key(block->get_src_idx()), block->get_root_hash(), block->get_file_hash(), td::BufferSlice(), - std::move(export_sigs), std::move(export_approve_sigs), std::move(cur_stats_)); + std::move(export_sigs), std::move(export_approve_sigs), std::move(stats)); } else { callback_->on_block_committed(cur_round_, description().get_source_public_key(block->get_src_idx()), block->get_root_hash(), block->get_file_hash(), it->second->data_.clone(), - std::move(export_sigs), std::move(export_approve_sigs), std::move(cur_stats_)); + std::move(export_sigs), std::move(export_approve_sigs), std::move(stats)); } } cur_round_++; if (have_block) { stats_init(); - } else { - stats_add_round(); } auto it2 = blocks_.begin(); while (it2 != blocks_.end()) { @@ -997,13 +1056,35 @@ void ValidatorSessionImpl::start_up() { } void ValidatorSessionImpl::stats_init() { + auto old_rounds = std::move(cur_stats_.rounds); + if (stats_inited_ && cur_stats_.first_round + old_rounds.size() > cur_round_) { + old_rounds.erase(old_rounds.begin(), old_rounds.end() - (cur_stats_.first_round + old_rounds.size() - cur_round_)); + } else { + old_rounds.clear(); + } cur_stats_ = ValidatorSessionStats(); + cur_stats_.rounds = std::move(old_rounds); cur_stats_.first_round = cur_round_; cur_stats_.session_id = unique_hash_; cur_stats_.total_validators = description().get_total_nodes(); cur_stats_.total_weight = description().get_total_weight(); cur_stats_.self = description().get_source_id(local_idx()); - stats_add_round(); + + for (auto it = stats_pending_approve_.begin(); it != stats_pending_approve_.end(); ) { + if (it->first.first < cur_round_) { + it = stats_pending_approve_.erase(it); + } else { + ++it; + } + } + for (auto it = stats_pending_sign_.begin(); it != stats_pending_sign_.end(); ) { + if (it->first.first < cur_round_) { + it = stats_pending_sign_.erase(it); + } else { + ++it; + } + } + stats_inited_ = true; } void ValidatorSessionImpl::stats_add_round() { @@ -1016,6 +1097,9 @@ void ValidatorSessionImpl::stats_add_round() { if (priority >= 0) { CHECK((size_t)priority < round.producers.size()); round.producers[priority].id = description().get_source_id(i); + round.producers[priority].is_ours = (local_idx() == i); + round.producers[priority].approvers.resize(description().get_total_nodes(), false); + round.producers[priority].signers.resize(description().get_total_nodes(), false); } } while (!round.producers.empty() && round.producers.back().id.is_zero()) { @@ -1023,26 +1107,93 @@ void ValidatorSessionImpl::stats_add_round() { } } -void ValidatorSessionImpl::stats_set_candidate_status(td::uint32 round, PublicKeyHash src, - ValidatorSessionCandidateId candidate_id, int status, - std::string comment) { - if (round < cur_stats_.first_round || round - cur_stats_.first_round >= cur_stats_.rounds.size()) { - return; +ValidatorSessionStats::Producer *ValidatorSessionImpl::stats_get_candidate_stat( + td::uint32 round, PublicKeyHash src, ValidatorSessionCandidateId candidate_id) { + if (round < cur_stats_.first_round || round > cur_round_ + 5) { + return nullptr; + } + while (round - cur_stats_.first_round >= cur_stats_.rounds.size()) { + stats_add_round(); } auto &stats_round = cur_stats_.rounds[round - cur_stats_.first_round]; auto it = std::find_if(stats_round.producers.begin(), stats_round.producers.end(), [&](const ValidatorSessionStats::Producer &p) { return p.id == src; }); if (it == stats_round.producers.end()) { - return; + return nullptr; + } + if (!candidate_id.is_zero()) { + it->candidate_id = candidate_id; } - it->candidate_id = candidate_id; - if (it->block_status == ValidatorSessionStats::status_none) { - it->block_timestamp = (td::uint64)td::Clocks::system(); + auto it2 = stats_pending_approve_.find({round, it->candidate_id}); + if (it2 != stats_pending_approve_.end()) { + for (td::uint32 node_id : it2->second) { + it->set_approved_by(node_id, description().get_node_weight(node_id), description().get_total_weight()); + } + stats_pending_approve_.erase(it2); + } + it2 = stats_pending_sign_.find({round, it->candidate_id}); + if (it2 != stats_pending_sign_.end()) { + for (td::uint32 node_id : it2->second) { + it->set_signed_by(node_id, description().get_node_weight(node_id), description().get_total_weight()); + } + stats_pending_sign_.erase(it2); + } + return &*it; +} + +ValidatorSessionStats::Producer *ValidatorSessionImpl::stats_get_candidate_stat_by_id( + td::uint32 round, ValidatorSessionCandidateId candidate_id) { + if (round < cur_stats_.first_round || round > cur_round_ + 5) { + return nullptr; } - it->block_status = status; - if (!comment.empty()) { - it->comment = std::move(comment); + while (round - cur_stats_.first_round >= cur_stats_.rounds.size()) { + stats_add_round(); } + auto &stats_round = cur_stats_.rounds[round - cur_stats_.first_round]; + auto it = std::find_if(stats_round.producers.begin(), stats_round.producers.end(), + [&](const ValidatorSessionStats::Producer &p) { return p.candidate_id == candidate_id; }); + if (it == stats_round.producers.end()) { + return nullptr; + } + return &*it; +} + +void ValidatorSessionImpl::stats_process_action(td::uint32 node_id, ton_api::validatorSession_round_Message &action) { + ton_api::downcast_call(action, td::overloaded( + [&](const ton_api::validatorSession_message_submittedBlock &obj) { + auto candidate_id = description().candidate_id( + node_id, obj.root_hash_, obj.file_hash_, obj.collated_data_file_hash_); + auto stat = stats_get_candidate_stat( + obj.round_, description().get_source_id(node_id), candidate_id); + if (stat && stat->got_submit_at <= 0.0) { + stat->got_submit_at = td::Clocks::system(); + } + }, + [&](const ton_api::validatorSession_message_approvedBlock &obj) { + if (obj.candidate_ == skip_round_candidate_id()) { + return; + } + auto stat = stats_get_candidate_stat_by_id(obj.round_, obj.candidate_); + if (stat) { + stat->set_approved_by(node_id, description().get_node_weight(node_id), + description().get_total_weight()); + } else { + stats_pending_approve_[{obj.round_, obj.candidate_}].push_back(node_id); + } + }, + [&](const ton_api::validatorSession_message_commit &obj) { + if (obj.candidate_ == skip_round_candidate_id()) { + return; + } + auto stat = stats_get_candidate_stat_by_id(obj.round_, obj.candidate_); + if (stat) { + stat->set_signed_by(node_id, description().get_node_weight(node_id), + description().get_total_weight()); + } else { + stats_pending_sign_[{obj.round_, obj.candidate_}].push_back(node_id); + } + }, + [](const auto &) {})); } td::actor::ActorOwn ValidatorSession::create( diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index 6300eaa49..2dd6625c1 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -56,6 +56,12 @@ class ValidatorSession : public td::actor::Actor { td::BufferSlice proof() const { return proof_.clone(); } + bool is_cached() const { + return is_cached_; + } + void set_is_cached(bool value = true) { + is_cached_ = value; + } CandidateDecision(td::uint32 ok_from) { ok_ = true; ok_from_ = ok_from; @@ -69,6 +75,12 @@ class ValidatorSession : public td::actor::Actor { td::uint32 ok_from_ = 0; std::string reason_; td::BufferSlice proof_; + bool is_cached_ = false; + }; + + struct GeneratedCandidate { + BlockCandidate candidate; + bool is_cached = false; }; class Callback { @@ -76,7 +88,7 @@ class ValidatorSession : public td::actor::Actor { virtual void on_candidate(td::uint32 round, PublicKey source, ValidatorSessionRootHash root_hash, td::BufferSlice data, td::BufferSlice collated_data, td::Promise promise) = 0; - virtual void on_generate_slot(td::uint32 round, td::Promise promise) = 0; + virtual void on_generate_slot(td::uint32 round, td::Promise promise) = 0; virtual void on_block_committed(td::uint32 round, PublicKey source, ValidatorSessionRootHash root_hash, ValidatorSessionFileHash file_hash, td::BufferSlice data, std::vector> signatures, diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 619f7178a..005292c03 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -159,10 +159,19 @@ class ValidatorSessionImpl : public ValidatorSession { bool compress_block_candidates_ = false; ValidatorSessionStats cur_stats_; + bool stats_inited_ = false; + std::map, std::vector> + stats_pending_approve_; // round, candidate_id -> approvers + std::map, std::vector> + stats_pending_sign_; // round, candidate_id -> signers void stats_init(); void stats_add_round(); - void stats_set_candidate_status(td::uint32 round, PublicKeyHash src, ValidatorSessionCandidateId candidate_id, - int status, std::string comment = ""); + ValidatorSessionStats::Producer *stats_get_candidate_stat( + td::uint32 round, PublicKeyHash src, + ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero()); + ValidatorSessionStats::Producer *stats_get_candidate_stat_by_id(td::uint32 round, + ValidatorSessionCandidateId candidate_id); + void stats_process_action(td::uint32 node_id, ton_api::validatorSession_round_Message &action); public: ValidatorSessionImpl(catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, @@ -190,17 +199,16 @@ class ValidatorSessionImpl : public ValidatorSession { void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise promise); void try_approve_block(const SentBlock *block); - void try_sign(); - void candidate_decision_fail(td::uint32 round, ValidatorSessionCandidateId hash, std::string result, - td::uint32 src, td::BufferSlice proof); + void candidate_decision_fail(td::uint32 round, ValidatorSessionCandidateId hash, std::string result, td::uint32 src, + td::BufferSlice proof, double validation_time, bool validation_cached); void candidate_decision_ok(td::uint32 round, ValidatorSessionCandidateId hash, RootHash root_hash, FileHash file_hash, - td::uint32 src, td::uint32 ok_from); + td::uint32 src, td::uint32 ok_from, double validation_time, bool validation_cached); void candidate_approved_signed(td::uint32 round, ValidatorSessionCandidateId hash, td::uint32 ok_from, td::BufferSlice signature); void generated_block(td::uint32 round, ValidatorSessionRootHash root_hash, td::BufferSlice data, - td::BufferSlice collated); + td::BufferSlice collated, double collation_time, bool collation_cached); void signed_block(td::uint32 round, ValidatorSessionCandidateId hash, td::BufferSlice signature); void end_request(td::uint32 round, ValidatorSessionCandidateId block_id) { diff --git a/validator/manager.cpp b/validator/manager.cpp index d88a50154..37281cc9a 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2714,8 +2714,12 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, std::vector> producers; for (const auto &producer : round.producers) { producers.push_back(create_tl_object( - producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.block_timestamp, - producer.comment)); + producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.comment, + producer.block_timestamp, producer.is_accepted, producer.is_ours, producer.got_submit_at, + producer.collation_time, producer.collated_at, producer.collation_cached, producer.validation_time, + producer.validated_at, producer.validation_cached, producer.gen_utime, producer.approved_weight, + producer.approved_33pct_at, producer.approved_66pct_at, producer.signed_weight, producer.signed_33pct_at, + producer.signed_66pct_at, producer.serialize_time, producer.deserialize_time, producer.serialized_size)); } rounds.push_back(create_tl_object(round.timestamp, std::move(producers))); } diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 05628ef5d..5193e5233 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -27,7 +27,8 @@ namespace ton { namespace validator { -void ValidatorGroup::generate_block_candidate(td::uint32 round_id, td::Promise promise) { +void ValidatorGroup::generate_block_candidate( + td::uint32 round_id, td::Promise promise) { if (round_id > last_known_round_id_) { last_known_round_id_ = round_id; } @@ -37,14 +38,18 @@ void ValidatorGroup::generate_block_candidate(td::uint32 round_id, td::Promiseresult) { - promise.set_result(cached_collated_block_->result.value().clone()); + promise.set_value({cached_collated_block_->result.value().clone(), true}); } else { - cached_collated_block_->promises.push_back(std::move(promise)); + cached_collated_block_->promises.push_back(promise.wrap([](BlockCandidate &&res) { + return validatorsession::ValidatorSession::GeneratedCandidate{std::move(res), true}; + })); } return; } cached_collated_block_ = std::make_shared(); - cached_collated_block_->promises.push_back(std::move(promise)); + cached_collated_block_->promises.push_back(promise.wrap([](BlockCandidate &&res) { + return validatorsession::ValidatorSession::GeneratedCandidate{std::move(res), false}; + })); run_collate_query( shard_, min_ts_, min_masterchain_block_id_, prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, manager_, td::Timestamp::in(10.0), @@ -73,7 +78,7 @@ void ValidatorGroup::generated_block_candidate(std::shared_ptr promise) { + td::Promise> promise) { if (round_id > last_known_round_id_) { last_known_round_id_ = round_id; } @@ -88,7 +93,7 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat CacheKey cache_key = block_to_cache_key(block); auto it = approved_candidates_cache_.find(cache_key); if (it != approved_candidates_cache_.end()) { - promise.set_result(it->second); + promise.set_value({it->second, true}); return; } @@ -113,7 +118,7 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat ts); td::actor::send_closure(SelfId, &ValidatorGroup::add_available_block_candidate, block.pubkey.as_bits256(), block.id, block.collated_file_hash); - promise.set_result(ts); + promise.set_value({ts, false}); }, [&](CandidateReject reject) { promise.set_error( @@ -247,15 +252,18 @@ std::unique_ptr ValidatorGroup::ma void on_candidate(td::uint32 round, PublicKey source, validatorsession::ValidatorSessionRootHash root_hash, td::BufferSlice data, td::BufferSlice collated_data, td::Promise promise) override { - auto P = td::PromiseCreator::lambda([id = id_, promise = std::move(promise)](td::Result R) mutable { - if (R.is_ok()) { - promise.set_value(validatorsession::ValidatorSession::CandidateDecision{R.move_as_ok()}); - } else { - auto S = R.move_as_error(); - promise.set_value( - validatorsession::ValidatorSession::CandidateDecision{S.message().c_str(), td::BufferSlice()}); - } - }); + auto P = + td::PromiseCreator::lambda([promise = std::move(promise)](td::Result> R) mutable { + if (R.is_ok()) { + validatorsession::ValidatorSession::CandidateDecision decision(R.ok().first); + decision.set_is_cached(R.ok().second); + promise.set_value(std::move(decision)); + } else { + auto S = R.move_as_error(); + promise.set_value( + validatorsession::ValidatorSession::CandidateDecision{S.message().c_str(), td::BufferSlice()}); + } + }); BlockCandidate candidate{Ed25519_PublicKey{source.ed25519_value().raw()}, BlockIdExt{0, 0, 0, root_hash, sha256_bits256(data.as_slice())}, @@ -264,7 +272,8 @@ std::unique_ptr ValidatorGroup::ma td::actor::send_closure(id_, &ValidatorGroup::validate_block_candidate, round, std::move(candidate), std::move(P)); } - void on_generate_slot(td::uint32 round, td::Promise promise) override { + void on_generate_slot(td::uint32 round, + td::Promise promise) override { td::actor::send_closure(id_, &ValidatorGroup::generate_block_candidate, round, std::move(promise)); } void on_block_committed(td::uint32 round, PublicKey source, validatorsession::ValidatorSessionRootHash root_hash, diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 2dbff8de2..cb3f7a630 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -34,8 +34,10 @@ class ValidatorManager; class ValidatorGroup : public td::actor::Actor { public: - void generate_block_candidate(td::uint32 round_id, td::Promise promise); - void validate_block_candidate(td::uint32 round_id, BlockCandidate block, td::Promise promise); + void generate_block_candidate(td::uint32 round_id, + td::Promise promise); + void validate_block_candidate(td::uint32 round_id, BlockCandidate block, + td::Promise> promise); void accept_block_candidate(td::uint32 round_id, PublicKeyHash src, td::BufferSlice block, RootHash root_hash, FileHash file_hash, std::vector signatures, std::vector approve_signatures, From 037053fffec58dedf8e72051fb695a4a42d3d9f3 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 29 Apr 2024 16:33:10 +0300 Subject: [PATCH 05/28] More verbose state serializer logs (#976) Co-authored-by: SpyCheese --- crypto/vm/large-boc-serializer.cpp | 60 ++++++++++++++++++++++-------- validator/state-serializer.cpp | 30 +++++++++------ 2 files changed, 63 insertions(+), 27 deletions(-) diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index fe16b7678..fbd065dc9 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -14,6 +14,9 @@ You should have received a copy of the GNU Lesser General Public License along with TON Blockchain Library. If not, see . */ +#include "td/utils/Time.h" +#include "td/utils/Timer.h" + #include #include "vm/boc.h" #include "vm/boc-writers.h" @@ -30,7 +33,8 @@ class LargeBocSerializer { public: using Hash = Cell::Hash; - explicit LargeBocSerializer(std::shared_ptr reader) : reader(std::move(reader)) {} + explicit LargeBocSerializer(std::shared_ptr reader) : reader(std::move(reader)) { + } void add_root(Hash root); td::Status import_cells(); @@ -65,7 +69,8 @@ class LargeBocSerializer { std::map cells; std::vector*> cell_list; struct RootInfo { - RootInfo(Hash hash, int idx) : hash(hash), idx(idx) {} + RootInfo(Hash hash, int idx) : hash(hash), idx(idx) { + } Hash hash; int idx; }; @@ -78,6 +83,10 @@ class LargeBocSerializer { void reorder_cells(); int revisit(int cell_idx, int force = 0); td::uint64 compute_sizes(int mode, int& r_size, int& o_size); + + td::Timestamp log_speed_at_; + size_t processed_cells_ = 0; + static constexpr double LOG_SPEED_PERIOD = 120.0; }; void LargeBocSerializer::add_root(Hash root) { @@ -85,12 +94,16 @@ void LargeBocSerializer::add_root(Hash root) { } td::Status LargeBocSerializer::import_cells() { + td::Timer timer; + log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD); + processed_cells_ = 0; for (auto& root : roots) { TRY_RESULT(idx, import_cell(root.hash)); root.idx = idx; } reorder_cells(); CHECK(!cell_list.empty()); + LOG(ERROR) << "serializer: import_cells took " << timer.elapsed() << "s, " << cell_count << " cells"; return td::Status::OK(); } @@ -98,6 +111,12 @@ td::Result LargeBocSerializer::import_cell(Hash hash, int depth) { if (depth > Cell::max_depth) { return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large"); } + ++processed_cells_; + if (log_speed_at_.is_in_past()) { + log_speed_at_ += LOG_SPEED_PERIOD; + LOG(WARNING) << "serializer: import_cells " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; + processed_cells_ = 0; + } auto it = cells.find(hash); if (it != cells.end()) { it->second.should_cache = true; @@ -282,6 +301,7 @@ td::uint64 LargeBocSerializer::compute_sizes(int mode, int& r_size, int& o_size) } td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { + td::Timer timer; using Mode = BagOfCells::Mode; BagOfCells::Info info; if ((mode & Mode::WithCacheBits) && !(mode & Mode::WithIndex)) { @@ -313,13 +333,9 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { return td::Status::Error("bag of cells is too large"); } - boc_writers::FileWriter writer{fd, (size_t) info.total_size}; - auto store_ref = [&](unsigned long long value) { - writer.store_uint(value, info.ref_byte_size); - }; - auto store_offset = [&](unsigned long long value) { - writer.store_uint(value, info.offset_byte_size); - }; + boc_writers::FileWriter writer{fd, (size_t)info.total_size}; + auto store_ref = [&](unsigned long long value) { writer.store_uint(value, info.ref_byte_size); }; + auto store_offset = [&](unsigned long long value) { writer.store_uint(value, info.offset_byte_size); }; writer.store_uint(info.magic, 4); @@ -371,6 +387,8 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { } DCHECK(writer.position() == info.data_offset); size_t keep_position = writer.position(); + log_speed_at_ = td::Timestamp::in(LOG_SPEED_PERIOD); + processed_cells_ = 0; for (int i = 0; i < cell_count; ++i) { auto hash = cell_list[cell_count - 1 - i]->first; const auto& dc_info = cell_list[cell_count - 1 - i]->second; @@ -389,6 +407,12 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { DCHECK(k > i && k < cell_count); store_ref(k); } + ++processed_cells_; + if (log_speed_at_.is_in_past()) { + log_speed_at_ += LOG_SPEED_PERIOD; + LOG(WARNING) << "serializer: serialize " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; + processed_cells_ = 0; + } } DCHECK(writer.position() - keep_position == info.data_size); if (info.has_crc32c) { @@ -396,17 +420,23 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { writer.store_uint(td::bswap32(crc), 4); } DCHECK(writer.empty()); - return writer.finalize(); -} + TRY_STATUS(writer.finalize()); + LOG(ERROR) << "serializer: serialize took " << timer.elapsed() << "s, " << cell_count << " cells, " + << writer.position() << " bytes"; + return td::Status::OK(); } +} // namespace -td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, - td::FileFd& fd, int mode) { +td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, + int mode) { + td::Timer timer; CHECK(reader != nullptr) LargeBocSerializer serializer(reader); serializer.add_root(root_hash); TRY_STATUS(serializer.import_cells()); - return serializer.serialize(fd, mode); + TRY_STATUS(serializer.serialize(fd, mode)); + LOG(ERROR) << "serialization took " << timer.elapsed() << "s"; + return td::Status::OK(); } -} +} // namespace vm diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index 8e1b1b575..93363d3b2 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -132,27 +132,33 @@ void AsyncStateSerializer::next_iteration() { if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno && need_serialize(masterchain_handle_)) { if (!have_masterchain_state_) { - LOG(INFO) << "started serializing persistent state for " << masterchain_handle_->id().id; + LOG(ERROR) << "started serializing persistent state for " << masterchain_handle_->id().id.to_str(); // block next attempts immediately, but send actual request later running_ = true; + double delay = td::Random::fast(0, 3600); + LOG(WARNING) << "serializer delay = " << delay << "s"; delay_action([SelfId = actor_id( this)]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_masterchain_state); }, - td::Timestamp::in(td::Random::fast(0, 3600))); + td::Timestamp::in(delay)); return; } while (next_idx_ < shards_.size()) { if (!need_monitor(shards_[next_idx_].shard_full())) { next_idx_++; } else { - // block next attempts immediately, but send actual request later - running_ = true; - delay_action( - [SelfId = actor_id(this), shard = shards_[next_idx_]]() { td::actor::send_closure(SelfId, &AsyncStateSerializer::request_shard_state, shard); }, - td::Timestamp::in(td::Random::fast(0, 1800))); + // block next attempts immediately, but send actual request later + running_ = true; + double delay = td::Random::fast(0, 1800); + LOG(WARNING) << "serializer delay = " << delay << "s"; + delay_action( + [SelfId = actor_id(this), shard = shards_[next_idx_]]() { + td::actor::send_closure(SelfId, &AsyncStateSerializer::request_shard_state, shard); + }, + td::Timestamp::in(delay)); return; } } - LOG(INFO) << "finished serializing persistent state for " << masterchain_handle_->id().id; + LOG(ERROR) << "finished serializing persistent state for " << masterchain_handle_->id().id.to_str(); last_key_block_ts_ = masterchain_handle_->unix_time(); last_key_block_id_ = masterchain_handle_->id(); } @@ -194,7 +200,7 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) { void AsyncStateSerializer::got_masterchain_state(td::Ref state, std::shared_ptr cell_db_reader) { - LOG(INFO) << "serializing masterchain state " << masterchain_handle_->id().id; + LOG(ERROR) << "serializing masterchain state " << masterchain_handle_->id().id.to_str(); have_masterchain_state_ = true; CHECK(next_idx_ == 0); CHECK(shards_.size() == 0); @@ -217,7 +223,7 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state } void AsyncStateSerializer::stored_masterchain_state() { - LOG(INFO) << "finished serializing masterchain state " << masterchain_handle_->id().id; + LOG(ERROR) << "finished serializing masterchain state " << masterchain_handle_->id().id.to_str(); running_ = false; next_iteration(); } @@ -247,13 +253,13 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) { void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref state, std::shared_ptr cell_db_reader) { - LOG(INFO) << "serializing shard state " << handle->id().id; + LOG(ERROR) << "serializing shard state " << handle->id().id.to_str(); auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader](td::FileFd& fd) { return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31); }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { R.ensure(); - LOG(INFO) << "finished serializing shard state " << handle->id().id; + LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str(); td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(), From 6fb2019a4fe16dcf298a46c9c719987b18411320 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Wed, 8 May 2024 17:41:15 +0300 Subject: [PATCH 06/28] Improve validator session stats (#982) * Add list of validators * Fix producer stats * Make round and stats timestamps more meaningful Co-authored-by: SpyCheese --- tl/generate/scheme/ton_api.tl | 8 +++- tl/generate/scheme/ton_api.tlo | Bin 88712 -> 89348 bytes validator-session/validator-session-types.h | 18 ++++++++- validator-session/validator-session.cpp | 39 ++++++++++++-------- validator/interfaces/validator-manager.h | 1 + validator/manager-disk.hpp | 3 ++ validator/manager-hardfork.hpp | 3 ++ validator/manager.cpp | 28 +++++++++++++- validator/manager.hpp | 1 + validator/validator-group.cpp | 19 ++++++++++ 10 files changed, 99 insertions(+), 21 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index af9df3fd0..3db2ace98 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -761,13 +761,17 @@ validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int co signed_weight:long signed_33pct_at:double signed_66pct_at:double serialize_time:double deserialize_time:double serialized_size:int = validatorSession.StatsProducer; -validatorSession.statsRound timestamp:long producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound; +validatorSession.statsRound timestamp:double producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound; -validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:long self:int256 session_id:int256 cc_seqno:int +validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self:int256 session_id:int256 cc_seqno:int creator:int256 total_validators:int total_weight:long signatures:int signatures_weight:long approve_signatures:int approve_signatures_weight:long first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats; +validatorSession.newValidatorGroupStats.node id:int256 weight:long = validatorSession.newValidatorGroupStats.Node; +validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int timestamp:double + self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats; + ---functions--- ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index c97fa2feefc0767775d31b5c072aaf671c979e35..b0d5f315b0603c43be7a61d0f9290c55a69a744a 100644 GIT binary patch delta 461 zcmeC!%G$DvmG{wXeJchiVBN^;Y^c8fyu=yxvc#Osl*E$!qTtlx;>`R!J-^iQFqoKo zQGRK`VABHA6KaB6_>R_0Vhi*b}Nn%OyP$9ux9mpKfT$C_VX1fWTynZ!VKpd{v#kmWxq%@`+gj zoE*ieIcf2kDHW49T1Zchd8Z-)avdYo$3@O;Ivlob2SFxHzro3v!wT|^*z|(}jAE<| zAolbWF2)#;N!u@QF<{9 delta 183 zcmZqK#oDozmG{wXeJchiVBW~sNXEq%U z+qQ!s9m|*SF|t5Ro^Hs)C^mh+5F;xmNcGKQLOx8>dwCe8Cs(8jO#dOoXu|>$-+o4z H(ZUk|G!Q%& diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index db06bbd9c..e13c36d24 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -129,7 +129,7 @@ struct ValidatorSessionStats { } }; struct Round { - td::uint64 timestamp = 0; + double timestamp = -1.0; std::vector producers; }; @@ -139,7 +139,7 @@ struct ValidatorSessionStats { bool success = false; ValidatorSessionId session_id = ValidatorSessionId::zero(); CatchainSeqno cc_seqno = 0; - td::uint64 timestamp = 0; + double timestamp = -1.0; PublicKeyHash self = PublicKeyHash::zero(); PublicKeyHash creator = PublicKeyHash::zero(); td::uint32 total_validators = 0; @@ -150,6 +150,20 @@ struct ValidatorSessionStats { ValidatorWeight approve_signatures_weight = 0; }; +struct NewValidatorGroupStats { + struct Node { + PublicKeyHash id = PublicKeyHash::zero(); + ValidatorWeight weight = 0; + }; + + ValidatorSessionId session_id = ValidatorSessionId::zero(); + ShardIdFull shard{masterchainId}; + CatchainSeqno cc_seqno = 0; + double timestamp = -1.0; + td::uint32 self_idx = 0; + std::vector nodes; +}; + } // namespace validatorsession } // namespace ton diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 215b6b12d..d2fe64589 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -871,7 +871,7 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { callback_->on_block_skipped(cur_round_); } else { cur_stats_.success = true; - cur_stats_.timestamp = (td::uint64)td::Clocks::system(); + cur_stats_.timestamp = td::Clocks::system(); cur_stats_.signatures = (td::uint32)export_sigs.size(); cur_stats_.signatures_weight = signatures_weight; cur_stats_.approve_signatures = (td::uint32)export_approve_sigs.size(); @@ -899,6 +899,12 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) { cur_round_++; if (have_block) { stats_init(); + } else { + size_t round_idx = cur_round_ - cur_stats_.first_round; + while (round_idx >= cur_stats_.rounds.size()) { + stats_add_round(); + } + cur_stats_.rounds[round_idx].timestamp = td::Clocks::system(); } auto it2 = blocks_.begin(); while (it2 != blocks_.end()) { @@ -988,9 +994,7 @@ void ValidatorSessionImpl::destroy() { } void ValidatorSessionImpl::get_current_stats(td::Promise promise) { - ValidatorSessionStats stats = cur_stats_; - stats.timestamp = (td::uint64)td::Clocks::system(); - promise.set_result(std::move(stats)); + promise.set_result(cur_stats_); } void ValidatorSessionImpl::get_validator_group_info_for_litequery( @@ -1084,26 +1088,31 @@ void ValidatorSessionImpl::stats_init() { ++it; } } + + if (cur_stats_.rounds.empty()) { + stats_add_round(); + } + cur_stats_.rounds[0].timestamp = td::Clocks::system(); stats_inited_ = true; } void ValidatorSessionImpl::stats_add_round() { + td::uint32 round = cur_stats_.first_round + cur_stats_.rounds.size(); cur_stats_.rounds.emplace_back(); - auto& round = cur_stats_.rounds.back(); - round.timestamp = (td::uint64)td::Clocks::system(); - round.producers.resize(description().get_max_priority() + 1); + auto& stat = cur_stats_.rounds.back(); + stat.producers.resize(description().get_max_priority() + 1); for (td::uint32 i = 0; i < description().get_total_nodes(); i++) { - td::int32 priority = description().get_node_priority(i, cur_round_); + td::int32 priority = description().get_node_priority(i, round); if (priority >= 0) { - CHECK((size_t)priority < round.producers.size()); - round.producers[priority].id = description().get_source_id(i); - round.producers[priority].is_ours = (local_idx() == i); - round.producers[priority].approvers.resize(description().get_total_nodes(), false); - round.producers[priority].signers.resize(description().get_total_nodes(), false); + CHECK((size_t)priority < stat.producers.size()); + stat.producers[priority].id = description().get_source_id(i); + stat.producers[priority].is_ours = (local_idx() == i); + stat.producers[priority].approvers.resize(description().get_total_nodes(), false); + stat.producers[priority].signers.resize(description().get_total_nodes(), false); } } - while (!round.producers.empty() && round.producers.back().id.is_zero()) { - round.producers.pop_back(); + while (!stat.producers.empty() && stat.producers.back().id.is_zero()) { + stat.producers.pop_back(); } } diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 41412fb17..278915f16 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -171,6 +171,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) = 0; virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0; + virtual void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) = 0; virtual void get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index d5a6e909f..eca6a7418 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -378,6 +378,9 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { UNREACHABLE(); } + void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { + UNREACHABLE(); + } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 7937729ce..f0cf83de5 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -439,6 +439,9 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override { UNREACHABLE(); } + void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { + UNREACHABLE(); + } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = diff --git a/validator/manager.cpp b/validator/manager.cpp index 37281cc9a..ddc582251 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2729,7 +2729,7 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, stats.cc_seqno, stats.creator.bits256_value(), stats.total_validators, stats.total_weight, stats.signatures, stats.signatures_weight, stats.approve_signatures, stats.approve_signatures_weight, stats.first_round, std::move(rounds)); - std::string s = td::json_encode(td::ToJson(*obj.get()), false); + auto s = td::json_encode(td::ToJson(*obj.get()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); std::ofstream file; @@ -2737,7 +2737,31 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, file << s << "\n"; file.close(); - LOG(INFO) << "Writing validator session stats for " << block_id.id; + LOG(INFO) << "Writing validator session stats for " << block_id.id.to_str(); +} + +void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) { + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { + return; + } + std::vector> nodes; + for (const auto &node : stats.nodes) { + nodes.push_back( + create_tl_object(node.id.bits256_value(), node.weight)); + } + auto obj = create_tl_object( + stats.session_id, stats.shard.workchain, stats.shard.shard, stats.cc_seqno, stats.timestamp, stats.self_idx, + std::move(nodes)); + auto s = td::json_encode(td::ToJson(*obj.get()), false); + s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); + + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); + + LOG(INFO) << "Writing new validator group stats for " << stats.shard.to_str(); } void ValidatorManagerImpl::get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 7e5930d37..d6d0307b3 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -565,6 +565,7 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_shard_client_state(BlockSeqno seqno, td::Timestamp timeout, td::Promise promise) override; void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override; + void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override; void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 5193e5233..6a71d306a 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -377,6 +377,22 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch prev_block_ids_ = std::vector{next_block_id}; } postponed_accept_.clear(); + + validatorsession::NewValidatorGroupStats stats; + stats.session_id = session_id_; + stats.shard = shard_; + stats.cc_seqno = validator_set_->get_catchain_seqno(); + stats.timestamp = td::Clocks::system(); + td::uint32 idx = 0; + for (const auto& node : validator_set_->export_vector()) { + PublicKeyHash id = ValidatorFullId{node.key}.compute_short_id(); + if (id == local_id_) { + stats.self_idx = idx; + } + stats.nodes.push_back(validatorsession::NewValidatorGroupStats::Node{id, node.weight}); + ++idx; + } + td::actor::send_closure(manager_, &ValidatorManager::log_new_validator_group_stats, std::move(stats)); } void ValidatorGroup::destroy() { @@ -390,6 +406,9 @@ void ValidatorGroup::destroy() { return; } auto stats = R.move_as_ok(); + if (stats.rounds.empty()) { + return; + } stats.cc_seqno = cc_seqno; td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats, block_id, std::move(stats)); From d5c09936cf019f57d5a3bb7246bc5445006db8ae Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 10 May 2024 17:04:49 +0300 Subject: [PATCH 07/28] Block broadcasts in custom overlays (#986) Co-authored-by: SpyCheese --- create-hardfork/create-hardfork.cpp | 2 +- test/test-ton-collator.cpp | 2 +- tl/generate/scheme/ton_api.tl | 2 +- tl/generate/scheme/ton_api.tlo | Bin 89348 -> 89388 bytes .../validator-engine-console-query.cpp | 7 +- validator-engine/validator-engine.cpp | 22 +-- validator/full-node-private-overlay.cpp | 77 ++++++++--- validator/full-node-private-overlay.hpp | 34 +++-- validator/full-node-shard.cpp | 21 +-- validator/full-node-shard.h | 2 +- validator/full-node-shard.hpp | 3 +- validator/full-node.cpp | 130 ++++++++++++------ validator/full-node.h | 17 ++- validator/full-node.hpp | 25 ++-- validator/impl/accept-block.cpp | 8 +- validator/interfaces/validator-manager.h | 2 +- validator/manager-disk.hpp | 2 +- validator/manager-hardfork.hpp | 2 +- validator/manager.cpp | 4 +- validator/manager.hpp | 2 +- validator/validator.h | 2 +- 21 files changed, 233 insertions(+), 133 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index c5f1add85..a9e7403a1 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -246,7 +246,7 @@ class HardforkCreator : public td::actor::Actor { } void send_shard_block_info(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::BufferSlice data) override { } - void send_broadcast(ton::BlockBroadcast broadcast) override { + void send_broadcast(ton::BlockBroadcast broadcast, bool custom_overlays_only) override { } void download_block(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override { diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index 286a5feaf..d36db5d24 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -347,7 +347,7 @@ class TestNode : public td::actor::Actor { } } } - void send_broadcast(ton::BlockBroadcast broadcast) override { + void send_broadcast(ton::BlockBroadcast broadcast, bool custom_overlays_only) override { } void download_block(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override { diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 3db2ace98..d0a063f7b 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -595,7 +595,7 @@ engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector eng liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) gc:engine.gc = engine.validator.Config; -engine.validator.customOverlayNode adnl_id:int256 msg_sender:Bool msg_sender_priority:int = engine.validator.CustomOverlayNode; +engine.validator.customOverlayNode adnl_id:int256 msg_sender:Bool msg_sender_priority:int block_sender:Bool = engine.validator.CustomOverlayNode; engine.validator.customOverlay name:string nodes:(vector engine.validator.customOverlayNode) = engine.validator.CustomOverlay; engine.validator.customOverlaysConfig overlays:(vector engine.validator.customOverlay) = engine.validator.CustomOverlaysConfig; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index b0d5f315b0603c43be7a61d0f9290c55a69a744a..fb7f219765fea2ff1aa4dee29f67166b388d01f9 100644 GIT binary patch delta 91 zcmZqK#kyt}>jnW4*8lh1HI*jc6qeg;EMnTg0^&~2nW6w;Oqim=2@<(^Ovs02^29mn mlb=lSnY_S~l_x1DKRG+TI5jUNwP^Cge0i|?%{Eg{90vgF<0sYt delta 78 zcmZ3pi?wAJ>jnW4)<<2JLzE`p6qeg;EMnTg0^&~2nW6w;Oqim=2@<(^Ovs0Ma^p(z Z$rt7bOnx%OXL5oS2UyAGl&L3<0{~H}Bq9I+ diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 956c23aa0..38f524fc0 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1171,9 +1171,10 @@ td::Status ShowCustomOverlaysQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "Overlay \"" << overlay->name_ << "\": " << overlay->nodes_.size() << " nodes\n"; for (const auto &node : overlay->nodes_) { td::TerminalIO::out() << " " << node->adnl_id_ - << (node->msg_sender_ ? (PSTRING() << " (sender, p=" << node->msg_sender_priority_ << ")") - : "") - << "\n"; + << (node->msg_sender_ + ? (PSTRING() << " (msg sender, p=" << node->msg_sender_priority_ << ")") + : "") + << (node->block_sender_ ? " (block sender)" : "") << "\n"; } td::TerminalIO::out() << "\n"; } diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 8ba991782..3c98a4ac6 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -2357,16 +2357,9 @@ void ValidatorEngine::load_custom_overlays_config() { } for (auto &overlay : custom_overlays_config_->overlays_) { - std::vector nodes; - std::map senders; - for (const auto &node : overlay->nodes_) { - nodes.emplace_back(node->adnl_id_); - if (node->msg_sender_) { - senders[ton::adnl::AdnlNodeIdShort{node->adnl_id_}] = node->msg_sender_priority_; - } - } - td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_ext_msg_overlay, std::move(nodes), - std::move(senders), overlay->name_, [](td::Result R) { R.ensure(); }); + td::actor::send_closure(full_node_, &ton::validator::fullnode::FullNode::add_custom_overlay, + ton::validator::fullnode::CustomOverlayParams::fetch(*overlay), + [](td::Result R) { R.ensure(); }); } } @@ -3571,11 +3564,10 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_addCustom senders[ton::adnl::AdnlNodeIdShort{node->adnl_id_}] = node->msg_sender_priority_; } } - std::string name = overlay->name_; + auto params = ton::validator::fullnode::CustomOverlayParams::fetch(*query.overlay_); td::actor::send_closure( - full_node_, &ton::validator::fullnode::FullNode::add_ext_msg_overlay, std::move(nodes), std::move(senders), - std::move(name), - [SelfId = actor_id(this), overlay = std::move(overlay), + full_node_, &ton::validator::fullnode::FullNode::add_custom_overlay, std::move(params), + [SelfId = actor_id(this), overlay = std::move(query.overlay_), promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); @@ -3605,7 +3597,7 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_delCustom return; } td::actor::send_closure( - full_node_, &ton::validator::fullnode::FullNode::del_ext_msg_overlay, query.name_, + full_node_, &ton::validator::fullnode::FullNode::del_custom_overlay, query.name_, [SelfId = actor_id(this), name = query.name_, promise = std::move(promise)](td::Result R) mutable { if (R.is_error()) { promise.set_value(create_control_query_error(R.move_as_error())); diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index a64c0e9b9..74bb75c52 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -38,17 +38,7 @@ void FullNodePrivateBlockOverlay::process_block_broadcast(PublicKeyHash src, ton } VLOG(FULL_NODE_DEBUG) << "Received block broadcast in private overlay from " << src << ": " << B.ok().block_id.to_str(); - auto P = td::PromiseCreator::lambda([](td::Result R) { - if (R.is_error()) { - if (R.error().code() == ErrorCode::notready) { - LOG(DEBUG) << "dropped broadcast: " << R.move_as_error(); - } else { - LOG(INFO) << "dropped broadcast: " << R.move_as_error(); - } - } - }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, B.move_as_ok(), - std::move(P)); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); } void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query) { @@ -60,6 +50,9 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api:: } void FullNodePrivateBlockOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + if (adnl::AdnlNodeIdShort{src} == local_id_) { + return; + } auto B = fetch_tl_object(std::move(broadcast), true); if (B.is_error()) { return; @@ -169,18 +162,47 @@ void FullNodePrivateBlockOverlay::tear_down() { } } +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { + process_block_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query) { + process_block_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { + VLOG(FULL_NODE_DEBUG) << "Dropping block broadcast in private overlay \"" << name_ << "\" from unauthorized sender " + << src; + return; + } + auto B = deserialize_block_broadcast(query, overlay::Overlays::max_fec_broadcast_size()); + if (B.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << B.move_as_error(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Received block broadcast in custom overlay \"" << name_ << "\" from " << src << ": " + << B.ok().block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); +} + void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query) { - auto it = senders_.find(adnl::AdnlNodeIdShort{src}); - if (it == senders_.end()) { + auto it = msg_senders_.find(adnl::AdnlNodeIdShort{src}); + if (it == msg_senders_.end()) { + VLOG(FULL_NODE_DEBUG) << "Dropping external message broadcast in custom overlay \"" << name_ + << "\" from unauthorized sender " << src; return; } - LOG(FULL_NODE_DEBUG) << "Got external message in private overlay \"" << name_ << "\" from " << src - << " (priority=" << it->second << ")"; + VLOG(FULL_NODE_DEBUG) << "Got external message in custom overlay \"" << name_ << "\" from " << src + << " (priority=" << it->second << ")"; td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_external_message, std::move(query.message_->data_), it->second); } void FullNodeCustomOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { + if (adnl::AdnlNodeIdShort{src} == local_id_) { + return; + } auto B = fetch_tl_object(std::move(broadcast), true); if (B.is_error()) { return; @@ -192,7 +214,7 @@ void FullNodeCustomOverlay::send_external_message(td::BufferSlice data) { if (!inited_ || config_.ext_messages_broadcast_disabled_) { return; } - LOG(FULL_NODE_DEBUG) << "Sending external message to private overlay \"" << name_ << "\""; + VLOG(FULL_NODE_DEBUG) << "Sending external message to custom overlay \"" << name_ << "\""; auto B = create_serialize_tl_object( create_tl_object(std::move(data))); if (B.size() <= overlay::Overlays::max_simple_broadcast_size()) { @@ -204,6 +226,21 @@ void FullNodeCustomOverlay::send_external_message(td::BufferSlice data) { } } +void FullNodeCustomOverlay::send_broadcast(BlockBroadcast broadcast) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending block broadcast to custom overlay \"" << name_ + << "\": " << broadcast.block_id.to_str(); + auto B = serialize_block_broadcast(broadcast, true); // compression_enabled = true + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block broadcast: " << B.move_as_error(); + return; + } + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); +} + void FullNodeCustomOverlay::start_up() { std::sort(nodes_.begin(), nodes_.end()); nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end()); @@ -234,7 +271,8 @@ void FullNodeCustomOverlay::try_init() { void FullNodeCustomOverlay::init() { LOG(FULL_NODE_WARNING) << "Creating custom overlay \"" << name_ << "\" for adnl id " << local_id_ << " : " - << nodes_.size() << " nodes, overlay_id=" << overlay_id_; + << nodes_.size() << " nodes, " << msg_senders_.size() << " msg senders, " + << block_senders_.size() << " block senders, overlay_id=" << overlay_id_; class Callback : public overlay::Overlays::Callback { public: void receive_message(adnl::AdnlNodeIdShort src, overlay::OverlayIdShort overlay_id, td::BufferSlice data) override { @@ -256,9 +294,12 @@ void FullNodeCustomOverlay::init() { }; std::map authorized_keys; - for (const auto &sender : senders_) { + for (const auto &sender : msg_senders_) { authorized_keys[sender.first.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); } + for (const auto &sender : block_senders_) { + authorized_keys[sender.pubkey_hash()] = overlay::Overlays::max_fec_broadcast_size(); + } overlay::OverlayPrivacyRules rules{overlay::Overlays::max_fec_broadcast_size(), 0, std::move(authorized_keys)}; td::actor::send_closure( overlays_, &overlay::Overlays::create_private_overlay, local_id_, overlay_id_full_.clone(), nodes_, diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index c651acefe..196d6da60 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -52,7 +52,8 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, - td::actor::ActorId validator_manager) + td::actor::ActorId validator_manager, + td::actor::ActorId full_node) : local_id_(local_id) , nodes_(std::move(nodes)) , zero_state_file_hash_(zero_state_file_hash) @@ -63,7 +64,8 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { , rldp_(rldp) , rldp2_(rldp2) , overlays_(overlays) - , validator_manager_(validator_manager) { + , validator_manager_(validator_manager) + , full_node_(full_node) { } private: @@ -79,6 +81,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { td::actor::ActorId rldp2_; td::actor::ActorId overlays_; td::actor::ActorId validator_manager_; + td::actor::ActorId full_node_; bool inited_ = false; overlay::OverlayIdFull overlay_id_full_; @@ -90,6 +93,10 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { class FullNodeCustomOverlay : public td::actor::Actor { public: + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcastCompressed &query); + void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); template void process_broadcast(PublicKeyHash, T &) { @@ -98,6 +105,7 @@ class FullNodeCustomOverlay : public td::actor::Actor { void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void send_external_message(td::BufferSlice data); + void send_broadcast(BlockBroadcast broadcast); void set_config(FullNodeConfig config) { config_ = std::move(config); @@ -106,16 +114,17 @@ class FullNodeCustomOverlay : public td::actor::Actor { void start_up() override; void tear_down() override; - FullNodeCustomOverlay(adnl::AdnlNodeIdShort local_id, std::vector nodes, - std::map senders, std::string name, FileHash zero_state_file_hash, + FullNodeCustomOverlay(adnl::AdnlNodeIdShort local_id, CustomOverlayParams params, FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, - td::actor::ActorId validator_manager) + td::actor::ActorId validator_manager, + td::actor::ActorId full_node) : local_id_(local_id) - , nodes_(std::move(nodes)) - , senders_(std::move(senders)) - , name_(std::move(name)) + , name_(std::move(params.name_)) + , nodes_(std::move(params.nodes_)) + , msg_senders_(std::move(params.msg_senders_)) + , block_senders_(std::move(params.block_senders_)) , zero_state_file_hash_(zero_state_file_hash) , config_(config) , keyring_(keyring) @@ -123,14 +132,16 @@ class FullNodeCustomOverlay : public td::actor::Actor { , rldp_(rldp) , rldp2_(rldp2) , overlays_(overlays) - , validator_manager_(validator_manager) { + , validator_manager_(validator_manager) + , full_node_(full_node) { } private: adnl::AdnlNodeIdShort local_id_; - std::vector nodes_; - std::map senders_; std::string name_; + std::vector nodes_; + std::map msg_senders_; + std::set block_senders_; FileHash zero_state_file_hash_; FullNodeConfig config_; @@ -140,6 +151,7 @@ class FullNodeCustomOverlay : public td::actor::Actor { td::actor::ActorId rldp2_; td::actor::ActorId overlays_; td::actor::ActorId validator_manager_; + td::actor::ActorId full_node_; bool inited_ = false; overlay::OverlayIdFull overlay_id_full_; diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 433d9469b..6e265cbc6 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -661,17 +661,7 @@ void FullNodeShardImpl::process_block_broadcast(PublicKeyHash src, ton_api::tonN return; } VLOG(FULL_NODE_DEBUG) << "Received block broadcast from " << src << ": " << B.ok().block_id.to_str(); - auto P = td::PromiseCreator::lambda([](td::Result R) { - if (R.is_error()) { - if (R.error().code() == ErrorCode::notready) { - LOG(DEBUG) << "dropped broadcast: " << R.move_as_error(); - } else { - LOG(INFO) << "dropped broadcast: " << R.move_as_error(); - } - } - }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, B.move_as_ok(), - std::move(P)); + td::actor::send_closure(full_node_, &FullNode::process_block_broadcast, B.move_as_ok()); } void FullNodeShardImpl::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { @@ -1137,7 +1127,8 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client) + td::actor::ActorId client, + td::actor::ActorId full_node) : shard_(shard) , local_id_(local_id) , adnl_id_(adnl_id) @@ -1149,6 +1140,7 @@ FullNodeShardImpl::FullNodeShardImpl(ShardIdFull shard, PublicKeyHash local_id, , overlays_(overlays) , validator_manager_(validator_manager) , client_(client) + , full_node_(full_node) , config_(config) { } @@ -1157,9 +1149,10 @@ td::actor::ActorOwn FullNodeShard::create( FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client) { + td::actor::ActorId client, td::actor::ActorId full_node) { return td::actor::create_actor("tonnode", shard, local_id, adnl_id, zero_state_file_hash, config, - keyring, adnl, rldp, rldp2, overlays, validator_manager, client); + keyring, adnl, rldp, rldp2, overlays, validator_manager, client, + full_node); } } // namespace fullnode diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 1b742fb98..2ed13d660 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -72,7 +72,7 @@ class FullNodeShard : public td::actor::Actor { FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client); + td::actor::ActorId client, td::actor::ActorId full_node); }; } // namespace fullnode diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index c8b5301ae..1fdbce1c1 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -214,7 +214,7 @@ class FullNodeShardImpl : public FullNodeShard { td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, td::actor::ActorId validator_manager, - td::actor::ActorId client); + td::actor::ActorId client, td::actor::ActorId full_node); private: bool use_new_download() const { @@ -236,6 +236,7 @@ class FullNodeShardImpl : public FullNodeShard { td::actor::ActorId overlays_; td::actor::ActorId validator_manager_; td::actor::ActorId client_; + td::actor::ActorId full_node_; td::uint32 attempt_ = 0; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index affb8bcc8..8d0f26bc9 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -36,8 +36,8 @@ void FullNodeImpl::add_permanent_key(PublicKeyHash key, td::Promise pr local_keys_.insert(key); create_private_block_overlay(key); - for (auto &p : private_custom_overlays_) { - update_ext_msg_overlay(p.first, p.second); + for (auto &p : custom_overlays_) { + update_custom_overlay(p.second); } if (!sign_cert_by_.is_zero()) { @@ -64,8 +64,8 @@ void FullNodeImpl::del_permanent_key(PublicKeyHash key, td::Promise pr } local_keys_.erase(key); private_block_overlays_.erase(key); - for (auto &p : private_custom_overlays_) { - update_ext_msg_overlay(p.first, p.second); + for (auto &p : custom_overlays_) { + update_custom_overlay(p.second); } if (sign_cert_by_ != key) { @@ -119,8 +119,8 @@ void FullNodeImpl::update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise nodes, - std::map senders, std::string name, - td::Promise promise) { - if (nodes.empty()) { +void FullNodeImpl::add_custom_overlay(CustomOverlayParams params, td::Promise promise) { + if (params.nodes_.empty()) { promise.set_error(td::Status::Error("list of nodes is empty")); return; } - if (private_custom_overlays_.count(name)) { - promise.set_error(td::Status::Error(PSTRING() << "duplicate overlay name \"" << name << "\"")); + std::string name = params.name_; + if (custom_overlays_.count(name)) { + promise.set_error(td::Status::Error(PSTRING() << "duplicate custom overlay name \"" << name << "\"")); return; } - VLOG(FULL_NODE_WARNING) << "Adding private overlay for external messages \"" << name << "\", " << nodes.size() - << " nodes"; - auto &p = private_custom_overlays_[name]; - p.nodes_ = nodes; - p.senders_ = senders; - update_ext_msg_overlay(name, p); + VLOG(FULL_NODE_WARNING) << "Adding custom overlay \"" << name << "\", " << params.nodes_.size() << " nodes"; + auto &p = custom_overlays_[name]; + p.params_ = std::move(params); + update_custom_overlay(p); promise.set_result(td::Unit()); } -void FullNodeImpl::del_ext_msg_overlay(std::string name, td::Promise promise) { - auto it = private_custom_overlays_.find(name); - if (it == private_custom_overlays_.end()) { +void FullNodeImpl::del_custom_overlay(std::string name, td::Promise promise) { + auto it = custom_overlays_.find(name); + if (it == custom_overlays_.end()) { promise.set_error(td::Status::Error(PSTRING() << "no such overlay \"" << name << "\"")); return; } - private_custom_overlays_.erase(it); + custom_overlays_.erase(it); promise.set_result(td::Unit()); } @@ -182,8 +179,9 @@ void FullNodeImpl::initial_read_complete(BlockHandle top_handle) { void FullNodeImpl::add_shard(ShardIdFull shard) { while (true) { if (shards_.count(shard) == 0) { - shards_.emplace(shard, FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, config_, keyring_, - adnl_, rldp_, rldp2_, overlays_, validator_manager_, client_)); + shards_.emplace(shard, + FullNodeShard::create(shard, local_id_, adnl_id_, zero_state_file_hash_, config_, keyring_, adnl_, + rldp_, rldp2_, overlays_, validator_manager_, client_, actor_id(this))); if (all_validators_.size() > 0) { td::actor::send_closure(shards_[shard], &FullNodeShard::update_validators, all_validators_, sign_cert_by_); } @@ -221,10 +219,10 @@ void FullNodeImpl::send_ext_message(AccountIdPrefixFull dst, td::BufferSlice dat VLOG(FULL_NODE_WARNING) << "dropping OUT ext message to unknown shard"; return; } - for (auto &private_overlay : private_custom_overlays_) { + for (auto &private_overlay : custom_overlays_) { for (auto &actor : private_overlay.second.actors_) { auto local_id = actor.first; - if (private_overlay.second.senders_.count(local_id)) { + if (private_overlay.second.params_.msg_senders_.count(local_id)) { td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_external_message, data.clone()); } } @@ -245,7 +243,11 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data)); } -void FullNodeImpl::send_broadcast(BlockBroadcast broadcast) { +void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) { + send_block_broadcast_to_custom_overlays(broadcast); + if (custom_overlays_only) { + return; + } auto shard = get_shard(ShardIdFull{masterchainId}); if (shard.empty()) { VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; @@ -404,7 +406,22 @@ void FullNodeImpl::new_key_block(BlockHandle handle) { std::move(P)); } +void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) { + send_block_broadcast_to_custom_overlays(broadcast); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::prevalidate_block, std::move(broadcast), + [](td::Result R) { + if (R.is_error()) { + if (R.error().code() == ErrorCode::notready) { + LOG(DEBUG) << "dropped broadcast: " << R.move_as_error(); + } else { + LOG(INFO) << "dropped broadcast: " << R.move_as_error(); + } + } + }); +} + void FullNodeImpl::start_up() { + add_shard(ShardIdFull{masterchainId}); if (local_id_.is_zero()) { if(adnl_id_.is_zero()) { auto pk = ton::PrivateKey{ton::privkeys::Ed25519::random()}; @@ -435,8 +452,8 @@ void FullNodeImpl::start_up() { void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_shard_block_info, block_id, cc_seqno, std::move(data)); } - void send_broadcast(BlockBroadcast broadcast) override { - td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast)); + void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override { + td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast), custom_overlays_only); } void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) override { @@ -488,8 +505,8 @@ void FullNodeImpl::start_up() { } void FullNodeImpl::update_private_overlays() { - for (auto &p : private_custom_overlays_) { - update_ext_msg_overlay(p.first, p.second); + for (auto &p : custom_overlays_) { + update_custom_overlay(p.second); } private_block_overlays_.clear(); @@ -520,28 +537,30 @@ void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { } private_block_overlays_[key] = td::actor::create_actor( "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, - private_block_overlays_enable_compression_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_); + private_block_overlays_enable_compression_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_, + actor_id(this)); } } -void FullNodeImpl::update_ext_msg_overlay(const std::string &name, ExtMsgOverlayInfo &overlay) { +void FullNodeImpl::update_custom_overlay(CustomOverlayInfo &overlay) { auto old_actors = std::move(overlay.actors_); overlay.actors_.clear(); + CustomOverlayParams ¶ms = overlay.params_; auto try_local_id = [&](const adnl::AdnlNodeIdShort &local_id) { - if (std::find(overlay.nodes_.begin(), overlay.nodes_.end(), local_id) != overlay.nodes_.end()) { + if (std::find(params.nodes_.begin(), params.nodes_.end(), local_id) != params.nodes_.end()) { auto it = old_actors.find(local_id); if (it != old_actors.end()) { overlay.actors_[local_id] = std::move(it->second); old_actors.erase(it); } else { overlay.actors_[local_id] = td::actor::create_actor( - "CustomOverlay", local_id, overlay.nodes_, overlay.senders_, name, zero_state_file_hash_, config_, - keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_); + "CustomOverlay", local_id, params, zero_state_file_hash_, config_, keyring_, adnl_, rldp_, rldp2_, + overlays_, validator_manager_, actor_id(this)); } } }; try_local_id(adnl_id_); - for (const PublicKeyHash &local_key: local_keys_) { + for (const PublicKeyHash &local_key : local_keys_) { auto it = current_validators_.find(local_key); if (it != current_validators_.end()) { try_local_id(it->second); @@ -549,6 +568,25 @@ void FullNodeImpl::update_ext_msg_overlay(const std::string &name, ExtMsgOverlay } } +void FullNodeImpl::send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast) { + if (!custom_overlays_sent_broadcasts_.insert(broadcast.block_id).second) { + return; + } + custom_overlays_sent_broadcasts_lru_.push(broadcast.block_id); + if (custom_overlays_sent_broadcasts_lru_.size() > 256) { + custom_overlays_sent_broadcasts_.erase(custom_overlays_sent_broadcasts_lru_.front()); + custom_overlays_sent_broadcasts_lru_.pop(); + } + for (auto &private_overlay : custom_overlays_) { + for (auto &actor : private_overlay.second.actors_) { + auto local_id = actor.first; + if (private_overlay.second.params_.block_senders_.count(local_id)) { + td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_broadcast, broadcast.clone()); + } + } + } +} + FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, @@ -569,7 +607,6 @@ FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id , client_(client) , db_root_(db_root) , config_(config) { - add_shard(ShardIdFull{masterchainId}); } td::actor::ActorOwn FullNode::create(ton::PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, @@ -598,6 +635,21 @@ bool FullNodeConfig::operator!=(const FullNodeConfig &rhs) const { return !(*this == rhs); } +CustomOverlayParams CustomOverlayParams::fetch(const ton_api::engine_validator_customOverlay& f) { + CustomOverlayParams c; + c.name_ = f.name_; + for (const auto &node : f.nodes_) { + c.nodes_.emplace_back(node->adnl_id_); + if (node->msg_sender_) { + c.msg_senders_[ton::adnl::AdnlNodeIdShort{node->adnl_id_}] = node->msg_sender_priority_; + } + if (node->block_sender_) { + c.block_senders_.emplace(node->adnl_id_); + } + } + return c; +} + } // namespace fullnode } // namespace validator diff --git a/validator/full-node.h b/validator/full-node.h index 82e0dd5c7..ac260dd78 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -55,6 +55,15 @@ struct FullNodeConfig { bool ext_messages_broadcast_disabled_ = false; }; +struct CustomOverlayParams { + std::string name_; + std::vector nodes_; + std::map msg_senders_; + std::set block_senders_; + + static CustomOverlayParams fetch(const ton_api::engine_validator_customOverlay& f); +}; + class FullNode : public td::actor::Actor { public: virtual ~FullNode() = default; @@ -74,10 +83,10 @@ class FullNode : public td::actor::Actor { virtual void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) = 0; virtual void set_config(FullNodeConfig config) = 0; - virtual void add_ext_msg_overlay(std::vector nodes, - std::map senders, std::string name, - td::Promise promise) = 0; - virtual void del_ext_msg_overlay(std::string name, td::Promise promise) = 0; + virtual void add_custom_overlay(CustomOverlayParams params, td::Promise promise) = 0; + virtual void del_custom_overlay(std::string name, td::Promise promise) = 0; + + virtual void process_block_broadcast(BlockBroadcast broadcast) = 0; static constexpr td::uint32 max_block_size() { return 4 << 20; diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 86664a3b6..946a1705f 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -27,6 +27,7 @@ #include #include +#include namespace ton { @@ -53,9 +54,8 @@ class FullNodeImpl : public FullNode { void update_adnl_id(adnl::AdnlNodeIdShort adnl_id, td::Promise promise) override; void set_config(FullNodeConfig config) override; - void add_ext_msg_overlay(std::vector nodes, std::map senders, - std::string name, td::Promise promise) override; - void del_ext_msg_overlay(std::string name, td::Promise promise) override; + void add_custom_overlay(CustomOverlayParams params, td::Promise promise) override; + void del_custom_overlay(std::string name, td::Promise promise) override; void add_shard(ShardIdFull shard); void del_shard(ShardIdFull shard); @@ -66,7 +66,7 @@ class FullNodeImpl : public FullNode { void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data); void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data); void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqnp, td::BufferSlice data); - void send_broadcast(BlockBroadcast broadcast); + void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only); void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); @@ -83,6 +83,8 @@ class FullNodeImpl : public FullNode { void got_key_block_state(td::Ref state); void new_key_block(BlockHandle handle); + void process_block_broadcast(BlockBroadcast broadcast) override; + void start_up() override; FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, @@ -123,18 +125,19 @@ class FullNodeImpl : public FullNode { std::map> private_block_overlays_; bool private_block_overlays_enable_compression_ = false; - struct ExtMsgOverlayInfo { - std::vector nodes_; - std::map senders_; - std::map> - actors_; // our local id -> actor + struct CustomOverlayInfo { + CustomOverlayParams params_; + std::map> actors_; // our local id -> actor }; - std::map private_custom_overlays_; + std::map custom_overlays_; + std::set custom_overlays_sent_broadcasts_; + std::queue custom_overlays_sent_broadcasts_lru_; void update_private_overlays(); void set_private_block_overlays_enable_compression(bool value); void create_private_block_overlay(PublicKeyHash key); - void update_ext_msg_overlay(const std::string& name, ExtMsgOverlayInfo& overlay); + void update_custom_overlay(CustomOverlayInfo& overlay); + void send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast); }; } // namespace fullnode diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index cda1c7871..3da1167aa 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -899,11 +899,6 @@ void AcceptBlockQuery::written_block_info_2() { } void AcceptBlockQuery::applied() { - if (!send_broadcast_) { - finish_query(); - return; - } - BlockBroadcast b; b.data = data_->data(); b.block_id = id_; @@ -923,7 +918,8 @@ void AcceptBlockQuery::applied() { } // do not wait for answer - td::actor::send_closure_later(manager_, &ValidatorManager::send_block_broadcast, std::move(b)); + td::actor::send_closure_later(manager_, &ValidatorManager::send_block_broadcast, std::move(b), + /* custom_overlays_only = */ !send_broadcast_); finish_query(); } diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 278915f16..d435b7569 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -133,7 +133,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void send_external_message(td::Ref message) = 0; virtual void send_ihr_message(td::Ref message) = 0; virtual void send_top_shard_block_description(td::Ref desc) = 0; - virtual void send_block_broadcast(BlockBroadcast broadcast) = 0; + virtual void send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) = 0; virtual void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) = 0; virtual void get_shard_client_state(bool from_db, td::Promise promise) = 0; diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index eca6a7418..5cf938847 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -251,7 +251,7 @@ class ValidatorManagerImpl : public ValidatorManager { new_ihr_message(message->serialize()); } void send_top_shard_block_description(td::Ref desc) override; - void send_block_broadcast(BlockBroadcast broadcast) override { + void send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override { } void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index f0cf83de5..fb5237d6c 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -317,7 +317,7 @@ class ValidatorManagerImpl : public ValidatorManager { void send_top_shard_block_description(td::Ref desc) override { UNREACHABLE(); } - void send_block_broadcast(BlockBroadcast broadcast) override { + void send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override { } void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override { diff --git a/validator/manager.cpp b/validator/manager.cpp index ddc582251..82cc10c23 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1503,8 +1503,8 @@ void ValidatorManagerImpl::send_top_shard_block_description(td::Refsend_broadcast(std::move(broadcast)); +void ValidatorManagerImpl::send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) { + callback_->send_broadcast(std::move(broadcast), custom_overlays_only); } void ValidatorManagerImpl::start_up() { diff --git a/validator/manager.hpp b/validator/manager.hpp index d6d0307b3..65636a94c 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -473,7 +473,7 @@ class ValidatorManagerImpl : public ValidatorManager { void send_external_message(td::Ref message) override; void send_ihr_message(td::Ref message) override; void send_top_shard_block_description(td::Ref desc) override; - void send_block_broadcast(BlockBroadcast broadcast) override; + void send_block_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override; void update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) override; void get_shard_client_state(bool from_db, td::Promise promise) override; diff --git a/validator/validator.h b/validator/validator.h index 9ede07110..17e55c70f 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -134,7 +134,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; - virtual void send_broadcast(BlockBroadcast broadcast) = 0; + virtual void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only = false) = 0; virtual void download_block(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; virtual void download_zero_state(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, From 136b99b8d10a0219a9616ac6e7fa1e3be2338c6c Mon Sep 17 00:00:00 2001 From: neodix42 Date: Sun, 12 May 2024 04:48:14 -0500 Subject: [PATCH 08/28] Use jemalloc (#987) * use jemalloc * install system jemalloc in github action * check if jemalloc is actually used --------- Co-authored-by: neodiX --- .github/workflows/build-ton-linux-x86-64-shared.yml | 2 +- CMake/{FindJeMalloc.cmake => Findjemalloc.cmake} | 0 CMakeLists.txt | 2 +- Dockerfile | 6 +++--- assembly/native/build-ubuntu-shared.sh | 6 ++++-- 5 files changed, 9 insertions(+), 7 deletions(-) rename CMake/{FindJeMalloc.cmake => Findjemalloc.cmake} (100%) diff --git a/.github/workflows/build-ton-linux-x86-64-shared.yml b/.github/workflows/build-ton-linux-x86-64-shared.yml index ce0ade646..34f92d93e 100644 --- a/.github/workflows/build-ton-linux-x86-64-shared.yml +++ b/.github/workflows/build-ton-linux-x86-64-shared.yml @@ -19,7 +19,7 @@ jobs: - name: Install system libraries run: | sudo apt-get update - sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev + sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev - name: Install clang-16 run: | diff --git a/CMake/FindJeMalloc.cmake b/CMake/Findjemalloc.cmake similarity index 100% rename from CMake/FindJeMalloc.cmake rename to CMake/Findjemalloc.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 89be32383..658eab70c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -234,7 +234,7 @@ if (THREADS_HAVE_PTHREAD_ARG) endif() if (TON_USE_JEMALLOC) - find_package(JeMalloc REQUIRED) + find_package(jemalloc REQUIRED) endif() set(MEMPROF "" CACHE STRING "Use one of \"ON\", \"FAST\" or \"SAFE\" to enable memory profiling. \ diff --git a/Dockerfile b/Dockerfile index 76c06b350..c5120e83e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM ubuntu:22.04 as builder RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential cmake clang openssl libssl-dev zlib1g-dev gperf wget git ninja-build libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev pkg-config autoconf automake libtool libjemalloc-dev && \ rm -rf /var/lib/apt/lists/* ENV CC clang ENV CXX clang++ @@ -14,12 +14,12 @@ COPY ./ ./ RUN mkdir build && \ cd build && \ - cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= .. && \ + cmake -GNinja -DCMAKE_BUILD_TYPE=Release -DPORTABLE=1 -DTON_ARCH= -DTON_USE_JEMALLOC=ON .. && \ ninja storage-daemon storage-daemon-cli tonlibjson fift func validator-engine validator-engine-console generate-random-id dht-server lite-client FROM ubuntu:22.04 RUN apt-get update && \ - apt-get install -y wget libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev && \ + apt-get install -y wget libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev && \ rm -rf /var/lib/apt/lists/* RUN mkdir -p /var/ton-work/db && \ diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index 7d5326a4f..ec868ecd2 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -1,7 +1,7 @@ #/bin/bash #sudo apt-get update -#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev +#sudo apt-get install -y build-essential git cmake ninja-build zlib1g-dev libsecp256k1-dev libmicrohttpd-dev libsodium-dev liblz4-dev libjemalloc-dev with_tests=false with_artifacts=false @@ -42,7 +42,7 @@ else echo "Using compiled openssl_3" fi -cmake -GNinja .. \ +cmake -GNinja -DTON_USE_JEMALLOC=ON .. \ -DCMAKE_BUILD_TYPE=Release \ -DOPENSSL_ROOT_DIR=$opensslPath \ -DOPENSSL_INCLUDE_DIR=$opensslPath/include \ @@ -96,6 +96,8 @@ test $? -eq 0 || { echo "Can't strip final binaries"; exit 1; } ./lite-client/lite-client -V || exit 1 ./crypto/fift -V || exit 1 +ldd ./validator-engine/validator-engine || exit 1 + cd .. if [ "$with_artifacts" = true ]; then From 1433f23effeafbe422d4cd9f540600cd52e8da2a Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 13 May 2024 12:48:18 +0300 Subject: [PATCH 09/28] Add option --celldb-cache-size (#988) Co-authored-by: SpyCheese --- tddb/td/db/RocksDb.cpp | 32 +++++++++++++-------------- tddb/td/db/RocksDb.h | 8 ++++++- validator-engine/validator-engine.cpp | 13 +++++++++++ validator-engine/validator-engine.hpp | 4 ++++ validator/db/archive-manager.cpp | 5 ++++- validator/db/archive-slice.cpp | 4 +++- validator/db/celldb.cpp | 8 ++++++- validator/validator-options.hpp | 7 ++++++ validator/validator.h | 2 ++ 9 files changed, 63 insertions(+), 20 deletions(-) diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index a84a804bb..d15b8e19f 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -59,38 +59,38 @@ RocksDb RocksDb::clone() const { return RocksDb{db_, statistics_}; } -Result RocksDb::open(std::string path, std::shared_ptr statistics) { +Result RocksDb::open(std::string path, RocksDbOptions options) { rocksdb::OptimisticTransactionDB *db; { - rocksdb::Options options; + rocksdb::Options db_options; - static auto cache = rocksdb::NewLRUCache(1 << 30); + static auto cache = rocksdb::NewLRUCache(options.block_cache_size); rocksdb::BlockBasedTableOptions table_options; table_options.block_cache = cache; - options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); - - options.manual_wal_flush = true; - options.create_if_missing = true; - options.max_background_compactions = 4; - options.max_background_flushes = 2; - options.bytes_per_sync = 1 << 20; - options.writable_file_max_buffer_size = 2 << 14; - options.statistics = statistics; + db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); + + db_options.manual_wal_flush = true; + db_options.create_if_missing = true; + db_options.max_background_compactions = 4; + db_options.max_background_flushes = 2; + db_options.bytes_per_sync = 1 << 20; + db_options.writable_file_max_buffer_size = 2 << 14; + db_options.statistics = options.statistics; rocksdb::OptimisticTransactionDBOptions occ_options; occ_options.validate_policy = rocksdb::OccValidationPolicy::kValidateSerial; - rocksdb::ColumnFamilyOptions cf_options(options); + rocksdb::ColumnFamilyOptions cf_options(db_options); std::vector column_families; column_families.push_back(rocksdb::ColumnFamilyDescriptor(rocksdb::kDefaultColumnFamilyName, cf_options)); std::vector handles; - TRY_STATUS(from_rocksdb( - rocksdb::OptimisticTransactionDB::Open(options, occ_options, std::move(path), column_families, &handles, &db))); + TRY_STATUS(from_rocksdb(rocksdb::OptimisticTransactionDB::Open(db_options, occ_options, std::move(path), + column_families, &handles, &db))); CHECK(handles.size() == 1); // i can delete the handle since DBImpl is always holding a reference to // default column family delete handles[0]; } - return RocksDb(std::shared_ptr(db), std::move(statistics)); + return RocksDb(std::shared_ptr(db), std::move(options.statistics)); } std::shared_ptr RocksDb::create_statistics() { diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index 1afba4cc4..44ef9de69 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -34,11 +34,17 @@ class Statistics; } // namespace rocksdb namespace td { + +struct RocksDbOptions { + std::shared_ptr statistics = nullptr; + uint64 block_cache_size = 1 << 30; +}; + class RocksDb : public KeyValue { public: static Status destroy(Slice path); RocksDb clone() const; - static Result open(std::string path, std::shared_ptr statistics = nullptr); + static Result open(std::string path, RocksDbOptions options = {}); Result get(Slice key, std::string &value) override; Status set(Slice key, Slice value) override; diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 3c98a4ac6..154085108 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1369,6 +1369,9 @@ td::Status ValidatorEngine::load_global_config() { validator_options_.write().set_archive_preload_period(archive_preload_period_); validator_options_.write().set_disable_rocksdb_stats(disable_rocksdb_stats_); validator_options_.write().set_nonfinal_ls_queries_enabled(nonfinal_ls_queries_enabled_); + if (celldb_cache_size_) { + validator_options_.write().set_celldb_cache_size(celldb_cache_size_.value()); + } std::vector h; for (auto &x : conf.validator_->hardforks_) { @@ -3967,6 +3970,16 @@ int main(int argc, char *argv[]) { p.add_option('\0', "nonfinal-ls", "enable special LS queries to non-finalized blocks", [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_nonfinal_ls_queries_enabled); }); }); + p.add_checked_option( + '\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 1G)", + [&](td::Slice s) -> td::Status { + TRY_RESULT(v, td::to_integer_safe(s)); + if (v == 0) { + return td::Status::Error("celldb-cache-size should be positive"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_cache_size, v); }); + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 0a99dfcc8..39685f9d6 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -209,6 +209,7 @@ class ValidatorEngine : public td::actor::Actor { double archive_preload_period_ = 0.0; bool disable_rocksdb_stats_ = false; bool nonfinal_ls_queries_enabled_ = false; + td::optional celldb_cache_size_; bool read_config_ = false; bool started_keyring_ = false; bool started_ = false; @@ -281,6 +282,9 @@ class ValidatorEngine : public td::actor::Actor { void set_nonfinal_ls_queries_enabled() { nonfinal_ls_queries_enabled_ = true; } + void set_celldb_cache_size(td::uint64 value) { + celldb_cache_size_ = value; + } void start_up() override; ValidatorEngine() { } diff --git a/validator/db/archive-manager.cpp b/validator/db/archive-manager.cpp index c806f4e54..b87d04f78 100644 --- a/validator/db/archive-manager.cpp +++ b/validator/db/archive-manager.cpp @@ -832,7 +832,10 @@ void ArchiveManager::start_up() { if (!opts_->get_disable_rocksdb_stats()) { statistics_.init(); } - index_ = std::make_shared(td::RocksDb::open(db_root_ + "/files/globalindex", statistics_.rocksdb_statistics).move_as_ok()); + td::RocksDbOptions db_options; + db_options.statistics = statistics_.rocksdb_statistics; + index_ = std::make_shared( + td::RocksDb::open(db_root_ + "/files/globalindex", std::move(db_options)).move_as_ok()); std::string value; auto v = index_->get(create_serialize_tl_object().as_slice(), value); v.ensure(); diff --git a/validator/db/archive-slice.cpp b/validator/db/archive-slice.cpp index daee546b0..d392431a9 100644 --- a/validator/db/archive-slice.cpp +++ b/validator/db/archive-slice.cpp @@ -554,7 +554,9 @@ void ArchiveSlice::get_archive_id(BlockSeqno masterchain_seqno, td::Promise(td::RocksDb::open(db_path_, statistics_.rocksdb_statistics).move_as_ok()); + td::RocksDbOptions db_options; + db_options.statistics = statistics_.rocksdb_statistics; + kv_ = std::make_unique(td::RocksDb::open(db_path_, std::move(db_options)).move_as_ok()); std::string value; auto R2 = kv_->get("status", value); R2.ensure(); diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 294515a31..007bedc89 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -88,7 +88,13 @@ void CellDbIn::start_up() { statistics_ = td::RocksDb::create_statistics(); statistics_flush_at_ = td::Timestamp::in(60.0); } - cell_db_ = std::make_shared(td::RocksDb::open(path_, statistics_).move_as_ok()); + td::RocksDbOptions db_options; + db_options.statistics = statistics_; + if (opts_->get_celldb_cache_size()) { + db_options.block_cache_size = opts_->get_celldb_cache_size().value(); + LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(db_options.block_cache_size); + } + cell_db_ = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); boc_ = vm::DynamicBagOfCellsDb::create(); diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 1b7c5b09c..2dd85b569 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -129,6 +129,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool nonfinal_ls_queries_enabled() const override { return nonfinal_ls_queries_enabled_; } + td::optional get_celldb_cache_size() const override { + return celldb_cache_size_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -197,6 +200,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_nonfinal_ls_queries_enabled(bool value) override { nonfinal_ls_queries_enabled_ = value; } + void set_celldb_cache_size(td::uint64 value) override { + celldb_cache_size_ = value; + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); @@ -244,6 +250,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { double archive_preload_period_ = 0.0; bool disable_rocksdb_stats_; bool nonfinal_ls_queries_enabled_ = false; + td::optional celldb_cache_size_; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 17e55c70f..92332e55f 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -86,6 +86,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual double get_archive_preload_period() const = 0; virtual bool get_disable_rocksdb_stats() const = 0; virtual bool nonfinal_ls_queries_enabled() const = 0; + virtual td::optional get_celldb_cache_size() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; @@ -110,6 +111,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_archive_preload_period(double value) = 0; virtual void set_disable_rocksdb_stats(bool value) = 0; virtual void set_nonfinal_ls_queries_enabled(bool value) = 0; + virtual void set_celldb_cache_size(td::uint64 value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, From c7fd75ce56dbdb6d8aece97415275c09efa9318b Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 13 May 2024 14:54:57 +0300 Subject: [PATCH 10/28] Fix creating rocksdb cache (#989) Co-authored-by: SpyCheese --- tddb/td/db/RocksDb.cpp | 8 ++++++-- tddb/td/db/RocksDb.h | 3 ++- validator-engine/validator-engine.cpp | 3 ++- validator/db/celldb.cpp | 2 +- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index d15b8e19f..91c5ca662 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -64,10 +64,14 @@ Result RocksDb::open(std::string path, RocksDbOptions options) { { rocksdb::Options db_options; - static auto cache = rocksdb::NewLRUCache(options.block_cache_size); + static auto cache = rocksdb::NewLRUCache(1 << 30); rocksdb::BlockBasedTableOptions table_options; - table_options.block_cache = cache; + if (options.block_cache_size) { + table_options.block_cache = rocksdb::NewLRUCache(options.block_cache_size.value()); + } else { + table_options.block_cache = cache; + } db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); db_options.manual_wal_flush = true; diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index 44ef9de69..babfadb73 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -24,6 +24,7 @@ #include "td/db/KeyValue.h" #include "td/utils/Status.h" +#include "td/utils/optional.h" namespace rocksdb { class OptimisticTransactionDB; @@ -37,7 +38,7 @@ namespace td { struct RocksDbOptions { std::shared_ptr statistics = nullptr; - uint64 block_cache_size = 1 << 30; + optional block_cache_size; // Default - one 1G cache for all RocksDb }; class RocksDb : public KeyValue { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 154085108..f52fe8cf7 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3971,7 +3971,8 @@ int main(int argc, char *argv[]) { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_nonfinal_ls_queries_enabled); }); }); p.add_checked_option( - '\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 1G)", + '\0', "celldb-cache-size", + "block cache size for RocksDb in CellDb, in bytes (default: 1G cache shared by archive DB)", [&](td::Slice s) -> td::Status { TRY_RESULT(v, td::to_integer_safe(s)); if (v == 0) { diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 007bedc89..bc315b599 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -92,7 +92,7 @@ void CellDbIn::start_up() { db_options.statistics = statistics_; if (opts_->get_celldb_cache_size()) { db_options.block_cache_size = opts_->get_celldb_cache_size().value(); - LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(db_options.block_cache_size); + LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(db_options.block_cache_size.value()); } cell_db_ = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); From 816dd9cf2d7348f7646ce2befb366ac6165557ce Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 13 May 2024 14:55:32 +0300 Subject: [PATCH 11/28] Add option --catchain-max-block-delay (#990) Co-authored-by: SpyCheese --- validator-engine/validator-engine.cpp | 13 +++++++++++++ validator-engine/validator-engine.hpp | 4 ++++ validator-session/validator-session.cpp | 4 ++-- validator-session/validator-session.h | 1 + validator-session/validator-session.hpp | 5 +++++ validator/manager.cpp | 2 +- validator/validator-group.cpp | 4 ++++ validator/validator-group.hpp | 6 ++++-- validator/validator-options.hpp | 7 +++++++ validator/validator.h | 2 ++ 10 files changed, 43 insertions(+), 5 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index f52fe8cf7..0c423f7a9 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1372,6 +1372,9 @@ td::Status ValidatorEngine::load_global_config() { if (celldb_cache_size_) { validator_options_.write().set_celldb_cache_size(celldb_cache_size_.value()); } + if (catchain_max_block_delay_) { + validator_options_.write().set_catchain_max_block_delay(catchain_max_block_delay_.value()); + } std::vector h; for (auto &x : conf.validator_->hardforks_) { @@ -3981,6 +3984,16 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_cache_size, v); }); return td::Status::OK(); }); + p.add_checked_option( + '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.5)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v < 0) { + return td::Status::Error("catchain-max-block-delay should be non-negative"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_catchain_max_block_delay, v); }); + return td::Status::OK(); + }); auto S = p.run(argc, argv); if (S.is_error()) { LOG(ERROR) << "failed to parse options: " << S.move_as_error(); diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 39685f9d6..265bb9e40 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -210,6 +210,7 @@ class ValidatorEngine : public td::actor::Actor { bool disable_rocksdb_stats_ = false; bool nonfinal_ls_queries_enabled_ = false; td::optional celldb_cache_size_; + td::optional catchain_max_block_delay_; bool read_config_ = false; bool started_keyring_ = false; bool started_ = false; @@ -285,6 +286,9 @@ class ValidatorEngine : public td::actor::Actor { void set_celldb_cache_size(td::uint64 value) { celldb_cache_size_ = value; } + void set_catchain_max_block_delay(double value) { + catchain_max_block_delay_ = value; + } void start_up() override; ValidatorEngine() { } diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index d2fe64589..46dd44403 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -807,8 +807,8 @@ void ValidatorSessionImpl::request_new_block(bool now) { } else { double lambda = 10.0 / description().get_total_nodes(); double x = -1 / lambda * log(td::Random::fast(1, 999) * 0.001); - if (x > 0.5) { - x = 0.5; + if (x > catchain_max_block_delay_) { // default = 0.5 + x = catchain_max_block_delay_; } td::actor::send_closure(catchain_, &catchain::CatChain::need_new_block, td::Timestamp::in(x)); } diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index 2dd6625c1..0870f6718 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -108,6 +108,7 @@ class ValidatorSession : public td::actor::Actor { virtual void get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) = 0; + virtual void set_catchain_max_block_delay(double value) = 0; static td::actor::ActorOwn create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 005292c03..b6a9ab0ca 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -90,6 +90,8 @@ class ValidatorSessionImpl : public ValidatorSession { td::actor::ActorOwn catchain_; std::unique_ptr description_; + double catchain_max_block_delay_ = 0.5; + void on_new_round(td::uint32 round); void on_catchain_started(); void check_vote_for_slot(td::uint32 att); @@ -188,6 +190,9 @@ class ValidatorSessionImpl : public ValidatorSession { void get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) override; + void set_catchain_max_block_delay(double value) override { + catchain_max_block_delay_ = value; + } void process_blocks(std::vector blocks); void finished_processing(); diff --git a/validator/manager.cpp b/validator/manager.cpp index 82cc10c23..3d5440037 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2135,7 +2135,7 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto G = td::actor::create_actor( "validatorgroup", shard, validator_id, session_id, validator_set, opts, keyring_, adnl_, rldp_, overlays_, db_root_, actor_id(this), init_session, - opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno())); + opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_); return G; } } diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 6a71d306a..68e2b07ec 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -348,6 +348,10 @@ void ValidatorGroup::create_session() { << ".", allow_unsafe_self_blocks_resync_); } + if (opts_->get_catchain_max_block_delay()) { + td::actor::send_closure(session_, &validatorsession::ValidatorSession::set_catchain_max_block_delay, + opts_->get_catchain_max_block_delay().value()); + } if (started_) { td::actor::send_closure(session_, &validatorsession::ValidatorSession::start); } diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index cb3f7a630..f99402647 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -69,7 +69,7 @@ class ValidatorGroup : public td::actor::Actor { td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId overlays, std::string db_root, td::actor::ActorId validator_manager, bool create_session, - bool allow_unsafe_self_blocks_resync) + bool allow_unsafe_self_blocks_resync, td::Ref opts) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) @@ -82,7 +82,8 @@ class ValidatorGroup : public td::actor::Actor { , db_root_(std::move(db_root)) , manager_(validator_manager) , init_(create_session) - , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) { + , allow_unsafe_self_blocks_resync_(allow_unsafe_self_blocks_resync) + , opts_(std::move(opts)) { } private: @@ -123,6 +124,7 @@ class ValidatorGroup : public td::actor::Actor { bool init_ = false; bool started_ = false; bool allow_unsafe_self_blocks_resync_; + td::Ref opts_; td::uint32 last_known_round_id_ = 0; struct CachedCollatedBlock { diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 2dd85b569..34f7fdf9f 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -132,6 +132,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { td::optional get_celldb_cache_size() const override { return celldb_cache_size_; } + td::optional get_catchain_max_block_delay() const override { + return catchain_max_block_delay_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -203,6 +206,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_celldb_cache_size(td::uint64 value) override { celldb_cache_size_ = value; } + void set_catchain_max_block_delay(double value) override { + catchain_max_block_delay_ = value; + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); @@ -251,6 +257,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool disable_rocksdb_stats_; bool nonfinal_ls_queries_enabled_ = false; td::optional celldb_cache_size_; + td::optional catchain_max_block_delay_; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 92332e55f..81dfe7a39 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -87,6 +87,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual bool get_disable_rocksdb_stats() const = 0; virtual bool nonfinal_ls_queries_enabled() const = 0; virtual td::optional get_celldb_cache_size() const = 0; + virtual td::optional get_catchain_max_block_delay() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; @@ -112,6 +113,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_disable_rocksdb_stats(bool value) = 0; virtual void set_nonfinal_ls_queries_enabled(bool value) = 0; virtual void set_celldb_cache_size(td::uint64 value) = 0; + virtual void set_catchain_max_block_delay(double value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, From 561f34263921852370417b9fb14d845f9479e400 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Tue, 14 May 2024 16:05:29 +0300 Subject: [PATCH 12/28] Add --celldb-direct-io and --celldb-preload-all (#993) Co-authored-by: SpyCheese --- tddb/td/db/KeyValue.h | 4 +++ tddb/td/db/RocksDb.cpp | 47 ++++++++++++++++++++------- tddb/td/db/RocksDb.h | 12 ++++--- validator-engine/validator-engine.cpp | 10 ++++++ validator-engine/validator-engine.hpp | 8 +++++ validator/db/celldb.cpp | 26 +++++++++++++-- validator/validator-options.hpp | 14 ++++++++ validator/validator.h | 4 +++ 8 files changed, 108 insertions(+), 17 deletions(-) diff --git a/tddb/td/db/KeyValue.h b/tddb/td/db/KeyValue.h index 4e0d85384..4f30a272b 100644 --- a/tddb/td/db/KeyValue.h +++ b/tddb/td/db/KeyValue.h @@ -19,6 +19,7 @@ #pragma once #include "td/utils/Status.h" #include "td/utils/logging.h" +#include namespace td { class KeyValueReader { public: @@ -27,6 +28,9 @@ class KeyValueReader { virtual Result get(Slice key, std::string &value) = 0; virtual Result count(Slice prefix) = 0; + virtual Status for_each(std::function f) { + return Status::Error("for_each is not supported"); + } }; class PrefixedKeyValueReader : public KeyValueReader { diff --git a/tddb/td/db/RocksDb.cpp b/tddb/td/db/RocksDb.cpp index 91c5ca662..f8688c006 100644 --- a/tddb/td/db/RocksDb.cpp +++ b/tddb/td/db/RocksDb.cpp @@ -56,7 +56,7 @@ RocksDb::~RocksDb() { } RocksDb RocksDb::clone() const { - return RocksDb{db_, statistics_}; + return RocksDb{db_, options_}; } Result RocksDb::open(std::string path, RocksDbOptions options) { @@ -64,16 +64,16 @@ Result RocksDb::open(std::string path, RocksDbOptions options) { { rocksdb::Options db_options; - static auto cache = rocksdb::NewLRUCache(1 << 30); + static auto default_cache = rocksdb::NewLRUCache(1 << 30); + if (options.block_cache == nullptr) { + options.block_cache = default_cache; + } rocksdb::BlockBasedTableOptions table_options; - if (options.block_cache_size) { - table_options.block_cache = rocksdb::NewLRUCache(options.block_cache_size.value()); - } else { - table_options.block_cache = cache; - } + table_options.block_cache = options.block_cache; db_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(table_options)); + db_options.use_direct_reads = options.use_direct_reads; db_options.manual_wal_flush = true; db_options.create_if_missing = true; db_options.max_background_compactions = 4; @@ -94,7 +94,7 @@ Result RocksDb::open(std::string path, RocksDbOptions options) { // default column family delete handles[0]; } - return RocksDb(std::shared_ptr(db), std::move(options.statistics)); + return RocksDb(std::shared_ptr(db), std::move(options)); } std::shared_ptr RocksDb::create_statistics() { @@ -109,6 +109,10 @@ void RocksDb::reset_statistics(const std::shared_ptr statis statistics->Reset(); } +std::shared_ptr RocksDb::create_cache(size_t capacity) { + return rocksdb::NewLRUCache(capacity); +} + std::unique_ptr RocksDb::snapshot() { auto res = std::make_unique(clone()); res->begin_snapshot().ensure(); @@ -120,7 +124,6 @@ std::string RocksDb::stats() const { db_->GetProperty("rocksdb.stats", &out); //db_->GetProperty("rocksdb.cur-size-all-mem-tables", &out); return out; - return statistics_->ToString(); } Result RocksDb::get(Slice key, std::string &value) { @@ -187,6 +190,28 @@ Result RocksDb::count(Slice prefix) { return res; } +Status RocksDb::for_each(std::function f) { + rocksdb::ReadOptions options; + options.snapshot = snapshot_.get(); + std::unique_ptr iterator; + if (snapshot_ || !transaction_) { + iterator.reset(db_->NewIterator(options)); + } else { + iterator.reset(transaction_->GetIterator(options)); + } + + iterator->SeekToFirst(); + for (; iterator->Valid(); iterator->Next()) { + auto key = from_rocksdb(iterator->key()); + auto value = from_rocksdb(iterator->value()); + TRY_STATUS(f(key, value)); + } + if (!iterator->status().ok()) { + return from_rocksdb(iterator->status()); + } + return Status::OK(); +} + Status RocksDb::begin_write_batch() { CHECK(!transaction_); write_batch_ = std::make_unique(); @@ -243,7 +268,7 @@ Status RocksDb::end_snapshot() { return td::Status::OK(); } -RocksDb::RocksDb(std::shared_ptr db, std::shared_ptr statistics) - : db_(std::move(db)), statistics_(std::move(statistics)) { +RocksDb::RocksDb(std::shared_ptr db, RocksDbOptions options) + : db_(std::move(db)), options_(options) { } } // namespace td diff --git a/tddb/td/db/RocksDb.h b/tddb/td/db/RocksDb.h index babfadb73..5efcd0f48 100644 --- a/tddb/td/db/RocksDb.h +++ b/tddb/td/db/RocksDb.h @@ -27,6 +27,7 @@ #include "td/utils/optional.h" namespace rocksdb { +class Cache; class OptimisticTransactionDB; class Transaction; class WriteBatch; @@ -38,7 +39,8 @@ namespace td { struct RocksDbOptions { std::shared_ptr statistics = nullptr; - optional block_cache_size; // Default - one 1G cache for all RocksDb + std::shared_ptr block_cache; // Default - one 1G cache for all RocksDb + bool use_direct_reads = false; }; class RocksDb : public KeyValue { @@ -51,6 +53,7 @@ class RocksDb : public KeyValue { Status set(Slice key, Slice value) override; Status erase(Slice key) override; Result count(Slice prefix) override; + Status for_each(std::function f) override; Status begin_write_batch() override; Status commit_write_batch() override; @@ -71,6 +74,8 @@ class RocksDb : public KeyValue { static std::string statistics_to_string(const std::shared_ptr statistics); static void reset_statistics(const std::shared_ptr statistics); + static std::shared_ptr create_cache(size_t capacity); + RocksDb(RocksDb &&); RocksDb &operator=(RocksDb &&); ~RocksDb(); @@ -81,7 +86,7 @@ class RocksDb : public KeyValue { private: std::shared_ptr db_; - std::shared_ptr statistics_; + RocksDbOptions options_; std::unique_ptr transaction_; std::unique_ptr write_batch_; @@ -94,7 +99,6 @@ class RocksDb : public KeyValue { }; std::unique_ptr snapshot_; - explicit RocksDb(std::shared_ptr db, - std::shared_ptr statistics); + explicit RocksDb(std::shared_ptr db, RocksDbOptions options); }; } // namespace td diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 0c423f7a9..06ca89b56 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1372,6 +1372,8 @@ td::Status ValidatorEngine::load_global_config() { if (celldb_cache_size_) { validator_options_.write().set_celldb_cache_size(celldb_cache_size_.value()); } + validator_options_.write().set_celldb_direct_io(celldb_direct_io_); + validator_options_.write().set_celldb_preload_all(celldb_preload_all_); if (catchain_max_block_delay_) { validator_options_.write().set_catchain_max_block_delay(catchain_max_block_delay_.value()); } @@ -3984,6 +3986,14 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_cache_size, v); }); return td::Status::OK(); }); + p.add_option('\0', "celldb-direct-io", "enable direct I/O mode for RocksDb in CellDb", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io); }); + }); + p.add_option( + '\0', "celldb-preload-all", + "preload all cells from CellDb on startup (recommended to use with big enough celldb-cache-size and " + "celldb-direct-io)", + [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all); }); }); p.add_checked_option( '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.5)", [&](td::Slice s) -> td::Status { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 265bb9e40..94ced8ba3 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -210,6 +210,8 @@ class ValidatorEngine : public td::actor::Actor { bool disable_rocksdb_stats_ = false; bool nonfinal_ls_queries_enabled_ = false; td::optional celldb_cache_size_; + bool celldb_direct_io_ = false; + bool celldb_preload_all_ = false; td::optional catchain_max_block_delay_; bool read_config_ = false; bool started_keyring_ = false; @@ -286,6 +288,12 @@ class ValidatorEngine : public td::actor::Actor { void set_celldb_cache_size(td::uint64 value) { celldb_cache_size_ = value; } + void set_celldb_direct_io() { + celldb_direct_io_ = true; + } + void set_celldb_preload_all() { + celldb_preload_all_ = true; + } void set_catchain_max_block_delay(double value) { catchain_max_block_delay_ = value; } diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index bc315b599..dfbee0a1a 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -91,9 +91,10 @@ void CellDbIn::start_up() { td::RocksDbOptions db_options; db_options.statistics = statistics_; if (opts_->get_celldb_cache_size()) { - db_options.block_cache_size = opts_->get_celldb_cache_size().value(); - LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(db_options.block_cache_size.value()); + db_options.block_cache = td::RocksDb::create_cache(opts_->get_celldb_cache_size().value()); + LOG(WARNING) << "Set CellDb block cache size to " << td::format::as_size(opts_->get_celldb_cache_size().value()); } + db_options.use_direct_reads = opts_->get_celldb_direct_io(); cell_db_ = std::make_shared(td::RocksDb::open(path_, std::move(db_options)).move_as_ok()); @@ -111,6 +112,27 @@ void CellDbIn::start_up() { set_block(empty, std::move(e)); cell_db_->commit_write_batch().ensure(); } + + if (opts_->get_celldb_preload_all()) { + // Iterate whole DB in a separate thread + delay_action([snapshot = cell_db_->snapshot()]() { + LOG(WARNING) << "CellDb: pre-loading all keys"; + td::uint64 total = 0; + td::Timer timer; + auto S = snapshot->for_each([&](td::Slice, td::Slice) { + ++total; + if (total % 1000000 == 0) { + LOG(INFO) << "CellDb: iterated " << total << " keys"; + } + return td::Status::OK(); + }); + if (S.is_error()) { + LOG(ERROR) << "CellDb: pre-load failed: " << S.move_as_error(); + } else { + LOG(WARNING) << "CellDb: iterated " << total << " keys in " << timer.elapsed() << "s"; + } + }, td::Timestamp::now()); + } } void CellDbIn::load_cell(RootHash hash, td::Promise> promise) { diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 34f7fdf9f..593c75bf6 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -132,6 +132,12 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { td::optional get_celldb_cache_size() const override { return celldb_cache_size_; } + bool get_celldb_direct_io() const override { + return celldb_direct_io_; + } + bool get_celldb_preload_all() const override { + return celldb_preload_all_; + } td::optional get_catchain_max_block_delay() const override { return catchain_max_block_delay_; } @@ -206,6 +212,12 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_celldb_cache_size(td::uint64 value) override { celldb_cache_size_ = value; } + void set_celldb_direct_io(bool value) override { + celldb_direct_io_ = value; + } + void set_celldb_preload_all(bool value) override { + celldb_preload_all_ = value; + } void set_catchain_max_block_delay(double value) override { catchain_max_block_delay_ = value; } @@ -257,6 +269,8 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool disable_rocksdb_stats_; bool nonfinal_ls_queries_enabled_ = false; td::optional celldb_cache_size_; + bool celldb_direct_io_ = false; + bool celldb_preload_all_ = false; td::optional catchain_max_block_delay_; }; diff --git a/validator/validator.h b/validator/validator.h index 81dfe7a39..245acbd49 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -87,6 +87,8 @@ struct ValidatorManagerOptions : public td::CntObject { virtual bool get_disable_rocksdb_stats() const = 0; virtual bool nonfinal_ls_queries_enabled() const = 0; virtual td::optional get_celldb_cache_size() const = 0; + virtual bool get_celldb_direct_io() const = 0; + virtual bool get_celldb_preload_all() const = 0; virtual td::optional get_catchain_max_block_delay() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; @@ -113,6 +115,8 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_disable_rocksdb_stats(bool value) = 0; virtual void set_nonfinal_ls_queries_enabled(bool value) = 0; virtual void set_celldb_cache_size(td::uint64 value) = 0; + virtual void set_celldb_direct_io(bool value) = 0; + virtual void set_celldb_preload_all(bool value) = 0; virtual void set_catchain_max_block_delay(double value) = 0; static td::Ref create( From 3a802fa67af1e08b0fa0f132b4be241a67c9c4ab Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Tue, 14 May 2024 17:11:01 +0300 Subject: [PATCH 13/28] Blockchain explorer: use liteServer.runSmcMethod (#994) Co-authored-by: SpyCheese --- .../blockchain-explorer-query.cpp | 166 ++++++------------ .../blockchain-explorer-query.hpp | 10 +- 2 files changed, 51 insertions(+), 125 deletions(-) diff --git a/blockchain-explorer/blockchain-explorer-query.cpp b/blockchain-explorer/blockchain-explorer-query.cpp index b53e79696..1808a3c46 100644 --- a/blockchain-explorer/blockchain-explorer-query.cpp +++ b/blockchain-explorer/blockchain-explorer-query.cpp @@ -50,32 +50,6 @@ #include "vm/vm.h" #include "vm/cp0.h" -namespace { - -td::Ref prepare_vm_c7(ton::UnixTime now, ton::LogicalTime lt, td::Ref my_addr, - const block::CurrencyCollection &balance) { - td::BitArray<256> rand_seed; - td::RefInt256 rand_seed_int{true}; - td::Random::secure_bytes(rand_seed.as_slice()); - if (!rand_seed_int.unique_write().import_bits(rand_seed.cbits(), 256, false)) { - return {}; - } - auto tuple = vm::make_tuple_ref(td::make_refint(0x076ef1ea), // [ magic:0x076ef1ea - td::make_refint(0), // actions:Integer - td::make_refint(0), // msgs_sent:Integer - td::make_refint(now), // unixtime:Integer - td::make_refint(lt), // block_lt:Integer - td::make_refint(lt), // trans_lt:Integer - std::move(rand_seed_int), // rand_seed:Integer - balance.as_vm_tuple(), // balance_remaining:[Integer (Maybe Cell)] - my_addr, // myself:MsgAddressInt - vm::StackEntry()); // global_config:(Maybe Cell) ] = SmartContractInfo; - LOG(DEBUG) << "SmartContractInfo initialized with " << vm::StackEntry(tuple).to_string(); - return vm::make_tuple_ref(std::move(tuple)); -} - -} // namespace - td::Result parse_block_id(std::map &opts, bool allow_empty) { if (allow_empty) { if (opts.count("workchain") == 0 && opts.count("shard") == 0 && opts.count("seqno") == 0) { @@ -1343,111 +1317,71 @@ void HttpQueryRunMethod::start_up_query() { if (R.is_error()) { td::actor::send_closure(SelfId, &HttpQueryRunMethod::abort_query, R.move_as_error_prefix("litequery failed: ")); } else { - td::actor::send_closure(SelfId, &HttpQueryRunMethod::got_account, R.move_as_ok()); + td::actor::send_closure(SelfId, &HttpQueryRunMethod::got_result, R.move_as_ok()); } }); + auto a = ton::create_tl_object(addr_.workchain, addr_.addr); - auto query = ton::serialize_tl_object(ton::create_tl_object( - ton::create_tl_lite_block_id(block_id_), std::move(a)), - true); + td::int64 method_id = (td::crc16(td::Slice{method_name_}) & 0xffff) | 0x10000; + + // serialize params + vm::CellBuilder cb; + td::Ref cell; + if (!(vm::Stack{params_}.serialize(cb) && cb.finalize_to(cell))) { + return abort_query(td::Status::Error("cannot serialize stack with get-method parameters")); + } + auto params_serialized = vm::std_boc_serialize(std::move(cell)); + if (params_serialized.is_error()) { + return abort_query(params_serialized.move_as_error_prefix("cannot serialize stack with get-method parameters : ")); + } + + auto query = ton::serialize_tl_object( + ton::create_tl_object( + 0x17, ton::create_tl_lite_block_id(block_id_), std::move(a), method_id, params_serialized.move_as_ok()), + true); td::actor::send_closure(CoreActorInterface::instance_actor_id(), &CoreActorInterface::send_lite_query, std::move(query), std::move(P)); } -void HttpQueryRunMethod::got_account(td::BufferSlice data) { - auto F = ton::fetch_tl_object(std::move(data), true); +void HttpQueryRunMethod::got_result(td::BufferSlice data) { + auto F = ton::fetch_tl_object(std::move(data), true); if (F.is_error()) { - abort_query(F.move_as_error()); - return; + return abort_query(F.move_as_error()); } - auto f = F.move_as_ok(); - data_ = std::move(f->state_); - proof_ = std::move(f->proof_); - shard_proof_ = std::move(f->shard_proof_); - block_id_ = ton::create_block_id(f->id_); - res_block_id_ = ton::create_block_id(f->shardblk_); - - finish_query(); -} - -void HttpQueryRunMethod::finish_query() { - if (promise_) { - auto page = [&]() -> std::string { - HttpAnswer A{"account", prefix_}; - A.set_account_id(addr_); - A.set_block_id(res_block_id_); + auto page = [&]() -> std::string { + HttpAnswer A{"account", prefix_}; + A.set_account_id(addr_); + A.set_block_id(ton::create_block_id(f->id_)); + if (f->exit_code_ != 0) { + A.abort(PSTRING() << "VM terminated with error code " << f->exit_code_); + return A.finish(); + } - block::AccountState account_state; - account_state.blk = block_id_; - account_state.shard_blk = res_block_id_; - account_state.shard_proof = std::move(shard_proof_); - account_state.proof = std::move(proof_); - account_state.state = std::move(data_); - auto r_info = account_state.validate(block_id_, addr_); - if (r_info.is_error()) { - A.abort(r_info.move_as_error()); - return A.finish(); - } - auto info = r_info.move_as_ok(); - if (info.root.is_null()) { - A.abort(PSTRING() << "account state of " << addr_ << " is empty (cannot run method `" << method_name_ << "`)"); - return A.finish(); - } - block::gen::Account::Record_account acc; - block::gen::AccountStorage::Record store; - block::CurrencyCollection balance; - if (!(tlb::unpack_cell(info.root, acc) && tlb::csr_unpack(acc.storage, store) && - balance.validate_unpack(store.balance))) { - A.abort("error unpacking account state"); + std::ostringstream os; + os << "result: "; + if (f->result_.empty()) { + os << ""; + } else { + auto r_cell = vm::std_boc_deserialize(f->result_); + if (r_cell.is_error()) { + A.abort(PSTRING() << "cannot deserialize VM result boc: " << r_cell.move_as_error()); return A.finish(); } - int tag = block::gen::t_AccountState.get_tag(*store.state); - switch (tag) { - case block::gen::AccountState::account_uninit: - A.abort(PSTRING() << "account " << addr_ << " not initialized yet (cannot run any methods)"); - return A.finish(); - case block::gen::AccountState::account_frozen: - A.abort(PSTRING() << "account " << addr_ << " frozen (cannot run any methods)"); - return A.finish(); - } - - CHECK(store.state.write().fetch_ulong(1) == 1); // account_init$1 _:StateInit = AccountState; - block::gen::StateInit::Record state_init; - CHECK(tlb::csr_unpack(store.state, state_init)); - auto code = state_init.code->prefetch_ref(); - auto data = state_init.data->prefetch_ref(); - auto stack = td::make_ref(std::move(params_)); - td::int64 method_id = (td::crc16(td::Slice{method_name_}) & 0xffff) | 0x10000; - stack.write().push_smallint(method_id); - long long gas_limit = vm::GasLimits::infty; - // OstreamLogger ostream_logger(ctx.error_stream); - // auto log = create_vm_log(ctx.error_stream ? &ostream_logger : nullptr); - vm::GasLimits gas{gas_limit}; - LOG(DEBUG) << "creating VM"; - vm::VmState vm{code, std::move(stack), gas, 1, data, vm::VmLog()}; - vm.set_c7(prepare_vm_c7(info.gen_utime, info.gen_lt, acc.addr, balance)); // tuple with SmartContractInfo - // vm.incr_stack_trace(1); // enable stack dump after each step - int exit_code = ~vm.run(); - if (exit_code != 0) { - A.abort(PSTRING() << "VM terminated with error code " << exit_code); + auto cs = vm::load_cell_slice(r_cell.move_as_ok()); + td::Ref stack; + if (!(vm::Stack::deserialize_to(cs, stack, 0) && cs.empty_ext())) { + A.abort("VM result boc cannot be deserialized"); return A.finish(); } - stack = vm.get_stack_ref(); - { - std::ostringstream os; - os << "result: "; - stack->dump(os, 3); - - A << HttpAnswer::CodeBlock{os.str()}; - } - - return A.finish(); - }(); - auto R = MHD_create_response_from_buffer(page.length(), const_cast(page.c_str()), MHD_RESPMEM_MUST_COPY); - MHD_add_response_header(R, "Content-Type", "text/html"); - promise_.set_value(std::move(R)); - } + stack->dump(os, 3); + } + A << HttpAnswer::CodeBlock{os.str()}; + return A.finish(); + }(); + auto R = MHD_create_response_from_buffer(page.length(), const_cast(page.c_str()), MHD_RESPMEM_MUST_COPY); + MHD_add_response_header(R, "Content-Type", "text/html"); + promise_.set_value(std::move(R)); stop(); } HttpQueryStatus::HttpQueryStatus(std::string prefix, td::Promise promise) diff --git a/blockchain-explorer/blockchain-explorer-query.hpp b/blockchain-explorer/blockchain-explorer-query.hpp index 0ce52af6e..29501265b 100644 --- a/blockchain-explorer/blockchain-explorer-query.hpp +++ b/blockchain-explorer/blockchain-explorer-query.hpp @@ -311,22 +311,14 @@ class HttpQueryRunMethod : public HttpQueryCommon { std::vector params, std::string prefix, td::Promise promise); HttpQueryRunMethod(std::map opts, std::string prefix, td::Promise promise); - void finish_query(); - void start_up_query() override; - void got_account(td::BufferSlice result); + void got_result(td::BufferSlice result); private: ton::BlockIdExt block_id_; block::StdAddress addr_; - std::string method_name_; std::vector params_; - - td::BufferSlice data_; - td::BufferSlice proof_; - td::BufferSlice shard_proof_; - ton::BlockIdExt res_block_id_; }; class HttpQueryStatus : public HttpQueryCommon { From 3827409957250a75ae86c4eef5894ba7dd35c38b Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Tue, 14 May 2024 20:33:53 +0300 Subject: [PATCH 14/28] Change default values for celldb flags (#996) Co-authored-by: SpyCheese --- validator-engine/validator-engine.cpp | 14 ++++++-------- validator-engine/validator-engine.hpp | 14 +++++++------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 06ca89b56..236535c9a 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3976,8 +3976,7 @@ int main(int argc, char *argv[]) { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_nonfinal_ls_queries_enabled); }); }); p.add_checked_option( - '\0', "celldb-cache-size", - "block cache size for RocksDb in CellDb, in bytes (default: 1G cache shared by archive DB)", + '\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 50G)", [&](td::Slice s) -> td::Status { TRY_RESULT(v, td::to_integer_safe(s)); if (v == 0) { @@ -3986,14 +3985,13 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_cache_size, v); }); return td::Status::OK(); }); - p.add_option('\0', "celldb-direct-io", "enable direct I/O mode for RocksDb in CellDb", [&]() { - acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io); }); + p.add_option('\0', "celldb-no-direct-io", "disable direct I/O mode for RocksDb in CellDb", [&]() { + acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, false); }); }); p.add_option( - '\0', "celldb-preload-all", - "preload all cells from CellDb on startup (recommended to use with big enough celldb-cache-size and " - "celldb-direct-io)", - [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all); }); }); + '\0', "celldb-no-preload-all", + "disable preloading all cells from CellDb on startup (enabled by default)", + [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, false); }); }); p.add_checked_option( '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.5)", [&](td::Slice s) -> td::Status { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 94ced8ba3..44fb1c27c 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -209,9 +209,9 @@ class ValidatorEngine : public td::actor::Actor { double archive_preload_period_ = 0.0; bool disable_rocksdb_stats_ = false; bool nonfinal_ls_queries_enabled_ = false; - td::optional celldb_cache_size_; - bool celldb_direct_io_ = false; - bool celldb_preload_all_ = false; + td::optional celldb_cache_size_ = 50LL << 30; + bool celldb_direct_io_ = true; + bool celldb_preload_all_ = true; td::optional catchain_max_block_delay_; bool read_config_ = false; bool started_keyring_ = false; @@ -288,11 +288,11 @@ class ValidatorEngine : public td::actor::Actor { void set_celldb_cache_size(td::uint64 value) { celldb_cache_size_ = value; } - void set_celldb_direct_io() { - celldb_direct_io_ = true; + void set_celldb_direct_io(bool value) { + celldb_direct_io_ = value; } - void set_celldb_preload_all() { - celldb_preload_all_ = true; + void set_celldb_preload_all(bool value) { + celldb_preload_all_ = value; } void set_catchain_max_block_delay(double value) { catchain_max_block_delay_ = value; From 7a74888d2fb33f7fb8011fdec81eb307e983ac6f Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Wed, 15 May 2024 16:06:25 +0300 Subject: [PATCH 15/28] Automatically disable celldb direct I/O if cache is small (#997) Co-authored-by: SpyCheese --- validator-engine/validator-engine.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 236535c9a..66c75827d 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1372,6 +1372,9 @@ td::Status ValidatorEngine::load_global_config() { if (celldb_cache_size_) { validator_options_.write().set_celldb_cache_size(celldb_cache_size_.value()); } + if (!celldb_cache_size_ || celldb_cache_size_.value() < (30ULL << 30)) { + celldb_direct_io_ = false; + } validator_options_.write().set_celldb_direct_io(celldb_direct_io_); validator_options_.write().set_celldb_preload_all(celldb_preload_all_); if (catchain_max_block_delay_) { @@ -3985,9 +3988,9 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_cache_size, v); }); return td::Status::OK(); }); - p.add_option('\0', "celldb-no-direct-io", "disable direct I/O mode for RocksDb in CellDb", [&]() { - acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, false); }); - }); + p.add_option( + '\0', "celldb-no-direct-io", "disable direct I/O mode for RocksDb in CellDb (forced when celldb cache is < 30G)", + [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, false); }); }); p.add_option( '\0', "celldb-no-preload-all", "disable preloading all cells from CellDb on startup (enabled by default)", From 539d5dd2def40ed1096d3745f78f5f4b67a2fbf4 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 24 May 2024 09:58:07 +0300 Subject: [PATCH 16/28] Add candidates cache (#1000) * Broadcast shardchain block candidates in private overlays, generate proof links from candidates * Disable shardchain block broadcasts in private overlays * Send block candidate broadcasts to custom overlays, allow non-validators to receive candidates --------- Co-authored-by: SpyCheese --- create-hardfork/create-hardfork.cpp | 3 + test/test-ton-collator.cpp | 3 + tl/generate/scheme/ton_api.tl | 3 + tl/generate/scheme/ton_api.tlo | Bin 89388 -> 89652 bytes ton/ton-types.h | 4 + validator/db/files-async.hpp | 2 +- validator/downloaders/wait-block-data.cpp | 81 ++++++++++++++++++++- validator/downloaders/wait-block-data.hpp | 6 +- validator/downloaders/wait-block-state.cpp | 3 +- validator/full-node-private-overlay.cpp | 65 +++++++++++++++++ validator/full-node-private-overlay.hpp | 6 ++ validator/full-node-shard.cpp | 32 ++++++++ validator/full-node-shard.h | 2 + validator/full-node-shard.hpp | 4 + validator/full-node.cpp | 56 +++++++++++++- validator/full-node.h | 2 + validator/full-node.hpp | 7 ++ validator/impl/collator.cpp | 13 ++-- validator/impl/validate-query.cpp | 3 +- validator/interfaces/validator-manager.h | 3 +- validator/manager-disk.cpp | 3 +- validator/manager-disk.hpp | 5 +- validator/manager-hardfork.hpp | 6 +- validator/manager.cpp | 71 +++++++++++++++++- validator/manager.hpp | 16 +++- validator/validator.h | 3 + 26 files changed, 382 insertions(+), 20 deletions(-) diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp index a9e7403a1..3110b1a85 100644 --- a/create-hardfork/create-hardfork.cpp +++ b/create-hardfork/create-hardfork.cpp @@ -246,6 +246,9 @@ class HardforkCreator : public td::actor::Actor { } void send_shard_block_info(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::BufferSlice data) override { } + void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override { + } void send_broadcast(ton::BlockBroadcast broadcast, bool custom_overlays_only) override { } void download_block(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp index d36db5d24..60476cf8f 100644 --- a/test/test-ton-collator.cpp +++ b/test/test-ton-collator.cpp @@ -347,6 +347,9 @@ class TestNode : public td::actor::Actor { } } } + void send_block_candidate(ton::BlockIdExt block_id, ton::CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override { + } void send_broadcast(ton::BlockBroadcast broadcast, bool custom_overlays_only) override { } void download_block(ton::BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index d0a063f7b..5d1e5b504 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -396,6 +396,9 @@ tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int valida tonNode.ihrMessageBroadcast message:tonNode.ihrMessage = tonNode.Broadcast; tonNode.externalMessageBroadcast message:tonNode.externalMessage = tonNode.Broadcast; tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast; +// signature may be empty, at least for now +tonNode.newBlockCandidateBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + collator_signature:tonNode.blockSignature data:bytes = tonNode.Broadcast; tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index fb7f219765fea2ff1aa4dee29f67166b388d01f9..89948bb4acb26d8a5255b77af3a6710b2739f3b4 100644 GIT binary patch delta 245 zcmZ3pi*?H$)(r==CExjW)e4s6=lSKQr0O{p block) { + if (data_.not_null()) { + return; + } + data_ = std::move(block); + if (handle_->received()) { + finish_query(); + return; + } + if (!handle_->id().is_masterchain() && !handle_->inited_proof_link()) { + // This can happen if we get block from candidates cache. + // Proof link can be derived from the block (but not for masterchain block). + auto r_proof_link = generate_proof_link(handle_->id(), data_->root_cell()); + if (r_proof_link.is_error()) { + abort_query(r_proof_link.move_as_error_prefix("failed to create proof link for block: ")); + return; + } + td::actor::send_closure(manager_, &ValidatorManager::validate_block_proof_link, handle_->id(), + r_proof_link.move_as_ok(), + [id = handle_->id().id, SelfId = actor_id(this)](td::Result R) { + if (R.is_error()) { + td::actor::send_closure(SelfId, &WaitBlockData::abort_query, + R.move_as_error_prefix("validate proof link error: ")); + return; + } + LOG(DEBUG) << "Created and validated proof link for " << id.to_str(); + td::actor::send_closure(SelfId, &WaitBlockData::checked_proof_link); + }); + return; + } + checked_proof_link(); +} + +void WaitBlockData::checked_proof_link() { CHECK(handle_->id().is_masterchain() ? handle_->inited_proof() : handle_->inited_proof_link()); if (!handle_->received()) { auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { @@ -198,6 +238,41 @@ void WaitBlockData::got_static_file(td::BufferSlice data) { run_hardfork_accept_block_query(handle_->id(), data_, manager_, std::move(P)); } +td::Result WaitBlockData::generate_proof_link(BlockIdExt id, td::Ref block_root) { + // Creating proof link. Similar to accept-block.cpp + if (id.is_masterchain()) { + return td::Status::Error("cannot create proof link for masterchain block"); + } + auto usage_tree = std::make_shared(); + auto usage_cell = vm::UsageCell::create(block_root, usage_tree->root_ptr()); + + block::gen::Block::Record blk; + block::gen::BlockInfo::Record info; + block::gen::BlockExtra::Record extra; + block::gen::ExtBlkRef::Record mcref{}; // _ ExtBlkRef = BlkMasterInfo; + ShardIdFull shard; + if (!(tlb::unpack_cell(usage_cell, blk) && tlb::unpack_cell(blk.info, info) && !info.version && + block::tlb::t_ShardIdent.unpack(info.shard.write(), shard) && + block::gen::BlkPrevInfo{info.after_merge}.validate_ref(info.prev_ref) && + tlb::unpack_cell(std::move(blk.extra), extra) && block::gen::t_ValueFlow.force_validate_ref(blk.value_flow) && + (!info.not_master || tlb::unpack_cell(info.master_ref, mcref)))) { + return td::Status::Error("cannot unpack block header"); + } + vm::CellSlice upd_cs{vm::NoVmSpec(), blk.state_update}; + + auto proof = vm::MerkleProof::generate(block_root, usage_tree.get()); + vm::CellBuilder cb; + td::Ref bs_cell; + if (!(cb.store_long_bool(0xc3, 8) // block_proof#c3 + && block::tlb::t_BlockIdExt.pack(cb, id) // proof_for:BlockIdExt + && cb.store_ref_bool(std::move(proof)) // proof:^Cell + && cb.store_bool_bool(false) // signatures:(Maybe ^BlockSignatures) + && cb.finalize_to(bs_cell))) { + return td::Status::Error("cannot serialize BlockProof"); + } + return std_boc_serialize(bs_cell, 0); +} + } // namespace validator } // namespace ton diff --git a/validator/downloaders/wait-block-data.hpp b/validator/downloaders/wait-block-data.hpp index 9a03b1cb6..229b4bfc8 100644 --- a/validator/downloaders/wait-block-data.hpp +++ b/validator/downloaders/wait-block-data.hpp @@ -57,11 +57,15 @@ class WaitBlockData : public td::actor::Actor { void set_is_hardfork(bool value); void start(); void got_block_data_from_db(td::Ref data); - void got_block_data_from_net(ReceivedBlock data); + void got_data_from_net(ReceivedBlock data); + void got_block_data_from_net(td::Ref block); + void checked_proof_link(); void failed_to_get_block_data_from_net(td::Status reason); void got_static_file(td::BufferSlice data); + static td::Result generate_proof_link(BlockIdExt id, td::Ref block_root); + private: BlockHandle handle_; diff --git a/validator/downloaders/wait-block-state.cpp b/validator/downloaders/wait-block-state.cpp index 42f5c791b..0ae82beaa 100644 --- a/validator/downloaders/wait-block-state.cpp +++ b/validator/downloaders/wait-block-state.cpp @@ -192,7 +192,8 @@ void WaitBlockState::got_proof_link(td::BufferSlice data) { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); } else { LOG(INFO) << "received bad proof link: " << R.move_as_error(); - td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); + delay_action([SelfId]() { td::actor::send_closure(SelfId, &WaitBlockState::after_get_proof_link); }, + td::Timestamp::in(0.1)); } }); run_check_proof_link_query(handle_->id(), R.move_as_ok(), manager_, timeout_, std::move(P)); diff --git a/validator/full-node-private-overlay.cpp b/validator/full-node-private-overlay.cpp index 74bb75c52..ffe5468d4 100644 --- a/validator/full-node-private-overlay.cpp +++ b/validator/full-node-private-overlay.cpp @@ -17,6 +17,7 @@ #include "full-node-private-overlay.hpp" #include "ton/ton-tl.hpp" #include "common/delay.h" +#include "common/checksum.h" #include "full-node-serializer.hpp" namespace ton::validator::fullnode { @@ -49,6 +50,22 @@ void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, ton_api:: query.block_->cc_seqno_, std::move(query.block_->data_)); } +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcast &query) { + if (query.data_.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + BlockIdExt block_id = create_block_id(query.id_); + if (td::sha256_bits256(query.data_.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in private overlay from " << src << ": " << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, query.catchain_seqno_, + query.validator_set_hash_, std::move(query.data_)); +} + void FullNodePrivateBlockOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { if (adnl::AdnlNodeIdShort{src} == local_id_) { return; @@ -77,6 +94,19 @@ void FullNodePrivateBlockOverlay::send_shard_block_info(BlockIdExt block_id, Cat } } +void FullNodePrivateBlockOverlay::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in private overlay: " << block_id.to_str(); + auto B = create_serialize_tl_object( + create_tl_block_id(block_id), cc_seqno, validator_set_hash, + create_tl_object(Bits256::zero(), td::BufferSlice()), std::move(data)); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); +} + void FullNodePrivateBlockOverlay::send_broadcast(BlockBroadcast broadcast) { if (!inited_) { return; @@ -199,6 +229,28 @@ void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNod std::move(query.message_->data_), it->second); } +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { + if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { + VLOG(FULL_NODE_DEBUG) << "Dropping block candidate broadcast in private overlay \"" << name_ + << "\" from unauthorized sender " << src; + return; + } + if (query.data_.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + BlockIdExt block_id = create_block_id(query.id_); + if (td::sha256_bits256(query.data_.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + // ignore cc_seqno and validator_hash for now + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in custom overlay \"" << name_ << "\" from " << src << ": " + << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, query.catchain_seqno_, + query.validator_set_hash_, std::move(query.data_)); +} + void FullNodeCustomOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { if (adnl::AdnlNodeIdShort{src} == local_id_) { return; @@ -241,6 +293,19 @@ void FullNodeCustomOverlay::send_broadcast(BlockBroadcast broadcast) { local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); } +void FullNodeCustomOverlay::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + if (!inited_) { + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in custom overlay \"" << name_ << "\": " << block_id.to_str(); + auto B = create_serialize_tl_object( + create_tl_block_id(block_id), cc_seqno, validator_set_hash, + create_tl_object(Bits256::zero(), td::BufferSlice()), std::move(data)); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); +} + void FullNodeCustomOverlay::start_up() { std::sort(nodes_.begin(), nodes_.end()); nodes_.erase(std::unique(nodes_.begin(), nodes_.end()), nodes_.end()); diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index 196d6da60..e310824f0 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -27,6 +27,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); template void process_broadcast(PublicKeyHash, T &) { VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; @@ -34,6 +35,8 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data); + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data); void send_broadcast(BlockBroadcast broadcast); void set_config(FullNodeConfig config) { @@ -98,6 +101,7 @@ class FullNodeCustomOverlay : public td::actor::Actor { void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); template void process_broadcast(PublicKeyHash, T &) { VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; @@ -106,6 +110,8 @@ class FullNodeCustomOverlay : public td::actor::Actor { void send_external_message(td::BufferSlice data); void send_broadcast(BlockBroadcast broadcast); + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data); void set_config(FullNodeConfig config) { config_ = std::move(config); diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 6e265cbc6..9b2b321e5 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -17,12 +17,14 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "auto/tl/ton_api.h" +#include "checksum.h" #include "overlays.h" #include "td/utils/SharedSlice.h" #include "full-node-shard.hpp" #include "full-node-shard-queries.hpp" #include "full-node-serializer.hpp" +#include "td/utils/buffer.h" #include "ton/ton-shard.h" #include "ton/ton-tl.hpp" #include "ton/ton-io.hpp" @@ -646,6 +648,22 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ne query.block_->cc_seqno_, std::move(query.block_->data_)); } +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { + if (query.data_.size() > FullNode::max_block_size()) { + VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; + return; + } + BlockIdExt block_id = create_block_id(query.id_); + if (td::sha256_bits256(query.data_.as_slice()) != block_id.file_hash) { + VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; + return; + } + // ignore cc_seqno and validator_hash for now + VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate from " << src << ": " << block_id.to_str(); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, query.catchain_seqno_, + query.validator_set_hash_, std::move(query.data_)); +} + void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { process_block_broadcast(src, query); } @@ -738,6 +756,20 @@ void FullNodeShardImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno } } +void FullNodeShardImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) { + if (!client_.empty()) { + UNREACHABLE(); + return; + } + VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate: " << block_id.to_str(); + auto B = create_serialize_tl_object( + create_tl_block_id(block_id), cc_seqno, validator_set_hash, + create_tl_object(Bits256::zero(), td::BufferSlice()), std::move(data)); + td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, adnl_id_, overlay_id_, local_id_, + overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); +} + void FullNodeShardImpl::send_broadcast(BlockBroadcast broadcast) { if (!client_.empty()) { UNREACHABLE(); diff --git a/validator/full-node-shard.h b/validator/full-node-shard.h index 2ed13d660..e89031fe9 100644 --- a/validator/full-node-shard.h +++ b/validator/full-node-shard.h @@ -41,6 +41,8 @@ class FullNodeShard : public td::actor::Actor { virtual void send_ihr_message(td::BufferSlice data) = 0; virtual void send_external_message(td::BufferSlice data) = 0; virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; + virtual void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) = 0; virtual void send_broadcast(BlockBroadcast broadcast) = 0; virtual void sign_overlay_certificate(PublicKeyHash signed_key, td::uint32 expiry_at, td::uint32 max_size, td::Promise promise) = 0; diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 1fdbce1c1..08f48f862 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -18,6 +18,7 @@ */ #pragma once +#include "auto/tl/ton_api.h" #include "full-node-shard.h" #include "td/actor/PromiseFuture.h" #include "td/utils/port/Poll.h" @@ -152,12 +153,15 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void check_broadcast(PublicKeyHash src, td::BufferSlice query, td::Promise promise); void send_ihr_message(td::BufferSlice data) override; void send_external_message(td::BufferSlice data) override; void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override; void send_broadcast(BlockBroadcast broadcast) override; void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 8d0f26bc9..ff8f1eca0 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -243,6 +243,24 @@ void FullNodeImpl::send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_s td::actor::send_closure(shard, &FullNodeShard::send_shard_block_info, block_id, cc_seqno, std::move(data)); } +void FullNodeImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) { + send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); + auto shard = get_shard(ShardIdFull{masterchainId, shardIdAll}); + if (shard.empty()) { + VLOG(FULL_NODE_WARNING) << "dropping OUT shard block info message to unknown shard"; + return; + } + if (!private_block_overlays_.empty()) { + td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_block_candidate, + block_id, cc_seqno, validator_set_hash, data.clone()); + } + if (broadcast_block_candidates_in_public_overlay_) { + td::actor::send_closure(shard, &FullNodeShard::send_block_candidate, block_id, cc_seqno, validator_set_hash, + std::move(data)); + } +} + void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) { send_block_broadcast_to_custom_overlays(broadcast); if (custom_overlays_only) { @@ -253,7 +271,7 @@ void FullNodeImpl::send_broadcast(BlockBroadcast broadcast, bool custom_overlays VLOG(FULL_NODE_WARNING) << "dropping OUT broadcast to unknown shard"; return; } - if (!private_block_overlays_.empty()) { + if (broadcast.block_id.is_masterchain() && !private_block_overlays_.empty()) { td::actor::send_closure(private_block_overlays_.begin()->second, &FullNodePrivateBlockOverlay::send_broadcast, broadcast.clone()); } @@ -420,6 +438,14 @@ void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) { }); } +void FullNodeImpl::process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) { + send_block_candidate_broadcast_to_custom_overlays(block_id, cc_seqno, validator_set_hash, data); + // ignore cc_seqno and validator_hash for now + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::new_block_candidate, block_id, + std::move(data)); +} + void FullNodeImpl::start_up() { add_shard(ShardIdFull{masterchainId}); if (local_id_.is_zero()) { @@ -452,6 +478,11 @@ void FullNodeImpl::start_up() { void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { td::actor::send_closure(id_, &FullNodeImpl::send_shard_block_info, block_id, cc_seqno, std::move(data)); } + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override { + td::actor::send_closure(id_, &FullNodeImpl::send_block_candidate, block_id, cc_seqno, validator_set_hash, + std::move(data)); + } void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only) override { td::actor::send_closure(id_, &FullNodeImpl::send_broadcast, std::move(broadcast), custom_overlays_only); } @@ -587,6 +618,29 @@ void FullNodeImpl::send_block_broadcast_to_custom_overlays(const BlockBroadcast& } } +void FullNodeImpl::send_block_candidate_broadcast_to_custom_overlays(const BlockIdExt &block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, + const td::BufferSlice &data) { + // Same cache of sent broadcasts as in send_block_broadcast_to_custom_overlays + if (!custom_overlays_sent_broadcasts_.insert(block_id).second) { + return; + } + custom_overlays_sent_broadcasts_lru_.push(block_id); + if (custom_overlays_sent_broadcasts_lru_.size() > 256) { + custom_overlays_sent_broadcasts_.erase(custom_overlays_sent_broadcasts_lru_.front()); + custom_overlays_sent_broadcasts_lru_.pop(); + } + for (auto &private_overlay : custom_overlays_) { + for (auto &actor : private_overlay.second.actors_) { + auto local_id = actor.first; + if (private_overlay.second.params_.block_senders_.count(local_id)) { + td::actor::send_closure(actor.second, &FullNodeCustomOverlay::send_block_candidate, block_id, cc_seqno, + validator_set_hash, data.clone()); + } + } + } +} + FullNodeImpl::FullNodeImpl(PublicKeyHash local_id, adnl::AdnlNodeIdShort adnl_id, FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, diff --git a/validator/full-node.h b/validator/full-node.h index ac260dd78..c3719f678 100644 --- a/validator/full-node.h +++ b/validator/full-node.h @@ -87,6 +87,8 @@ class FullNode : public td::actor::Actor { virtual void del_custom_overlay(std::string name, td::Promise promise) = 0; virtual void process_block_broadcast(BlockBroadcast broadcast) = 0; + virtual void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::BufferSlice data) = 0; static constexpr td::uint32 max_block_size() { return 4 << 20; diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 946a1705f..9b78b3e01 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -66,6 +66,8 @@ class FullNodeImpl : public FullNode { void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data); void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data); void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqnp, td::BufferSlice data); + void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data); void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only); void download_block(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, td::Promise promise); void download_zero_state(BlockIdExt id, td::uint32 priority, td::Timestamp timeout, @@ -84,6 +86,8 @@ class FullNodeImpl : public FullNode { void new_key_block(BlockHandle handle); void process_block_broadcast(BlockBroadcast broadcast) override; + void process_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) override; void start_up() override; @@ -124,6 +128,7 @@ class FullNodeImpl : public FullNode { std::map> private_block_overlays_; bool private_block_overlays_enable_compression_ = false; + bool broadcast_block_candidates_in_public_overlay_ = false; struct CustomOverlayInfo { CustomOverlayParams params_; @@ -138,6 +143,8 @@ class FullNodeImpl : public FullNode { void create_private_block_overlay(PublicKeyHash key); void update_custom_overlay(CustomOverlayInfo& overlay); void send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast); + void send_block_candidate_broadcast_to_custom_overlays(const BlockIdExt& block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, const td::BufferSlice& data); }; } // namespace fullnode diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index fd4ddd341..ba38ba423 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -5012,12 +5012,13 @@ bool Collator::create_block_candidate() { } // 4. save block candidate LOG(INFO) << "saving new BlockCandidate"; - td::actor::send_closure_later(manager, &ValidatorManager::set_block_candidate, block_candidate->id, - block_candidate->clone(), [self = get_self()](td::Result saved) -> void { - LOG(DEBUG) << "got answer to set_block_candidate"; - td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, - std::move(saved)); - }); + td::actor::send_closure_later( + manager, &ValidatorManager::set_block_candidate, block_candidate->id, block_candidate->clone(), + validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), + [self = get_self()](td::Result saved) -> void { + LOG(DEBUG) << "got answer to set_block_candidate"; + td::actor::send_closure_later(std::move(self), &Collator::return_block_candidate, std::move(saved)); + }); // 5. communicate about bad and delayed external messages if (!bad_ext_msgs_.empty() || !delay_ext_msgs_.empty()) { LOG(INFO) << "sending complete_external_messages() to Manager"; diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 8b2723c33..88bc61634 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -6330,7 +6330,8 @@ bool ValidateQuery::save_candidate() { } }); - td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(), std::move(P)); + td::actor::send_closure(manager, &ValidatorManager::set_block_candidate, id_, block_candidate.clone(), + validator_set_->get_catchain_seqno(), validator_set_->get_validator_set_hash(), std::move(P)); return true; } diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index d435b7569..0e9fab73b 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -93,7 +93,8 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) = 0; - virtual void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) = 0; + virtual void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) = 0; virtual void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 42e440811..17b793c7a 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -775,7 +775,8 @@ void ValidatorManagerImpl::set_next_block(BlockIdExt block_id, BlockIdExt next, get_block_handle(block_id, true, std::move(P)); } -void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) { +void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) { td::actor::send_closure(db_, &Db::store_block_candidate, std::move(candidate), std::move(promise)); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 5cf938847..12bd68ef2 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -130,6 +130,8 @@ class ValidatorManagerImpl : public ValidatorManager { } void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; + void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override { + } void add_ext_server_id(adnl::AdnlNodeIdShort id) override { UNREACHABLE(); @@ -177,7 +179,8 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) override; - void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) override; + void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) override; void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index fb5237d6c..3a124f827 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -152,6 +152,9 @@ class ValidatorManagerImpl : public ValidatorManager { void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override { UNREACHABLE(); } + void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override { + UNREACHABLE(); + } void add_ext_server_id(adnl::AdnlNodeIdShort id) override { UNREACHABLE(); @@ -215,7 +218,8 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) override; - void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) override { + void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) override { promise.set_value(td::Unit()); } diff --git a/validator/manager.cpp b/validator/manager.cpp index 3d5440037..f91f04a17 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -17,6 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #include "manager.hpp" +#include "checksum.h" +#include "td/utils/buffer.h" #include "validator-group.hpp" #include "adnl/utils.hpp" #include "downloaders/wait-block-state.hpp" @@ -464,6 +466,17 @@ void ValidatorManagerImpl::new_shard_block(BlockIdExt block_id, CatchainSeqno cc actor_id(this), td::Timestamp::in(2.0), std::move(P)); } +void ValidatorManagerImpl::new_block_candidate(BlockIdExt block_id, td::BufferSlice data) { + if (!last_masterchain_block_handle_) { + VLOG(VALIDATOR_DEBUG) << "dropping top shard block broadcast: not inited"; + return; + } + if (!started_) { + return; + } + add_cached_block_candidate(ReceivedBlock{block_id, std::move(data)}); +} + void ValidatorManagerImpl::add_shard_block_description(td::Ref desc) { if (desc->may_be_valid(last_masterchain_block_handle_, last_masterchain_state_)) { auto it = shard_blocks_.find(ShardTopBlockDescriptionId{desc->shard(), desc->catchain_seqno()}); @@ -495,6 +508,36 @@ void ValidatorManagerImpl::add_shard_block_description(td::Refsecond.actor_, &WaitBlockData::got_block_data_from_net, r_block.move_as_ok()); + } + } + } + { + auto it = wait_state_.find(id); + if (it != wait_state_.end()) { + // Proof link is not ready at this point, but this will force WaitBlockState to redo send_get_proof_link_request + td::actor::send_closure(it->second.actor_, &WaitBlockState::after_get_proof_link); + } + } + } + if (cached_block_candidates_lru_.size() > max_cached_candidates()) { + CHECK(cached_block_candidates_.erase(cached_block_candidates_lru_.front())); + cached_block_candidates_lru_.pop_front(); + } +} + void ValidatorManagerImpl::add_ext_server_id(adnl::AdnlNodeIdShort id) { class Cb : public adnl::Adnl::Callback { private: @@ -1189,11 +1232,16 @@ void ValidatorManagerImpl::set_next_block(BlockIdExt block_id, BlockIdExt next, get_block_handle(block_id, true, std::move(P)); } -void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) { +void ValidatorManagerImpl::set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) { if (!candidates_buffer_.empty()) { td::actor::send_closure(candidates_buffer_, &CandidatesBuffer::add_new_candidate, id, PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}, candidate.collated_file_hash); } + if (!id.is_masterchain()) { + add_cached_block_candidate(ReceivedBlock{id, candidate.data.clone()}); + callback_->send_block_candidate(id, cc_seqno, validator_set_hash, candidate.data.clone()); + } td::actor::send_closure(db_, &Db::store_block_candidate, std::move(candidate), std::move(promise)); } @@ -1450,6 +1498,13 @@ void ValidatorManagerImpl::get_last_liteserver_state_block( void ValidatorManagerImpl::send_get_block_request(BlockIdExt id, td::uint32 priority, td::Promise promise) { + { + auto it = cached_block_candidates_.find(id); + if (it != cached_block_candidates_.end()) { + LOG(DEBUG) << "send_get_block_request: got result from candidates cache for " << id.to_str(); + return promise.set_value(it->second.clone()); + } + } callback_->download_block(id, priority, td::Timestamp::in(10.0), std::move(promise)); } @@ -1472,6 +1527,20 @@ void ValidatorManagerImpl::send_get_block_proof_request(BlockIdExt block_id, td: void ValidatorManagerImpl::send_get_block_proof_link_request(BlockIdExt block_id, td::uint32 priority, td::Promise promise) { + if (!block_id.is_masterchain()) { + auto it = cached_block_candidates_.find(block_id); + if (it != cached_block_candidates_.end()) { + // Proof link can be created from the cached block candidate + LOG(DEBUG) << "send_get_block_proof_link_request: creating proof link from cached caniddate for " + << block_id.to_str(); + TRY_RESULT_PROMISE_PREFIX(promise, block_root, vm::std_boc_deserialize(it->second.data), + "failed to create proof link: "); + TRY_RESULT_PROMISE_PREFIX(promise, proof_link, WaitBlockData::generate_proof_link(it->second.id, block_root), + "failed to create proof link: "); + promise.set_result(std::move(proof_link)); + return; + } + } callback_->download_block_proof_link(block_id, priority, td::Timestamp::in(10.0), std::move(promise)); } diff --git a/validator/manager.hpp b/validator/manager.hpp index 65636a94c..be7f18db9 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -18,10 +18,14 @@ */ #pragma once +#include "common/refcnt.hpp" #include "interfaces/validator-manager.h" #include "interfaces/db.h" #include "td/actor/PromiseFuture.h" +#include "td/utils/SharedSlice.h" +#include "td/utils/buffer.h" #include "td/utils/port/Poll.h" +#include "td/utils/port/StdStreams.h" #include "validator-group.hpp" #include "shard-client.hpp" #include "manager-init.h" @@ -220,6 +224,10 @@ class ValidatorManagerImpl : public ValidatorManager { }; // DATA FOR COLLATOR std::map> shard_blocks_; + + std::map cached_block_candidates_; + std::list cached_block_candidates_lru_; + struct ExtMessages { std::map, std::unique_ptr>> ext_messages_; std::map, std::map>> @@ -365,6 +373,7 @@ class ValidatorManagerImpl : public ValidatorManager { void new_ihr_message(td::BufferSlice data) override; void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) override; + void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) override; void add_ext_server_id(adnl::AdnlNodeIdShort id) override; void add_ext_server_port(td::uint16 port) override; @@ -409,7 +418,8 @@ class ValidatorManagerImpl : public ValidatorManager { void wait_block_signatures_short(BlockIdExt id, td::Timestamp timeout, td::Promise> promise) override; - void set_block_candidate(BlockIdExt id, BlockCandidate candidate, td::Promise promise) override; + void set_block_candidate(BlockIdExt id, BlockCandidate candidate, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Promise promise) override; void wait_block_state_merge(BlockIdExt left_id, BlockIdExt right_id, td::uint32 priority, td::Timestamp timeout, td::Promise> promise) override; @@ -503,6 +513,7 @@ class ValidatorManagerImpl : public ValidatorManager { } void add_shard_block_description(td::Ref desc); + void add_cached_block_candidate(ReceivedBlock block); void register_block_handle(BlockHandle handle); @@ -664,6 +675,9 @@ class ValidatorManagerImpl : public ValidatorManager { double max_mempool_num() const { return opts_->max_mempool_num(); } + size_t max_cached_candidates() const { + return 128; + } private: std::map> shard_client_waiters_; diff --git a/validator/validator.h b/validator/validator.h index 245acbd49..4e3390cc5 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -142,6 +142,8 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void send_ihr_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_ext_message(AccountIdPrefixFull dst, td::BufferSlice data) = 0; virtual void send_shard_block_info(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; + virtual void send_block_candidate(BlockIdExt block_id, CatchainSeqno cc_seqno, td::uint32 validator_set_hash, + td::BufferSlice data) = 0; virtual void send_broadcast(BlockBroadcast broadcast, bool custom_overlays_only = false) = 0; virtual void download_block(BlockIdExt block_id, td::uint32 priority, td::Timestamp timeout, td::Promise promise) = 0; @@ -210,6 +212,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void check_external_message(td::BufferSlice data, td::Promise> promise) = 0; virtual void new_ihr_message(td::BufferSlice data) = 0; virtual void new_shard_block(BlockIdExt block_id, CatchainSeqno cc_seqno, td::BufferSlice data) = 0; + virtual void new_block_candidate(BlockIdExt block_id, td::BufferSlice data) = 0; virtual void add_ext_server_id(adnl::AdnlNodeIdShort id) = 0; virtual void add_ext_server_port(td::uint16 port) = 0; From d80ce8d3eb34054a85c0e499fe987109b6eafdce Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Mon, 27 May 2024 17:32:11 +0300 Subject: [PATCH 17/28] Fix choosing neighbours in private overlays (#1004) Co-authored-by: SpyCheese --- overlay/overlay-peers.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index 409f09935..e81fecc70 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -229,7 +229,7 @@ void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) { continue; } - if (X->get_version() <= td::Clocks::system() - 600) { + if (public_ && X->get_version() <= td::Clocks::system() - 600) { if (X->is_neighbour()) { bool found = false; for (auto &n : neighbours_) { @@ -301,7 +301,7 @@ void OverlayImpl::get_overlay_random_peers(td::uint32 max_peers, auto t = td::Clocks::system(); while (v.size() < max_peers && v.size() < peers_.size() - bad_peers_.size()) { auto P = peers_.get_random(); - if (P->get_version() + 3600 < t) { + if (public_ && P->get_version() + 3600 < t) { VLOG(OVERLAY_INFO) << this << ": deleting outdated peer " << P->get_id(); del_peer(P->get_id()); } else if (P->is_alive()) { From ceefac74cf3802f6bce95d05b191555549de7c30 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Tue, 28 May 2024 13:31:13 +0300 Subject: [PATCH 18/28] Limit checked external messages per address (#1005) * Limit checked external messages per address * Change max_ext_msg_per_addr_time_window; cleanup mempool by timer --------- Co-authored-by: SpyCheese --- validator/fabric.h | 4 +- validator/impl/external-message.cpp | 20 +++------ validator/impl/external-message.hpp | 3 +- validator/impl/fabric.cpp | 5 +-- validator/manager.cpp | 67 +++++++++++++++++++++++++++-- validator/manager.hpp | 16 +++++++ 6 files changed, 92 insertions(+), 23 deletions(-) diff --git a/validator/fabric.h b/validator/fabric.h index 326b17aec..6bb668452 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -49,8 +49,8 @@ td::Result>> create_new_shard_bloc td::Ref create_signature_set(std::vector sig_set); -void run_check_external_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits, - td::actor::ActorId manager, td::Promise> promise); +void run_check_external_message(td::Ref message, td::actor::ActorId manager, + td::Promise> promise); void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector prev, td::Ref validator_set, td::Ref signatures, diff --git a/validator/impl/external-message.cpp b/validator/impl/external-message.cpp index 073e7360e..2fdb491bc 100644 --- a/validator/impl/external-message.cpp +++ b/validator/impl/external-message.cpp @@ -86,24 +86,18 @@ td::Result> ExtMessageQ::create_ext_message(td::BufferSlice dat return Ref{true, std::move(data), std::move(ext_msg), dest_prefix, wc, addr}; } -void ExtMessageQ::run_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits, - td::actor::ActorId manager, +void ExtMessageQ::run_message(td::Ref message, td::actor::ActorId manager, td::Promise> promise) { - auto R = create_ext_message(std::move(data), limits); - if (R.is_error()) { - return promise.set_error(R.move_as_error_prefix("failed to parse external message ")); - } - auto M = R.move_as_ok(); - auto root = M->root_cell(); + auto root = message->root_cell(); block::gen::CommonMsgInfo::Record_ext_in_msg_info info; tlb::unpack_cell_inexact(root, info); // checked in create message - ton::StdSmcAddress addr = M->addr(); - ton::WorkchainId wc = M->wc(); + ton::StdSmcAddress addr = message->addr(); + ton::WorkchainId wc = message->wc(); run_fetch_account_state( wc, addr, manager, - [promise = std::move(promise), msg_root = root, wc, addr, - M](td::Result, UnixTime, LogicalTime, std::unique_ptr>> + [promise = std::move(promise), msg_root = root, wc, addr, message]( + td::Result, UnixTime, LogicalTime, std::unique_ptr>> res) mutable { if (res.is_error()) { promise.set_error(td::Status::Error(PSLICE() << "Failed to get account state")); @@ -120,7 +114,7 @@ void ExtMessageQ::run_message(td::BufferSlice data, block::SizeLimitsConfig::Ext } else { auto status = run_message_on_account(wc, &acc, utime, lt + 1, msg_root, std::move(config)); if (status.is_ok()) { - promise.set_value(std::move(M)); + promise.set_value(std::move(message)); } else { promise.set_error(td::Status::Error(PSLICE() << "External message was not accepted\n" << status.message())); diff --git a/validator/impl/external-message.hpp b/validator/impl/external-message.hpp index d50847617..ad7ecc74e 100644 --- a/validator/impl/external-message.hpp +++ b/validator/impl/external-message.hpp @@ -61,8 +61,7 @@ class ExtMessageQ : public ExtMessage { ton::StdSmcAddress addr); static td::Result> create_ext_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits); - static void run_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits, - td::actor::ActorId manager, + static void run_message(td::Ref message, td::actor::ActorId manager, td::Promise> promise); static td::Status run_message_on_account(ton::WorkchainId wc, block::Account* acc, diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index e3478594b..997fa9a18 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -119,10 +119,9 @@ td::Result> create_ext_message(td::BufferSlice data, return std::move(res); } -void run_check_external_message(td::BufferSlice data, block::SizeLimitsConfig::ExtMsgLimits limits, - td::actor::ActorId manager, +void run_check_external_message(Ref message, td::actor::ActorId manager, td::Promise> promise) { - ExtMessageQ::run_message(std::move(data), limits, std::move(manager), std::move(promise)); + ExtMessageQ::run_message(std::move(message), std::move(manager), std::move(promise)); } td::Result> create_ihr_message(td::BufferSlice data) { diff --git a/validator/manager.cpp b/validator/manager.cpp index f91f04a17..0dead122c 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -412,14 +412,42 @@ void ValidatorManagerImpl::add_external_message(td::Ref msg, int pri ext_messages_hashes_[id.hash] = {priority, id}; } void ValidatorManagerImpl::check_external_message(td::BufferSlice data, td::Promise> promise) { - ++ls_stats_check_ext_messages_; auto state = do_get_last_liteserver_state(); if (state.is_null()) { promise.set_error(td::Status::Error(ErrorCode::notready, "not ready")); return; } - run_check_external_message(std::move(data), state->get_ext_msg_limits(), actor_id(this), - std::move(promise)); + auto R = create_ext_message(std::move(data), state->get_ext_msg_limits()); + if (R.is_error()) { + promise.set_error(R.move_as_error_prefix("failed to parse external message: ")); + return; + } + auto message = R.move_as_ok(); + WorkchainId wc = message->wc(); + StdSmcAddress addr = message->addr(); + if (checked_ext_msg_counter_.get_msg_count(wc, addr) >= max_ext_msg_per_addr()) { + promise.set_error( + td::Status::Error(PSTRING() << "too many external messages to address " << wc << ":" << addr.to_hex())); + return; + } + + promise = [self = this, wc, addr, promise = std::move(promise), + SelfId = actor_id(this)](td::Result> R) mutable { + if (R.is_error()) { + promise.set_error(R.move_as_error()); + return; + } + td::actor::send_lambda(SelfId, [=, promise = std::move(promise), message = R.move_as_ok()]() mutable { + if (self->checked_ext_msg_counter_.inc_msg_count(wc, addr) > max_ext_msg_per_addr()) { + promise.set_error( + td::Status::Error(PSTRING() << "too many external messages to address " << wc << ":" << addr.to_hex())); + return; + } + promise.set_result(std::move(message)); + }); + }; + ++ls_stats_check_ext_messages_; + run_check_external_message(std::move(message), actor_id(this), std::move(promise)); } void ValidatorManagerImpl::new_ihr_message(td::BufferSlice data) { @@ -2592,6 +2620,16 @@ void ValidatorManagerImpl::alarm() { log_ls_stats_at_ = td::Timestamp::in(60.0); } alarm_timestamp().relax(log_ls_stats_at_); + if (cleanup_mempool_at_.is_in_past()) { + if (is_validator()) { + get_external_messages(ShardIdFull{masterchainId, shardIdAll}, + [](td::Result, int>>>) {}); + get_external_messages(ShardIdFull{basechainId, shardIdAll}, + [](td::Result, int>>>) {}); + } + cleanup_mempool_at_ = td::Timestamp::in(250.0); + } + alarm_timestamp().relax(cleanup_mempool_at_); } void ValidatorManagerImpl::update_shard_client_state(BlockIdExt masterchain_block_id, td::Promise promise) { @@ -3102,6 +3140,29 @@ td::actor::ActorOwn ValidatorManagerFactory::create( rldp, overlays); } +size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) { + before_query(); + auto it1 = counter_cur_.find({wc, addr}); + auto it2 = counter_prev_.find({wc, addr}); + return (it1 == counter_cur_.end() ? 0 : it1->second) + (it2 == counter_prev_.end() ? 0 : it2->second); +} +size_t ValidatorManagerImpl::CheckedExtMsgCounter::inc_msg_count(WorkchainId wc, StdSmcAddress addr) { + before_query(); + auto it2 = counter_prev_.find({wc, addr}); + return (it2 == counter_prev_.end() ? 0 : it2->second) + ++counter_cur_[{wc, addr}]; +} +void ValidatorManagerImpl::CheckedExtMsgCounter::before_query() { + while (cleanup_at_.is_in_past()) { + counter_prev_ = std::move(counter_cur_); + counter_cur_.clear(); + if (counter_prev_.empty()) { + cleanup_at_ = td::Timestamp::in(max_ext_msg_per_addr_time_window() / 2.0); + break; + } + cleanup_at_ += max_ext_msg_per_addr_time_window() / 2.0; + } +} + } // namespace validator } // namespace ton diff --git a/validator/manager.hpp b/validator/manager.hpp index be7f18db9..a78ed0693 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -241,10 +241,20 @@ class ValidatorManagerImpl : public ValidatorManager { }; std::map ext_msgs_; // priority -> messages std::map>> ext_messages_hashes_; // hash -> priority + td::Timestamp cleanup_mempool_at_; // IHR ? std::map, std::unique_ptr>> ihr_messages_; std::map> ihr_messages_hashes_; + struct CheckedExtMsgCounter { + std::map, size_t> counter_cur_, counter_prev_; + td::Timestamp cleanup_at_ = td::Timestamp::now(); + + size_t get_msg_count(WorkchainId wc, StdSmcAddress addr); + size_t inc_msg_count(WorkchainId wc, StdSmcAddress addr); + void before_query(); + } checked_ext_msg_counter_; + private: // VALIDATOR GROUPS ValidatorSessionId get_validator_set_id(ShardIdFull shard, td::Ref val_set, td::Bits256 opts_hash, @@ -678,6 +688,12 @@ class ValidatorManagerImpl : public ValidatorManager { size_t max_cached_candidates() const { return 128; } + static double max_ext_msg_per_addr_time_window() { + return 10.0; + } + static size_t max_ext_msg_per_addr() { + return 3 * 10; + } private: std::map> shard_client_waiters_; From 8a4d44de1ecaaa33e8fe8200d153e0686dae0778 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Wed, 29 May 2024 15:10:42 +0300 Subject: [PATCH 19/28] Compress block candidate broadcasts (#1007) Co-authored-by: SpyCheese --- tl/generate/scheme/ton_api.tl | 2 + tl/generate/scheme/ton_api.tlo | Bin 89652 -> 89968 bytes validator/full-node-private-overlay.cpp | 80 ++++++++++++++++++------ validator/full-node-private-overlay.hpp | 8 +++ validator/full-node-serializer.cpp | 59 +++++++++++++++++ validator/full-node-serializer.hpp | 7 +++ validator/full-node-shard.cpp | 36 ++++++++--- validator/full-node-shard.hpp | 4 ++ 8 files changed, 167 insertions(+), 29 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 5d1e5b504..f9f726497 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -399,6 +399,8 @@ tonNode.newShardBlockBroadcast block:tonNode.newShardBlock = tonNode.Broadcast; // signature may be empty, at least for now tonNode.newBlockCandidateBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int collator_signature:tonNode.blockSignature data:bytes = tonNode.Broadcast; +tonNode.newBlockCandidateBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + collator_signature:tonNode.blockSignature flags:# compressed:bytes = tonNode.Broadcast; tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 89948bb4acb26d8a5255b77af3a6710b2739f3b4..8616657ee34cb0c8efa793e76c7ba0f85c6557d8 100644 GIT binary patch delta 212 zcmdn8hxNlg)(r==d8$-{3I$8@^ZfEtQYZhAR@wYPyMmpuV{@i){B{ FullNode::max_block_size()) { + process_block_candidate_broadcast(src, query); +} + +void FullNodePrivateBlockOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodePrivateBlockOverlay::process_block_candidate_broadcast(PublicKeyHash src, + ton_api::tonNode_Broadcast &query) { + BlockIdExt block_id; + CatchainSeqno cc_seqno; + td::uint32 validator_set_hash; + td::BufferSlice data; + auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, + overlay::Overlays::max_fec_broadcast_size()); + if (S.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << S; + return; + } + if (data.size() > FullNode::max_block_size()) { VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; return; } - BlockIdExt block_id = create_block_id(query.id_); - if (td::sha256_bits256(query.data_.as_slice()) != block_id.file_hash) { + if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; return; } VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in private overlay from " << src << ": " << block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, query.catchain_seqno_, - query.validator_set_hash_, std::move(query.data_)); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, + validator_set_hash, std::move(data)); } void FullNodePrivateBlockOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { @@ -99,12 +118,15 @@ void FullNodePrivateBlockOverlay::send_block_candidate(BlockIdExt block_id, Catc if (!inited_) { return; } + auto B = + serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); + return; + } VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in private overlay: " << block_id.to_str(); - auto B = create_serialize_tl_object( - create_tl_block_id(block_id), cc_seqno, validator_set_hash, - create_tl_object(Bits256::zero(), td::BufferSlice()), std::move(data)); td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); } void FullNodePrivateBlockOverlay::send_broadcast(BlockBroadcast broadcast) { @@ -230,25 +252,42 @@ void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNod } void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeCustomOverlay::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { if (!block_senders_.count(adnl::AdnlNodeIdShort(src))) { VLOG(FULL_NODE_DEBUG) << "Dropping block candidate broadcast in private overlay \"" << name_ << "\" from unauthorized sender " << src; return; } - if (query.data_.size() > FullNode::max_block_size()) { + BlockIdExt block_id; + CatchainSeqno cc_seqno; + td::uint32 validator_set_hash; + td::BufferSlice data; + auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, + overlay::Overlays::max_fec_broadcast_size()); + if (S.is_error()) { + LOG(DEBUG) << "dropped broadcast: " << S; + return; + } + if (data.size() > FullNode::max_block_size()) { VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; return; } - BlockIdExt block_id = create_block_id(query.id_); - if (td::sha256_bits256(query.data_.as_slice()) != block_id.file_hash) { + if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; return; } - // ignore cc_seqno and validator_hash for now VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate in custom overlay \"" << name_ << "\" from " << src << ": " << block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, query.catchain_seqno_, - query.validator_set_hash_, std::move(query.data_)); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, + validator_set_hash, std::move(data)); } void FullNodeCustomOverlay::receive_broadcast(PublicKeyHash src, td::BufferSlice broadcast) { @@ -298,12 +337,15 @@ void FullNodeCustomOverlay::send_block_candidate(BlockIdExt block_id, CatchainSe if (!inited_) { return; } + auto B = + serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); + return; + } VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate in custom overlay \"" << name_ << "\": " << block_id.to_str(); - auto B = create_serialize_tl_object( - create_tl_block_id(block_id), cc_seqno, validator_set_hash, - create_tl_object(Bits256::zero(), td::BufferSlice()), std::move(data)); td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, local_id_, overlay_id_, - local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + local_id_.pubkey_hash(), overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); } void FullNodeCustomOverlay::start_up() { diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index e310824f0..0f76c5ae9 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -27,7 +27,11 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + template void process_broadcast(PublicKeyHash, T &) { VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; @@ -101,7 +105,11 @@ class FullNodeCustomOverlay : public td::actor::Actor { void process_block_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + template void process_broadcast(PublicKeyHash, T &) { VLOG(FULL_NODE_WARNING) << "dropping unknown broadcast"; diff --git a/validator/full-node-serializer.cpp b/validator/full-node-serializer.cpp index 42e682864..94dc2155e 100644 --- a/validator/full-node-serializer.cpp +++ b/validator/full-node-serializer.cpp @@ -152,4 +152,63 @@ td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id return S; } +td::Result serialize_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Slice data, + bool compression_enabled) { + if (!compression_enabled) { + return create_serialize_tl_object( + create_tl_block_id(block_id), cc_seqno, validator_set_hash, + create_tl_object(Bits256::zero(), td::BufferSlice()), td::BufferSlice(data)); + } + TRY_RESULT(root, vm::std_boc_deserialize(data)); + TRY_RESULT(data_new, vm::std_boc_serialize(root, 2)); + td::BufferSlice compressed = td::lz4_compress(data_new); + VLOG(FULL_NODE_DEBUG) << "Compressing block candidate broadcast: " << data.size() << " -> " << compressed.size(); + return create_serialize_tl_object( + create_tl_block_id(block_id), cc_seqno, validator_set_hash, + create_tl_object(Bits256::zero(), td::BufferSlice()), 0, std::move(compressed)); +} + +static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBlockCandidateBroadcast& obj, + BlockIdExt& block_id, CatchainSeqno& cc_seqno, + td::uint32& validator_set_hash, td::BufferSlice& data) { + block_id = create_block_id(obj.id_); + cc_seqno = obj.catchain_seqno_; + validator_set_hash = obj.validator_set_hash_; + data = std::move(obj.data_); + return td::Status::OK(); +} + +static td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_newBlockCandidateBroadcastCompressed& obj, + BlockIdExt& block_id, CatchainSeqno& cc_seqno, + td::uint32& validator_set_hash, td::BufferSlice& data, + int max_decompressed_data_size) { + block_id = create_block_id(obj.id_); + cc_seqno = obj.catchain_seqno_; + validator_set_hash = obj.validator_set_hash_; + TRY_RESULT(decompressed, td::lz4_decompress(obj.compressed_, max_decompressed_data_size)); + TRY_RESULT(root, vm::std_boc_deserialize(decompressed)); + TRY_RESULT_ASSIGN(data, vm::std_boc_serialize(root, 31)); + VLOG(FULL_NODE_DEBUG) << "Decompressing block candidate broadcast: " << obj.compressed_.size() << " -> " + << data.size(); + return td::Status::OK(); +} + +td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_Broadcast& obj, BlockIdExt& block_id, + CatchainSeqno& cc_seqno, td::uint32& validator_set_hash, + td::BufferSlice& data, int max_decompressed_data_size) { + td::Status S; + ton_api::downcast_call(obj, td::overloaded( + [&](ton_api::tonNode_newBlockCandidateBroadcast& f) { + S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, + data); + }, + [&](ton_api::tonNode_newBlockCandidateBroadcastCompressed& f) { + S = deserialize_block_candidate_broadcast(f, block_id, cc_seqno, validator_set_hash, + data, max_decompressed_data_size); + }, + [&](auto&) { S = td::Status::Error("unknown data type"); })); + return S; +} + } // namespace ton::validator::fullnode diff --git a/validator/full-node-serializer.hpp b/validator/full-node-serializer.hpp index a5c73cbc2..f6751689c 100644 --- a/validator/full-node-serializer.hpp +++ b/validator/full-node-serializer.hpp @@ -28,4 +28,11 @@ td::Result serialize_block_full(const BlockIdExt& id, td::Slice td::Status deserialize_block_full(ton_api::tonNode_DataFull& obj, BlockIdExt& id, td::BufferSlice& proof, td::BufferSlice& data, bool& is_proof_link, int max_decompressed_data_size); +td::Result serialize_block_candidate_broadcast(BlockIdExt block_id, CatchainSeqno cc_seqno, + td::uint32 validator_set_hash, td::Slice data, + bool compression_enabled); +td::Status deserialize_block_candidate_broadcast(ton_api::tonNode_Broadcast& obj, BlockIdExt& block_id, + CatchainSeqno& cc_seqno, td::uint32& validator_set_hash, + td::BufferSlice& data, int max_decompressed_data_size); + } // namespace ton::validator::fullnode diff --git a/validator/full-node-shard.cpp b/validator/full-node-shard.cpp index 9b2b321e5..fa0c3e62a 100644 --- a/validator/full-node-shard.cpp +++ b/validator/full-node-shard.cpp @@ -649,19 +649,32 @@ void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_ne } void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query) { - if (query.data_.size() > FullNode::max_block_size()) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeShardImpl::process_broadcast(PublicKeyHash src, + ton_api::tonNode_newBlockCandidateBroadcastCompressed &query) { + process_block_candidate_broadcast(src, query); +} + +void FullNodeShardImpl::process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query) { + BlockIdExt block_id; + CatchainSeqno cc_seqno; + td::uint32 validator_set_hash; + td::BufferSlice data; + auto S = deserialize_block_candidate_broadcast(query, block_id, cc_seqno, validator_set_hash, data, + overlay::Overlays::max_fec_broadcast_size()); + if (data.size() > FullNode::max_block_size()) { VLOG(FULL_NODE_WARNING) << "received block candidate with too big size from " << src; return; } - BlockIdExt block_id = create_block_id(query.id_); - if (td::sha256_bits256(query.data_.as_slice()) != block_id.file_hash) { + if (td::sha256_bits256(data.as_slice()) != block_id.file_hash) { VLOG(FULL_NODE_WARNING) << "received block candidate with incorrect file hash from " << src; return; } - // ignore cc_seqno and validator_hash for now VLOG(FULL_NODE_DEBUG) << "Received newBlockCandidate from " << src << ": " << block_id.to_str(); - td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, query.catchain_seqno_, - query.validator_set_hash_, std::move(query.data_)); + td::actor::send_closure(full_node_, &FullNode::process_block_candidate_broadcast, block_id, cc_seqno, + validator_set_hash, std::move(data)); } void FullNodeShardImpl::process_broadcast(PublicKeyHash src, ton_api::tonNode_blockBroadcast &query) { @@ -762,12 +775,15 @@ void FullNodeShardImpl::send_block_candidate(BlockIdExt block_id, CatchainSeqno UNREACHABLE(); return; } + auto B = + serialize_block_candidate_broadcast(block_id, cc_seqno, validator_set_hash, data, true); // compression enabled + if (B.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to serialize block candidate broadcast: " << B.move_as_error(); + return; + } VLOG(FULL_NODE_DEBUG) << "Sending newBlockCandidate: " << block_id.to_str(); - auto B = create_serialize_tl_object( - create_tl_block_id(block_id), cc_seqno, validator_set_hash, - create_tl_object(Bits256::zero(), td::BufferSlice()), std::move(data)); td::actor::send_closure(overlays_, &overlay::Overlays::send_broadcast_fec_ex, adnl_id_, overlay_id_, local_id_, - overlay::Overlays::BroadcastFlagAnySender(), std::move(B)); + overlay::Overlays::BroadcastFlagAnySender(), B.move_as_ok()); } void FullNodeShardImpl::send_broadcast(BlockBroadcast broadcast) { diff --git a/validator/full-node-shard.hpp b/validator/full-node-shard.hpp index 08f48f862..cec7c6494 100644 --- a/validator/full-node-shard.hpp +++ b/validator/full-node-shard.hpp @@ -153,7 +153,11 @@ class FullNodeShardImpl : public FullNodeShard { void process_broadcast(PublicKeyHash src, ton_api::tonNode_ihrMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_externalMessageBroadcast &query); void process_broadcast(PublicKeyHash src, ton_api::tonNode_newShardBlockBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcast &query); + void process_broadcast(PublicKeyHash src, ton_api::tonNode_newBlockCandidateBroadcastCompressed &query); + void process_block_candidate_broadcast(PublicKeyHash src, ton_api::tonNode_Broadcast &query); + void receive_broadcast(PublicKeyHash src, td::BufferSlice query); void check_broadcast(PublicKeyHash src, td::BufferSlice query, td::Promise promise); From 229d6a8ee9c74560d18fb54b875d27591a4ff210 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Wed, 29 May 2024 15:11:09 +0300 Subject: [PATCH 20/28] Fix TL scheme for liteServer.nonfinal.getValidatorGroups (#1008) Co-authored-by: SpyCheese --- tl/generate/scheme/lite_api.tl | 2 +- tl/generate/scheme/lite_api.tlo | Bin 17584 -> 17584 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/tl/generate/scheme/lite_api.tl b/tl/generate/scheme/lite_api.tl index d74e89536..6d91be8f7 100644 --- a/tl/generate/scheme/lite_api.tl +++ b/tl/generate/scheme/lite_api.tl @@ -98,7 +98,7 @@ liteServer.getLibrariesWithProof id:tonNode.blockIdExt mode:# library_list:(vect liteServer.getShardBlockProof id:tonNode.blockIdExt = liteServer.ShardBlockProof; liteServer.getOutMsgQueueSizes mode:# wc:mode.0?int shard:mode.0?long = liteServer.OutMsgQueueSizes; -liteServer.nonfinal.getValidatorGroups mode:# wc:mode.0?int shard:mode.1?long = liteServer.nonfinal.ValidatorGroups; +liteServer.nonfinal.getValidatorGroups mode:# wc:mode.0?int shard:mode.0?long = liteServer.nonfinal.ValidatorGroups; liteServer.nonfinal.getCandidate id:liteServer.nonfinal.candidateId = liteServer.nonfinal.Candidate; liteServer.queryPrefix = Object; diff --git a/tl/generate/scheme/lite_api.tlo b/tl/generate/scheme/lite_api.tlo index ccae66f367f17a192e3864d0c9935547f046d58c..d6e65b1846084b658e9e370c14620b9670ce3984 100644 GIT binary patch delta 36 rcmdnc$+!WCIviOai_ToCHrY@|Wb#7C>d73IGLsYZ1vYy)WiSB%2TTnN delta 32 mcmdnc$+!WCIviOWbvO2_O*YgKnY_@kda{YS$Yukl1||T`=?kO) From 74801d00b8b1d16826bccdd8f709e142de43ea4e Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 31 May 2024 20:50:31 +0300 Subject: [PATCH 21/28] Command to disable state serializer (#1011) Co-authored-by: SpyCheese --- crypto/vm/boc.h | 6 ++- crypto/vm/large-boc-serializer.cpp | 14 +++-- dht-server/dht-server.cpp | 2 +- tdutils/td/utils/CancellationToken.h | 7 +++ tl/generate/scheme/ton_api.tl | 6 ++- tl/generate/scheme/ton_api.tlo | Bin 89968 -> 90304 bytes .../validator-engine-console-query.cpp | 23 +++++++++ .../validator-engine-console-query.h | 22 ++++++++ .../validator-engine-console.cpp | 1 + validator-engine/validator-engine.cpp | 43 +++++++++++++++- validator-engine/validator-engine.hpp | 4 ++ validator/manager-disk.hpp | 4 ++ validator/manager-hardfork.hpp | 3 ++ validator/manager.cpp | 8 +++ validator/manager.hpp | 2 + validator/state-serializer.cpp | 48 +++++++++++++++--- validator/state-serializer.hpp | 3 ++ validator/validator-options.hpp | 7 +++ validator/validator.h | 3 ++ 19 files changed, 190 insertions(+), 16 deletions(-) diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index 09ae1b661..a5f87774d 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -17,6 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "td/utils/CancellationToken.h" + #include #include #include "vm/db/DynamicBagOfCellsDb.h" @@ -331,7 +333,7 @@ td::Result>> std_boc_deserialize_multi(td::Slice data, int max_roots = BagOfCells::default_max_roots); td::Result std_boc_serialize_multi(std::vector> root, int mode = 0); -td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, - td::FileFd& fd, int mode = 0); +td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, + int mode = 0, td::CancellationToken cancellation_token = {}); } // namespace vm diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index fbd065dc9..a7dae1b08 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -33,7 +33,8 @@ class LargeBocSerializer { public: using Hash = Cell::Hash; - explicit LargeBocSerializer(std::shared_ptr reader) : reader(std::move(reader)) { + explicit LargeBocSerializer(std::shared_ptr reader, td::CancellationToken cancellation_token = {}) + : reader(std::move(reader)), cancellation_token(std::move(cancellation_token)) { } void add_root(Hash root); @@ -84,6 +85,7 @@ class LargeBocSerializer { int revisit(int cell_idx, int force = 0); td::uint64 compute_sizes(int mode, int& r_size, int& o_size); + td::CancellationToken cancellation_token; td::Timestamp log_speed_at_; size_t processed_cells_ = 0; static constexpr double LOG_SPEED_PERIOD = 120.0; @@ -112,6 +114,9 @@ td::Result LargeBocSerializer::import_cell(Hash hash, int depth) { return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large"); } ++processed_cells_; + if (processed_cells_ % 1000 == 0) { + TRY_STATUS(cancellation_token.check()); + } if (log_speed_at_.is_in_past()) { log_speed_at_ += LOG_SPEED_PERIOD; LOG(WARNING) << "serializer: import_cells " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; @@ -408,6 +413,9 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { store_ref(k); } ++processed_cells_; + if (processed_cells_ % 1000 == 0) { + TRY_STATUS(cancellation_token.check()); + } if (log_speed_at_.is_in_past()) { log_speed_at_ += LOG_SPEED_PERIOD; LOG(WARNING) << "serializer: serialize " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; @@ -428,10 +436,10 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { } // namespace td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, - int mode) { + int mode, td::CancellationToken cancellation_token) { td::Timer timer; CHECK(reader != nullptr) - LargeBocSerializer serializer(reader); + LargeBocSerializer serializer(reader, std::move(cancellation_token)); serializer.add_root(root_hash); TRY_STATUS(serializer.import_cells()); TRY_STATUS(serializer.serialize(fd, mode)); diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index 37a158ebb..025cf7d51 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -170,7 +170,7 @@ ton::tl_object_ptr Config::tl() const { return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), - nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); + nullptr, nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, diff --git a/tdutils/td/utils/CancellationToken.h b/tdutils/td/utils/CancellationToken.h index 9f30d204c..7ef304979 100644 --- a/tdutils/td/utils/CancellationToken.h +++ b/tdutils/td/utils/CancellationToken.h @@ -20,6 +20,7 @@ #include #include +#include "Status.h" namespace td { @@ -38,6 +39,12 @@ class CancellationToken { } return token_->is_cancelled_.load(std::memory_order_acquire); } + Status check() const { + if (*this) { + return Status::Error(653, "cancelled"); // cancelled = 653 + } + return Status::OK(); + } CancellationToken() = default; explicit CancellationToken(std::shared_ptr token) : token_(std::move(token)) { } diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index f9f726497..58691ffd7 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -592,11 +592,13 @@ engine.dht.config dht:(vector engine.dht) gc:engine.gc = engine.dht.Config; engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNodeMaster; engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig; -engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) +engine.validator.extraConfig state_serializer_enabled:Bool = engine.validator.ExtraConfig; +engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) validators:(vector engine.validator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) fullnodemasters:(vector engine.validator.fullNodeMaster) fullnodeconfig:engine.validator.fullNodeConfig + extraconfig:engine.validator.extraConfig liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) gc:engine.gc = engine.validator.Config; @@ -711,6 +713,8 @@ engine.validator.addCustomOverlay overlay:engine.validator.customOverlay = engin engine.validator.delCustomOverlay name:string = engine.validator.Success; engine.validator.showCustomOverlays = engine.validator.CustomOverlaysConfig; +engine.validator.setStateSerializerEnabled enabled:Bool = engine.validator.Success; + ---types--- storage.pong = storage.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 8616657ee34cb0c8efa793e76c7ba0f85c6557d8..ad6bb0c99ba1a7218a7336911fd6902cbd09cfaf 100644 GIT binary patch delta 334 zcmeyckM+PoR^CUm^{p77fNdjhurSMhUl!iYDZ+0U72L!O0%cP3(lhf?^~w@+GE)*u z@{9CbD@uwIo%8e3GAAc6%4`l0m5^iX+?;7#9m@*RDn5Oq7NZDHT4p+kck`H#5AXDY zYK%IYO(#xa6i)?dP6laC2U*Ht+jbCS%j7+iW1X#fC5lTDOH$*DQ;RZzPOVBUicig(e!-kkg$1N?^Ofllt&DZsb!8c^OF3WVoKvNR x;j!Y>l3Fo|>1Q znU|_pmY9>7l30>oq?eqZmzJ3hlDm0K$cJb0$A0b21rw(-PS%{<4KZlP()); + if (value != 0 && value != 1) { + return td::Status::Error("expected 0 or 1"); + } + TRY_STATUS(tokenizer_.check_endl()); + enabled_ = value; + return td::Status::OK(); +} + +td::Status SetStateSerializerEnabledQuery::send() { + auto b = ton::create_serialize_tl_object(enabled_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SetStateSerializerEnabledQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 34b516d6c..3047350fe 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1207,3 +1207,25 @@ class ShowCustomOverlaysQuery : public Query { return get_name(); } }; + +class SetStateSerializerEnabledQuery : public Query { + public: + SetStateSerializerEnabledQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "setstateserializerenabled"; + } + static std::string get_help() { + return "setstateserializerenabled \tdisable or enable persistent state serializer; value is 0 or 1"; + } + std::string name() const override { + return get_name(); + } + + private: + bool enabled_; +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 263bca3c0..4878a292f 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -146,6 +146,7 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 66c75827d..f83f978ed 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -73,6 +73,7 @@ #include "block-parse.h" #include "common/delay.h" #include "block/precompiled-smc/PrecompiledSmartContract.h" +#include "interfaces/validator-manager.h" Config::Config() { out_port = 3278; @@ -155,6 +156,11 @@ Config::Config(ton::ton_api::engine_validator_config &config) { if (config.fullnodeconfig_) { full_node_config = ton::validator::fullnode::FullNodeConfig(config.fullnodeconfig_); } + if (config.extraconfig_) { + state_serializer_enabled = config.extraconfig_->state_serializer_enabled_; + } else { + state_serializer_enabled = true; + } for (auto &serv : config.liteservers_) { config_add_lite_server(ton::PublicKeyHash{serv->id_}, serv->port_).ensure(); @@ -231,6 +237,12 @@ ton::tl_object_ptr Config::tl() const { full_node_config_obj = full_node_config.tl(); } + ton::tl_object_ptr extra_config_obj = {}; + if (!state_serializer_enabled) { + // Non-default values + extra_config_obj = ton::create_tl_object(state_serializer_enabled); + } + std::vector> liteserver_vec; for (auto &x : liteservers) { liteserver_vec.push_back(ton::create_tl_object(x.second.tl(), x.first)); @@ -253,7 +265,7 @@ ton::tl_object_ptr Config::tl() const { return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), std::move(full_node_config_obj), - std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); + std::move(extra_config_obj), std::move(liteserver_vec), std::move(control_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -1399,6 +1411,7 @@ td::Status ValidatorEngine::load_global_config() { h.push_back(b); } validator_options_.write().set_hardforks(std::move(h)); + validator_options_.write().set_state_serializer_enabled(config_.state_serializer_enabled); return td::Status::OK(); } @@ -3642,6 +3655,34 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCusto custom_overlays_config_, true)); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + if (query.enabled_ == validator_options_->get_state_serializer_enabled()) { + promise.set_value(ton::create_serialize_tl_object()); + return; + } + validator_options_.write().set_state_serializer_enabled(query.enabled_); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + config_.state_serializer_enabled = query.enabled_; + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::create_serialize_tl_object()); + } + }); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 44fb1c27c..877a2be52 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -90,6 +90,8 @@ struct Config { std::map controls; std::set gc; + bool state_serializer_enabled = true; + void decref(ton::PublicKeyHash key); void incref(ton::PublicKeyHash key) { keys_refcnt[key]++; @@ -473,6 +475,8 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 12bd68ef2..389c7c0de 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -422,6 +422,10 @@ class ValidatorManagerImpl : public ValidatorManager { promise.set_result(td::Status::Error("not implemented")); } + void update_options(td::Ref opts) override { + opts_ = std::move(opts); + } + private: PublicKeyHash local_id_; diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 3a124f827..7bf95b3f7 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -483,6 +483,9 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { promise.set_result(td::Status::Error("not implemented")); } + void update_options(td::Ref opts) override { + opts_ = std::move(opts); + } private: td::Ref opts_; diff --git a/validator/manager.cpp b/validator/manager.cpp index 0dead122c..2af818e6f 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3132,6 +3132,14 @@ void ValidatorManagerImpl::get_validator_groups_info_for_litequery( td::actor::create_actor("get-validator-groups-info", std::move(groups), std::move(promise)).release(); } +void ValidatorManagerImpl::update_options(td::Ref opts) { + // Currently options can be updated only to change state_serializer_enabled flag + if (!serializer_.empty()) { + td::actor::send_closure(serializer_, &AsyncStateSerializer::update_options, opts); + } + opts_ = std::move(opts); +} + td::actor::ActorOwn ValidatorManagerFactory::create( td::Ref opts, std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, diff --git a/validator/manager.hpp b/validator/manager.hpp index a78ed0693..f76900a9e 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -588,6 +588,8 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override; void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override; + void update_options(td::Ref opts) override; + void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { if (last_masterchain_state_.is_null()) { diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index 93363d3b2..4f10d959e 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -27,6 +27,9 @@ namespace ton { namespace validator { void AsyncStateSerializer::start_up() { + if (!opts_->get_state_serializer_enabled()) { + LOG(ERROR) << "Persistent state serializer is disabled"; + } alarm_timestamp() = td::Timestamp::in(1.0 + td::Random::fast(0, 10) * 1.0); running_ = true; @@ -130,7 +133,7 @@ void AsyncStateSerializer::next_iteration() { } CHECK(masterchain_handle_->id() == last_block_id_); if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno && - need_serialize(masterchain_handle_)) { + need_serialize(masterchain_handle_) && opts_->get_state_serializer_enabled()) { if (!have_masterchain_state_) { LOG(ERROR) << "started serializing persistent state for " << masterchain_handle_->id().id.to_str(); // block next attempts immediately, but send actual request later @@ -174,6 +177,9 @@ void AsyncStateSerializer::next_iteration() { return; } if (masterchain_handle_->inited_next_left()) { + if (need_serialize(masterchain_handle_) && !opts_->get_state_serializer_enabled()) { + LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str(); + } last_block_id_ = masterchain_handle_->one_next(true); have_masterchain_state_ = false; masterchain_handle_ = nullptr; @@ -200,6 +206,10 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) { void AsyncStateSerializer::got_masterchain_state(td::Ref state, std::shared_ptr cell_db_reader) { + if (!opts_->get_state_serializer_enabled()) { + stored_masterchain_state(); + return; + } LOG(ERROR) << "serializing masterchain state " << masterchain_handle_->id().id.to_str(); have_masterchain_state_ = true; CHECK(next_idx_ == 0); @@ -210,11 +220,16 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state shards_.push_back(v->top_block_id()); } - auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader](td::FileFd& fd) { - return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31); + auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader, + cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { + return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token)); }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); + if (R.is_error() && R.error().code() == cancelled) { + LOG(ERROR) << "Persistent state serialization cancelled"; + } else { + R.ensure(); + } td::actor::send_closure(SelfId, &AsyncStateSerializer::stored_masterchain_state); }); @@ -253,13 +268,22 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) { void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref state, std::shared_ptr cell_db_reader) { + if (!opts_->get_state_serializer_enabled()) { + success_handler(); + return; + } LOG(ERROR) << "serializing shard state " << handle->id().id.to_str(); - auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader](td::FileFd& fd) { - return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31); + auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader, + cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { + return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token)); }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { - R.ensure(); - LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str(); + if (R.is_error() && R.error().code() == cancelled) { + LOG(ERROR) << "Persistent state serialization cancelled"; + } else { + R.ensure(); + LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str(); + } td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(), @@ -285,6 +309,14 @@ void AsyncStateSerializer::success_handler() { next_iteration(); } +void AsyncStateSerializer::update_options(td::Ref opts) { + opts_ = std::move(opts); + if (!opts_->get_state_serializer_enabled()) { + cancellation_token_source_.cancel(); + } +} + + bool AsyncStateSerializer::need_monitor(ShardIdFull shard) { return opts_->need_monitor(shard); } diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index ee2aace01..0bee70315 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -37,6 +37,7 @@ class AsyncStateSerializer : public td::actor::Actor { bool saved_to_db_ = true; td::Ref opts_; + td::CancellationTokenSource cancellation_token_source_; td::actor::ActorId manager_; @@ -89,6 +90,8 @@ class AsyncStateSerializer : public td::actor::Actor { void fail_handler(td::Status reason); void fail_handler_cont(); void success_handler(); + + void update_options(td::Ref opts); }; } // namespace validator diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 593c75bf6..37006bdad 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -141,6 +141,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { td::optional get_catchain_max_block_delay() const override { return catchain_max_block_delay_; } + bool get_state_serializer_enabled() const override { + return state_serializer_enabled_; + } void set_zero_block_id(BlockIdExt block_id) override { zero_block_id_ = block_id; @@ -221,6 +224,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_catchain_max_block_delay(double value) override { catchain_max_block_delay_ = value; } + void set_state_serializer_enabled(bool value) override { + state_serializer_enabled_ = value; + } ValidatorManagerOptionsImpl *make_copy() const override { return new ValidatorManagerOptionsImpl(*this); @@ -272,6 +278,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool celldb_direct_io_ = false; bool celldb_preload_all_ = false; td::optional catchain_max_block_delay_; + bool state_serializer_enabled_ = true; }; } // namespace validator diff --git a/validator/validator.h b/validator/validator.h index 4e3390cc5..b64cfb51d 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -90,6 +90,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual bool get_celldb_direct_io() const = 0; virtual bool get_celldb_preload_all() const = 0; virtual td::optional get_catchain_max_block_delay() const = 0; + virtual bool get_state_serializer_enabled() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; virtual void set_init_block_id(BlockIdExt block_id) = 0; @@ -118,6 +119,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_celldb_direct_io(bool value) = 0; virtual void set_celldb_preload_all(bool value) = 0; virtual void set_catchain_max_block_delay(double value) = 0; + virtual void set_state_serializer_enabled(bool value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, @@ -249,6 +251,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void add_perf_timer_stat(std::string name, double duration) = 0; virtual void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) = 0; + virtual void update_options(td::Ref opts) = 0; }; } // namespace validator From 3d7e85bafd649508ee3cc6a634cd47931fb21cc6 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Sun, 2 Jun 2024 13:41:02 +0300 Subject: [PATCH 22/28] Rollback celldb default flag values and decrease default TTLs (#1012) * Rollback celldb default flag values and decrease default TTLs * Fix description --- validator-engine/validator-engine.cpp | 16 ++++++++-------- validator-engine/validator-engine.hpp | 6 +++--- validator/validator.h | 4 ++-- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index f83f978ed..897e3c53b 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3897,7 +3897,7 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); }); return td::Status::OK(); }); - p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=7*86400", + p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=86400", [&](td::Slice fname) { auto v = td::to_double(fname); if (v <= 0) { @@ -3907,7 +3907,7 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); p.add_checked_option( - 'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=365*86400", + 'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=7*86400", [&](td::Slice fname) { auto v = td::to_double(fname); if (v <= 0) { @@ -4020,7 +4020,7 @@ int main(int argc, char *argv[]) { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_nonfinal_ls_queries_enabled); }); }); p.add_checked_option( - '\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 50G)", + '\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 1G)", [&](td::Slice s) -> td::Status { TRY_RESULT(v, td::to_integer_safe(s)); if (v == 0) { @@ -4030,12 +4030,12 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); p.add_option( - '\0', "celldb-no-direct-io", "disable direct I/O mode for RocksDb in CellDb (forced when celldb cache is < 30G)", - [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, false); }); }); + '\0', "celldb-direct-io", "enable direct I/O mode for RocksDb in CellDb (doesn't apply when celldb cache is < 30G)", + [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, true); }); }); p.add_option( - '\0', "celldb-no-preload-all", - "disable preloading all cells from CellDb on startup (enabled by default)", - [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, false); }); }); + '\0', "celldb-preload-all", + "preload all cells from CellDb on startup (recommended to use with big enough celldb-cache-size and celldb-direct-io)", + [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, true); }); }); p.add_checked_option( '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.5)", [&](td::Slice s) -> td::Status { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 877a2be52..8adc8d9a7 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -211,9 +211,9 @@ class ValidatorEngine : public td::actor::Actor { double archive_preload_period_ = 0.0; bool disable_rocksdb_stats_ = false; bool nonfinal_ls_queries_enabled_ = false; - td::optional celldb_cache_size_ = 50LL << 30; - bool celldb_direct_io_ = true; - bool celldb_preload_all_ = true; + td::optional celldb_cache_size_ = 1LL << 30; + bool celldb_direct_io_ = false; + bool celldb_preload_all_ = false; td::optional catchain_max_block_delay_; bool read_config_ = false; bool started_keyring_ = false; diff --git a/validator/validator.h b/validator/validator.h index b64cfb51d..9082fd882 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -125,8 +125,8 @@ struct ValidatorManagerOptions : public td::CntObject { BlockIdExt zero_block_id, BlockIdExt init_block_id, std::function check_shard = [](ShardIdFull, CatchainSeqno, ShardCheckMode) { return true; }, - bool allow_blockchain_init = false, double sync_blocks_before = 86400, double block_ttl = 86400 * 7, - double state_ttl = 3600, double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, + bool allow_blockchain_init = false, double sync_blocks_before = 3600, double block_ttl = 86400, + double state_ttl = 3600, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650, double max_mempool_num = 999999, bool initial_sync_disabled = false); }; From 0301e9fbb771d1124b61519be752efbe181cb4b5 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Tue, 4 Jun 2024 13:03:58 +0300 Subject: [PATCH 23/28] Fix loading key block config in FullNodeImpl (#1013) Co-authored-by: SpyCheese --- validator/full-node-private-overlay.hpp | 9 +--- validator/full-node.cpp | 57 +++++++++++++------------ validator/full-node.hpp | 4 +- 3 files changed, 32 insertions(+), 38 deletions(-) diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index 0f76c5ae9..a0022fa03 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -47,15 +47,11 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { config_ = std::move(config); } - void set_enable_compression(bool value) { - enable_compression_ = value; - } - void start_up() override; void tear_down() override; FullNodePrivateBlockOverlay(adnl::AdnlNodeIdShort local_id, std::vector nodes, - FileHash zero_state_file_hash, FullNodeConfig config, bool enable_compression, + FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, @@ -65,7 +61,6 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { , nodes_(std::move(nodes)) , zero_state_file_hash_(zero_state_file_hash) , config_(config) - , enable_compression_(enable_compression) , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) @@ -80,7 +75,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { std::vector nodes_; FileHash zero_state_file_hash_; FullNodeConfig config_; - bool enable_compression_; + bool enable_compression_ = true; td::actor::ActorId keyring_; td::actor::ActorId adnl_; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index ff8f1eca0..47c379a56 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -369,14 +369,12 @@ td::actor::ActorId FullNodeImpl::get_shard(AccountIdPrefixFull ds return get_shard(shard_prefix(dst, 60)); } -void FullNodeImpl::got_key_block_state(td::Ref state) { - auto m = td::Ref{std::move(state)}; - +void FullNodeImpl::got_key_block_config(td::Ref config) { PublicKeyHash l = PublicKeyHash::zero(); std::vector keys; std::map current_validators; for (td::int32 i = -1; i <= 1; i++) { - auto r = m->get_total_validator_set(i < 0 ? i : 1 - i); + auto r = config->get_total_validator_set(i < 0 ? i : 1 - i); if (r.not_null()) { auto vec = r->export_vector(); for (auto &el : vec) { @@ -392,8 +390,6 @@ void FullNodeImpl::got_key_block_state(td::Ref state) { } } - set_private_block_overlays_enable_compression(m->get_consensus_config().proto_version >= 3); - if (current_validators != current_validators_) { current_validators_ = std::move(current_validators); update_private_overlays(); @@ -413,15 +409,31 @@ void FullNodeImpl::got_key_block_state(td::Ref state) { } void FullNodeImpl::new_key_block(BlockHandle handle) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - VLOG(FULL_NODE_WARNING) << "failed to get key block state: " << R.move_as_error(); - } else { - td::actor::send_closure(SelfId, &FullNodeImpl::got_key_block_state, R.move_as_ok()); - } - }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_shard_state_from_db, handle, - std::move(P)); + if (handle->id().seqno() == 0) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to get zero state: " << R.move_as_error(); + } else { + auto s = td::Ref{R.move_as_ok()}; + CHECK(s.not_null()); + td::actor::send_closure(SelfId, &FullNodeImpl::got_key_block_config, s->get_config_holder().move_as_ok()); + } + }); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_shard_state_from_db, handle, + std::move(P)); + } else { + CHECK(handle->is_key_block()); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to get key block proof: " << R.move_as_error(); + } else { + td::actor::send_closure(SelfId, &FullNodeImpl::got_key_block_config, + R.ok()->get_key_block_config().move_as_ok()); + } + }); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_block_proof_link_from_db, handle, + std::move(P)); + } } void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) { @@ -549,16 +561,6 @@ void FullNodeImpl::update_private_overlays() { } } -void FullNodeImpl::set_private_block_overlays_enable_compression(bool value) { - if (private_block_overlays_enable_compression_ == value) { - return; - } - private_block_overlays_enable_compression_ = true; - for (auto &p : private_block_overlays_) { - td::actor::send_closure(p.second, &FullNodePrivateBlockOverlay::set_enable_compression, value); - } -} - void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { CHECK(local_keys_.count(key)); if (current_validators_.count(key)) { @@ -567,9 +569,8 @@ void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { nodes.push_back(p.second); } private_block_overlays_[key] = td::actor::create_actor( - "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, - private_block_overlays_enable_compression_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_, - actor_id(this)); + "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, keyring_, + adnl_, rldp_, rldp2_, overlays_, validator_manager_, actor_id(this)); } } diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 9b78b3e01..3dfa17fdd 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -82,7 +82,7 @@ class FullNodeImpl : public FullNode { void download_archive(BlockSeqno masterchain_seqno, std::string tmp_dir, td::Timestamp timeout, td::Promise promise); - void got_key_block_state(td::Ref state); + void got_key_block_config(td::Ref config); void new_key_block(BlockHandle handle); void process_block_broadcast(BlockBroadcast broadcast) override; @@ -127,7 +127,6 @@ class FullNodeImpl : public FullNode { FullNodeConfig config_; std::map> private_block_overlays_; - bool private_block_overlays_enable_compression_ = false; bool broadcast_block_candidates_in_public_overlay_ = false; struct CustomOverlayInfo { @@ -139,7 +138,6 @@ class FullNodeImpl : public FullNode { std::queue custom_overlays_sent_broadcasts_lru_; void update_private_overlays(); - void set_private_block_overlays_enable_compression(bool value); void create_private_block_overlay(PublicKeyHash key); void update_custom_overlay(CustomOverlayInfo& overlay); void send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast); From f9b6d21620f839734166da67fa4b641facd85054 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Tue, 4 Jun 2024 13:09:06 +0300 Subject: [PATCH 24/28] Update validator list on each keyblock --- validator/full-node.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/validator/full-node.cpp b/validator/full-node.cpp index 47c379a56..a72be3ff4 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -395,9 +395,10 @@ void FullNodeImpl::got_key_block_config(td::Ref config) { update_private_overlays(); } - if (keys == all_validators_) { - return; - } + // Let's turn off this optimization, since keyblocks are rare enough to update on each keyblock + // if (keys == all_validators_) { + // return; + // } all_validators_ = keys; sign_cert_by_ = l; From 5186c4755c39205260bc2b01b3eef325d9feb09c Mon Sep 17 00:00:00 2001 From: neodix42 Date: Fri, 7 Jun 2024 17:50:11 +0200 Subject: [PATCH 25/28] Change fift path separator for FIFTPATH and -I argument (#1014) * add github action for macOS 14 (arm64, M1) * add github action (portable) for macOS 14 (arm64, M1) * rename macOS arm64 output artifact * Colon cannot be used as a path separator for FIFTPATH or -I argument in fift on Windows when absolute paths are used (e.g. C:\path\lib:C:\path\smartcont). Suggestion to use @ as a new path separator on Windows. --------- Co-authored-by: neodiX --- .../build-ton-macos-arm64-shared.yml | 25 ++++ .github/workflows/create-release.yml | 107 ++++++++++++++++++ .github/workflows/ton-arm64-macos.yml | 37 ++++++ crypto/CMakeLists.txt | 9 +- crypto/block/create-state.cpp | 9 +- crypto/fift/fift-main.cpp | 11 +- 6 files changed, 192 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/build-ton-macos-arm64-shared.yml create mode 100644 .github/workflows/ton-arm64-macos.yml diff --git a/.github/workflows/build-ton-macos-arm64-shared.yml b/.github/workflows/build-ton-macos-arm64-shared.yml new file mode 100644 index 000000000..2a68272cc --- /dev/null +++ b/.github/workflows/build-ton-macos-arm64-shared.yml @@ -0,0 +1,25 @@ +name: MacOS TON build (shared, arm64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: macos-14 + + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Build TON + run: | + cp assembly/native/build-macos-shared.sh . + chmod +x build-macos-shared.sh + ./build-macos-shared.sh -t -a + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-binaries-macos-14 + path: artifacts diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 50d2661ba..367dd6638 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -35,6 +35,14 @@ jobs: workflow_conclusion: success skip_unpack: true + - name: Download Mac arm64 artifacts + uses: dawidd6/action-download-artifact@v2 + with: + workflow: ton-arm64-macos.yml + path: artifacts + workflow_conclusion: success + skip_unpack: true + - name: Download and unzip Mac x86-64 artifacts uses: dawidd6/action-download-artifact@v2 with: @@ -43,6 +51,14 @@ jobs: workflow_conclusion: success skip_unpack: false + - name: Download and unzip arm64 artifacts + uses: dawidd6/action-download-artifact@v2 + with: + workflow: ton-arm64-macos.yml + path: artifacts + workflow_conclusion: success + skip_unpack: false + - name: Download Windows artifacts uses: dawidd6/action-download-artifact@v2 with: @@ -297,6 +313,97 @@ jobs: asset_name: tonlib-cli-mac-x86-64 tag: ${{ steps.tag.outputs.TAG }} + + # mac arm64 + + - name: Upload Mac arm64 artifacts + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries.zip + asset_name: ton-mac-arm64.zip + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - fift + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/fift + asset_name: fift-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - func + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/func + asset_name: func-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - lite-client + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/lite-client + asset_name: lite-client-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - rldp-http-proxy + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/rldp-http-proxy + asset_name: rldp-http-proxy-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - http-proxy + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/http-proxy + asset_name: http-proxy-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - storage-daemon-cli + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/storage-daemon-cli + asset_name: storage-daemon-cli-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - storage-daemon + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/storage-daemon + asset_name: storage-daemon-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - tonlibjson + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/libtonlibjson.dylib + asset_name: tonlibjson-mac-arm64.dylib + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - libemulator + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/libemulator.dylib + asset_name: libemulator-mac-arm64.dylib + tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Mac arm64 single artifact - tonlib-cli + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-arm64-macos-binaries/tonlib-cli + asset_name: tonlib-cli-mac-arm64 + tag: ${{ steps.tag.outputs.TAG }} + # linux x86-64 - name: Upload Linux x86-64 artifacts diff --git a/.github/workflows/ton-arm64-macos.yml b/.github/workflows/ton-arm64-macos.yml new file mode 100644 index 000000000..9e8302e80 --- /dev/null +++ b/.github/workflows/ton-arm64-macos.yml @@ -0,0 +1,37 @@ +name: MacOS TON build (portable, arm64) + +on: [push,workflow_dispatch,workflow_call] + +jobs: + build: + runs-on: macos-14 + + steps: + - uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - uses: cachix/install-nix-action@v23 + with: + extra_nix_config: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + + - name: Build TON + run: | + cp assembly/nix/build-macos-nix.sh . + chmod +x build-macos-nix.sh + ./build-macos-nix.sh -t + + - name: Simple binaries test + run: | + sudo mv /nix/store /nix/store2 + artifacts/validator-engine -V + artifacts/lite-client -V + artifacts/fift -V + artifacts/func -V + + - name: Upload artifacts + uses: actions/upload-artifact@master + with: + name: ton-arm64-macos-binaries + path: artifacts diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index 29b954664..306194408 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -479,10 +479,17 @@ if (NOT CMAKE_CROSSCOMPILING OR USE_EMSCRIPTEN) OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/${ARG_DEST_FIF} ) set(ARG_DEST_CPP "${ARG_DEST}.cpp") + + if (WIN32) + set(ARG_LIB_DIR "fift/lib@smartcont") + else() + set(ARG_LIB_DIR "fift/lib:smartcont") + endif() + add_custom_command( COMMENT "Generate ${ARG_DEST_CPP}" WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} - COMMAND fift -Ifift/lib:smartcont -s asm-to-cpp.fif ${ARG_DEST_FIF} ${ARG_DEST_CPP} ${ARG_NAME} + COMMAND fift -I${ARG_LIB_DIR} -s asm-to-cpp.fif ${ARG_DEST_FIF} ${ARG_DEST_CPP} ${ARG_NAME} MAIN_DEPENDENCY ${ARG_SOURCE} DEPENDS fift ${ARG_DEST_FIF} smartcont/asm-to-cpp.fif fift/lib/Asm.fif OUTPUT ${CMAKE_CURRENT_SOURCE_DIR}/${ARG_DEST_CPP} diff --git a/crypto/block/create-state.cpp b/crypto/block/create-state.cpp index 183da0a73..348377e94 100644 --- a/crypto/block/create-state.cpp +++ b/crypto/block/create-state.cpp @@ -814,11 +814,16 @@ void usage(const char* progname) { void parse_include_path_set(std::string include_path_set, std::vector& res) { td::Parser parser(include_path_set); while (!parser.empty()) { - auto path = parser.read_till_nofail(':'); + #if TD_WINDOWS + auto path_separator = '@'; + #else + auto path_separator = ':'; + #endif + auto path = parser.read_till_nofail(path_separator); if (!path.empty()) { res.push_back(path.str()); } - parser.skip_nofail(':'); + parser.skip_nofail(path_separator); } } diff --git a/crypto/fift/fift-main.cpp b/crypto/fift/fift-main.cpp index ef833f43e..fd424e8cf 100644 --- a/crypto/fift/fift-main.cpp +++ b/crypto/fift/fift-main.cpp @@ -62,7 +62,7 @@ void usage(const char* progname) { << " [-i] [-n] [-I ] {-L } ...\n"; std::cerr << "\t-n\tDo not preload standard preamble file `Fift.fif`\n" "\t-i\tForce interactive mode even if explicit source file names are indicated\n" - "\t-I\tSets colon-separated library source include path. If not indicated, " + "\t-I\tSets colon-separated (unix) or at-separated (windows) library source include path. If not indicated, " "$FIFTPATH is used instead.\n" "\t-L\tPre-loads a library source file\n" "\t-d\tUse a ton database\n" @@ -75,11 +75,16 @@ void usage(const char* progname) { void parse_include_path_set(std::string include_path_set, std::vector& res) { td::Parser parser(include_path_set); while (!parser.empty()) { - auto path = parser.read_till_nofail(':'); + #if TD_WINDOWS + auto path_separator = '@'; + #else + auto path_separator = ':'; + #endif + auto path = parser.read_till_nofail(path_separator); if (!path.empty()) { res.push_back(path.str()); } - parser.skip_nofail(':'); + parser.skip_nofail(path_separator); } } From 0a0a92c6b04c77b9a59d65ba503c3c847dce885f Mon Sep 17 00:00:00 2001 From: neodix42 Date: Fri, 7 Jun 2024 17:52:11 +0200 Subject: [PATCH 26/28] Use jemalloc in portable artifacts (#1003) * use jemalloc in portable builds; also avoid mixing musl with glibc artifacts in nix builds; * minor fix for ubuntu arm nix build --- assembly/nix/build-linux-arm64-nix.sh | 4 +-- assembly/nix/build-linux-x86-64-nix.sh | 4 +-- assembly/nix/build-macos-nix.sh | 2 ++ assembly/nix/linux-arm64-static.nix | 44 +++++++++++++++-------- assembly/nix/linux-arm64-tonlib.nix | 26 ++++++++++---- assembly/nix/linux-x86-64-static.nix | 44 +++++++++++++++-------- assembly/nix/linux-x86-64-tonlib.nix | 50 ++++++++++++++++++-------- assembly/nix/macos-static.nix | 4 +-- assembly/nix/microhttpd.nix | 28 --------------- assembly/nix/openssl.nix | 30 ---------------- 10 files changed, 123 insertions(+), 113 deletions(-) delete mode 100644 assembly/nix/microhttpd.nix delete mode 100644 assembly/nix/openssl.nix diff --git a/assembly/nix/build-linux-arm64-nix.sh b/assembly/nix/build-linux-arm64-nix.sh index 8e5c367c9..7e85a8712 100644 --- a/assembly/nix/build-linux-arm64-nix.sh +++ b/assembly/nix/build-linux-arm64-nix.sh @@ -15,8 +15,6 @@ while getopts 't' flag; do done cp assembly/nix/linux-arm64* . -cp assembly/nix/microhttpd.nix . -cp assembly/nix/openssl.nix . export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz if [ "$with_tests" = true ]; then @@ -30,7 +28,9 @@ cp ./result/bin/* artifacts/ test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; } chmod +x artifacts/* rm -rf result + nix-build linux-arm64-tonlib.nix + cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so cp ./result/lib/libemulator.so artifacts/ cp ./result/lib/fift/* artifacts/lib/ diff --git a/assembly/nix/build-linux-x86-64-nix.sh b/assembly/nix/build-linux-x86-64-nix.sh index 38431ca43..c1f1dcf37 100644 --- a/assembly/nix/build-linux-x86-64-nix.sh +++ b/assembly/nix/build-linux-x86-64-nix.sh @@ -15,8 +15,6 @@ while getopts 't' flag; do done cp assembly/nix/linux-x86-64* . -cp assembly/nix/microhttpd.nix . -cp assembly/nix/openssl.nix . export NIX_PATH=nixpkgs=https://github.com/nixOS/nixpkgs/archive/23.05.tar.gz if [ "$with_tests" = true ]; then @@ -30,7 +28,9 @@ cp ./result/bin/* artifacts/ test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; } chmod +x artifacts/* rm -rf result + nix-build linux-x86-64-tonlib.nix + cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so cp ./result/lib/libemulator.so artifacts/ cp ./result/lib/fift/* artifacts/lib/ diff --git a/assembly/nix/build-macos-nix.sh b/assembly/nix/build-macos-nix.sh index 12977745b..c75ca0428 100644 --- a/assembly/nix/build-macos-nix.sh +++ b/assembly/nix/build-macos-nix.sh @@ -28,7 +28,9 @@ cp ./result-bin/bin/* artifacts/ test $? -eq 0 || { echo "No artifacts have been built..."; exit 1; } chmod +x artifacts/* rm -rf result-bin + nix-build macos-tonlib.nix + cp ./result/lib/libtonlibjson.dylib artifacts/ cp ./result/lib/libemulator.dylib artifacts/ cp ./result/lib/fift/* artifacts/lib/ diff --git a/assembly/nix/linux-arm64-static.nix b/assembly/nix/linux-arm64-static.nix index 8c2749b07..536152d26 100644 --- a/assembly/nix/linux-arm64-static.nix +++ b/assembly/nix/linux-arm64-static.nix @@ -6,9 +6,23 @@ , testing ? false }: let - microhttpdmy = (import ./microhttpd.nix) {}; + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + jemallocStatic = (staticOptions pkgs.jemalloc); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); + in -with import microhttpdmy; stdenv.mkDerivation { pname = "ton"; version = "dev-bin"; @@ -16,31 +30,33 @@ stdenv.mkDerivation { src = ./.; nativeBuildInputs = with pkgs; - [ - cmake ninja git pkg-config - ]; + [ cmake ninja git pkg-config ]; buildInputs = with pkgs; [ - pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static pkgsStatic.lz4 + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + jemallocStatic + secp256k1Static + libsodiumStatic.dev + glibc.static ]; - makeStatic = true; - doCheck = testing; - cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" "-DNIX=ON" "-DBUILD_SHARED_LIBS=OFF" "-DCMAKE_LINK_SEARCH_START_STATIC=ON" "-DCMAKE_LINK_SEARCH_END_STATIC=ON" - "-DMHD_FOUND=1" - "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include" - "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a" - "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800" + "-DTON_USE_JEMALLOC=ON" ]; + makeStatic = true; + doCheck = testing; + LDFLAGS = [ - "-static-libgcc" "-static-libstdc++" "-static" + "-static-libgcc" "-static-libstdc++" "-static" ]; } diff --git a/assembly/nix/linux-arm64-tonlib.nix b/assembly/nix/linux-arm64-tonlib.nix index ae62ca263..a051e34cd 100644 --- a/assembly/nix/linux-arm64-tonlib.nix +++ b/assembly/nix/linux-arm64-tonlib.nix @@ -5,9 +5,21 @@ , stdenv ? pkgs.stdenv }: let - microhttpdmy = (import ./microhttpd.nix) {}; + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); in -with import microhttpdmy; pkgs.llvmPackages_16.stdenv.mkDerivation { pname = "ton"; version = "dev-lib"; @@ -21,7 +33,12 @@ pkgs.llvmPackages_16.stdenv.mkDerivation { buildInputs = with pkgs; [ - pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 pkgsStatic.lz4 + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + secp256k1Static + libsodiumStatic.dev ]; dontAddStaticConfigureFlags = false; @@ -29,9 +46,6 @@ pkgs.llvmPackages_16.stdenv.mkDerivation { cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" "-DNIX=ON" - "-DMHD_FOUND=1" - "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include" - "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a" ]; LDFLAGS = [ diff --git a/assembly/nix/linux-x86-64-static.nix b/assembly/nix/linux-x86-64-static.nix index 8c2749b07..a96a9e867 100644 --- a/assembly/nix/linux-x86-64-static.nix +++ b/assembly/nix/linux-x86-64-static.nix @@ -6,9 +6,23 @@ , testing ? false }: let - microhttpdmy = (import ./microhttpd.nix) {}; + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + jemallocStatic = (staticOptions pkgs.jemalloc); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); + in -with import microhttpdmy; stdenv.mkDerivation { pname = "ton"; version = "dev-bin"; @@ -16,31 +30,33 @@ stdenv.mkDerivation { src = ./.; nativeBuildInputs = with pkgs; - [ - cmake ninja git pkg-config - ]; + [ cmake ninja git pkg-config ]; buildInputs = with pkgs; [ - pkgsStatic.openssl microhttpdmy pkgsStatic.zlib pkgsStatic.libsodium.dev pkgsStatic.secp256k1 glibc.static pkgsStatic.lz4 + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + jemallocStatic + secp256k1Static + libsodiumStatic.dev + glibc.static ]; - makeStatic = true; - doCheck = testing; - cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" "-DNIX=ON" "-DBUILD_SHARED_LIBS=OFF" "-DCMAKE_LINK_SEARCH_START_STATIC=ON" "-DCMAKE_LINK_SEARCH_END_STATIC=ON" - "-DMHD_FOUND=1" - "-DMHD_INCLUDE_DIR=${microhttpdmy}/usr/local/include" - "-DMHD_LIBRARY=${microhttpdmy}/usr/local/lib/libmicrohttpd.a" - "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800" + "-DTON_USE_JEMALLOC=ON" ]; + makeStatic = true; + doCheck = testing; + LDFLAGS = [ - "-static-libgcc" "-static-libstdc++" "-static" + "-static-libgcc" "-static-libstdc++" "-fPIC" ]; } diff --git a/assembly/nix/linux-x86-64-tonlib.nix b/assembly/nix/linux-x86-64-tonlib.nix index 5a6e43e8f..afcbe3ba6 100644 --- a/assembly/nix/linux-x86-64-tonlib.nix +++ b/assembly/nix/linux-x86-64-tonlib.nix @@ -7,20 +7,35 @@ , stdenv ? pkgs.stdenv }: let - system = builtins.currentSystem; - - nixos1909 = (import (builtins.fetchTarball { - url = "https://channels.nixos.org/nixos-19.09/nixexprs.tar.xz"; - sha256 = "1vp1h2gkkrckp8dzkqnpcc6xx5lph5d2z46sg2cwzccpr8ay58zy"; - }) { inherit system; }); - glibc227 = nixos1909.glibc // { pname = "glibc"; }; - stdenv227 = let - cc = pkgs.wrapCCWith { - cc = nixos1909.buildPackages.gcc-unwrapped; - libc = glibc227; - bintools = pkgs.binutils.override { libc = glibc227; }; - }; - in (pkgs.overrideCC pkgs.stdenv cc); + system = builtins.currentSystem; + + staticOptions = pkg: pkg.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--without-shared" "--disable-shared" "--disable-tests" ]; + }); + + secp256k1Static = (staticOptions pkgs.secp256k1); + libsodiumStatic = (staticOptions pkgs.libsodium); + + microhttpdStatic = pkgs.libmicrohttpd.overrideAttrs(oldAttrs: { + dontDisableStatic = true; + enableSharedExecutables = false; + configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-tests" "--disable-benchmark" "--disable-shared" "--disable-https" "--with-pic" ]; + }); + + nixos1909 = (import (builtins.fetchTarball { + url = "https://channels.nixos.org/nixos-19.09/nixexprs.tar.xz"; + sha256 = "1vp1h2gkkrckp8dzkqnpcc6xx5lph5d2z46sg2cwzccpr8ay58zy"; + }) { inherit system; }); + glibc227 = nixos1909.glibc // { pname = "glibc"; }; + stdenv227 = let + cc = pkgs.wrapCCWith { + cc = nixos1909.buildPackages.gcc-unwrapped; + libc = glibc227; + bintools = pkgs.binutils.override { libc = glibc227; }; + }; + in (pkgs.overrideCC pkgs.stdenv cc); in stdenv227.mkDerivation { @@ -34,7 +49,12 @@ stdenv227.mkDerivation { buildInputs = with pkgs; [ - pkgsStatic.openssl pkgsStatic.zlib pkgsStatic.libmicrohttpd.dev pkgsStatic.libsodium.dev pkgsStatic.secp256k1 pkgsStatic.lz4 + (openssl.override { static = true; }).dev + microhttpdStatic.dev + (zlib.override { shared = false; }).dev + (lz4.override { enableStatic = true; enableShared = false; }).dev + secp256k1Static + libsodiumStatic.dev ]; dontAddStaticConfigureFlags = false; diff --git a/assembly/nix/macos-static.nix b/assembly/nix/macos-static.nix index be15579ca..2fd0b3a66 100644 --- a/assembly/nix/macos-static.nix +++ b/assembly/nix/macos-static.nix @@ -17,7 +17,7 @@ pkgs.llvmPackages_14.stdenv.mkDerivation { buildInputs = with pkgs; lib.forEach [ - secp256k1 libsodium.dev libmicrohttpd.dev gmp.dev nettle.dev libtasn1.dev libidn2.dev libunistring.dev gettext (gnutls.override { withP11-kit = false; }).dev + secp256k1 libsodium.dev libmicrohttpd.dev gmp.dev nettle.dev libtasn1.dev libidn2.dev libunistring.dev gettext jemalloc (gnutls.override { withP11-kit = false; }).dev ] (x: x.overrideAttrs(oldAttrs: rec { configureFlags = (oldAttrs.configureFlags or []) ++ [ "--enable-static" "--disable-shared" "--disable-tests" ]; dontDisableStatic = true; })) ++ [ @@ -38,13 +38,13 @@ pkgs.llvmPackages_14.stdenv.mkDerivation { cmakeFlags = [ "-DTON_USE_ABSEIL=OFF" "-DNIX=ON" + "-DTON_USE_JEMALLOC=ON" "-DCMAKE_CROSSCOMPILING=OFF" "-DCMAKE_LINK_SEARCH_START_STATIC=ON" "-DCMAKE_LINK_SEARCH_END_STATIC=ON" "-DBUILD_SHARED_LIBS=OFF" "-DCMAKE_CXX_FLAGS=-stdlib=libc++" "-DCMAKE_OSX_DEPLOYMENT_TARGET:STRING=11.3" - "-DCMAKE_CTEST_ARGUMENTS=--timeout;1800" ]; LDFLAGS = [ diff --git a/assembly/nix/microhttpd.nix b/assembly/nix/microhttpd.nix deleted file mode 100644 index 4f871425a..000000000 --- a/assembly/nix/microhttpd.nix +++ /dev/null @@ -1,28 +0,0 @@ -{ pkgs ? import { system = builtins.currentSystem; } -, stdenv ? pkgs.stdenv -, fetchgit ? pkgs.fetchgit -}: - -stdenv.mkDerivation rec { - name = "microhttpdmy"; - - - src = fetchgit { - url = "https://git.gnunet.org/libmicrohttpd.git"; - rev = "refs/tags/v0.9.77"; - sha256 = "sha256-x+nfB07PbZwBlFc6kZZFYiRpk0a3QN/ByHB+hC8na/o="; - }; - - nativeBuildInputs = with pkgs; [ automake libtool autoconf texinfo ]; - - buildInputs = with pkgs; [ ]; - - configurePhase = '' - ./autogen.sh - ./configure --enable-static --disable-tests --disable-benchmark --disable-shared --disable-https --with-pic - ''; - - installPhase = '' - make install DESTDIR=$out - ''; -} diff --git a/assembly/nix/openssl.nix b/assembly/nix/openssl.nix deleted file mode 100644 index 8d30aa504..000000000 --- a/assembly/nix/openssl.nix +++ /dev/null @@ -1,30 +0,0 @@ -{ pkgs ? import { system = builtins.currentSystem; } -, stdenv ? pkgs.stdenv -, fetchFromGitHub ? pkgs.fetchFromGitHub -}: - -stdenv.mkDerivation rec { - name = "opensslmy"; - - src = fetchFromGitHub { - owner = "openssl"; - repo = "openssl"; - rev = "refs/tags/openssl-3.1.4"; - sha256 = "sha256-Vvf1wiNb4ikg1lIS9U137aodZ2JzM711tSWMJFYWtWI="; - }; - - nativeBuildInputs = with pkgs; [ perl ]; - - buildInputs = with pkgs; [ ]; - - postPatch = '' - patchShebangs Configure - ''; - - configurePhase = '' - ./Configure no-shared - ''; - installPhase = '' - make install DESTDIR=$out - ''; -} From ec00ccc48236aaf2d985c531c6a04b9c85795ee1 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Fri, 7 Jun 2024 17:52:43 +0200 Subject: [PATCH 27/28] Add github actions for macOS 14 (arm64, shared and portable) (#1001) * add github action for macOS 14 (arm64, M1) * add github action (portable) for macOS 14 (arm64, M1) * rename macOS arm64 output artifact From 7841d751c08d218597c933628bcd0d0b751f6be8 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Sat, 8 Jun 2024 17:57:38 +0300 Subject: [PATCH 28/28] Add changelog --- Changelog.md | 13 +++++++++++++ recent_changelog.md | 20 +++++++++----------- 2 files changed, 22 insertions(+), 11 deletions(-) diff --git a/Changelog.md b/Changelog.md index a02410557..effe339f4 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,16 @@ +## 2024.06 Update + +1. Make Jemalloc default allocator +2. Add candidate broadcasting and caching +3. Limit per address speed for external messages broadcast by reasonably large number +4. Overlay improvements: fix dropping peers in small custom overlays, fix wrong certificate on missed keyblocks +5. Extended statistics and logs for celldb usage, session stats, persistent state serialization +6. Tonlib and explorer fixes +7. Flags for precize control of Celldb: `--celldb-cache-size`, `--celldb-direct-io` and `--celldb-preload-all` +8. Add valiator-console command to stop persistent state serialization +9. Use `@` path separator for defining include path in fift and create-state utilities on Windows only. + + ## 2024.04 Update 1. Emulator: Single call optimized runGetMethod added diff --git a/recent_changelog.md b/recent_changelog.md index f0b029eea..25a93c189 100644 --- a/recent_changelog.md +++ b/recent_changelog.md @@ -1,13 +1,11 @@ ## 2024.04 Update -1. Emulator: Single call optimized runGetMethod added -2. Tonlib: a series of proof improvements, also breaking Change in `liteServer.getAllShardsInfo` method (see below) -3. DB: usage statistics now collected, outdated persistent states are not serialized -4. LS: fast `getOutMsgQueueSizes` added, preliminary support of non-final block requests -5. Network: lz4 compression of block candidates (disabled by default). -6. Overlays: add custom overlays -7. Transaction Executor: fixed issue with due_payment collection - -* `liteServer.getAllShardsInfo` method was updated for better efficiency. Previously, field proof contained BoC with two roots: one for BlockState from block's root and another for ShardHashes from BlockState. Now, it returns a single-root proof BoC, specifically the merkle proof of ShardHashes directly from the block's root, streamlining data access and integrity. Checking of the proof requires to check that ShardHashes in the `data` correspond to ShardHashes from the block. - -Besides the work of the core team, this update is based on the efforts of @akifoq (due_payment issue). +1. Make Jemalloc default allocator +2. Add candidate broadcasting and caching +3. Limit per address speed for external messages broadcast by reasonably large number +4. Overlay improvements: fix dropping peers in small custom overlays, fix wrong certificate on missed keyblocks +5. Extended statistics and logs for celldb usage, session stats, persistent state serialization +6. Tonlib and explorer fixes +7. Flags for precize control of Celldb: `--celldb-cache-size`, `--celldb-direct-io` and `--celldb-preload-all` +8. Add valiator-console command to stop persistent state serialization +9. Use `@` path separator for defining include path in fift and create-state utilities on Windows only.