diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index f81928cd1..bdd0a06f1 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -17,6 +17,8 @@ Copyright 2017-2020 Telegram Systems LLP */ #pragma once +#include "td/utils/CancellationToken.h" + #include #include #include "vm/db/DynamicBagOfCellsDb.h" @@ -343,7 +345,7 @@ td::Result>> std_boc_deserialize_multi(td::Slice data, int max_roots = BagOfCells::default_max_roots); td::Result std_boc_serialize_multi(std::vector> root, int mode = 0); -td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, - td::FileFd& fd, int mode = 0); +td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, + int mode = 0, td::CancellationToken cancellation_token = {}); } // namespace vm diff --git a/crypto/vm/large-boc-serializer.cpp b/crypto/vm/large-boc-serializer.cpp index fbd065dc9..a7dae1b08 100644 --- a/crypto/vm/large-boc-serializer.cpp +++ b/crypto/vm/large-boc-serializer.cpp @@ -33,7 +33,8 @@ class LargeBocSerializer { public: using Hash = Cell::Hash; - explicit LargeBocSerializer(std::shared_ptr reader) : reader(std::move(reader)) { + explicit LargeBocSerializer(std::shared_ptr reader, td::CancellationToken cancellation_token = {}) + : reader(std::move(reader)), cancellation_token(std::move(cancellation_token)) { } void add_root(Hash root); @@ -84,6 +85,7 @@ class LargeBocSerializer { int revisit(int cell_idx, int force = 0); td::uint64 compute_sizes(int mode, int& r_size, int& o_size); + td::CancellationToken cancellation_token; td::Timestamp log_speed_at_; size_t processed_cells_ = 0; static constexpr double LOG_SPEED_PERIOD = 120.0; @@ -112,6 +114,9 @@ td::Result LargeBocSerializer::import_cell(Hash hash, int depth) { return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large"); } ++processed_cells_; + if (processed_cells_ % 1000 == 0) { + TRY_STATUS(cancellation_token.check()); + } if (log_speed_at_.is_in_past()) { log_speed_at_ += LOG_SPEED_PERIOD; LOG(WARNING) << "serializer: import_cells " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; @@ -408,6 +413,9 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { store_ref(k); } ++processed_cells_; + if (processed_cells_ % 1000 == 0) { + TRY_STATUS(cancellation_token.check()); + } if (log_speed_at_.is_in_past()) { log_speed_at_ += LOG_SPEED_PERIOD; LOG(WARNING) << "serializer: serialize " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s"; @@ -428,10 +436,10 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) { } // namespace td::Status std_boc_serialize_to_file_large(std::shared_ptr reader, Cell::Hash root_hash, td::FileFd& fd, - int mode) { + int mode, td::CancellationToken cancellation_token) { td::Timer timer; CHECK(reader != nullptr) - LargeBocSerializer serializer(reader); + LargeBocSerializer serializer(reader, std::move(cancellation_token)); serializer.add_root(root_hash); TRY_STATUS(serializer.import_cells()); TRY_STATUS(serializer.serialize(fd, mode)); diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index 49d2bfbc4..6e7c9b8b2 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -172,7 +172,7 @@ ton::tl_object_ptr Config::tl() const { return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), - nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec)); + nullptr, nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, diff --git a/tdutils/td/utils/CancellationToken.h b/tdutils/td/utils/CancellationToken.h index 9f30d204c..7ef304979 100644 --- a/tdutils/td/utils/CancellationToken.h +++ b/tdutils/td/utils/CancellationToken.h @@ -20,6 +20,7 @@ #include #include +#include "Status.h" namespace td { @@ -38,6 +39,12 @@ class CancellationToken { } return token_->is_cancelled_.load(std::memory_order_acquire); } + Status check() const { + if (*this) { + return Status::Error(653, "cancelled"); // cancelled = 653 + } + return Status::OK(); + } CancellationToken() = default; explicit CancellationToken(std::shared_ptr token) : token_(std::move(token)) { } diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index 9fec39d1c..f7dfb93fa 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -609,13 +609,14 @@ engine.dht.config dht:(vector engine.dht) gc:engine.gc = engine.dht.Config; engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNodeMaster; engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave; engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig; - +engine.validator.extraConfig state_serializer_enabled:Bool = engine.validator.ExtraConfig; engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl) dht:(vector engine.dht) validators:(vector engine.validator) collators:(vector engine.collator) fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave) fullnodemasters:(vector engine.validator.fullNodeMaster) fullnodeconfig:engine.validator.fullNodeConfig + extraconfig:engine.validator.extraConfig liteservers:(vector engine.liteServer) control:(vector engine.controlInterface) shards_to_monitor:(vector tonNode.shardId) gc:engine.gc = engine.validator.Config; @@ -741,6 +742,8 @@ engine.validator.addCustomOverlay overlay:engine.validator.customOverlay = engin engine.validator.delCustomOverlay name:string = engine.validator.Success; engine.validator.showCustomOverlays = engine.validator.CustomOverlaysConfig; +engine.validator.setStateSerializerEnabled enabled:Bool = engine.validator.Success; + engine.validator.getValidatorSessionsInfo = engine.validator.ValidatorSessionsInfo; engine.validator.addCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 3073390b7..83f7d32f9 100644 Binary files a/tl/generate/scheme/ton_api.tlo and b/tl/generate/scheme/ton_api.tlo differ diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 8fa5739ed..660915bb7 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1192,6 +1192,29 @@ td::Status ShowCustomOverlaysQuery::receive(td::BufferSlice data) { return td::Status::OK(); } +td::Status SetStateSerializerEnabledQuery::run() { + TRY_RESULT(value, tokenizer_.get_token()); + if (value != 0 && value != 1) { + return td::Status::Error("expected 0 or 1"); + } + TRY_STATUS(tokenizer_.check_endl()); + enabled_ = value; + return td::Status::OK(); +} + +td::Status SetStateSerializerEnabledQuery::send() { + auto b = ton::create_serialize_tl_object(enabled_); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status SetStateSerializerEnabledQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::TerminalIO::out() << "success\n"; + return td::Status::OK(); +} + td::Status GetValidatorSessionsInfoQuery::run() { TRY_STATUS(tokenizer_.check_endl()); return td::Status::OK(); diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 9bf408299..ecdfa5e2b 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1208,6 +1208,28 @@ class ShowCustomOverlaysQuery : public Query { } }; +class SetStateSerializerEnabledQuery : public Query { + public: + SetStateSerializerEnabledQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "setstateserializerenabled"; + } + static std::string get_help() { + return "setstateserializerenabled \tdisable or enable persistent state serializer; value is 0 or 1"; + } + std::string name() const override { + return get_name(); + } + + private: + bool enabled_; +}; + class GetValidatorSessionsInfoQuery : public Query { public: GetValidatorSessionsInfoQuery(td::actor::ActorId console, Tokenizer tokenizer) diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index 063a3926d..cd4c7c36b 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -146,6 +146,7 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index c66f59a79..585c4c74e 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -73,6 +73,7 @@ #include "block-parse.h" #include "common/delay.h" #include "block/precompiled-smc/PrecompiledSmartContract.h" +#include "interfaces/validator-manager.h" Config::Config() { out_port = 3278; @@ -160,6 +161,11 @@ Config::Config(const ton::ton_api::engine_validator_config &config) { if (config.fullnodeconfig_) { full_node_config = ton::validator::fullnode::FullNodeConfig(config.fullnodeconfig_); } + if (config.extraconfig_) { + state_serializer_enabled = config.extraconfig_->state_serializer_enabled_; + } else { + state_serializer_enabled = true; + } for (auto &serv : config.liteservers_) { config_add_lite_server(ton::PublicKeyHash{serv->id_}, serv->port_).ensure(); @@ -245,6 +251,12 @@ ton::tl_object_ptr Config::tl() const { full_node_config_obj = full_node_config.tl(); } + ton::tl_object_ptr extra_config_obj = {}; + if (!state_serializer_enabled) { + // Non-default values + extra_config_obj = ton::create_tl_object(state_serializer_enabled); + } + std::vector> liteserver_vec; for (auto &x : liteservers) { liteserver_vec.push_back(ton::create_tl_object(x.second.tl(), x.first)); @@ -273,8 +285,8 @@ ton::tl_object_ptr Config::tl() const { return ton::create_tl_object( out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec), full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec), - std::move(full_node_config_obj), std::move(liteserver_vec), std::move(control_vec), std::move(shards_vec), - std::move(gc_vec)); + std::move(full_node_config_obj), std::move(extra_config_obj), std::move(liteserver_vec), std::move(control_vec), + std::move(shards_vec), std::move(gc_vec)); } td::Result Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip, @@ -1459,6 +1471,7 @@ td::Status ValidatorEngine::load_global_config() { h.push_back(b); } validator_options_.write().set_hardforks(std::move(h)); + validator_options_.write().set_state_serializer_enabled(config_.state_serializer_enabled); validator_options_.write().set_validator_mode(validator_mode_); return td::Status::OK(); @@ -3751,6 +3764,34 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCusto custom_overlays_config_, true)); } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, + td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, + td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_modify)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (!started_) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + if (query.enabled_ == validator_options_->get_state_serializer_enabled()) { + promise.set_value(ton::create_serialize_tl_object()); + return; + } + validator_options_.write().set_state_serializer_enabled(query.enabled_); + td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options, + validator_options_); + config_.state_serializer_enabled = query.enabled_; + write_config([promise = std::move(promise)](td::Result R) mutable { + if (R.is_error()) { + promise.set_value(create_control_query_error(R.move_as_error())); + } else { + promise.set_value(ton::create_serialize_tl_object()); + } + }); +} + void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { @@ -4150,7 +4191,7 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); }); return td::Status::OK(); }); - p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=7*86400", + p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=86400", [&](td::Slice fname) { auto v = td::to_double(fname); if (v <= 0) { @@ -4160,7 +4201,7 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); p.add_checked_option( - 'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=365*86400", + 'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=7*86400", [&](td::Slice fname) { auto v = td::to_double(fname); if (v <= 0) { @@ -4290,7 +4331,7 @@ int main(int argc, char *argv[]) { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_nonfinal_ls_queries_enabled); }); }); p.add_checked_option( - '\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 50G)", + '\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 1G)", [&](td::Slice s) -> td::Status { TRY_RESULT(v, td::to_integer_safe(s)); if (v == 0) { @@ -4300,12 +4341,12 @@ int main(int argc, char *argv[]) { return td::Status::OK(); }); p.add_option( - '\0', "celldb-no-direct-io", "disable direct I/O mode for RocksDb in CellDb (forced when celldb cache is < 30G)", - [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, false); }); }); + '\0', "celldb-direct-io", "enable direct I/O mode for RocksDb in CellDb (doesn't apply when celldb cache is < 30G)", + [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, true); }); }); p.add_option( - '\0', "celldb-no-preload-all", - "disable preloading all cells from CellDb on startup (enabled by default)", - [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, false); }); }); + '\0', "celldb-preload-all", + "preload all cells from CellDb on startup (recommended to use with big enough celldb-cache-size and celldb-direct-io)", + [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, true); }); }); p.add_checked_option( '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.5)", [&](td::Slice s) -> td::Status { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index cc9da22e9..ebf8fb581 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -100,6 +100,8 @@ struct Config { std::set gc; std::vector shards_to_monitor; + bool state_serializer_enabled = true; + void decref(ton::PublicKeyHash key); void incref(ton::PublicKeyHash key) { keys_refcnt[key]++; @@ -223,9 +225,9 @@ class ValidatorEngine : public td::actor::Actor { double archive_preload_period_ = 0.0; bool disable_rocksdb_stats_ = false; bool nonfinal_ls_queries_enabled_ = false; - td::optional celldb_cache_size_ = 50LL << 30; - bool celldb_direct_io_ = true; - bool celldb_preload_all_ = true; + td::optional celldb_cache_size_ = 1LL << 30; + bool celldb_direct_io_ = false; + bool celldb_preload_all_ = false; td::optional catchain_max_block_delay_; bool read_config_ = false; bool started_keyring_ = false; @@ -510,6 +512,8 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { diff --git a/validator/full-node-private-overlay.hpp b/validator/full-node-private-overlay.hpp index 0f76c5ae9..a0022fa03 100644 --- a/validator/full-node-private-overlay.hpp +++ b/validator/full-node-private-overlay.hpp @@ -47,15 +47,11 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { config_ = std::move(config); } - void set_enable_compression(bool value) { - enable_compression_ = value; - } - void start_up() override; void tear_down() override; FullNodePrivateBlockOverlay(adnl::AdnlNodeIdShort local_id, std::vector nodes, - FileHash zero_state_file_hash, FullNodeConfig config, bool enable_compression, + FileHash zero_state_file_hash, FullNodeConfig config, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId rldp, td::actor::ActorId rldp2, td::actor::ActorId overlays, @@ -65,7 +61,6 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { , nodes_(std::move(nodes)) , zero_state_file_hash_(zero_state_file_hash) , config_(config) - , enable_compression_(enable_compression) , keyring_(keyring) , adnl_(adnl) , rldp_(rldp) @@ -80,7 +75,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor { std::vector nodes_; FileHash zero_state_file_hash_; FullNodeConfig config_; - bool enable_compression_; + bool enable_compression_ = true; td::actor::ActorId keyring_; td::actor::ActorId adnl_; diff --git a/validator/full-node.cpp b/validator/full-node.cpp index a6eb121fb..9f5fbf698 100644 --- a/validator/full-node.cpp +++ b/validator/full-node.cpp @@ -531,14 +531,12 @@ td::actor::ActorId FullNodeImpl::get_shard(AccountIdPrefixFull ds return get_shard(shard_prefix(dst, max_shard_pfx_len)); } -void FullNodeImpl::got_key_block_state(td::Ref state) { - auto m = td::Ref{std::move(state)}; - +void FullNodeImpl::got_key_block_config(td::Ref config) { PublicKeyHash l = PublicKeyHash::zero(); std::vector keys; std::map current_validators; for (td::int32 i = -1; i <= 1; i++) { - auto r = m->get_total_validator_set(i < 0 ? i : 1 - i); + auto r = config->get_total_validator_set(i < 0 ? i : 1 - i); if (r.not_null()) { auto vec = r->export_vector(); for (auto &el : vec) { @@ -554,16 +552,15 @@ void FullNodeImpl::got_key_block_state(td::Ref state) { } } - set_private_block_overlays_enable_compression(m->get_consensus_config().proto_version >= 3); - if (current_validators != current_validators_) { current_validators_ = std::move(current_validators); update_private_overlays(); } - if (keys == all_validators_) { - return; - } + // Let's turn off this optimization, since keyblocks are rare enough to update on each keyblock + // if (keys == all_validators_) { + // return; + // } all_validators_ = keys; sign_cert_by_ = l; @@ -577,15 +574,31 @@ void FullNodeImpl::got_key_block_state(td::Ref state) { } void FullNodeImpl::new_key_block(BlockHandle handle) { - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { - if (R.is_error()) { - VLOG(FULL_NODE_WARNING) << "failed to get key block state: " << R.move_as_error(); - } else { - td::actor::send_closure(SelfId, &FullNodeImpl::got_key_block_state, R.move_as_ok()); - } - }); - td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_shard_state_from_db, handle, - std::move(P)); + if (handle->id().seqno() == 0) { + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to get zero state: " << R.move_as_error(); + } else { + auto s = td::Ref{R.move_as_ok()}; + CHECK(s.not_null()); + td::actor::send_closure(SelfId, &FullNodeImpl::got_key_block_config, s->get_config_holder().move_as_ok()); + } + }); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_shard_state_from_db, handle, + std::move(P)); + } else { + CHECK(handle->is_key_block()); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result> R) { + if (R.is_error()) { + VLOG(FULL_NODE_WARNING) << "failed to get key block proof: " << R.move_as_error(); + } else { + td::actor::send_closure(SelfId, &FullNodeImpl::got_key_block_config, + R.ok()->get_key_block_config().move_as_ok()); + } + }); + td::actor::send_closure(validator_manager_, &ValidatorManagerInterface::get_block_proof_link_from_db, handle, + std::move(P)); + } } void FullNodeImpl::process_block_broadcast(BlockBroadcast broadcast) { @@ -717,16 +730,6 @@ void FullNodeImpl::update_private_overlays() { } } -void FullNodeImpl::set_private_block_overlays_enable_compression(bool value) { - if (private_block_overlays_enable_compression_ == value) { - return; - } - private_block_overlays_enable_compression_ = true; - for (auto &p : private_block_overlays_) { - td::actor::send_closure(p.second, &FullNodePrivateBlockOverlay::set_enable_compression, value); - } -} - void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { if (!use_old_private_overlays_) { return; @@ -738,9 +741,8 @@ void FullNodeImpl::create_private_block_overlay(PublicKeyHash key) { nodes.push_back(p.second); } private_block_overlays_[key] = td::actor::create_actor( - "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, - private_block_overlays_enable_compression_, keyring_, adnl_, rldp_, rldp2_, overlays_, validator_manager_, - actor_id(this)); + "BlocksPrivateOverlay", current_validators_[key], std::move(nodes), zero_state_file_hash_, config_, keyring_, + adnl_, rldp_, rldp2_, overlays_, validator_manager_, actor_id(this)); } } diff --git a/validator/full-node.hpp b/validator/full-node.hpp index 16f6fffbe..76daf1159 100644 --- a/validator/full-node.hpp +++ b/validator/full-node.hpp @@ -88,7 +88,7 @@ class FullNodeImpl : public FullNode { block::ImportedMsgQueueLimits limits, td::Timestamp timeout, td::Promise>> promise); - void got_key_block_state(td::Ref state); + void got_key_block_config(td::Ref config); void new_key_block(BlockHandle handle); void process_block_broadcast(BlockBroadcast broadcast) override; @@ -149,7 +149,6 @@ class FullNodeImpl : public FullNode { // New overlays (v2) - overlay per shard (monitor_min_split depth). bool use_old_private_overlays_ = false; // TODO: set from config std::map> private_block_overlays_; - bool private_block_overlays_enable_compression_ = false; bool broadcast_block_candidates_in_public_overlay_ = false; FullNodePrivateBlockOverlaysV2 private_block_overlays_v2_; @@ -162,7 +161,6 @@ class FullNodeImpl : public FullNode { std::queue custom_overlays_sent_broadcasts_lru_; void update_private_overlays(); - void set_private_block_overlays_enable_compression(bool value); void create_private_block_overlay(PublicKeyHash key); void update_custom_overlay(CustomOverlayInfo& overlay); void send_block_broadcast_to_custom_overlays(const BlockBroadcast& broadcast); diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index 681c1f4f2..f85b2f641 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -493,6 +493,9 @@ class ValidatorManagerImpl : public ValidatorManager { td::Promise> promise) override { promise.set_result(td::Status::Error("not implemented")); } + void update_options(td::Ref opts) override { + opts_ = std::move(opts); + } void validated_new_block(BlockIdExt block_id) override { } void add_persistent_state_description(td::Ref desc) override { @@ -508,9 +511,6 @@ class ValidatorManagerImpl : public ValidatorManager { void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override { UNREACHABLE(); } - void update_options(td::Ref opts) override { - opts_ = std::move(opts); - } private: td::Ref opts_; diff --git a/validator/manager.cpp b/validator/manager.cpp index c2233e388..d79267eaf 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -3334,6 +3334,22 @@ void ValidatorManagerImpl::get_validator_groups_info_for_litequery( td::actor::create_actor("get-validator-groups-info", std::move(groups), std::move(promise)).release(); } +void ValidatorManagerImpl::update_options(td::Ref opts) { + if (!shard_client_.empty()) { + td::actor::send_closure(shard_client_, &ShardClient::update_options, opts); + } + if (!serializer_.empty()) { + td::actor::send_closure(serializer_, &AsyncStateSerializer::update_options, opts); + } + if (!out_msg_queue_importer_.empty()) { + td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::update_options, opts); + } + if (!queue_size_counter_.empty()) { + td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::update_options, opts); + } + opts_ = std::move(opts); +} + void ValidatorManagerImpl::get_validator_sessions_info( td::Promise> promise) { std::vector> groups; @@ -3400,22 +3416,6 @@ void ValidatorManagerImpl::del_collator(adnl::AdnlNodeIdShort id, ShardIdFull sh } } -void ValidatorManagerImpl::update_options(td::Ref opts) { - if (!shard_client_.empty()) { - td::actor::send_closure(shard_client_, &ShardClient::update_options, opts); - } - if (!serializer_.empty()) { - td::actor::send_closure(serializer_, &AsyncStateSerializer::update_options, opts); - } - if (!out_msg_queue_importer_.empty()) { - td::actor::send_closure(out_msg_queue_importer_, &OutMsgQueueImporter::update_options, opts); - } - if (!queue_size_counter_.empty()) { - td::actor::send_closure(queue_size_counter_, &QueueSizeCounter::update_options, opts); - } - opts_ = std::move(opts); -} - void ValidatorManagerImpl::add_persistent_state_description(td::Ref desc) { auto now = (UnixTime)td::Clocks::system(); if (desc->end_time <= now) { diff --git a/validator/manager.hpp b/validator/manager.hpp index 8c434c094..37b447cc7 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -615,6 +615,8 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override; void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override; + void update_options(td::Ref opts) override; + void get_validator_sessions_info( td::Promise> promise) override; @@ -627,7 +629,6 @@ class ValidatorManagerImpl : public ValidatorManager { void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) override; - void update_options(td::Ref opts) override; void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (last_masterchain_state_.is_null()) { diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index cb125a0d4..d40bf39df 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -26,6 +26,9 @@ namespace ton { namespace validator { void AsyncStateSerializer::start_up() { + if (!opts_->get_state_serializer_enabled()) { + LOG(ERROR) << "Persistent state serializer is disabled"; + } alarm_timestamp() = td::Timestamp::in(1.0 + td::Random::fast(0, 10) * 1.0); running_ = true; @@ -129,7 +132,7 @@ void AsyncStateSerializer::next_iteration() { } CHECK(masterchain_handle_->id() == last_block_id_); if (attempt_ < max_attempt() && last_key_block_id_.id.seqno < last_block_id_.id.seqno && - need_serialize(masterchain_handle_)) { + need_serialize(masterchain_handle_) && opts_->get_state_serializer_enabled()) { if (!stored_persistent_state_description_) { LOG(INFO) << "storing persistent state description for " << masterchain_handle_->id().id; running_ = true; @@ -184,6 +187,9 @@ void AsyncStateSerializer::next_iteration() { return; } if (masterchain_handle_->inited_next_left()) { + if (need_serialize(masterchain_handle_) && !opts_->get_state_serializer_enabled()) { + LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str(); + } last_block_id_ = masterchain_handle_->one_next(true); have_masterchain_state_ = false; stored_persistent_state_description_ = false; @@ -229,6 +235,10 @@ void AsyncStateSerializer::got_masterchain_handle(BlockHandle handle) { void AsyncStateSerializer::got_masterchain_state(td::Ref state, std::shared_ptr cell_db_reader) { + if (!opts_->get_state_serializer_enabled()) { + stored_masterchain_state(); + return; + } LOG(ERROR) << "serializing masterchain state " << masterchain_handle_->id().id.to_str(); have_masterchain_state_ = true; CHECK(next_idx_ == 0); @@ -241,11 +251,16 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state } } - auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader](td::FileFd& fd) { - return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31); + auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader, + cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { + return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token)); }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { - R.ensure(); + if (R.is_error() && R.error().code() == cancelled) { + LOG(ERROR) << "Persistent state serialization cancelled"; + } else { + R.ensure(); + } td::actor::send_closure(SelfId, &AsyncStateSerializer::stored_masterchain_state); }); @@ -284,13 +299,22 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) { void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref state, std::shared_ptr cell_db_reader) { + if (!opts_->get_state_serializer_enabled()) { + success_handler(); + return; + } LOG(ERROR) << "serializing shard state " << handle->id().id.to_str(); - auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader](td::FileFd& fd) { - return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31); + auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader, + cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { + return vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token)); }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { - R.ensure(); - LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str(); + if (R.is_error() && R.error().code() == cancelled) { + LOG(ERROR) << "Persistent state serialization cancelled"; + } else { + R.ensure(); + LOG(ERROR) << "finished serializing shard state " << handle->id().id.to_str(); + } td::actor::send_closure(SelfId, &AsyncStateSerializer::success_handler); }); td::actor::send_closure(manager_, &ValidatorManager::store_persistent_state_file_gen, handle->id(), @@ -316,6 +340,13 @@ void AsyncStateSerializer::success_handler() { next_iteration(); } +void AsyncStateSerializer::update_options(td::Ref opts) { + opts_ = std::move(opts); + if (!opts_->get_state_serializer_enabled()) { + cancellation_token_source_.cancel(); + } +} + bool AsyncStateSerializer::need_serialize(BlockHandle handle) { if (handle->id().id.seqno == 0 || !handle->is_key_block()) { return false; diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index 060c3f966..6ce0d015c 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -37,6 +37,7 @@ class AsyncStateSerializer : public td::actor::Actor { bool saved_to_db_ = true; td::Ref opts_; + td::CancellationTokenSource cancellation_token_source_; td::actor::ActorId manager_; @@ -91,9 +92,7 @@ class AsyncStateSerializer : public td::actor::Actor { void fail_handler_cont(); void success_handler(); - void update_options(td::Ref opts) { - opts_ = std::move(opts); - } + void update_options(td::Ref opts); }; } // namespace validator diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 6c644e5bd..55b40a339 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -139,6 +139,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { td::optional get_catchain_max_block_delay() const override { return catchain_max_block_delay_; } + bool get_state_serializer_enabled() const override { + return state_serializer_enabled_; + } ValidatorMode validator_mode() const override { return validator_mode_; } @@ -222,6 +225,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_catchain_max_block_delay(double value) override { catchain_max_block_delay_ = value; } + void set_state_serializer_enabled(bool value) override { + state_serializer_enabled_ = value; + } void set_validator_mode(ValidatorMode value) override { validator_mode_ = value; } @@ -274,6 +280,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { bool celldb_direct_io_ = false; bool celldb_preload_all_ = false; td::optional catchain_max_block_delay_; + bool state_serializer_enabled_ = true; ValidatorMode validator_mode_ = validator_normal; }; diff --git a/validator/validator.h b/validator/validator.h index 7b155dc6a..afd884acc 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -90,6 +90,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual bool get_celldb_direct_io() const = 0; virtual bool get_celldb_preload_all() const = 0; virtual td::optional get_catchain_max_block_delay() const = 0; + virtual bool get_state_serializer_enabled() const = 0; virtual ValidatorMode validator_mode() const = 0; virtual void set_zero_block_id(BlockIdExt block_id) = 0; @@ -118,15 +119,16 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_celldb_direct_io(bool value) = 0; virtual void set_celldb_preload_all(bool value) = 0; virtual void set_catchain_max_block_delay(double value) = 0; + virtual void set_state_serializer_enabled(bool value) = 0; virtual void set_validator_mode(ValidatorMode value) = 0; static td::Ref create( BlockIdExt zero_block_id, BlockIdExt init_block_id, + std::function check_shard = [](ShardIdFull) { return true; }, - bool allow_blockchain_init = false, double sync_blocks_before = 86400, double block_ttl = 86400 * 7, - double state_ttl = 3600, double archive_ttl = 86400 * 365, double key_proof_ttl = 86400 * 3650, - double max_mempool_num = 999999, - bool initial_sync_disabled = false); + bool allow_blockchain_init = false, double sync_blocks_before = 3600, double block_ttl = 86400, + double state_ttl = 3600, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650, + double max_mempool_num = 999999, bool initial_sync_disabled = false); }; class ValidatorManagerInterface : public td::actor::Actor { @@ -262,13 +264,13 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void add_perf_timer_stat(std::string name, double duration) = 0; virtual void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) = 0; + virtual void update_options(td::Ref opts) = 0; virtual void get_validator_sessions_info( td::Promise> promise) = 0; virtual void add_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; virtual void del_collator(adnl::AdnlNodeIdShort id, ShardIdFull shard) = 0; - virtual void update_options(td::Ref opts) = 0; }; } // namespace validator