Merge branch 'testnet' into block-generation

crypto/vm/boc.h

@@ -17,6 +17,8 @@
     Copyright 2017-2020 Telegram Systems LLP
 */
 #pragma once
+#include "td/utils/CancellationToken.h"
+
 #include <set>
 #include <map>
 #include "vm/db/DynamicBagOfCellsDb.h"
@@ -343,7 +345,7 @@ td::Result<std::vector<Ref<Cell>>> std_boc_deserialize_multi(td::Slice data,
                                                              int max_roots = BagOfCells::default_max_roots);
 td::Result<td::BufferSlice> std_boc_serialize_multi(std::vector<Ref<Cell>> root, int mode = 0);
 
-td::Status std_boc_serialize_to_file_large(std::shared_ptr<CellDbReader> reader, Cell::Hash root_hash,
-                                           td::FileFd& fd, int mode = 0);
+td::Status std_boc_serialize_to_file_large(std::shared_ptr<CellDbReader> reader, Cell::Hash root_hash, td::FileFd& fd,
+                                           int mode = 0, td::CancellationToken cancellation_token = {});
 
 }  // namespace vm

crypto/vm/large-boc-serializer.cpp

@@ -33,7 +33,8 @@ class LargeBocSerializer {
  public:
   using Hash = Cell::Hash;
 
-  explicit LargeBocSerializer(std::shared_ptr<CellDbReader> reader) : reader(std::move(reader)) {
+  explicit LargeBocSerializer(std::shared_ptr<CellDbReader> reader, td::CancellationToken cancellation_token = {})
+      : reader(std::move(reader)), cancellation_token(std::move(cancellation_token)) {
   }
 
   void add_root(Hash root);
@@ -84,6 +85,7 @@ class LargeBocSerializer {
   int revisit(int cell_idx, int force = 0);
   td::uint64 compute_sizes(int mode, int& r_size, int& o_size);
 
+  td::CancellationToken cancellation_token;
   td::Timestamp log_speed_at_;
   size_t processed_cells_ = 0;
   static constexpr double LOG_SPEED_PERIOD = 120.0;
@@ -112,6 +114,9 @@ td::Result<int> LargeBocSerializer::import_cell(Hash hash, int depth) {
     return td::Status::Error("error while importing a cell into a bag of cells: cell depth too large");
   }
   ++processed_cells_;
+  if (processed_cells_ % 1000 == 0) {
+    TRY_STATUS(cancellation_token.check());
+  }
   if (log_speed_at_.is_in_past()) {
     log_speed_at_ += LOG_SPEED_PERIOD;
     LOG(WARNING) << "serializer: import_cells " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s";
@@ -408,6 +413,9 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) {
       store_ref(k);
     }
     ++processed_cells_;
+    if (processed_cells_ % 1000 == 0) {
+      TRY_STATUS(cancellation_token.check());
+    }
     if (log_speed_at_.is_in_past()) {
       log_speed_at_ += LOG_SPEED_PERIOD;
       LOG(WARNING) << "serializer: serialize " << (double)processed_cells_ / LOG_SPEED_PERIOD << " cells/s";
@@ -428,10 +436,10 @@ td::Status LargeBocSerializer::serialize(td::FileFd& fd, int mode) {
 }  // namespace
 
 td::Status std_boc_serialize_to_file_large(std::shared_ptr<CellDbReader> reader, Cell::Hash root_hash, td::FileFd& fd,
-                                           int mode) {
+                                           int mode, td::CancellationToken cancellation_token) {
   td::Timer timer;
   CHECK(reader != nullptr)
-  LargeBocSerializer serializer(reader);
+  LargeBocSerializer serializer(reader, std::move(cancellation_token));
   serializer.add_root(root_hash);
   TRY_STATUS(serializer.import_cells());
   TRY_STATUS(serializer.serialize(fd, mode));

dht-server/dht-server.cpp

@@ -172,7 +172,7 @@ ton::tl_object_ptr<ton::ton_api::engine_validator_config> Config::tl() const {
   return ton::create_tl_object<ton::ton_api::engine_validator_config>(
       out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec),
       ton::PublicKeyHash::zero().tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec),
-      nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec));
+      nullptr, nullptr, std::move(liteserver_vec), std::move(control_vec), std::move(shard_vec), std::move(gc_vec));
 }
 
 td::Result<bool> Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip,

tdutils/td/utils/CancellationToken.h

@@ -20,6 +20,7 @@
 
 #include <atomic>
 #include <memory>
+#include "Status.h"
 
 namespace td {
 
@@ -38,6 +39,12 @@ class CancellationToken {
     }
     return token_->is_cancelled_.load(std::memory_order_acquire);
   }
+  Status check() const {
+    if (*this) {
+      return Status::Error(653, "cancelled");  // cancelled = 653
+    }
+    return Status::OK();
+  }
   CancellationToken() = default;
   explicit CancellationToken(std::shared_ptr<detail::RawCancellationToken> token) : token_(std::move(token)) {
   }

tl/generate/scheme/ton_api.tl

@@ -609,13 +609,14 @@ engine.dht.config dht:(vector engine.dht) gc:engine.gc = engine.dht.Config;
 engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNodeMaster;
 engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave;
 engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig;
-
+engine.validator.extraConfig state_serializer_enabled:Bool = engine.validator.ExtraConfig;
 engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl)
         dht:(vector engine.dht)
         validators:(vector engine.validator) collators:(vector engine.collator)
         fullnode:int256 fullnodeslaves:(vector engine.validator.fullNodeSlave)
         fullnodemasters:(vector engine.validator.fullNodeMaster)
         fullnodeconfig:engine.validator.fullNodeConfig
+        extraconfig:engine.validator.extraConfig
         liteservers:(vector engine.liteServer) control:(vector engine.controlInterface)
         shards_to_monitor:(vector tonNode.shardId)
         gc:engine.gc = engine.validator.Config;
@@ -741,6 +742,8 @@ engine.validator.addCustomOverlay overlay:engine.validator.customOverlay = engin
 engine.validator.delCustomOverlay name:string = engine.validator.Success;
 engine.validator.showCustomOverlays = engine.validator.CustomOverlaysConfig;
 
+engine.validator.setStateSerializerEnabled enabled:Bool = engine.validator.Success;
+
 engine.validator.getValidatorSessionsInfo = engine.validator.ValidatorSessionsInfo;
 
 engine.validator.addCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success;

validator-engine-console/validator-engine-console-query.cpp

@@ -1192,6 +1192,29 @@ td::Status ShowCustomOverlaysQuery::receive(td::BufferSlice data) {
   return td::Status::OK();
 }
 
+td::Status SetStateSerializerEnabledQuery::run() {
+  TRY_RESULT(value, tokenizer_.get_token<int>());
+  if (value != 0 && value != 1) {
+    return td::Status::Error("expected 0 or 1");
+  }
+  TRY_STATUS(tokenizer_.check_endl());
+  enabled_ = value;
+  return td::Status::OK();
+}
+
+td::Status SetStateSerializerEnabledQuery::send() {
+  auto b = ton::create_serialize_tl_object<ton::ton_api::engine_validator_setStateSerializerEnabled>(enabled_);
+  td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise());
+  return td::Status::OK();
+}
+
+td::Status SetStateSerializerEnabledQuery::receive(td::BufferSlice data) {
+  TRY_RESULT_PREFIX(f, ton::fetch_tl_object<ton::ton_api::engine_validator_success>(data.as_slice(), true),
+                    "received incorrect answer: ");
+  td::TerminalIO::out() << "success\n";
+  return td::Status::OK();
+}
+
 td::Status GetValidatorSessionsInfoQuery::run() {
   TRY_STATUS(tokenizer_.check_endl());
   return td::Status::OK();

validator-engine-console/validator-engine-console-query.h

@@ -1208,6 +1208,28 @@ class ShowCustomOverlaysQuery : public Query {
   }
 };
 
+class SetStateSerializerEnabledQuery : public Query {
+ public:
+  SetStateSerializerEnabledQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)
+      : Query(console, std::move(tokenizer)) {
+  }
+  td::Status run() override;
+  td::Status send() override;
+  td::Status receive(td::BufferSlice data) override;
+  static std::string get_name() {
+    return "setstateserializerenabled";
+  }
+  static std::string get_help() {
+    return "setstateserializerenabled <value>\tdisable or enable persistent state serializer; value is 0 or 1";
+  }
+  std::string name() const override {
+    return get_name();
+  }
+
+ private:
+  bool enabled_;
+};
+
 class GetValidatorSessionsInfoQuery : public Query {
  public:
   GetValidatorSessionsInfoQuery(td::actor::ActorId<ValidatorEngineConsole> console, Tokenizer tokenizer)

validator-engine-console/validator-engine-console.cpp

@@ -146,6 +146,7 @@ void ValidatorEngineConsole::run() {
   add_query_runner(std::make_unique<QueryRunnerImpl<AddCustomOverlayQuery>>());
   add_query_runner(std::make_unique<QueryRunnerImpl<DelCustomOverlayQuery>>());
   add_query_runner(std::make_unique<QueryRunnerImpl<ShowCustomOverlaysQuery>>());
+  add_query_runner(std::make_unique<QueryRunnerImpl<SetStateSerializerEnabledQuery>>());
   add_query_runner(std::make_unique<QueryRunnerImpl<GetValidatorSessionsInfoQuery>>());
   add_query_runner(std::make_unique<QueryRunnerImpl<AddCollatorQuery>>());
   add_query_runner(std::make_unique<QueryRunnerImpl<AddShardQuery>>());

validator-engine/validator-engine.cpp

@@ -73,6 +73,7 @@
 #include "block-parse.h"
 #include "common/delay.h"
 #include "block/precompiled-smc/PrecompiledSmartContract.h"
+#include "interfaces/validator-manager.h"
 
 Config::Config() {
   out_port = 3278;
@@ -160,6 +161,11 @@ Config::Config(const ton::ton_api::engine_validator_config &config) {
   if (config.fullnodeconfig_) {
     full_node_config = ton::validator::fullnode::FullNodeConfig(config.fullnodeconfig_);
   }
+  if (config.extraconfig_) {
+    state_serializer_enabled = config.extraconfig_->state_serializer_enabled_;
+  } else {
+    state_serializer_enabled = true;
+  }
 
   for (auto &serv : config.liteservers_) {
     config_add_lite_server(ton::PublicKeyHash{serv->id_}, serv->port_).ensure();
@@ -245,6 +251,12 @@ ton::tl_object_ptr<ton::ton_api::engine_validator_config> Config::tl() const {
     full_node_config_obj = full_node_config.tl();
   }
 
+  ton::tl_object_ptr<ton::ton_api::engine_validator_extraConfig> extra_config_obj = {};
+  if (!state_serializer_enabled) {
+    // Non-default values
+    extra_config_obj = ton::create_tl_object<ton::ton_api::engine_validator_extraConfig>(state_serializer_enabled);
+  }
+
   std::vector<ton::tl_object_ptr<ton::ton_api::engine_liteServer>> liteserver_vec;
   for (auto &x : liteservers) {
     liteserver_vec.push_back(ton::create_tl_object<ton::ton_api::engine_liteServer>(x.second.tl(), x.first));
@@ -273,8 +285,8 @@ ton::tl_object_ptr<ton::ton_api::engine_validator_config> Config::tl() const {
   return ton::create_tl_object<ton::ton_api::engine_validator_config>(
       out_port, std::move(addrs_vec), std::move(adnl_vec), std::move(dht_vec), std::move(val_vec), std::move(col_vec),
       full_node.tl(), std::move(full_node_slaves_vec), std::move(full_node_masters_vec),
-      std::move(full_node_config_obj), std::move(liteserver_vec), std::move(control_vec), std::move(shards_vec),
-      std::move(gc_vec));
+      std::move(full_node_config_obj), std::move(extra_config_obj), std::move(liteserver_vec), std::move(control_vec),
+      std::move(shards_vec), std::move(gc_vec));
 }
 
 td::Result<bool> Config::config_add_network_addr(td::IPAddress in_ip, td::IPAddress out_ip,
@@ -1459,6 +1471,7 @@ td::Status ValidatorEngine::load_global_config() {
     h.push_back(b);
   }
   validator_options_.write().set_hardforks(std::move(h));
+  validator_options_.write().set_state_serializer_enabled(config_.state_serializer_enabled);
   validator_options_.write().set_validator_mode(validator_mode_);
 
   return td::Status::OK();
@@ -3751,6 +3764,34 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_showCusto
       custom_overlays_config_, true));
 }
 
+void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query,
+                                        td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm,
+                                        td::Promise<td::BufferSlice> promise) {
+  if (!(perm & ValidatorEnginePermissions::vep_modify)) {
+    promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized")));
+    return;
+  }
+  if (!started_) {
+    promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started")));
+    return;
+  }
+  if (query.enabled_ == validator_options_->get_state_serializer_enabled()) {
+    promise.set_value(ton::create_serialize_tl_object<ton::ton_api::engine_validator_success>());
+    return;
+  }
+  validator_options_.write().set_state_serializer_enabled(query.enabled_);
+  td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::update_options,
+                          validator_options_);
+  config_.state_serializer_enabled = query.enabled_;
+  write_config([promise = std::move(promise)](td::Result<td::Unit> R) mutable {
+    if (R.is_error()) {
+      promise.set_value(create_control_query_error(R.move_as_error()));
+    } else {
+      promise.set_value(ton::create_serialize_tl_object<ton::ton_api::engine_validator_success>());
+    }
+  });
+}
+
 void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getValidatorSessionsInfo &query,
                                         td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm,
                                         td::Promise<td::BufferSlice> promise) {
@@ -4150,7 +4191,7 @@ int main(int argc, char *argv[]) {
     acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_max_mempool_num, v); });
     return td::Status::OK();
   });
-  p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=7*86400",
+  p.add_checked_option('b', "block-ttl", "blocks will be gc'd after this time (in seconds) default=86400",
                        [&](td::Slice fname) {
                          auto v = td::to_double(fname);
                          if (v <= 0) {
@@ -4160,7 +4201,7 @@ int main(int argc, char *argv[]) {
                          return td::Status::OK();
                        });
   p.add_checked_option(
-      'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=365*86400",
+      'A', "archive-ttl", "archived blocks will be deleted after this time (in seconds) default=7*86400",
       [&](td::Slice fname) {
         auto v = td::to_double(fname);
         if (v <= 0) {
@@ -4290,7 +4331,7 @@ int main(int argc, char *argv[]) {
     acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_nonfinal_ls_queries_enabled); });
   });
   p.add_checked_option(
-      '\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 50G)",
+      '\0', "celldb-cache-size", "block cache size for RocksDb in CellDb, in bytes (default: 1G)",
       [&](td::Slice s) -> td::Status {
         TRY_RESULT(v, td::to_integer_safe<td::uint64>(s));
         if (v == 0) {
@@ -4300,12 +4341,12 @@ int main(int argc, char *argv[]) {
         return td::Status::OK();
       });
   p.add_option(
-      '\0', "celldb-no-direct-io", "disable direct I/O mode for RocksDb in CellDb (forced when celldb cache is < 30G)",
-      [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, false); }); });
+      '\0', "celldb-direct-io", "enable direct I/O mode for RocksDb in CellDb (doesn't apply when celldb cache is < 30G)",
+      [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_direct_io, true); }); });
   p.add_option(
-      '\0', "celldb-no-preload-all",
-      "disable preloading all cells from CellDb on startup (enabled by default)",
-      [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, false); }); });
+      '\0', "celldb-preload-all",
+      "preload all cells from CellDb on startup (recommended to use with big enough celldb-cache-size and celldb-direct-io)",
+      [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, true); }); });
   p.add_checked_option(
       '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.5)",
       [&](td::Slice s) -> td::Status {

validator-engine/validator-engine.hpp

@@ -100,6 +100,8 @@ struct Config {
   std::set<ton::PublicKeyHash> gc;
   std::vector<ton::ShardIdFull> shards_to_monitor;
 
+  bool state_serializer_enabled = true;
+
   void decref(ton::PublicKeyHash key);
   void incref(ton::PublicKeyHash key) {
     keys_refcnt[key]++;
@@ -223,9 +225,9 @@ class ValidatorEngine : public td::actor::Actor {
   double archive_preload_period_ = 0.0;
   bool disable_rocksdb_stats_ = false;
   bool nonfinal_ls_queries_enabled_ = false;
-  td::optional<td::uint64> celldb_cache_size_ = 50LL << 30;
-  bool celldb_direct_io_ = true;
-  bool celldb_preload_all_ = true;
+  td::optional<td::uint64> celldb_cache_size_ = 1LL << 30;
+  bool celldb_direct_io_ = false;
+  bool celldb_preload_all_ = false;
   td::optional<double> catchain_max_block_delay_;
   bool read_config_ = false;
   bool started_keyring_ = false;
@@ -510,6 +512,8 @@ class ValidatorEngine : public td::actor::Actor {
                          ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
   void run_control_query(ton::ton_api::engine_validator_showCustomOverlays &query, td::BufferSlice data,
                          ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
+  void run_control_query(ton::ton_api::engine_validator_setStateSerializerEnabled &query, td::BufferSlice data,
+                         ton::PublicKeyHash src, td::uint32 perm, td::Promise<td::BufferSlice> promise);
   template <class T>
   void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm,
                          td::Promise<td::BufferSlice> promise) {

validator/full-node-private-overlay.hpp

@@ -47,15 +47,11 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor {
     config_ = std::move(config);
   }
 
-  void set_enable_compression(bool value) {
-    enable_compression_ = value;
-  }
-
   void start_up() override;
   void tear_down() override;
 
   FullNodePrivateBlockOverlay(adnl::AdnlNodeIdShort local_id, std::vector<adnl::AdnlNodeIdShort> nodes,
-                              FileHash zero_state_file_hash, FullNodeConfig config, bool enable_compression,
+                              FileHash zero_state_file_hash, FullNodeConfig config,
                               td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
                               td::actor::ActorId<rldp::Rldp> rldp, td::actor::ActorId<rldp2::Rldp> rldp2,
                               td::actor::ActorId<overlay::Overlays> overlays,
@@ -65,7 +61,6 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor {
       , nodes_(std::move(nodes))
       , zero_state_file_hash_(zero_state_file_hash)
       , config_(config)
-      , enable_compression_(enable_compression)
       , keyring_(keyring)
       , adnl_(adnl)
       , rldp_(rldp)
@@ -80,7 +75,7 @@ class FullNodePrivateBlockOverlay : public td::actor::Actor {
   std::vector<adnl::AdnlNodeIdShort> nodes_;
   FileHash zero_state_file_hash_;
   FullNodeConfig config_;
-  bool enable_compression_;
+  bool enable_compression_ = true;
 
   td::actor::ActorId<keyring::Keyring> keyring_;
   td::actor::ActorId<adnl::Adnl> adnl_;