Merge branch 'accelerator' into accelerator-test

SpyCheese · Nov 26, 2024 · c06e436 · c06e436
2 parents 6eddf5a + d9aeab0
commit c06e436
Show file tree

Hide file tree

Showing 54 changed files with 1,850 additions and 365 deletions.
diff --git a/create-hardfork/create-hardfork.cpp b/create-hardfork/create-hardfork.cpp
@@ -279,6 +279,9 @@ class HardforkCreator : public td::actor::Actor {
 
       void new_key_block(ton::validator::BlockHandle handle) override {
       }
+      void send_validator_telemetry(ton::PublicKeyHash key,
+                                    ton::tl_object_ptr<ton::ton_api::validator_telemetry> telemetry) override {
+      }
     };
 
     td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::install_callback,

diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb
@@ -801,11 +801,6 @@ misbehaviour_punishment_config_v1#01
    = MisbehaviourPunishmentConfig;
 _ MisbehaviourPunishmentConfig = ConfigParam 40;
 
-// collator_nodes: each collator is (workchain:int32 shard:uint64 adnl_id:uint256)
-collator_info#00 = CollatorInfo;
-colator_config#a0 full_collated_data:Bool collator_nodes:(HashmapE 352 CollatorInfo) = CollatorConfig;
-_ CollatorConfig = ConfigParam 41;
-
 size_limits_config#01 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16
   max_ext_msg_size:uint32 max_ext_msg_depth:uint16 = SizeLimitsConfig;
 size_limits_config_v2#02 max_msg_bits:uint32 max_msg_cells:uint32 max_library_cells:uint32 max_vm_data_depth:uint16

diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp
@@ -2339,28 +2339,4 @@ td::optional<PrecompiledContractsConfig::Contract> PrecompiledContractsConfig::g
   return c;
 }
 
-CollatorConfig Config::get_collator_config(bool need_collator_nodes) const {
-  CollatorConfig collator_config;
-  gen::CollatorConfig::Record rec;
-  auto cell = get_config_param(41, -41);
-  if (cell.is_null() || !tlb::unpack_cell(std::move(cell), rec)) {
-    return collator_config;
-  }
-  collator_config.full_collated_data = rec.full_collated_data;
-  if (need_collator_nodes) {
-    vm::Dictionary dict{rec.collator_nodes->prefetch_ref(), 32 + 64 + 256};
-    dict.check_for_each([&](Ref<vm::CellSlice> value, td::ConstBitPtr key, int n) {
-      CHECK(n == 32 + 64 + 256);
-      auto workchain = (td::int32)key.get_int(32);
-      key.advance(32);
-      td::uint64 shard = key.get_uint(64);
-      key.advance(64);
-      td::Bits256 adnl_id(key);
-      collator_config.collator_nodes.push_back({ton::ShardIdFull(workchain, shard), adnl_id});
-      return true;
-    });
-  }
-  return collator_config;
-}
-
 }  // namespace block
diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h
@@ -543,11 +543,6 @@ struct CollatorNodeDescr {
   ton::NodeIdShort adnl_id;
 };
 
-struct CollatorConfig {
-  bool full_collated_data = false;
-  std::vector<CollatorNodeDescr> collator_nodes;
-};
-
 class Config {
   enum {
     default_mc_catchain_lifetime = 200,
@@ -664,7 +659,6 @@ class Config {
   std::vector<ton::ValidatorDescr> compute_validator_set(ton::ShardIdFull shard, ton::UnixTime time,
                                                          ton::CatchainSeqno cc_seqno) const;
   std::vector<ton::ValidatorDescr> compute_total_validator_set(int next) const;
-  CollatorConfig get_collator_config(bool need_collator_nodes) const;
   td::Result<SizeLimitsConfig> get_size_limits_config() const;
   static td::Result<SizeLimitsConfig> do_get_size_limits_config(td::Ref<vm::CellSlice> cs);
   std::unique_ptr<vm::Dictionary> get_suspended_addresses(ton::UnixTime now) const;

diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp
@@ -68,6 +68,9 @@ void OverlayManager::register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdS
   }
   overlays_[local_id][overlay_id] = OverlayDescription{std::move(overlay), std::move(cert)};
 
+  if (!with_db_) {
+    return;
+  }
   auto P =
       td::PromiseCreator::lambda([id = overlays_[local_id][overlay_id].overlay.get()](td::Result<DbType::GetResult> R) {
         R.ensure();
@@ -417,13 +420,19 @@ OverlayManager::OverlayManager(std::string db_root, td::actor::ActorId<keyring::
 }
 
 void OverlayManager::start_up() {
-  std::shared_ptr<td::KeyValue> kv =
-      std::make_shared<td::RocksDb>(td::RocksDb::open(PSTRING() << db_root_ << "/overlays").move_as_ok());
-  db_ = DbType{std::move(kv)};
+  if (!db_root_.empty()) {
+    with_db_ = true;
+    std::shared_ptr<td::KeyValue> kv =
+        std::make_shared<td::RocksDb>(td::RocksDb::open(PSTRING() << db_root_ << "/overlays").move_as_ok());
+    db_ = DbType{std::move(kv)};
+  }
 }
 
 void OverlayManager::save_to_db(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id,
                                 std::vector<OverlayNode> nodes) {
+  if (!with_db_) {
+    return;
+  }
   std::vector<tl_object_ptr<ton_api::overlay_node>> nodes_vec;
   for (auto &n : nodes) {
     nodes_vec.push_back(n.tl());

diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h
@@ -131,6 +131,7 @@ class OverlayManager : public Overlays {
   td::actor::ActorId<dht::Dht> dht_node_;
 
   using DbType = td::KeyValueAsync<td::Bits256, td::BufferSlice>;
+  bool with_db_ = false;
   DbType db_;
 
   class AdnlCallback : public adnl::Adnl::Callback {

diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp
@@ -213,7 +213,7 @@ void OverlayImpl::add_peer(OverlayNode node) {
     peer_list_.peers_.insert(id, OverlayPeer(std::move(node)));
     del_some_peers();
     auto X = peer_list_.peers_.get(id);
-    if (X != nullptr && peer_list_.neighbours_.size() < max_neighbours() &&
+    if (X != nullptr && !X->is_neighbour() && peer_list_.neighbours_.size() < max_neighbours() &&
         !(X->get_node()->flags() & OverlayMemberFlags::DoNotReceiveBroadcasts) && X->get_id() != local_id_) {
       peer_list_.neighbours_.push_back(X->get_id());
       X->set_neighbour(true);
@@ -440,7 +440,7 @@ void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) {
       VLOG(OVERLAY_INFO) << this << ": adding new neighbour " << X->get_id();
       peer_list_.neighbours_.push_back(X->get_id());
       X->set_neighbour(true);
-    } else {
+    } else if (X->is_alive()) {
       CHECK(nodes_to_change > 0);
       auto i = td::Random::fast(0, static_cast<td::uint32>(peer_list_.neighbours_.size()) - 1);
       auto Y = peer_list_.peers_.get(peer_list_.neighbours_[i]);

diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp
@@ -347,7 +347,12 @@ void OverlayImpl::alarm() {
       update_db_at_ = td::Timestamp::in(60.0);
     }
 
-    update_neighbours(0);
+    if (update_neighbours_at_.is_in_past()) {
+      update_neighbours(2);
+      update_neighbours_at_ = td::Timestamp::in(td::Random::fast(30.0, 120.0));
+    } else {
+      update_neighbours(0);
+    }
     alarm_timestamp() = td::Timestamp::in(1.0);
   } else {
     update_neighbours(0);

diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp
@@ -391,6 +391,7 @@ class OverlayImpl : public Overlay {
   td::Timestamp next_dht_store_query_ = td::Timestamp::in(1.0);
   td::Timestamp update_db_at_;
   td::Timestamp update_throughput_at_;
+  td::Timestamp update_neighbours_at_;
   td::Timestamp last_throughput_update_;
 
   std::unique_ptr<Overlays::Callback> callback_;

diff --git a/tdutils/td/utils/port/Stat.cpp b/tdutils/td/utils/port/Stat.cpp
@@ -472,4 +472,45 @@ Result<TotalMemStat> get_total_mem_stat() {
 #endif
 }
 
+Result<uint32> get_cpu_cores() {
+#if TD_LINUX
+  uint32 result = 0;
+  TRY_RESULT(fd, FileFd::open("/proc/cpuinfo", FileFd::Read));
+  SCOPE_EXIT {
+    fd.close();
+  };
+  std::string data;
+  char buf[10000];
+  while (true) {
+    TRY_RESULT(size, fd.read(MutableSlice{buf, sizeof(buf) - 1}));
+    if (size == 0) {
+      break;
+    }
+    buf[size] = '\0';
+    data += buf;
+  }
+  size_t i = 0;
+  while (i < data.size()) {
+    const char *line_begin = data.data() + i;
+    while (i < data.size() && data[i] != '\n') {
+      ++i;
+    }
+    auto line_end = data.data() + i;
+    ++i;
+    Slice line{line_begin, line_end};
+    size_t j = 0;
+    while (j < line.size() && line[j] != ' ' && line[j] != '\t' && line[j] != ':') {
+      ++j;
+    }
+    Slice name = line.substr(0, j);
+    if (name == "processor") {
+      ++result;
+    }
+  }
+  return result;
+#else
+  return Status::Error("Not supported");
+#endif
+}
+
 }  // namespace td
diff --git a/tdutils/td/utils/port/Stat.h b/tdutils/td/utils/port/Stat.h
@@ -70,4 +70,6 @@ struct TotalMemStat {
 };
 Result<TotalMemStat> get_total_mem_stat() TD_WARN_UNUSED_RESULT;
 
+Result<uint32> get_cpu_cores() TD_WARN_UNUSED_RESULT;
+
 }  // namespace td
diff --git a/test/test-ton-collator.cpp b/test/test-ton-collator.cpp
@@ -380,6 +380,9 @@ class TestNode : public td::actor::Actor {
 
       void new_key_block(ton::validator::BlockHandle handle) override {
       }
+      void send_validator_telemetry(ton::PublicKeyHash key,
+                                    ton::tl_object_ptr<ton::ton_api::validator_telemetry> telemetry) override {
+      }
     };
 
     td::actor::send_closure(validator_manager_, &ton::validator::ValidatorManagerInterface::install_callback,

diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl
@@ -595,6 +595,10 @@ validator.group workchain:int shard:long catchain_seqno:int config_hash:int256 m
 validator.groupEx workchain:int shard:long vertical_seqno:int catchain_seqno:int config_hash:int256 members:(vector validator.groupMember) = validator.Group; 
 validator.groupNew workchain:int shard:long vertical_seqno:int last_key_block_seqno:int catchain_seqno:int config_hash:int256 members:(vector validator.groupMember) = validator.Group; 
 
+validator.telemetry flags:# timestamp:double adnl_id:int256
+    node_version:string os_version:string node_started_at:int
+    ram_size:long cpu_cores:int node_threads:int = validator.Telemetry;
+
 ---functions---
 
 
@@ -646,8 +650,9 @@ engine.validator.fullNodeMaster port:int adnl:int256 = engine.validator.FullNode
 engine.validator.fullNodeSlave ip:int port:int adnl:PublicKey = engine.validator.FullNodeSlave;
 engine.validator.fullNodeConfig ext_messages_broadcast_disabled:Bool = engine.validator.FullNodeConfig;
 engine.validator.fastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.FastSyncMemberCertificate;
+engine.validator.collatorNodeWhitelist enabled:Bool adnl_ids:(vector int256) = engine.validator.CollatorNodeWhitelist;
 engine.validator.extraConfig state_serializer_enabled:Bool fast_sync_member_certificates:(vector engine.validator.fastSyncMemberCertificate)
-    = engine.validator.ExtraConfig;
+    collator_node_whitelist:engine.validator.collatorNodeWhitelist = engine.validator.ExtraConfig;
 engine.validator.config out_port:int addrs:(vector engine.Addr) adnl:(vector engine.adnl)
         dht:(vector engine.dht)
         validators:(vector engine.validator) collators:(vector engine.collator)
@@ -668,13 +673,13 @@ engine.validator.collatorOptions
   deferring_enabled:Bool defer_messages_after:int defer_out_queue_size_limit:long
   dispatch_phase_2_max_total:int dispatch_phase_3_max_total:int
   dispatch_phase_2_max_per_initiator:int dispatch_phase_3_max_per_initiator:int
-  whitelist:(vector string) prioritylist:(vector string) = engine.validator.CollatorOptions;
+  whitelist:(vector string) prioritylist:(vector string)
+  force_full_collated_data:Bool ignore_collated_data_limits:Bool = engine.validator.CollatorOptions;
 
-engine.validator.collatorsList.collator adnl_id:int256 trusted:Bool = engine.validator.collatorsList.Collator;
+engine.validator.collatorsList.collator adnl_id:int256 = engine.validator.collatorsList.Collator;
 engine.validator.collatorsList.shard shard_id:tonNode.shardId collators:(vector engine.validator.collatorsList.collator)
-    = engine.validator.collatorsList.Shard;
-engine.validator.collatorsList self_collate:Bool use_config_41:Bool shards:(vector engine.validator.collatorsList.shard)
-    = engine.validator.CollatorsList;
+    self_collate:Bool select_mode:string = engine.validator.collatorsList.Shard;
+engine.validator.collatorsList shards:(vector engine.validator.collatorsList.shard) = engine.validator.CollatorsList;
 
 ---functions---
 ---types---
@@ -740,6 +745,11 @@ engine.validator.perfTimerStats stats:(vector engine.validator.PerfTimerStatsByN
 
 engine.validator.shardOutQueueSize size:long = engine.validator.ShardOutQueueSize;
 
+engine.validator.collationManagerStats.shard shard_id:tonNode.shardId self_collate:Bool select_mode:string active:Bool collators:(vector int256) = engine.validator.collationManagerStats.Shard;
+engine.validator.collationManagerStats.collator adnl_id:int256 active:Bool alive:Bool ping_in:double = engine.validator.collationManagerStats.Collator;
+engine.validator.collationManagerStats.localId adnl_id:int256 shards:(vector engine.validator.collationManagerStats.shard)
+    collators:(vector engine.validator.collationManagerStats.collator) = engine.validator.collationManagerStats.LocalId;
+engine.validator.collationManagerStats local_ids:(vector engine.validator.collationManagerStats.localId) = engine.validator.CollationManagerStats;
 
 ---functions---
 
@@ -811,8 +821,14 @@ engine.validator.addShard shard:tonNode.shardId = engine.validator.Success;
 engine.validator.delCollator adnl_id:int256 shard:tonNode.shardId = engine.validator.Success;
 engine.validator.delShard shard:tonNode.shardId = engine.validator.Success;
 
+engine.validator.collatorNodeSetWhitelistedValidator adnl_id:int256 add:Bool = engine.validator.Success;
+engine.validator.collatorNodeSetWhitelistEnabled enabled:Bool = engine.validator.Success;
+engine.validator.showCollatorNodeWhitelist = engine.validator.CollatorNodeWhitelist;
+
 engine.validator.setCollatorsList list:engine.validator.collatorsList = engine.validator.Success;
+engine.validator.clearCollatorsList = engine.validator.Success;
 engine.validator.showCollatorsList = engine.validator.CollatorsList;
+engine.validator.getCollationManagerStats = engine.validator.CollationManagerStats;
 
 engine.validator.signOverlayMemberCertificate sign_by:int256 adnl_id:int256 slot:int expire_at:int = overlay.MemberCertificate;
 engine.validator.importFastSyncMemberCertificate adnl_id:int256 certificate:overlay.MemberCertificate = engine.validator.Success;
@@ -863,8 +879,12 @@ http.server.config dhs:(vector http.server.dnsEntry) local_hosts:(vector http.se
 
 ---types---
 
-validatorSession.collationStats bytes:int gas:int lt_delta:int cat_bytes:int cat_gas:int cat_lt_delta:int
-  limits_log:string ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validadorSession.CollationStats;
+validatorSession.collationStats actual_bytes:int actual_collated_data_bytes:int
+  bytes:int gas:int lt_delta:int collated_data_bytes:int
+  cat_bytes:int cat_gas:int cat_lt_delta:int cat_collated_data_bytes:int
+  limits_log:string ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int
+  work_time:double cpu_work_time:double
+  serialized_size:int serialized_size_no_collated_data:int = validadorSession.CollationStats;
 
 validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int root_hash:int256 file_hash:int256
   comment:string block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double
@@ -885,10 +905,12 @@ validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self:
     signatures:int signatures_weight:long approve_signatures:int approve_signatures_weight:long
     first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats;
 
+validatorSession.statsCollatedBlock timestamp:double id:tonNode.blockIdExt stats:validatorSession.collationStats = validatorSession.StatsCollatedBlock;
+
 collatorNode.candidate source:PublicKey id:tonNode.blockIdExt data:bytes collated_data:bytes = collatorNode.Candidate;
 collatorNode.compressedCandidate flags:# source:PublicKey id:tonNode.blockIdExt decompressed_size:int data:bytes = collatorNode.Candidate;
-collatorNode.generateBlockSuccess candidate:collatorNode.Candidate = collatorNode.GenerateBlockResult;
-collatorNode.generateBlockError code:int message:string = collatorNode.GenerateBlockResult;
+collatorNode.pong flags:# = collatorNode.Pong;
+collatorNode.error code:int message:string = collatorNode.Error;
 
 validatorSession.newValidatorGroupStats.node id:int256 weight:long = validatorSession.newValidatorGroupStats.Node;
 validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int
@@ -901,7 +923,8 @@ validatorSession.endValidatorGroupStats session_id:int256 timestamp:double
 
 ---functions---
 collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt)
-    creator:int256 = collatorNode.GenerateBlockResult;
+    creator:int256 = collatorNode.Candidate;
+collatorNode.ping flags:# = collatorNode.Pong;
 
 ---types---
 

diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo
diff --git a/ton/ton-types.h b/ton/ton-types.h
@@ -62,7 +62,8 @@ enum GlobalCapabilities {
   capShortDequeue = 32,
   capStoreOutMsgQueueSize = 64,
   capMsgMetadata = 128,
-  capDeferMessages = 256
+  capDeferMessages = 256,
+  capFullCollatedData = 512
 };
 
 inline int shard_pfx_len(ShardId shard) {