Accelerator, part 1 (#1119)

This commit contains some parts of https://github.com/ton-blockchain/ton/tree/accelerator
This is auxiliary code that mostly does not change node behavior.

1) Semiprivate overlays and other improvements in overlays code
2) Rename actual_min_split -> monitor_min_split, fix building shard overlays
3) Loading block candidates by block id from DB, fix accept_block after validator restart
4) Cells: ProofStorageStat and changes in CellUsageTree
5) Remove some unused code, other minor changes

CMakeLists.txt

@@ -529,6 +529,8 @@ target_link_libraries(test-rldp2 adnl adnltest dht rldp2 tl_api)
 add_executable(test-validator-session-state test/test-validator-session-state.cpp)
 target_link_libraries(test-validator-session-state adnl dht rldp validatorsession tl_api)
 
+add_executable(test-overlay test/test-overlay.cpp)
+target_link_libraries(test-overlay overlay tdutils tdactor adnl adnltest tl_api dht )
 add_executable(test-catchain test/test-catchain.cpp)
 target_link_libraries(test-catchain overlay tdutils tdactor adnl adnltest rldp tl_api dht
   catchain )

adnl/adnl-query.cpp

@@ -25,7 +25,7 @@ namespace ton {
 namespace adnl {
 
 void AdnlQuery::alarm() {
-  set_error(td::Status::Error(ErrorCode::timeout, "adnl query timeout"));
+  set_error(td::Status::Error(ErrorCode::timeout, PSTRING() << "timeout for adnl query " << name_));
 }
 void AdnlQuery::result(td::BufferSlice data) {
   promise_.set_value(std::move(data));

crypto/block/block.tlb

@@ -666,15 +666,15 @@ wc_split_merge_timings#0
 //workchain#a5 enabled_since:uint32 min_split:(## 8) max_split:(## 8)
 //  { min_split <= max_split } { max_split <= 60 }
 
-workchain#a6 enabled_since:uint32 actual_min_split:(## 8) 
-  min_split:(## 8) max_split:(## 8) { actual_min_split <= min_split }
+workchain#a6 enabled_since:uint32 monitor_min_split:(## 8)
+  min_split:(## 8) max_split:(## 8) { monitor_min_split <= min_split }
   basic:(## 1) active:Bool accept_msgs:Bool flags:(## 13) { flags = 0 }
   zerostate_root_hash:bits256 zerostate_file_hash:bits256
   version:uint32 format:(WorkchainFormat basic)
   = WorkchainDescr;
 
-workchain_v2#a7 enabled_since:uint32 actual_min_split:(## 8)
-  min_split:(## 8) max_split:(## 8) { actual_min_split <= min_split }
+workchain_v2#a7 enabled_since:uint32 monitor_min_split:(## 8)
+  min_split:(## 8) max_split:(## 8) { monitor_min_split <= min_split }
   basic:(## 1) active:Bool accept_msgs:Bool flags:(## 13) { flags = 0 }
   zerostate_root_hash:bits256 zerostate_file_hash:bits256
   version:uint32 format:(WorkchainFormat basic)

crypto/block/mc-config.cpp

@@ -2075,7 +2075,7 @@ bool WorkchainInfo::unpack(ton::WorkchainId wc, vm::CellSlice& cs) {
   }
   auto unpack_v1 = [this](auto& info) {
     enabled_since = info.enabled_since;
-    actual_min_split = info.actual_min_split;
+    monitor_min_split = info.monitor_min_split;
     min_split = info.min_split;
     max_split = info.max_split;
     basic = info.basic;

crypto/block/mc-config.h

@@ -414,7 +414,7 @@ struct CatchainValidatorsConfig {
 struct WorkchainInfo : public td::CntObject {
   ton::WorkchainId workchain{ton::workchainInvalid};
   ton::UnixTime enabled_since;
-  td::uint32 actual_min_split;
+  td::uint32 monitor_min_split;
   td::uint32 min_split, max_split;
   bool basic;
   bool active;

crypto/vm/boc.cpp

@@ -1214,4 +1214,35 @@ bool VmStorageStat::add_storage(const CellSlice& cs) {
   return true;
 }
 
+static td::uint64 estimate_prunned_size() {
+  return 41;
+}
+
+static td::uint64 estimate_serialized_size(const Ref<DataCell>& cell) {
+  return cell->get_serialized_size() + cell->size_refs() * 3 + 3;
+}
+
+void ProofStorageStat::add_cell(const Ref<DataCell>& cell) {
+  auto& status = cells_[cell->get_hash()];
+  if (status == c_loaded) {
+    return;
+  }
+  if (status == c_prunned) {
+    proof_size_ -= estimate_prunned_size();
+  }
+  status = c_loaded;
+  proof_size_ += estimate_serialized_size(cell);
+  for (unsigned i = 0; i < cell->size_refs(); ++i) {
+    auto& child_status = cells_[cell->get_ref(i)->get_hash()];
+    if (child_status == c_none) {
+      child_status = c_prunned;
+      proof_size_ += estimate_prunned_size();
+    }
+  }
+}
+
+td::uint64 ProofStorageStat::estimate_proof_size() const {
+  return proof_size_;
+}
+
 }  // namespace vm

crypto/vm/boc.h

@@ -163,6 +163,18 @@ struct VmStorageStat {
   }
 };
 
+class ProofStorageStat {
+ public:
+  void add_cell(const Ref<DataCell>& cell);
+  td::uint64 estimate_proof_size() const;
+ private:
+  enum CellStatus {
+    c_none = 0, c_prunned = 1, c_loaded = 2
+  };
+  std::map<vm::Cell::Hash, CellStatus> cells_;
+  td::uint64 proof_size_ = 0;
+};
+
 struct CellSerializationInfo {
   bool special;
   Cell::LevelMask level_mask;

crypto/vm/cells/CellUsageTree.cpp

@@ -22,12 +22,12 @@ namespace vm {
 //
 // CellUsageTree::NodePtr
 //
-bool CellUsageTree::NodePtr::on_load() const {
+bool CellUsageTree::NodePtr::on_load(const td::Ref<vm::DataCell>& cell) const {
   auto tree = tree_weak_.lock();
   if (!tree) {
     return false;
   }
-  tree->on_load(node_id_);
+  tree->on_load(node_id_, cell);
   return true;
 }
 
@@ -111,8 +111,14 @@ void CellUsageTree::set_use_mark_for_is_loaded(bool use_mark) {
   use_mark_ = use_mark;
 }
 
-void CellUsageTree::on_load(NodeId node_id) {
+void CellUsageTree::on_load(NodeId node_id, const td::Ref<vm::DataCell>& cell) {
+  if (nodes_[node_id].is_loaded) {
+    return;
+  }
   nodes_[node_id].is_loaded = true;
+  if (cell_load_callback_) {
+    cell_load_callback_(cell);
+  }
 }
 
 CellUsageTree::NodeId CellUsageTree::create_child(NodeId node_id, unsigned ref_id) {

crypto/vm/cells/CellUsageTree.h

@@ -22,8 +22,12 @@
 
 #include "td/utils/int_types.h"
 #include "td/utils/logging.h"
+#include <functional>
 
 namespace vm {
+
+class DataCell;
+
 class CellUsageTree : public std::enable_shared_from_this<CellUsageTree> {
  public:
   using NodeId = td::uint32;
@@ -38,7 +42,7 @@ class CellUsageTree : public std::enable_shared_from_this<CellUsageTree> {
       return node_id_ == 0 || tree_weak_.expired();
     }
 
-    bool on_load() const;
+    bool on_load(const td::Ref<vm::DataCell>& cell) const;
     NodePtr create_child(unsigned ref_id) const;
     bool mark_path(CellUsageTree* master_tree) const;
     bool is_from_tree(const CellUsageTree* master_tree) const;
@@ -59,6 +63,10 @@ class CellUsageTree : public std::enable_shared_from_this<CellUsageTree> {
   void set_use_mark_for_is_loaded(bool use_mark = true);
   NodeId create_child(NodeId node_id, unsigned ref_id);
 
+  void set_cell_load_callback(std::function<void(const td::Ref<vm::DataCell>&)> f) {
+    cell_load_callback_ = std::move(f);
+  }
+
  private:
   struct Node {
     bool is_loaded{false};
@@ -68,8 +76,9 @@ class CellUsageTree : public std::enable_shared_from_this<CellUsageTree> {
   };
   bool use_mark_{false};
   std::vector<Node> nodes_{2};
+  std::function<void(const td::Ref<vm::DataCell>&)> cell_load_callback_;
 
-  void on_load(NodeId node_id);
+  void on_load(NodeId node_id, const td::Ref<vm::DataCell>& cell);
   NodeId create_node(NodeId parent);
 };
 }  // namespace vm

crypto/vm/cells/MerkleProof.h

@@ -66,6 +66,10 @@ class MerkleProofBuilder {
   td::Result<Ref<Cell>> extract_proof() const;
   bool extract_proof_to(Ref<Cell> &proof_root) const;
   td::Result<td::BufferSlice> extract_proof_boc() const;
+
+  void set_cell_load_callback(std::function<void(const td::Ref<vm::DataCell>&)> f) {
+    usage_tree->set_cell_load_callback(std::move(f));
+  }
 };
 
 }  // namespace vm

crypto/vm/cells/UsageCell.h

@@ -39,7 +39,7 @@ class UsageCell : public Cell {
   // load interface
   td::Result<LoadedCell> load_cell() const override {
     TRY_RESULT(loaded_cell, cell_->load_cell());
-    if (tree_node_.on_load()) {
+    if (tree_node_.on_load(loaded_cell.data_cell)) {
       CHECK(loaded_cell.tree_node.empty());
       loaded_cell.tree_node = tree_node_;
     }

keys/keys.hpp

@@ -260,6 +260,10 @@ class PublicKey {
   td::BufferSlice export_as_slice() const;
   static td::Result<PublicKey> import(td::Slice s);
 
+  bool is_ed25519() const {
+    return pub_key_.get_offset() == pub_key_.offset<pubkeys::Ed25519>();
+  }
+
   pubkeys::Ed25519 ed25519_value() const {
     CHECK(pub_key_.get_offset() == pub_key_.offset<pubkeys::Ed25519>());
     return pub_key_.get<pubkeys::Ed25519>();

overlay/overlay-broadcast.cpp

@@ -68,7 +68,7 @@ td::Status BroadcastSimple::run_checks() {
 
 td::Status BroadcastSimple::distribute() {
   auto B = serialize();
-  auto nodes = overlay_->get_neighbours(3);
+  auto nodes = overlay_->get_neighbours(overlay_->propagate_broadcast_to());
 
   auto manager = overlay_->overlay_manager();
   for (auto &n : nodes) {
@@ -115,7 +115,8 @@ td::Status BroadcastSimple::run() {
   return run_continue();
 }
 
-td::Status BroadcastSimple::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr<ton_api::overlay_broadcast> broadcast) {
+td::Status BroadcastSimple::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id,
+                                   tl_object_ptr<ton_api::overlay_broadcast> broadcast) {
   auto src = PublicKey{broadcast->src_};
   auto data_hash = sha256_bits256(broadcast->data_.as_slice());
   auto broadcast_hash = compute_broadcast_id(src, data_hash, broadcast->flags_);

overlay/overlay-fec-broadcast.cpp

@@ -112,7 +112,7 @@ td::Status BroadcastFec::distribute_part(td::uint32 seqno) {
   td::BufferSlice data_short = std::move(tls.first);
   td::BufferSlice data = std::move(tls.second);
 
-  auto nodes = overlay_->get_neighbours(5);
+  auto nodes = overlay_->get_neighbours(overlay_->propagate_broadcast_to());
   auto manager = overlay_->overlay_manager();
 
   for (auto &n : nodes) {

overlay/overlay-id.hpp

@@ -21,27 +21,45 @@
 #include "auto/tl/ton_api.h"
 #include "adnl/adnl-node-id.hpp"
 #include "overlay/overlays.h"
+#include "td/utils/SharedSlice.h"
+#include "td/utils/buffer.h"
 #include "td/utils/overloaded.h"
 #include "keys/encryptor.h"
+#include "td/utils/port/StdStreams.h"
+#include "td/utils/unique_ptr.h"
+#include <limits>
+#include <memory>
 
 namespace ton {
 
 namespace overlay {
 
 class OverlayNode {
  public:
-  explicit OverlayNode(adnl::AdnlNodeIdShort self_id, OverlayIdShort overlay) {
+  explicit OverlayNode(adnl::AdnlNodeIdShort self_id, OverlayIdShort overlay, td::uint32 flags) {
     source_ = self_id;
     overlay_ = overlay;
+    flags_ = flags;
     version_ = static_cast<td::int32>(td::Clocks::system());
   }
   static td::Result<OverlayNode> create(const tl_object_ptr<ton_api::overlay_node> &node) {
     TRY_RESULT(source, adnl::AdnlNodeIdFull::create(node->id_));
-    return OverlayNode{source, OverlayIdShort{node->overlay_}, node->version_, node->signature_.as_slice()};
+    return OverlayNode{source, OverlayIdShort{node->overlay_}, 0, node->version_, node->signature_.as_slice()};
   }
-  OverlayNode(td::Variant<adnl::AdnlNodeIdFull, adnl::AdnlNodeIdShort> source, OverlayIdShort overlay,
+  static td::Result<OverlayNode> create(const tl_object_ptr<ton_api::overlay_nodeV2> &node) {
+    TRY_RESULT(source, adnl::AdnlNodeIdFull::create(node->id_));
+    auto res = OverlayNode{source, OverlayIdShort{node->overlay_}, (td::uint32)node->flags_, node->version_,
+                           node->signature_.as_slice()};
+    res.update_certificate(OverlayMemberCertificate(node->certificate_.get()));
+    return res;
+  }
+  OverlayNode(td::Variant<adnl::AdnlNodeIdFull, adnl::AdnlNodeIdShort> source, OverlayIdShort overlay, td::uint32 flags,
               td::int32 version, td::Slice signature)
-      : source_(std::move(source)), overlay_(overlay), version_(version), signature_(td::SharedSlice(signature)) {
+      : source_(std::move(source))
+      , overlay_(overlay)
+      , flags_(flags)
+      , version_(version)
+      , signature_(td::SharedSlice(signature)) {
   }
   OverlayNode(td::Variant<adnl::AdnlNodeIdFull, adnl::AdnlNodeIdShort> source, OverlayIdShort overlay,
               td::int32 version, td::SharedSlice signature)
@@ -64,10 +82,17 @@ class OverlayNode {
   }
 
   td::BufferSlice to_sign() const {
-    auto obj = create_tl_object<ton_api::overlay_node_toSign>(nullptr, overlay_.tl(), version_);
-    source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { obj->id_ = id.tl(); },
-                                 [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.compute_short_id().tl(); }));
-    return serialize_tl_object(obj, true);
+    if (flags_ == 0) {
+      auto obj = create_tl_object<ton_api::overlay_node_toSign>(nullptr, overlay_.tl(), version_);
+      source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { obj->id_ = id.tl(); },
+                                   [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.compute_short_id().tl(); }));
+      return serialize_tl_object(obj, true);
+    } else {
+      auto obj = create_tl_object<ton_api::overlay_node_toSignEx>(nullptr, overlay_.tl(), flags_, version_);
+      source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { obj->id_ = id.tl(); },
+                                   [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.compute_short_id().tl(); }));
+      return serialize_tl_object(obj, true);
+    }
   }
   void update_adnl_id(adnl::AdnlNodeIdFull node_id) {
     source_ = node_id;
@@ -81,6 +106,9 @@ class OverlayNode {
   td::int32 version() const {
     return version_;
   }
+  td::uint32 flags() const {
+    return flags_;
+  }
   td::BufferSlice signature() const {
     return signature_.clone_as_buffer_slice();
   }
@@ -103,15 +131,69 @@ class OverlayNode {
                                  [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.tl(); }));
     return obj;
   }
+  tl_object_ptr<ton_api::overlay_nodeV2> tl_v2() const {
+    tl_object_ptr<ton_api::overlay_MemberCertificate> cert;
+    if (cert_ && !cert_->empty()) {
+      cert = cert_->tl();
+    } else {
+      cert = create_tl_object<ton_api::overlay_emptyMemberCertificate>();
+    }
+    auto obj = create_tl_object<ton_api::overlay_nodeV2>(nullptr, overlay_.tl(), flags_, version_,
+                                                         signature_.clone_as_buffer_slice(), std::move(cert));
+    source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { UNREACHABLE(); },
+                                 [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.tl(); }));
+    return obj;
+  }
   OverlayNode clone() const {
-    return OverlayNode{source_, overlay_, version_, signature_.clone()};
+    auto res = OverlayNode{source_, overlay_, version_, signature_.clone()};
+    if (cert_) {
+      res.cert_ = td::make_unique<OverlayMemberCertificate>(*cert_);
+    }
+    return res;
+  }
+
+  const OverlayMemberCertificate *certificate() const {
+    if (cert_) {
+      return cert_.get();
+    }
+    return &empty_certificate_;
+  }
+
+  void update_certificate(OverlayMemberCertificate cert) {
+    if (!cert_ || cert_->empty() || cert_->is_expired() || cert.is_newer(*cert_)) {
+      cert_ = td::make_unique<OverlayMemberCertificate>(std::move(cert));
+    }
+  }
+
+  void update(OverlayNode from) {
+    if (version_ < from.version_) {
+      source_ = from.source_;
+      overlay_ = from.overlay_;
+      flags_ = from.flags_;
+      version_ = from.version_;
+      signature_ = from.signature_.clone();
+    }
+    if (from.cert_ && !from.cert_->empty()) {
+      update_certificate(std::move(*from.cert_));
+    }
+  }
+
+  void clear_certificate() {
+    cert_ = nullptr;
+  }
+
+  bool has_full_id() const {
+    return source_.get_offset() == source_.offset<adnl::AdnlNodeIdFull>();
   }
 
  private:
   td::Variant<adnl::AdnlNodeIdFull, adnl::AdnlNodeIdShort> source_;
   OverlayIdShort overlay_;
+  td::uint32 flags_;
   td::int32 version_;
+  td::unique_ptr<OverlayMemberCertificate> cert_;
   td::SharedSlice signature_;
+  static const OverlayMemberCertificate empty_certificate_;
 };
 
 }  // namespace overlay