Merge branch 'accelerator' into accelerator-test

lite-client/lite-client.cpp

@@ -1627,27 +1627,93 @@ void TestNode::send_compute_complaint_price_query(ton::StdSmcAddress elector_add
 }
 
 bool TestNode::get_msg_queue_sizes() {
-  auto q = ton::serialize_tl_object(ton::create_tl_object<ton::lite_api::liteServer_getOutMsgQueueSizes>(0, 0, 0), true);
-  return envelope_send_query(std::move(q), [Self = actor_id(this)](td::Result<td::BufferSlice> res) -> void {
-    if (res.is_error()) {
-      LOG(ERROR) << "liteServer.getOutMsgQueueSizes error: " << res.move_as_error();
+  ton::BlockIdExt blkid = mc_last_id_;
+  if (!blkid.is_valid_full()) {
+    return set_error("must obtain last block information before making other queries");
+  }
+  if (!(ready_ && !client_.empty())) {
+    return set_error("server connection not ready");
+  }
+  auto b =
+      ton::create_serialize_tl_object<ton::lite_api::liteServer_getAllShardsInfo>(ton::create_tl_lite_block_id(blkid));
+  LOG(INFO) << "requesting recent shard configuration";
+  return envelope_send_query(std::move(b), [Self = actor_id(this), blkid](td::Result<td::BufferSlice> R) -> void {
+    if (R.is_error()) {
       return;
     }
-    auto F = ton::fetch_tl_object<ton::lite_api::liteServer_outMsgQueueSizes>(res.move_as_ok(), true);
+    auto F = ton::fetch_tl_object<ton::lite_api::liteServer_allShardsInfo>(R.move_as_ok(), true);
     if (F.is_error()) {
-      LOG(ERROR) << "cannot parse answer to liteServer.getOutMsgQueueSizes";
-      return;
+      LOG(ERROR) << "cannot parse answer to liteServer.getAllShardsInfo";
+    } else {
+      auto f = F.move_as_ok();
+      td::actor::send_closure_later(Self, &TestNode::get_msg_queue_sizes_cont, blkid, std::move(f->data_));
     }
-    td::actor::send_closure_later(Self, &TestNode::got_msg_queue_sizes, F.move_as_ok());
   });
 }
 
-void TestNode::got_msg_queue_sizes(ton::tl_object_ptr<ton::lite_api::liteServer_outMsgQueueSizes> f) {
+void TestNode::get_msg_queue_sizes_cont(ton::BlockIdExt mc_blkid, td::BufferSlice data) {
+  LOG(INFO) << "got shard configuration with respect to block " << mc_blkid.to_str();
+  std::vector<ton::BlockIdExt> blocks;
+  blocks.push_back(mc_blkid);
+  auto R = vm::std_boc_deserialize(data.clone());
+  if (R.is_error()) {
+    set_error(R.move_as_error_prefix("cannot deserialize shard configuration: "));
+    return;
+  }
+  auto root = R.move_as_ok();
+  block::ShardConfig sh_conf;
+  if (!sh_conf.unpack(vm::load_cell_slice_ref(root))) {
+    set_error("cannot extract shard block list from shard configuration");
+    return;
+  }
+  auto ids = sh_conf.get_shard_hash_ids(true);
+  for (auto id : ids) {
+    auto ref = sh_conf.get_shard_hash(ton::ShardIdFull(id));
+    if (ref.not_null()) {
+      blocks.push_back(ref->top_block_id());
+    }
+  }
+
+  struct QueryInfo {
+    std::vector<ton::BlockIdExt> blocks;
+    std::vector<td::uint64> sizes;
+    size_t pending;
+  };
+  auto info = std::make_shared<QueryInfo>();
+  info->blocks = std::move(blocks);
+  info->sizes.resize(info->blocks.size(), 0);
+  info->pending = info->blocks.size();
+
+  for (size_t i = 0; i < info->blocks.size(); ++i) {
+    ton::BlockIdExt block_id = info->blocks[i];
+    auto b = ton::create_serialize_tl_object<ton::lite_api::liteServer_getBlockOutMsgQueueSize>(
+        0, ton::create_tl_lite_block_id(block_id), false);
+    LOG(DEBUG) << "requesting queue size for block " << block_id.to_str();
+    envelope_send_query(std::move(b), [=, this](td::Result<td::BufferSlice> R) -> void {
+      if (R.is_error()) {
+        return;
+      }
+      auto F = ton::fetch_tl_object<ton::lite_api::liteServer_blockOutMsgQueueSize>(R.move_as_ok(), true);
+      if (F.is_error()) {
+        set_error(F.move_as_error_prefix("failed to get queue size: "));
+        return;
+      }
+      auto f = F.move_as_ok();
+      LOG(DEBUG) << "got queue size for block " << block_id.to_str() << " : " << f->size_;
+      info->sizes[i] = f->size_;
+      if (--info->pending == 0) {
+        get_msg_queue_sizes_finish(std::move(info->blocks), std::move(info->sizes));
+      }
+    });
+  }
+}
+
+void TestNode::get_msg_queue_sizes_finish(std::vector<ton::BlockIdExt> blocks, std::vector<td::uint64> sizes) {
+  CHECK(blocks.size() == sizes.size());
   td::TerminalIO::out() << "Outbound message queue sizes:" << std::endl;
-  for (auto &x : f->shards_) {
-    td::TerminalIO::out() << ton::create_block_id(x->id_).id.to_str() << "    " << x->size_ << std::endl;
+  for (size_t i = 0; i < blocks.size(); ++i) {
+    td::TerminalIO::out() << blocks[i].id.to_str() << "    " << sizes[i] << std::endl;
   }
-  td::TerminalIO::out() << "External message queue size limit: " << f->ext_msg_queue_size_limit_ << std::endl;
 }
 
 bool TestNode::get_dispatch_queue_info(ton::BlockIdExt block_id) {

lite-client/lite-client.h

@@ -324,7 +324,8 @@ class TestNode : public td::actor::Actor {
   void send_compute_complaint_price_query(ton::StdSmcAddress elector_addr, unsigned expires_in, unsigned bits,
                                           unsigned refs, td::Bits256 chash, std::string filename);
   bool get_msg_queue_sizes();
-  void got_msg_queue_sizes(ton::tl_object_ptr<ton::lite_api::liteServer_outMsgQueueSizes> f);
+  void get_msg_queue_sizes_cont(ton::BlockIdExt mc_blkid, td::BufferSlice data);
+  void get_msg_queue_sizes_finish(std::vector<ton::BlockIdExt> blocks, std::vector<td::uint64> sizes);
   bool get_dispatch_queue_info(ton::BlockIdExt block_id);
   bool get_dispatch_queue_info_cont(ton::BlockIdExt block_id, bool first, td::Bits256 after_addr);
   void got_dispatch_queue_info(ton::BlockIdExt block_id,

tdutils/td/utils/StringBuilder.h

@@ -149,4 +149,19 @@ std::enable_if_t<std::is_arithmetic<T>::value, string> to_string(const T &x) {
   return sb.as_cslice().str();
 }
 
+template <class SB>
+struct LambdaPrintHelper {
+  SB& sb;
+};
+template <class SB, class F>
+SB& operator<<(const LambdaPrintHelper<SB>& helper, F&& f) {
+  f(helper.sb);
+  return helper.sb;
+}
+struct LambdaPrint {};
+
+inline LambdaPrintHelper<td::StringBuilder> operator<<(td::StringBuilder& sb, const LambdaPrint&) {
+  return LambdaPrintHelper<td::StringBuilder>{sb};
+}
+
 }  // namespace td

tdutils/td/utils/Timer.cpp

@@ -22,6 +22,8 @@
 #include "td/utils/logging.h"
 #include "td/utils/Time.h"
 
+#include <numeric>
+
 namespace td {
 
 Timer::Timer(bool is_paused) : is_paused_(is_paused) {
@@ -60,12 +62,15 @@ StringBuilder &operator<<(StringBuilder &string_builder, const Timer &timer) {
   return string_builder << format::as_time(timer.elapsed());
 }
 
-PerfWarningTimer::PerfWarningTimer(string name, double max_duration, std::function<void(double)>&& callback)
+PerfWarningTimer::PerfWarningTimer(string name, double max_duration, std::function<void(double)> &&callback)
     : name_(std::move(name)), start_at_(Time::now()), max_duration_(max_duration), callback_(std::move(callback)) {
 }
 
 PerfWarningTimer::PerfWarningTimer(PerfWarningTimer &&other)
-    : name_(std::move(other.name_)), start_at_(other.start_at_), max_duration_(other.max_duration_), callback_(std::move(other.callback_)) {
+    : name_(std::move(other.name_))
+    , start_at_(other.start_at_)
+    , max_duration_(other.max_duration_)
+    , callback_(std::move(other.callback_)) {
   other.start_at_ = 0;
 }
 
@@ -134,4 +139,34 @@ double ThreadCpuTimer::elapsed() const {
   return res;
 }
 
+PerfLogAction PerfLog::start_action(std::string name) {
+  auto i = entries_.size();
+  entries_.push_back({.name = std::move(name), .begin = td::Timestamp::now().at()});
+  return PerfLogAction{i, std::unique_ptr<PerfLog, EmptyDeleter>(this)};
+}
+td::StringBuilder &operator<<(StringBuilder &sb, const PerfLog &log) {
+  sb << "{";
+  std::vector<size_t> ids(log.entries_.size());
+  std::iota(ids.begin(), ids.end(), 0);
+  std::sort(ids.begin(), ids.end(), [&](auto a, auto b) {
+    return log.entries_[a].end - log.entries_[a].begin > log.entries_[b].end - log.entries_[b].begin;
+  });
+  sb << "{";
+  for (size_t i = 0; i < log.entries_.size(); i++) {
+    sb << "\n\t";
+    auto &entry = log.entries_[ids[i]];
+    sb << "{" << entry.name << ":" << entry.begin << "->" << entry.end << "(" << entry.end - entry.begin << ")"
+       << td::format::cond(entry.status.is_error(), entry.status, "") << "}";
+  }
+  sb << "\n}";
+  return sb;
+}
+
+double PerfLog::finish_action(size_t i, td::Status status) {
+  auto &entry = entries_[i];
+  CHECK(entry.end == 0);
+  entry.end = td::Timestamp::now().at();
+  entry.status = std::move(status);
+  return entry.end - entry.begin;
+}
 }  // namespace td

tdutils/td/utils/Timer.h

@@ -19,6 +19,7 @@
 #pragma once
 
 #include "td/utils/StringBuilder.h"
+#include "td/utils/Status.h"
 
 #include <functional>
 
@@ -46,7 +47,7 @@ class Timer {
 
 class PerfWarningTimer {
  public:
-  explicit PerfWarningTimer(string name, double max_duration = 0.1, std::function<void(double)>&& callback = {});
+  explicit PerfWarningTimer(string name, double max_duration = 0.1, std::function<void(double)> &&callback = {});
   PerfWarningTimer(const PerfWarningTimer &) = delete;
   PerfWarningTimer &operator=(const PerfWarningTimer &) = delete;
   PerfWarningTimer(PerfWarningTimer &&other);
@@ -80,4 +81,44 @@ class ThreadCpuTimer {
   bool is_paused_{false};
 };
 
+class PerfLog;
+struct EmptyDeleter {
+  template <class T>
+  void operator()(T *) {
+  }
+};
+class PerfLogAction {
+ public:
+  template <class T>
+  double finish(const T &result);
+  size_t i_{0};
+  std::unique_ptr<PerfLog, EmptyDeleter> perf_log_;
+};
+
+class PerfLog {
+ public:
+  PerfLogAction start_action(std::string name);
+  friend td::StringBuilder &operator<<(td::StringBuilder &sb, const PerfLog &log);
+
+ private:
+  struct Entry {
+    std::string name;
+    double begin{};
+    double end{};
+    td::Status status;
+  };
+  std::vector<Entry> entries_;
+  friend class PerfLogAction;
+
+  double finish_action(size_t i, td::Status status);
+};
+template <class T>
+double PerfLogAction::finish(const T &result) {
+  if (result.is_ok()) {
+    return perf_log_->finish_action(i_, td::Status::OK());
+  } else {
+    return perf_log_->finish_action(i_, result.error().clone());
+  }
+}
+
 }  // namespace td

tdutils/td/utils/logging.h

@@ -74,6 +74,7 @@
 
 #define LOG(level) LOG_IMPL(level, level, true, ::td::Slice())
 #define LOG_IF(level, condition) LOG_IMPL(level, level, condition, #condition)
+#define FLOG(level) LOG_IMPL(level, level, true, ::td::Slice()) << td::LambdaPrint{} << [&](auto &sb)
 
 #define VLOG(level) LOG_IMPL(DEBUG, level, true, TD_DEFINE_STR(level))
 #define VLOG_IF(level, condition) LOG_IMPL(DEBUG, level, condition, TD_DEFINE_STR(level) " " #condition)
@@ -95,13 +96,13 @@ inline bool no_return_func() {
 #define DUMMY_LOG_CHECK(condition) LOG_IF(NEVER, !(condition))
 
 #ifdef TD_DEBUG
-  #if TD_MSVC
+#if TD_MSVC
     #define LOG_CHECK(condition)        \
       __analysis_assume(!!(condition)); \
       LOG_IMPL(FATAL, FATAL, !(condition), #condition)
-  #else
+#else
     #define LOG_CHECK(condition) LOG_IMPL(FATAL, FATAL, !(condition) && no_return_func(), #condition)
-  #endif
+#endif
 #else
   #define LOG_CHECK DUMMY_LOG_CHECK
 #endif
@@ -263,6 +264,9 @@ class Logger {
     sb_ << other;
     return *this;
   }
+  LambdaPrintHelper<td::Logger> operator<<(const LambdaPrint &) {
+    return LambdaPrintHelper<td::Logger>{*this};
+  }
 
   MutableCSlice as_cslice() {
     return sb_.as_cslice();

tl/generate/scheme/ton_api.tl

@@ -433,6 +433,10 @@ tonNode.newBlockCandidateBroadcast id:tonNode.blockIdExt catchain_seqno:int vali
 tonNode.newBlockCandidateBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int
               collator_signature:tonNode.blockSignature flags:# compressed:bytes = tonNode.Broadcast;
 
+// optimistic broadcast of response to tonNode.getOutMsgQueueProof with dst_shard, block and limits arguments
+tonNode.outMsgQueueProofBroadcast dst_shard:tonNode.shardId block:tonNode.blockIdExt
+              limits:ImportedMsgQueueLimits proof:tonNode.OutMsgQueueProof = tonNode.Broadcast;
+
 tonNode.shardPublicOverlayId workchain:int shard:long zero_state_file_hash:int256 = tonNode.ShardPublicOverlayId;
 
 tonNode.privateBlockOverlayId zero_state_file_hash:int256 nodes:(vector int256) = tonNode.PrivateBlockOverlayId;
@@ -746,7 +750,7 @@ engine.validator.perfTimerStats stats:(vector engine.validator.PerfTimerStatsByN
 engine.validator.shardOutQueueSize size:long = engine.validator.ShardOutQueueSize;
 
 engine.validator.collationManagerStats.shard shard_id:tonNode.shardId self_collate:Bool select_mode:string active:Bool collators:(vector int256) = engine.validator.collationManagerStats.Shard;
-engine.validator.collationManagerStats.collator adnl_id:int256 active:Bool alive:Bool ping_in:double = engine.validator.collationManagerStats.Collator;
+engine.validator.collationManagerStats.collator adnl_id:int256 active:Bool alive:Bool ping_in:double last_ping_ago:double last_ping_status:string = engine.validator.collationManagerStats.Collator;
 engine.validator.collationManagerStats.localId adnl_id:int256 shards:(vector engine.validator.collationManagerStats.shard)
     collators:(vector engine.validator.collationManagerStats.collator) = engine.validator.collationManagerStats.LocalId;
 engine.validator.collationManagerStats local_ids:(vector engine.validator.collationManagerStats.localId) = engine.validator.CollationManagerStats;
@@ -924,7 +928,7 @@ validatorSession.endValidatorGroupStats session_id:int256 timestamp:double
 
 ---functions---
 collatorNode.generateBlock shard:tonNode.shardId cc_seqno:int prev_blocks:(vector tonNode.blockIdExt)
-    creator:int256 = collatorNode.Candidate;
+    creator:int256 round:int first_block_round:int priority:int = collatorNode.Candidate;
 collatorNode.ping flags:# = collatorNode.Pong;
 
 ---types---

ton/ton-types.h

@@ -425,26 +425,68 @@ struct Ed25519_PublicKey {
 };
 
 // represents (the contents of) a block
+
+struct OutMsgQueueProofBroadcast : public td::CntObject {
+  OutMsgQueueProofBroadcast(ShardIdFull dst_shard, BlockIdExt block_id, td::int32 max_bytes, td::int32 max_msgs,
+                            td::BufferSlice queue_proof, td::BufferSlice block_state_proof, int msg_count)
+      : dst_shard(std::move(dst_shard))
+      , block_id(block_id)
+      , max_bytes(max_bytes)
+      , max_msgs(max_msgs)
+      , queue_proofs(std::move(queue_proof))
+      , block_state_proofs(std::move(block_state_proof))
+      , msg_count(std::move(msg_count)) {
+  }
+  ShardIdFull dst_shard;
+  BlockIdExt block_id;
+
+  // importedMsgQueueLimits
+  td::uint32 max_bytes;
+  td::uint32 max_msgs;
+
+  // outMsgQueueProof
+  td::BufferSlice queue_proofs;
+  td::BufferSlice block_state_proofs;
+  int msg_count;
+
+  OutMsgQueueProofBroadcast* make_copy() const override {
+    return new OutMsgQueueProofBroadcast(dst_shard, block_id, max_bytes, max_msgs, queue_proofs.clone(),
+                                         block_state_proofs.clone(), msg_count);
+  }
+};
+
 struct BlockCandidate {
   BlockCandidate(Ed25519_PublicKey pubkey, BlockIdExt id, FileHash collated_file_hash, td::BufferSlice data,
-                 td::BufferSlice collated_data)
+                 td::BufferSlice collated_data,
+                 std::vector<td::Ref<OutMsgQueueProofBroadcast>> out_msg_queue_broadcasts = {})
       : pubkey(pubkey)
       , id(id)
       , collated_file_hash(collated_file_hash)
       , data(std::move(data))
-      , collated_data(std::move(collated_data)) {
+      , collated_data(std::move(collated_data))
+      , out_msg_queue_proof_broadcasts(std::move(out_msg_queue_broadcasts)) {
   }
   Ed25519_PublicKey pubkey;
   BlockIdExt id;
   FileHash collated_file_hash;
   td::BufferSlice data;
   td::BufferSlice collated_data;
 
+  // used only locally
+  std::vector<td::Ref<OutMsgQueueProofBroadcast>> out_msg_queue_proof_broadcasts;
+
   BlockCandidate clone() const {
-    return BlockCandidate{pubkey, id, collated_file_hash, data.clone(), collated_data.clone()};
+    return BlockCandidate{
+        pubkey, id, collated_file_hash, data.clone(), collated_data.clone(), out_msg_queue_proof_broadcasts};
   }
 };
 
+struct BlockCandidatePriority {
+  td::uint32 round{};
+  td::uint32 first_block_round{};
+  td::int32 priority{};
+};
+
 struct ValidatorDescr {
   /* ton::validator::ValidatorFullId */ Ed25519_PublicKey key;
   ValidatorWeight weight;