Improve validator session stats (ton-blockchain#1117)

* Improve validator session stats

* Collator stats: block limits, number of processed external messages
* Collator and validator work time
* Last key block seqno
* Approvers and signers

* End validator session stats

catchain/catchain.h

@@ -96,6 +96,7 @@ class CatChain : public td::actor::Actor {
   virtual void send_query_via(const PublicKeyHash &dst, std::string name, td::Promise<td::BufferSlice> promise,
                               td::Timestamp timeout, td::BufferSlice query, td::uint64 max_answer_size,
                               td::actor::ActorId<adnl::AdnlSenderInterface> via) = 0;
+  virtual void get_source_heights(td::Promise<std::vector<CatChainBlockHeight>> promise) = 0;
   virtual void destroy() = 0;
 
   static td::actor::ActorOwn<CatChain> create(std::unique_ptr<Callback> callback, const CatChainOptions &opts,

catchain/catchain.hpp

@@ -115,6 +115,15 @@ class CatChainImpl : public CatChain {
     td::actor::send_closure(receiver_, &CatChainReceiverInterface::send_custom_query_data_via, dst, name,
                             std::move(promise), timeout, std::move(query), max_answer_size, via);
   }
+  void get_source_heights(td::Promise<std::vector<CatChainBlockHeight>> promise) override {
+    std::vector<CatChainBlockHeight> heights(top_source_blocks_.size(), 0);
+    for (size_t i = 0; i < top_source_blocks_.size(); ++i) {
+      if (top_source_blocks_[i]) {
+        heights[i] = top_source_blocks_[i]->height();
+      }
+    }
+    promise.set_result(std::move(heights));
+  }
   void destroy() override;
   CatChainImpl(std::unique_ptr<Callback> callback, const CatChainOptions &opts,
                td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,

tdutils/td/utils/Timer.cpp

@@ -91,4 +91,47 @@ double PerfWarningTimer::elapsed() const {
   return Time::now() - start_at_;
 }
 
+static double thread_cpu_clock() {
+#if defined(CLOCK_THREAD_CPUTIME_ID)
+  timespec ts;
+  int result = clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts);
+  CHECK(result == 0);
+  return (double)ts.tv_sec + (double)ts.tv_nsec * 1e-9;
+#else
+  return 0.0;  // TODO: MacOS and Windows support (currently cpu timer is used only in validators)
+#endif
+}
+
+ThreadCpuTimer::ThreadCpuTimer(bool is_paused) : is_paused_(is_paused) {
+  if (is_paused_) {
+    start_time_ = 0;
+  } else {
+    start_time_ = thread_cpu_clock();
+  }
+}
+
+void ThreadCpuTimer::pause() {
+  if (is_paused_) {
+    return;
+  }
+  elapsed_ += thread_cpu_clock() - start_time_;
+  is_paused_ = true;
+}
+
+void ThreadCpuTimer::resume() {
+  if (!is_paused_) {
+    return;
+  }
+  start_time_ = thread_cpu_clock();
+  is_paused_ = false;
+}
+
+double ThreadCpuTimer::elapsed() const {
+  double res = elapsed_;
+  if (!is_paused_) {
+    res += thread_cpu_clock() - start_time_;
+  }
+  return res;
+}
+
 }  // namespace td

tdutils/td/utils/Timer.h

@@ -62,4 +62,22 @@ class PerfWarningTimer {
   std::function<void(double)> callback_;
 };
 
+class ThreadCpuTimer {
+ public:
+  ThreadCpuTimer() : ThreadCpuTimer(false) {
+  }
+  explicit ThreadCpuTimer(bool is_paused);
+  ThreadCpuTimer(const ThreadCpuTimer &other) = default;
+  ThreadCpuTimer &operator=(const ThreadCpuTimer &other) = default;
+
+  double elapsed() const;
+  void pause();
+  void resume();
+
+ private:
+  double elapsed_{0};
+  double start_time_;
+  bool is_paused_{false};
+};
+
 }  // namespace td

tl/generate/scheme/ton_api.tl

@@ -388,8 +388,8 @@ tonNode.newShardBlock block:tonNode.blockIdExt cc_seqno:int data:bytes = tonNode
 
 tonNode.blockBroadcastCompressed.data signatures:(vector tonNode.blockSignature) proof_data:bytes = tonNode.blockBroadcaseCompressed.Data;
 
-tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int 
-              signatures:(vector tonNode.blockSignature) 
+tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int
+              signatures:(vector tonNode.blockSignature)
               proof:bytes data:bytes = tonNode.Broadcast;
 tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int
               flags:# compressed:bytes = tonNode.Broadcast;
@@ -769,13 +769,19 @@ http.server.config dhs:(vector http.server.dnsEntry) local_hosts:(vector http.se
 
 ---types---
 
-validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int comment:string
-  block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double
+validatorSession.collationStats bytes:int gas:int lt_delta:int cat_bytes:int cat_gas:int cat_lt_delta:int
+  limits_log:string ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validadorSession.CollationStats;
+
+validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int root_hash:int256 file_hash:int256
+  comment:string block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double
   collation_time:double collated_at:double collation_cached:Bool
+  collation_work_time:double collation_cpu_work_time:double
+  collation_stats:validatorSession.collationStats
   validation_time:double validated_at:double validation_cached:Bool
+  validation_work_time:double validation_cpu_work_time:double
   gen_utime:double
-  approved_weight:long approved_33pct_at:double approved_66pct_at:double
-  signed_weight:long signed_33pct_at:double signed_66pct_at:double
+  approved_weight:long approved_33pct_at:double approved_66pct_at:double approvers:string
+  signed_weight:long signed_33pct_at:double signed_66pct_at:double signers:string
   serialize_time:double deserialize_time:double serialized_size:int = validatorSession.StatsProducer;
 
 validatorSession.statsRound timestamp:double producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound;
@@ -786,9 +792,14 @@ validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self:
     first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats;
 
 validatorSession.newValidatorGroupStats.node id:int256 weight:long = validatorSession.newValidatorGroupStats.Node;
-validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int timestamp:double
+validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int
+    last_key_block_seqno:int timestamp:double
     self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats;
 
+validatorSession.endValidatorGroupStats.node id:int256 catchain_blocks:int = validatorSession.endValidatorGroupStats.Node;
+validatorSession.endValidatorGroupStats session_id:int256 timestamp:double
+    nodes:(vector validatorSession.endValidatorGroupStats.node) = validatorSession.EndValidatorGroupStats;
+
 ---functions---
 
 ---types---

validator-session/validator-session-types.h

@@ -77,6 +77,8 @@ struct ValidatorSessionStats {
     ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero();
     int block_status = status_none;
     double block_timestamp = -1.0;
+    td::Bits256 root_hash = td::Bits256::zero();
+    td::Bits256 file_hash = td::Bits256::zero();
     std::string comment;
 
     bool is_accepted = false;
@@ -159,11 +161,23 @@ struct NewValidatorGroupStats {
   ValidatorSessionId session_id = ValidatorSessionId::zero();
   ShardIdFull shard{masterchainId};
   CatchainSeqno cc_seqno = 0;
+  BlockSeqno last_key_block_seqno = 0;
   double timestamp = -1.0;
   td::uint32 self_idx = 0;
   std::vector<Node> nodes;
 };
 
+struct EndValidatorGroupStats {
+  struct Node {
+    PublicKeyHash id = PublicKeyHash::zero();
+    td::uint32 catchain_blocks = 0;
+  };
+
+  ValidatorSessionId session_id = ValidatorSessionId::zero();
+  double timestamp = -1.0;
+  std::vector<Node> nodes;
+};
+
 }  // namespace validatorsession
 
 }  // namespace ton

validator-session/validator-session.cpp

@@ -270,6 +270,8 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice
     }
     stat->deserialize_time = deserialize_time;
     stat->serialized_size = data.size();
+    stat->root_hash = candidate->root_hash_;
+    stat->file_hash = file_hash;
   }
 
   if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK ||
@@ -468,6 +470,8 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCan
     stat->collated_at = td::Clocks::system();
     stat->block_timestamp = td::Clocks::system();
     stat->collation_cached = collation_cached;
+    stat->root_hash = root_hash;
+    stat->file_hash = file_hash;
   }
   if (round != cur_round_) {
     return;
@@ -602,6 +606,8 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) {
         if (stat->block_timestamp <= 0.0) {
           stat->block_timestamp = td::Clocks::system();
         }
+        stat->root_hash = B->root_hash_;
+        stat->file_hash = td::sha256_bits256(B->data_);
       }
 
       auto P = td::PromiseCreator::lambda([round = cur_round_, hash = block_id, root_hash = block->get_root_hash(),
@@ -997,6 +1003,29 @@ void ValidatorSessionImpl::get_current_stats(td::Promise<ValidatorSessionStats>
   promise.set_result(cur_stats_);
 }
 
+void ValidatorSessionImpl::get_end_stats(td::Promise<EndValidatorGroupStats> promise) {
+  if (!started_) {
+    promise.set_error(td::Status::Error(ErrorCode::notready, "not started"));
+    return;
+  }
+  EndValidatorGroupStats stats;
+  stats.session_id = unique_hash_;
+  stats.timestamp = td::Clocks::system();
+  stats.nodes.resize(description().get_total_nodes());
+  for (size_t i = 0; i < stats.nodes.size(); ++i) {
+    stats.nodes[i].id = description().get_source_id(i);
+  }
+  td::actor::send_closure(catchain_, &catchain::CatChain::get_source_heights,
+                          [promise = std::move(promise),
+                           stats = std::move(stats)](td::Result<std::vector<catchain::CatChainBlockHeight>> R) mutable {
+                            TRY_RESULT_PROMISE(promise, heights, std::move(R));
+                            for (size_t i = 0; i < std::min(heights.size(), stats.nodes.size()); ++i) {
+                              stats.nodes[i].catchain_blocks = heights[i];
+                            }
+                            promise.set_result(std::move(stats));
+                          });
+}
+
 void ValidatorSessionImpl::get_validator_group_info_for_litequery(
     td::uint32 cur_round,
     td::Promise<std::vector<tl_object_ptr<lite_api::liteServer_nonfinal_candidateInfo>>> promise) {

validator-session/validator-session.h

@@ -105,6 +105,7 @@ class ValidatorSession : public td::actor::Actor {
   virtual void start() = 0;
   virtual void destroy() = 0;
   virtual void get_current_stats(td::Promise<ValidatorSessionStats> promise) = 0;
+  virtual void get_end_stats(td::Promise<EndValidatorGroupStats> promise) = 0;
   virtual void get_validator_group_info_for_litequery(
       td::uint32 cur_round,
       td::Promise<std::vector<tl_object_ptr<lite_api::liteServer_nonfinal_candidateInfo>>> promise) = 0;

validator-session/validator-session.hpp

@@ -187,6 +187,7 @@ class ValidatorSessionImpl : public ValidatorSession {
   void start() override;
   void destroy() override;
   void get_current_stats(td::Promise<ValidatorSessionStats> promise) override;
+  void get_end_stats(td::Promise<EndValidatorGroupStats> promise) override;
   void get_validator_group_info_for_litequery(
       td::uint32 cur_round,
       td::Promise<std::vector<tl_object_ptr<lite_api::liteServer_nonfinal_candidateInfo>>> promise) override;

validator/impl/collator-impl.h

@@ -356,6 +356,11 @@ class Collator final : public td::actor::Actor {
 
  public:
   static td::uint32 get_skip_externals_queue_size();
+
+ private:
+  td::Timer work_timer_{true};
+  td::ThreadCpuTimer cpu_work_timer_{true};
+  CollationStats stats_;
 };
 
 }  // namespace validator