Improve validator session stats

tdutils/td/utils/Timer.cpp

@@ -87,4 +87,8 @@ void PerfWarningTimer::reset() {
   start_at_ = 0;
 }
 
+double PerfWarningTimer::elapsed() const {
+  return Time::now() - start_at_;
+}
+
 }  // namespace td

tdutils/td/utils/Timer.h

@@ -53,6 +53,7 @@ class PerfWarningTimer {
   PerfWarningTimer &operator=(PerfWarningTimer &&) = delete;
   ~PerfWarningTimer();
   void reset();
+  double elapsed() const;
 
  private:
   string name_;

tl/generate/scheme/ton_api.tl

@@ -752,7 +752,12 @@ http.server.config dhs:(vector http.server.dnsEntry) local_hosts:(vector http.se
 
 ---types---
 
-validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int block_timestamp:long comment:string = validatorSession.StatsProducer;
+validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int comment:string
+  block_timestamp:double is_accepted:Bool is_ours:Bool collation_time:double collated_at:double
+  validation_time:double validated_at:double
+  gen_utime:double
+  approved_weight:long approved_33pct_at:double approved_66pct_at:double
+  signed_weight:long signed_33pct_at:double signed_66pct_at:double = validatorSession.StatsProducer;
 
 validatorSession.statsRound timestamp:long producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound;
 

validator-session/validator-session-types.h

@@ -76,8 +76,50 @@ struct ValidatorSessionStats {
     PublicKeyHash id = PublicKeyHash::zero();
     ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero();
     int block_status = status_none;
-    td::uint64 block_timestamp = 0;
+    double block_timestamp = -1.0;
     std::string comment;
+
+    bool is_accepted = false;
+    bool is_ours = false;
+    double collation_time = -1.0;
+    double validation_time = -1.0;
+    double collated_at = -1.0;
+    double validated_at = -1.0;
+    double gen_utime = -1.0;
+
+    std::vector<bool> approvers, signers;
+    ValidatorWeight approved_weight = 0;
+    ValidatorWeight signed_weight = 0;
+    double approved_33pct_at = -1.0;
+    double approved_66pct_at = -1.0;
+    double signed_33pct_at = -1.0;
+    double signed_66pct_at = -1.0;
+
+    void set_approved_by(td::uint32 id, ValidatorWeight weight, ValidatorWeight total_weight) {
+      if (!approvers.at(id)) {
+        approvers.at(id) = true;
+        approved_weight += weight;
+        if (approved_33pct_at <= 0.0 && approved_weight >= total_weight / 3 + 1) {
+          approved_33pct_at = td::Clocks::system();
+        }
+        if (approved_66pct_at <= 0.0 && approved_weight >= (total_weight * 2) / 3 + 1) {
+          approved_66pct_at = td::Clocks::system();
+        }
+      }
+    }
+
+    void set_signed_by(td::uint32 id, ValidatorWeight weight, ValidatorWeight total_weight) {
+      if (!signers.at(id)) {
+        signers.at(id) = true;
+        signed_weight += weight;
+        if (signed_33pct_at <= 0.0 && signed_weight >= total_weight / 3 + 1) {
+          signed_33pct_at = td::Clocks::system();
+        }
+        if (signed_66pct_at <= 0.0 && signed_weight >= (total_weight * 2) / 3 + 1) {
+          signed_66pct_at = td::Clocks::system();
+        }
+      }
+    }
   };
   struct Round {
     td::uint64 timestamp = 0;

validator-session/validator-session.cpp

@@ -20,6 +20,7 @@
 #include "td/utils/Random.h"
 #include "td/utils/crypto.h"
 #include "candidate-serializer.h"
+#include "td/utils/overloaded.h"
 
 namespace ton {
 
@@ -86,6 +87,7 @@ void ValidatorSessionImpl::process_blocks(std::vector<catchain::CatChainBlock *>
 
   for (auto &msg : msgs) {
     VLOG(VALIDATOR_SESSION_INFO) << this << ": applying action: " << msg.get();
+    stats_process_actions(local_idx(), *msg);
     real_state_ = ValidatorSessionState::action(description(), real_state_, local_idx(), att, msg.get());
   }
 
@@ -167,6 +169,7 @@ void ValidatorSessionImpl::preprocess_block(catchain::CatChainBlock *block) {
       for (auto &msg : B->actions_) {
         VLOG(VALIDATOR_SESSION_INFO) << this << "[node " << description().get_source_id(block->source()) << "][block "
                                      << block->hash() << "]: applying action " << msg.get();
+        stats_process_actions(block->source(), *msg);
         state = ValidatorSessionState::action(description(), state, block->source(), att, msg.get());
       }
       state = ValidatorSessionState::make_all(description(), state, block->source(), att);
@@ -291,7 +294,10 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice
   CHECK(!pending_reject_.count(block_id));
   CHECK(!rejected_.count(block_id));
 
-  stats_set_candidate_status(cur_round_, src, block_id, ValidatorSessionStats::status_received);
+  auto stat = stats_get_candidate_stat(cur_round_, src, block_id);
+  if (stat && stat->block_status == ValidatorSessionStats::status_none) {
+    stat->block_status = ValidatorSessionStats::status_received;
+  }
   auto v = virtual_state_->choose_blocks_to_approve(description(), local_idx());
   for (auto &b : v) {
     if (b && SentBlock::get_block_id(b) == block_id) {
@@ -363,9 +369,15 @@ void ValidatorSessionImpl::process_query(PublicKeyHash src, td::BufferSlice data
 }
 
 void ValidatorSessionImpl::candidate_decision_fail(td::uint32 round, ValidatorSessionCandidateId hash,
-                                                   std::string result, td::uint32 src, td::BufferSlice proof) {
-  stats_set_candidate_status(round, description().get_source_id(src), hash, ValidatorSessionStats::status_rejected,
-                             result);
+                                                   std::string result, td::uint32 src, td::BufferSlice proof,
+                                                   double validation_time) {
+  auto stat = stats_get_candidate_stat(round, description().get_source_id(src), hash);
+  if (stat) {
+    stat->block_status = ValidatorSessionStats::status_rejected;
+    stat->comment = result;
+    stat->validation_time = validation_time;
+    stat->validated_at = td::Clocks::system();
+  }
   if (round != cur_round_) {
     return;
   }
@@ -379,9 +391,16 @@ void ValidatorSessionImpl::candidate_decision_fail(td::uint32 round, ValidatorSe
 }
 
 void ValidatorSessionImpl::candidate_decision_ok(td::uint32 round, ValidatorSessionCandidateId hash, RootHash root_hash,
-                                                 FileHash file_hash, td::uint32 src, td::uint32 ok_from) {
-  stats_set_candidate_status(round, description().get_source_id(src), hash, ValidatorSessionStats::status_approved,
-                             PSTRING() << "ts=" << ok_from);
+                                                 FileHash file_hash, td::uint32 src, td::uint32 ok_from,
+                                                 double validation_time) {
+  auto stat = stats_get_candidate_stat(round, description().get_source_id(src), hash);
+  if (stat) {
+    stat->block_status = ValidatorSessionStats::status_approved;
+    stat->comment = PSTRING() << "ts=" << ok_from;
+    stat->validation_time = validation_time;
+    stat->gen_utime = (double)ok_from;
+    stat->validated_at = td::Clocks::system();
+  }
   if (round != cur_round_) {
     return;
   }
@@ -418,10 +437,7 @@ void ValidatorSessionImpl::candidate_approved_signed(td::uint32 round, Validator
 }
 
 void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCandidateId root_hash,
-                                           td::BufferSlice data, td::BufferSlice collated_data) {
-  if (round != cur_round_) {
-    return;
-  }
+                                           td::BufferSlice data, td::BufferSlice collated_data, double collation_time) {
   if (data.size() > description().opts().max_block_size ||
       collated_data.size() > description().opts().max_collated_data_size) {
     LOG(ERROR) << this << ": generated candidate is too big. Dropping. size=" << data.size() << " "
@@ -430,14 +446,21 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCan
   }
   auto file_hash = sha256_bits256(data.as_slice());
   auto collated_data_file_hash = sha256_bits256(collated_data.as_slice());
+  auto block_id = description().candidate_id(local_idx(), root_hash, file_hash, collated_data_file_hash);
 
+  auto stat = stats_get_candidate_stat(round, local_id(), block_id);
+  if (stat) {
+    stat->block_status = ValidatorSessionStats::status_received;
+    stat->collation_time = collation_time;
+    stat->collated_at = td::Clocks::system();
+  }
+  if (round != cur_round_) {
+    return;
+  }
   auto b = create_tl_object<ton_api::validatorSession_candidate>(local_id().tl(), round, root_hash, std::move(data),
                                                                  std::move(collated_data));
-
   auto B = serialize_candidate(b, compress_block_candidates_).move_as_ok();
 
-  auto block_id = description().candidate_id(local_idx(), root_hash, file_hash, collated_data_file_hash);
-
   td::actor::send_closure(catchain_, &catchain::CatChain::send_broadcast, std::move(B));
 
   blocks_.emplace(block_id, std::move(b));
@@ -500,7 +523,7 @@ void ValidatorSessionImpl::check_generate_slot() {
           if (R.is_ok()) {
             auto c = R.move_as_ok();
             td::actor::send_closure(SelfId, &ValidatorSessionImpl::generated_block, round, c.id.root_hash,
-                                    c.data.clone(), c.collated_data.clone());
+                                    c.data.clone(), c.collated_data.clone(), timer.elapsed());
           } else {
             LOG(WARNING) << print_id << ": failed to generate block candidate: " << R.move_as_error();
           }
@@ -563,10 +586,10 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) {
         auto R = res.move_as_ok();
         if (R.is_ok()) {
           td::actor::send_closure(SelfId, &ValidatorSessionImpl::candidate_decision_ok, round, hash, root_hash,
-                                  file_hash, src, R.ok_from());
+                                  file_hash, src, R.ok_from(), timer.elapsed());
         } else {
           td::actor::send_closure(SelfId, &ValidatorSessionImpl::candidate_decision_fail, round, hash, R.reason(),
-                                  src, R.proof());
+                                  src, R.proof(), timer.elapsed());
         }
       });
       pending_approve_.insert(block_id);
@@ -824,6 +847,10 @@ void ValidatorSessionImpl::on_new_round(td::uint32 round) {
       cur_stats_.approve_signatures = (td::uint32)export_approve_sigs.size();
       cur_stats_.approve_signatures_weight = approve_signatures_weight;
       cur_stats_.creator = description().get_source_id(block->get_src_idx());
+      auto stat = stats_get_candidate_stat(cur_round_, cur_stats_.creator);
+      if (stat) {
+        stat->is_accepted = true;
+      }
 
       if (it == blocks_.end()) {
         callback_->on_block_committed(cur_round_, description().get_source_public_key(block->get_src_idx()),
@@ -1003,6 +1030,8 @@ void ValidatorSessionImpl::stats_init() {
   cur_stats_.total_validators = description().get_total_nodes();
   cur_stats_.total_weight = description().get_total_weight();
   cur_stats_.self = description().get_source_id(local_idx());
+  stats_pending_approve_.clear();
+  stats_pending_sign_.clear();
   stats_add_round();
 }
 
@@ -1016,33 +1045,91 @@ void ValidatorSessionImpl::stats_add_round() {
     if (priority >= 0) {
       CHECK((size_t)priority < round.producers.size());
       round.producers[priority].id = description().get_source_id(i);
+      round.producers[priority].is_ours = (local_idx() == i);
+      round.producers[priority].approvers.resize(description().get_total_nodes(), false);
+      round.producers[priority].signers.resize(description().get_total_nodes(), false);
     }
   }
   while (!round.producers.empty() && round.producers.back().id.is_zero()) {
     round.producers.pop_back();
   }
 }
 
-void ValidatorSessionImpl::stats_set_candidate_status(td::uint32 round, PublicKeyHash src,
-                                                      ValidatorSessionCandidateId candidate_id, int status,
-                                                      std::string comment) {
+ValidatorSessionStats::Producer *ValidatorSessionImpl::stats_get_candidate_stat(
+    td::uint32 round, PublicKeyHash src, ValidatorSessionCandidateId candidate_id) {
   if (round < cur_stats_.first_round || round - cur_stats_.first_round >= cur_stats_.rounds.size()) {
-    return;
+    return nullptr;
   }
   auto &stats_round = cur_stats_.rounds[round - cur_stats_.first_round];
   auto it = std::find_if(stats_round.producers.begin(), stats_round.producers.end(),
                          [&](const ValidatorSessionStats::Producer &p) { return p.id == src; });
   if (it == stats_round.producers.end()) {
-    return;
+    return nullptr;
+  }
+  if (!candidate_id.is_zero()) {
+    it->candidate_id = candidate_id;
   }
-  it->candidate_id = candidate_id;
-  if (it->block_status == ValidatorSessionStats::status_none) {
-    it->block_timestamp = (td::uint64)td::Clocks::system();
+  if (it->block_timestamp <= 0.0) {
+    it->block_timestamp = td::Clocks::system();
+  }
+  auto it2 = stats_pending_approve_.find({round, it->candidate_id});
+  if (it2 != stats_pending_approve_.end()) {
+    for (td::uint32 node_id : it2->second) {
+      it->set_approved_by(node_id, description().get_node_weight(node_id), description().get_total_weight());
+    }
+    stats_pending_approve_.erase(it2);
   }
-  it->block_status = status;
-  if (!comment.empty()) {
-    it->comment = std::move(comment);
+  it2 = stats_pending_sign_.find({round, it->candidate_id});
+  if (it2 != stats_pending_sign_.end()) {
+    for (td::uint32 node_id : it2->second) {
+      it->set_signed_by(node_id, description().get_node_weight(node_id), description().get_total_weight());
+    }
+    stats_pending_sign_.erase(it2);
   }
+  return &*it;
+}
+
+ValidatorSessionStats::Producer *ValidatorSessionImpl::stats_get_candidate_stat_by_id(
+    td::uint32 round, ValidatorSessionCandidateId candidate_id) {
+  if (round < cur_stats_.first_round || round - cur_stats_.first_round >= cur_stats_.rounds.size()) {
+    return nullptr;
+  }
+  auto &stats_round = cur_stats_.rounds[round - cur_stats_.first_round];
+  auto it = std::find_if(stats_round.producers.begin(), stats_round.producers.end(),
+                         [&](const ValidatorSessionStats::Producer &p) { return p.candidate_id == candidate_id; });
+  if (it == stats_round.producers.end()) {
+    return nullptr;
+  }
+  return &*it;
+}
+
+void ValidatorSessionImpl::stats_process_actions(td::uint32 node_id, ton_api::validatorSession_round_Message &action) {
+  ton_api::downcast_call(action, td::overloaded(
+                                     [&](const ton_api::validatorSession_message_approvedBlock &obj) {
+                                       if (obj.candidate_ == skip_round_candidate_id()) {
+                                         return;
+                                       }
+                                       auto stat = stats_get_candidate_stat_by_id(obj.round_, obj.candidate_);
+                                       if (stat) {
+                                         stat->set_approved_by(node_id, description().get_node_weight(node_id),
+                                                               description().get_total_weight());
+                                       } else {
+                                         stats_pending_approve_[{obj.round_, obj.candidate_}].push_back(node_id);
+                                       }
+                                     },
+                                     [&](const ton_api::validatorSession_message_commit &obj) {
+                                       if (obj.candidate_ == skip_round_candidate_id()) {
+                                         return;
+                                       }
+                                       auto stat = stats_get_candidate_stat_by_id(obj.round_, obj.candidate_);
+                                       if (stat) {
+                                         stat->set_signed_by(node_id, description().get_node_weight(node_id),
+                                                             description().get_total_weight());
+                                       } else {
+                                         stats_pending_sign_[{obj.round_, obj.candidate_}].push_back(node_id);
+                                       }
+                                     },
+                                     [](const auto &) {}));
 }
 
 td::actor::ActorOwn<ValidatorSession> ValidatorSession::create(

validator-session/validator-session.hpp

@@ -159,10 +159,18 @@ class ValidatorSessionImpl : public ValidatorSession {
   bool compress_block_candidates_ = false;
 
   ValidatorSessionStats cur_stats_;
+  std::map<std::pair<td::uint32, ValidatorSessionCandidateId>, std::vector<td::uint32>>
+      stats_pending_approve_;  // round, candidate_id -> approvers
+  std::map<std::pair<td::uint32, ValidatorSessionCandidateId>, std::vector<td::uint32>>
+      stats_pending_sign_;  // round, candidate_id -> signers
   void stats_init();
   void stats_add_round();
-  void stats_set_candidate_status(td::uint32 round, PublicKeyHash src, ValidatorSessionCandidateId candidate_id,
-                                  int status, std::string comment = "");
+  ValidatorSessionStats::Producer *stats_get_candidate_stat(
+      td::uint32 round, PublicKeyHash src,
+      ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero());
+  ValidatorSessionStats::Producer *stats_get_candidate_stat_by_id(td::uint32 round,
+                                                                  ValidatorSessionCandidateId candidate_id);
+  void stats_process_actions(td::uint32 node_id, ton_api::validatorSession_round_Message &action);
 
  public:
   ValidatorSessionImpl(catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id,
@@ -190,17 +198,16 @@ class ValidatorSessionImpl : public ValidatorSession {
   void process_query(PublicKeyHash src, td::BufferSlice data, td::Promise<td::BufferSlice> promise);
 
   void try_approve_block(const SentBlock *block);
-  void try_sign();
 
   void candidate_decision_fail(td::uint32 round, ValidatorSessionCandidateId hash, std::string result,
-                               td::uint32 src, td::BufferSlice proof);
+                               td::uint32 src, td::BufferSlice proof, double validation_time);
   void candidate_decision_ok(td::uint32 round, ValidatorSessionCandidateId hash, RootHash root_hash, FileHash file_hash,
-                             td::uint32 src, td::uint32 ok_from);
+                             td::uint32 src, td::uint32 ok_from, double validation_time);
   void candidate_approved_signed(td::uint32 round, ValidatorSessionCandidateId hash, td::uint32 ok_from,
                                  td::BufferSlice signature);
 
   void generated_block(td::uint32 round, ValidatorSessionRootHash root_hash, td::BufferSlice data,
-                       td::BufferSlice collated);
+                       td::BufferSlice collated, double collation_time);
   void signed_block(td::uint32 round, ValidatorSessionCandidateId hash, td::BufferSlice signature);
 
   void end_request(td::uint32 round, ValidatorSessionCandidateId block_id) {

validator/manager.cpp

@@ -2714,8 +2714,11 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id,
     std::vector<tl_object_ptr<ton_api::validatorSession_statsProducer>> producers;
     for (const auto &producer : round.producers) {
       producers.push_back(create_tl_object<ton_api::validatorSession_statsProducer>(
-          producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.block_timestamp,
-          producer.comment));
+          producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.comment,
+          producer.block_timestamp, producer.is_accepted, producer.is_ours, producer.collation_time,
+          producer.collated_at, producer.validation_time, producer.validated_at, producer.gen_utime,
+          producer.approved_weight, producer.approved_33pct_at, producer.approved_66pct_at, producer.signed_weight,
+          producer.signed_33pct_at, producer.signed_66pct_at));
     }
     rounds.push_back(create_tl_object<ton_api::validatorSession_statsRound>(round.timestamp, std::move(producers)));
   }