Skip to content

Commit

Permalink
Improve overlay stats (ton-blockchain#1242)
Browse files Browse the repository at this point in the history
  • Loading branch information
@SpyCheese
SpyCheese authored Oct 4, 2024
1 parent f94d1be commit 8364a24
Show file tree
Hide file tree
Showing 8 changed files with 179 additions and 104 deletions.
23 changes: 18 additions & 5 deletions overlay/overlay-manager.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -188,7 +188,7 @@ void OverlayManager::receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeId
return;
}

td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), false);
td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_in_ctr, src, data.size(), false, false);
td::actor::send_closure(it2->second.overlay, &Overlay::receive_message, src, std::move(extra), std::move(data));
}

Expand Down Expand Up @@ -225,7 +225,14 @@ void OverlayManager::receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdSh
return;
}

td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), true);
td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_in_ctr, src, data.size(), true, false);
promise = [overlay = it2->second.overlay.get(), promise = std::move(promise),
src](td::Result<td::BufferSlice> R) mutable {
if (R.is_ok()) {
td::actor::send_closure(overlay, &Overlay::update_throughput_out_ctr, src, R.ok().size(), false, true);
}
promise.set_result(std::move(R));
};
td::actor::send_closure(it2->second.overlay, &Overlay::receive_query, src, std::move(extra), std::move(data),
std::move(promise));
}
Expand All @@ -243,8 +250,14 @@ void OverlayManager::send_query_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdS
if (it != overlays_.end()) {
auto it2 = it->second.find(overlay_id);
if (it2 != it->second.end()) {
td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, (td::uint32)query.size(),
true);
td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, query.size(), true, false);
promise = [overlay = it2->second.overlay.get(), promise = std::move(promise),
dst](td::Result<td::BufferSlice> R) mutable {
if (R.is_ok()) {
td::actor::send_closure(overlay, &Overlay::update_throughput_in_ctr, dst, R.ok().size(), false, true);
}
promise.set_result(std::move(R));
};
if (!it2->second.member_certificate.empty()) {
extra->flags_ |= 1;
extra->certificate_ = it2->second.member_certificate.tl();
Expand Down Expand Up @@ -273,7 +286,7 @@ void OverlayManager::send_message_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeI
if (it != overlays_.end()) {
auto it2 = it->second.find(overlay_id);
if (it2 != it->second.end()) {
td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, (td::uint32)object.size(),
td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, object.size(), false,
false);
if (!it2->second.member_certificate.empty()) {
// do not send certificate here, we hope that all our neighbours already know of out certificate
Expand Down
28 changes: 20 additions & 8 deletions overlay/overlay-peers.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -574,28 +574,40 @@ void OverlayImpl::update_peer_err_ctr(adnl::AdnlNodeIdShort peer_id, bool is_fec
}
}

void OverlayImpl::update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) {
void OverlayImpl::update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query,
bool is_response) {
auto out_peer = peer_list_.peers_.get(peer_id);
if (out_peer) {
out_peer->throughput_out_bytes_ctr += msg_size;
out_peer->throughput_out_packets_ctr++;

out_peer->traffic_ctr.add_packet(msg_size, false);
if (is_response) {
out_peer->traffic_responses_ctr.add_packet(msg_size, false);
}
if (is_query) {
out_peer->last_out_query_at = td::Timestamp::now();
}
}
total_traffic_ctr.add_packet(msg_size, false);
if (is_response) {
total_traffic_responses_ctr.add_packet(msg_size, false);
}
}

void OverlayImpl::update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) {
void OverlayImpl::update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query,
bool is_response) {
auto in_peer = peer_list_.peers_.get(peer_id);
if (in_peer) {
in_peer->throughput_in_bytes_ctr += msg_size;
in_peer->throughput_in_packets_ctr++;

in_peer->traffic_ctr.add_packet(msg_size, true);
if (is_response) {
in_peer->traffic_responses_ctr.add_packet(msg_size, true);
}
if (is_query) {
in_peer->last_in_query_at = td::Timestamp::now();
}
}
total_traffic_ctr.add_packet(msg_size, true);
if (is_response) {
total_traffic_responses_ctr.add_packet(msg_size, true);
}
}

void OverlayImpl::update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) {
Expand Down
85 changes: 53 additions & 32 deletions overlay/overlay.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,30 +38,32 @@ namespace overlay {

const OverlayMemberCertificate OverlayNode::empty_certificate_{};

static std::string overlay_actor_name(const OverlayIdFull &overlay_id) {
return PSTRING() << "overlay." << overlay_id.compute_short_id().bits256_value().to_hex().substr(0, 4);
}

td::actor::ActorOwn<Overlay> Overlay::create_public(td::actor::ActorId<keyring::Keyring> keyring,
td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<OverlayManager> manager,
td::actor::ActorId<dht::Dht> dht_node,
adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id,
std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules, td::string scope, OverlayOptions opts) {
auto R = td::actor::create_actor<OverlayImpl>(
"overlay", keyring, adnl, manager, dht_node, local_id, std::move(overlay_id), OverlayType::Public,
std::vector<adnl::AdnlNodeIdShort>(), std::vector<PublicKeyHash>(), OverlayMemberCertificate{}, std::move(callback),
std::move(rules), std::move(scope), std::move(opts));
return td::actor::ActorOwn<Overlay>(std::move(R));
return td::actor::create_actor<OverlayImpl>(
overlay_actor_name(overlay_id), keyring, adnl, manager, dht_node, local_id, std::move(overlay_id),
OverlayType::Public, std::vector<adnl::AdnlNodeIdShort>(), std::vector<PublicKeyHash>(),
OverlayMemberCertificate{}, std::move(callback), std::move(rules), std::move(scope), std::move(opts));
}

td::actor::ActorOwn<Overlay> Overlay::create_private(
td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
td::actor::ActorId<OverlayManager> manager, td::actor::ActorId<dht::Dht> dht_node, adnl::AdnlNodeIdShort local_id,
OverlayIdFull overlay_id, std::vector<adnl::AdnlNodeIdShort> nodes, std::unique_ptr<Overlays::Callback> callback,
OverlayPrivacyRules rules, std::string scope, OverlayOptions opts) {
auto R = td::actor::create_actor<OverlayImpl>("overlay", keyring, adnl, manager, dht_node, local_id,
std::move(overlay_id), OverlayType::FixedMemberList, std::move(nodes),
std::vector<PublicKeyHash>(), OverlayMemberCertificate{},
std::move(callback), std::move(rules), std::move(scope));
return td::actor::ActorOwn<Overlay>(std::move(R));
return td::actor::create_actor<OverlayImpl>(
overlay_actor_name(overlay_id), keyring, adnl, manager, dht_node, local_id, std::move(overlay_id),
OverlayType::FixedMemberList, std::move(nodes), std::vector<PublicKeyHash>(), OverlayMemberCertificate{},
std::move(callback), std::move(rules), std::move(scope));
}

td::actor::ActorOwn<Overlay> Overlay::create_semiprivate(
Expand All @@ -70,11 +72,10 @@ td::actor::ActorOwn<Overlay> Overlay::create_semiprivate(
OverlayIdFull overlay_id, std::vector<adnl::AdnlNodeIdShort> nodes, std::vector<PublicKeyHash> root_public_keys,
OverlayMemberCertificate cert, std::unique_ptr<Overlays::Callback> callback, OverlayPrivacyRules rules,
std::string scope, OverlayOptions opts) {
auto R = td::actor::create_actor<OverlayImpl>(
"overlay", keyring, adnl, manager, dht_node, local_id, std::move(overlay_id), OverlayType::CertificatedMembers,
std::move(nodes), std::move(root_public_keys), std::move(cert), std::move(callback), std::move(rules),
std::move(scope), std::move(opts));
return td::actor::ActorOwn<Overlay>(std::move(R));
return td::actor::create_actor<OverlayImpl>(overlay_actor_name(overlay_id), keyring, adnl, manager, dht_node,
local_id, std::move(overlay_id), OverlayType::CertificatedMembers,
std::move(nodes), std::move(root_public_keys), std::move(cert),
std::move(callback), std::move(rules), std::move(scope), std::move(opts));
}

OverlayImpl::OverlayImpl(td::actor::ActorId<keyring::Keyring> keyring, td::actor::ActorId<adnl::Adnl> adnl,
Expand Down Expand Up @@ -281,17 +282,12 @@ void OverlayImpl::alarm() {

auto SelfId = actor_id(this);
iterate_all_peers([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) {
peer.throughput_out_bytes = static_cast<td::uint32>(peer.throughput_out_bytes_ctr / t_elapsed);
peer.throughput_in_bytes = static_cast<td::uint32>(peer.throughput_in_bytes_ctr / t_elapsed);

peer.throughput_out_packets = static_cast<td::uint32>(peer.throughput_out_packets_ctr / t_elapsed);
peer.throughput_in_packets = static_cast<td::uint32>(peer.throughput_in_packets_ctr / t_elapsed);

peer.throughput_out_bytes_ctr = 0;
peer.throughput_in_bytes_ctr = 0;

peer.throughput_out_packets_ctr = 0;
peer.throughput_in_packets_ctr = 0;
peer.traffic = peer.traffic_ctr;
peer.traffic.normalize(t_elapsed);
peer.traffic_ctr = {};
peer.traffic_responses = peer.traffic_responses_ctr;
peer.traffic_responses.normalize(t_elapsed);
peer.traffic_responses_ctr = {};

auto P = td::PromiseCreator::lambda([SelfId, peer_id = key](td::Result<td::string> result) {
result.ensure();
Expand All @@ -300,6 +296,12 @@ void OverlayImpl::alarm() {

td::actor::send_closure(adnl_, &adnl::AdnlSenderInterface::get_conn_ip_str, local_id_, key, std::move(P));
});
total_traffic = total_traffic_ctr;
total_traffic.normalize(t_elapsed);
total_traffic_ctr = {};
total_traffic_responses = total_traffic_responses_ctr;
total_traffic_responses.normalize(t_elapsed);
total_traffic_responses_ctr = {};

update_throughput_at_ = td::Timestamp::in(50.0);
last_throughput_update_ = td::Timestamp::now();
Expand Down Expand Up @@ -691,12 +693,8 @@ void OverlayImpl::get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_
iterate_all_peers([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) {
auto node_obj = create_tl_object<ton_api::engine_validator_overlayStatsNode>();
node_obj->adnl_id_ = key.bits256_value();
node_obj->t_out_bytes_ = peer.throughput_out_bytes;
node_obj->t_in_bytes_ = peer.throughput_in_bytes;

node_obj->t_out_pckts_ = peer.throughput_out_packets;
node_obj->t_in_pckts_ = peer.throughput_in_packets;

node_obj->traffic_ = peer.traffic.tl();
node_obj->traffic_responses_ = peer.traffic_responses.tl();
node_obj->ip_addr_ = peer.ip_addr_str;

node_obj->last_in_query_ = static_cast<td::uint32>(peer.last_in_query_at.at_unix());
Expand All @@ -712,6 +710,8 @@ void OverlayImpl::get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_
res->nodes_.push_back(std::move(node_obj));
});

res->total_traffic_ = total_traffic.tl();
res->total_traffic_responses_ = total_traffic_responses.tl();
res->stats_.push_back(
create_tl_object<ton_api::engine_validator_oneStat>("neighbours_cnt", PSTRING() << neighbours_cnt()));

Expand All @@ -732,6 +732,27 @@ bool OverlayImpl::has_valid_broadcast_certificate(const PublicKeyHash &source, s
BroadcastCheckResult::Forbidden;
}

void TrafficStats::add_packet(td::uint64 size, bool in) {
if (in) {
++in_packets;
in_bytes += size;
} else {
++out_packets;
out_bytes += size;
}
}

void TrafficStats::normalize(double elapsed) {
out_bytes = static_cast<td::uint64>(out_bytes / elapsed);
in_bytes = static_cast<td::uint64>(in_bytes / elapsed);
out_packets = static_cast<td::uint32>(out_packets / elapsed);
in_packets = static_cast<td::uint32>(in_packets / elapsed);
}

tl_object_ptr<ton_api::engine_validator_overlayStatsTraffic> TrafficStats::tl() const {
return create_tl_object<ton_api::engine_validator_overlayStatsTraffic>(out_bytes, in_bytes, out_packets, in_packets);
}

} // namespace overlay

} // namespace ton
6 changes: 4 additions & 2 deletions overlay/overlay.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,8 +72,10 @@ class Overlay : public td::actor::Actor {
virtual void receive_nodes_from_db(tl_object_ptr<ton_api::overlay_nodes> nodes) = 0;
virtual void receive_nodes_from_db_v2(tl_object_ptr<ton_api::overlay_nodesV2> nodes) = 0;
virtual void get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_overlayStats>> promise) = 0;
virtual void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) = 0;
virtual void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) = 0;
virtual void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query,
bool is_response) = 0;
virtual void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query,
bool is_response) = 0;
virtual void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) = 0;
virtual void update_member_certificate(OverlayMemberCertificate cert) = 0;
virtual void update_root_member_list(std::vector<adnl::AdnlNodeIdShort> nodes,
Expand Down
33 changes: 20 additions & 13 deletions overlay/overlay.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,18 @@ namespace overlay {

class OverlayImpl;

struct TrafficStats {
td::uint64 out_bytes = 0;
td::uint64 in_bytes = 0;

td::uint32 out_packets = 0;
td::uint32 in_packets = 0;

void add_packet(td::uint64 size, bool in);
void normalize(double elapsed);
tl_object_ptr<ton_api::engine_validator_overlayStatsTraffic> tl() const;
};

class OverlayPeer {
public:
adnl::AdnlNodeIdShort get_id() const {
Expand Down Expand Up @@ -126,17 +138,8 @@ class OverlayPeer {
return node_.has_full_id();
}

td::uint32 throughput_out_bytes = 0;
td::uint32 throughput_in_bytes = 0;

td::uint32 throughput_out_packets = 0;
td::uint32 throughput_in_packets = 0;

td::uint32 throughput_out_bytes_ctr = 0;
td::uint32 throughput_in_bytes_ctr = 0;

td::uint32 throughput_out_packets_ctr = 0;
td::uint32 throughput_in_packets_ctr = 0;
TrafficStats traffic, traffic_ctr;
TrafficStats traffic_responses, traffic_responses_ctr;

td::uint32 broadcast_errors = 0;
td::uint32 fec_broadcast_errors = 0;
Expand Down Expand Up @@ -269,9 +272,11 @@ class OverlayImpl : public Overlay {

void get_stats(td::Promise<tl_object_ptr<ton_api::engine_validator_overlayStats>> promise) override;

void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override;
void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query,
bool is_response) override;

void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override;
void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint64 msg_size, bool is_query,
bool is_response) override;

void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) override;

Expand Down Expand Up @@ -457,6 +462,8 @@ class OverlayImpl : public Overlay {
td::Timestamp local_cert_is_valid_until_;
td::uint32 local_member_flags_{0};
} peer_list_;
TrafficStats total_traffic, total_traffic_ctr;
TrafficStats total_traffic_responses, total_traffic_responses_ctr;

OverlayOptions opts_;
};
Expand Down
10 changes: 8 additions & 2 deletions tl/generate/scheme/ton_api.tl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -683,10 +683,16 @@ engine.validator.proposalVote perm_key:int256 to_send:bytes = engine.validator.P
engine.validator.dhtServerStatus id:int256 status:int = engine.validator.DhtServerStatus;
engine.validator.dhtServersStatus servers:(vector engine.validator.dhtServerStatus) = engine.validator.DhtServersStatus;

engine.validator.overlayStatsTraffic t_out_bytes:long t_in_bytes:long t_out_pckts:int t_in_pckts:int = engine.validator.OverlayStatsTraffic;

engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string is_neighbour:Bool is_alive:Bool node_flags:int
bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int t_out_bytes:int t_in_bytes:int t_out_pckts:int t_in_pckts:int = engine.validator.OverlayStatsNode;
bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int
traffic:engine.validator.overlayStatsTraffic traffic_responses:engine.validator.overlayStatsTraffic = engine.validator.OverlayStatsNode;

engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) extra:string = engine.validator.OverlayStats;
engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string
nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat)
total_traffic:engine.validator.overlayStatsTraffic total_traffic_responses:engine.validator.overlayStatsTraffic
extra:string = engine.validator.OverlayStats;
engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats;

engine.validator.onePerfTimerStat time:int min:double avg:double max:double = engine.validator.OnePerfTimerStat;
Expand Down
Binary file modified tl/generate/scheme/ton_api.tlo
View file
Binary file not shown.
Loading

0 comments on commit 8364a24

Please sign in to comment.